From ddde71a7c5b19fe7e6069452e2435bddfaf64edc Mon Sep 17 00:00:00 2001 From: zhouwg Date: Fri, 14 Feb 2025 21:50:53 +0800 Subject: [PATCH 01/76] ggml-qnn: add Qualcomm QNN backend for GGML --- ggml/CMakeLists.txt | 2 + ggml/include/ggml-qnn.h | 68 + ggml/src/CMakeLists.txt | 1 + ggml/src/ggml-backend-reg.cpp | 8 + ggml/src/ggml-qnn/CMakeLists.txt | 33 + ggml/src/ggml-qnn/ggml-qnn.cpp | 3932 ++++++++++++++++++++++++++++++ scripts/build-run-android.sh | 202 ++ 7 files changed, 4246 insertions(+) create mode 100644 ggml/include/ggml-qnn.h create mode 100644 ggml/src/ggml-qnn/CMakeLists.txt create mode 100644 ggml/src/ggml-qnn/ggml-qnn.cpp create mode 100755 scripts/build-run-android.sh diff --git a/ggml/CMakeLists.txt b/ggml/CMakeLists.txt index 740f9f69cf2ed..7a717a9983c3b 100644 --- a/ggml/CMakeLists.txt +++ b/ggml/CMakeLists.txt @@ -198,6 +198,7 @@ option(GGML_OPENCL_EMBED_KERNELS "ggml: embed kernels" option(GGML_OPENCL_USE_ADRENO_KERNELS "ggml: use optimized kernels for Adreno" ON) set (GGML_OPENCL_TARGET_VERSION "300" CACHE STRING "gmml: OpenCL API version to target") +option(GGML_QNN "ggml: use QNN" OFF) # toolchain for vulkan-shaders-gen set (GGML_VULKAN_SHADERS_GEN_TOOLCHAIN "" CACHE FILEPATH "ggml: toolchain file for vulkan-shaders-gen") @@ -263,6 +264,7 @@ set(GGML_PUBLIC_HEADERS include/ggml-rpc.h include/ggml-sycl.h include/ggml-vulkan.h + include/ggml-qnn.h include/gguf.h) set_target_properties(ggml PROPERTIES PUBLIC_HEADER "${GGML_PUBLIC_HEADERS}") diff --git a/ggml/include/ggml-qnn.h b/ggml/include/ggml-qnn.h new file mode 100644 index 0000000000000..06f143546ad24 --- /dev/null +++ b/ggml/include/ggml-qnn.h @@ -0,0 +1,68 @@ + /* + * Copyright (c) 2023-2024 The ggml authors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ +#pragma once + +#include "ggml.h" +#include "ggml-backend.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define GGML_QNN_MAX_DEVICES 3 +#define GGML_QNN_BACKEND_NAME "qnn" + +enum QNNBackend { + QNN_BACKEND_CPU, + QNN_BACKEND_GPU, + QNN_BACKEND_NPU, + QNN_BACKEND_GGML, //"fake" QNN backend for compare performance between QNN backend and cpu backend +}; + +GGML_BACKEND_API ggml_backend_t ggml_backend_qnn_init(size_t dev_num, const char * qnn_lib_path); + +GGML_BACKEND_API bool ggml_backend_is_qnn(ggml_backend_t backend); + +GGML_BACKEND_API void ggml_backend_qnn_set_n_threads(ggml_backend_t backend, int thread_counts); + +GGML_BACKEND_API int ggml_backend_qnn_get_device_count(void); + +GGML_BACKEND_API ggml_backend_reg_t ggml_backend_qnn_reg(void); + +inline const char * ggml_backend_qnn_get_devname(size_t dev_num) { + switch (dev_num) { + case QNN_BACKEND_CPU: + return "QNN-CPU"; + case QNN_BACKEND_GPU: + return "QNN-GPU"; + case QNN_BACKEND_NPU: + return "QNN-NPU"; + case QNN_BACKEND_GGML: + return "ggml"; //"fake" QNN backend, used for compare performance between QNN backend and original GGML + default: + return "unknown"; + } +} + +#ifdef __cplusplus +} +#endif diff --git a/ggml/src/CMakeLists.txt b/ggml/src/CMakeLists.txt index 1e4c2422756ac..73a20a11bff0b 100644 --- a/ggml/src/CMakeLists.txt +++ b/ggml/src/CMakeLists.txt @@ -310,6 +310,7 @@ ggml_add_backend(RPC) ggml_add_backend(SYCL) ggml_add_backend(Vulkan) ggml_add_backend(OpenCL) +ggml_add_backend(QNN) foreach (target ggml-base ggml) target_include_directories(${target} PUBLIC $ $) diff --git a/ggml/src/ggml-backend-reg.cpp b/ggml/src/ggml-backend-reg.cpp index 405d8e31514b5..9030de3cfeef9 100644 --- a/ggml/src/ggml-backend-reg.cpp +++ b/ggml/src/ggml-backend-reg.cpp @@ -65,6 +65,10 @@ #include "ggml-kompute.h" #endif +#ifdef GGML_USE_QNN +#include "ggml-qnn.h" +#endif + // disable C++17 deprecation warning for std::codecvt_utf8 #if defined(__clang__) # pragma clang diagnostic push @@ -187,6 +191,9 @@ struct ggml_backend_registry { #ifdef GGML_USE_KOMPUTE register_backend(ggml_backend_kompute_reg()); #endif +#ifdef GGML_USE_QNN + register_backend(ggml_backend_qnn_reg()); +#endif #ifdef GGML_USE_CPU register_backend(ggml_backend_cpu_reg()); #endif @@ -577,6 +584,7 @@ void ggml_backend_load_all_from_path(const char * dir_path) { ggml_backend_load_best("vulkan", silent, dir_path); ggml_backend_load_best("opencl", silent, dir_path); ggml_backend_load_best("musa", silent, dir_path); + ggml_backend_load_best("qnn", silent, dir_path); ggml_backend_load_best("cpu", silent, dir_path); // check the environment variable GGML_BACKEND_PATH to load an out-of-tree backend const char * backend_path = std::getenv("GGML_BACKEND_PATH"); diff --git a/ggml/src/ggml-qnn/CMakeLists.txt b/ggml/src/ggml-qnn/CMakeLists.txt new file mode 100644 index 0000000000000..7bbb9be76b4f6 --- /dev/null +++ b/ggml/src/ggml-qnn/CMakeLists.txt @@ -0,0 +1,33 @@ +message(STATUS "Using QNN backend") + +if(CMAKE_SYSTEM_NAME STREQUAL "Android") + find_library(LOG_LIB log) + set(QNN_LINK_LIBRARIES ${LOG_LIB}) + set(QNN_DEFAULT_LIB_SEARCH_PATH "/data/local/tmp/" CACHE STRING "customized library search path for QNN backend") +else() + message(FATAL_ERROR "QNN now only available on Android") +endif() + +if(NOT DEFINED GGML_QNN_SDK_PATH) + # try read from environment variable + if(DEFINED ENV{QNN_SDK_PATH}) + set(GGML_QNN_SDK_PATH $ENV{QNN_SDK_PATH}) + else() + message(FATAL_ERROR "GGML_QNN_SDK_PATH not defined") + endif() +endif() + +message("QNN_SDK_PATH: ${GGML_QNN_SDK_PATH}") + +set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3") + +file(GLOB QNN_SOURCES "${CMAKE_CURRENT_LIST_DIR}/*.cpp") +ggml_add_backend_library(ggml-qnn + ${QNN_SOURCES} +) + +target_include_directories(ggml-qnn PRIVATE ${GGML_QNN_SDK_PATH}/include/QNN ${CMAKE_CURRENT_LIST_DIR}) +target_link_libraries(ggml-qnn PRIVATE ${QNN_LINK_LIBRARIES}) + +string(REGEX REPLACE "/$" "" GGML_QNN_DEFAULT_LIB_SEARCH_PATH "${QNN_DEFAULT_LIB_SEARCH_PATH}") +target_compile_definitions(ggml-qnn PRIVATE GGML_QNN_DEFAULT_LIB_SEARCH_PATH="${QNN_DEFAULT_LIB_SEARCH_PATH}/") diff --git a/ggml/src/ggml-qnn/ggml-qnn.cpp b/ggml/src/ggml-qnn/ggml-qnn.cpp new file mode 100644 index 0000000000000..d29c6cb6f9222 --- /dev/null +++ b/ggml/src/ggml-qnn/ggml-qnn.cpp @@ -0,0 +1,3932 @@ +/* + * Copyright (c) 2023-2024 The ggml authors + * + * this is implementation of ggml-qnn(ggml-qnn backend of Qualcomm QNN(Qualcomm Neural Network, + * aka Qualcomm AI Engine Direct) + * + * Qualcomm QNN SDK and reference tech guides could be found at: + * https://www.qualcomm.com/developer/software/qualcomm-ai-engine-direct-sdk + * https://developer.qualcomm.com/software/hexagon-dsp-sdk/tools + * + * the implementation of ggml-qnn backend has six sections: + * section-1 does forward/external declaration, + * section-2 defines ggml-qnn internal log function + * section-3 does general helper macro / data structure / function + * section-4 does QNN helper macro / data structure / function + * section-5 does ggml-qnn backend helper macro / data structure / function / class + * section-6 does implementation of ggml-qnn backend according to ggml's backend subsystem + * + * currently only provide GGML_OP_ADD's QNN backend implementation: + * - GGML_OP_ADD: this is skeleton, can expand other ggml ops as expertise + * + * of course, can porting ggml-qnn to Windows on ARM as need. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if (defined __ANDROID__) || (defined ANDROID) +#include "android/log.h" +#endif + +#include "QnnTypes.h" +#include "QnnCommon.h" +#include "QnnContext.h" +#include "QnnBackend.h" +#include "QnnGraph.h" +#include "QnnProperty.h" +#include "QnnTensor.h" +#include "QnnInterface.h" +#include "Saver/QnnSaver.h" +#include "System/QnnSystemInterface.h" +#include "HTP/QnnHtpDevice.h" +#include "HTP/QnnHtpGraph.h" + +#include "ggml-qnn.h" +#include "ggml-impl.h" +#include "ggml-backend-impl.h" + +// ================================================================================================= +// section-1: forward/external declaration +// ================================================================================================= +class qnn_instance; +struct ggml_backend_qnn_context; +static int free_qnn_tensor(Qnn_Tensor_t * tensor); +static enum ggml_status ggml_backend_qnn_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph); + +#if (defined __ANDROID__) || (defined ANDROID) +extern "C" int __android_log_print(int prio, const char * tag, const char * fmt, ...) __attribute__((__format__(printf, 3, 4))); +#endif +static void ggmlqnn_log_internal(ggml_log_level level, const char * file, const char * func, int line, const char * format, ...); + +// ================================================================================================= +// section-2: ggml-qnn internal troubleshooting function +// ================================================================================================= +#define GGMLQNN_DEBUG 1 // for troubleshooting QNN backend +#define GGML_QNN_LOGBUF_LEN 4096 +#define ENABLE_QNNBACKEND_PERF 1 // enable/disable op's perf info +#define GGMLQNN_PRINT_QNN_INTERNAL_LOG 0 // enable/disable QNN's internal log +#define GGMLQNN_PRINT_OP_ADD_LOG 1 // GGML_OP_ADD already verified with QNN-CPU / QNN-GPU / QNN-NPU +#define GGMLQNN_PRINT_OP_MUL_MAT_LOG 1 + +#define GGMLQNN_LOG_ERROR(...) ggmlqnn_log_internal(GGML_LOG_LEVEL_DEBUG, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) +#define GGMLQNN_LOG_WARN(...) ggmlqnn_log_internal(GGML_LOG_LEVEL_DEBUG , __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) +#define GGMLQNN_LOG_INFO(...) ggmlqnn_log_internal(GGML_LOG_LEVEL_DEBUG , __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) + +#if GGMLQNN_DEBUG +#define GGMLQNN_LOG_DEBUG(...) ggmlqnn_log_internal(GGML_LOG_LEVEL_DEBUG, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) +#else +#define GGMLQNN_LOG_DEBUG(...) +#endif +static void ggmlqnn_log_internal(ggml_log_level level, const char * file, const char * func, int line, const char * format, ...) { + static std::mutex ggmlqnn_log_internal_mutex; + static char s_ggmlqnn_log_internal_buf[GGML_QNN_LOGBUF_LEN]; + + { + std::lock_guard lock(ggmlqnn_log_internal_mutex); + va_list args; + va_start(args, format); + int len_prefix = snprintf(s_ggmlqnn_log_internal_buf, GGML_QNN_LOGBUF_LEN, "[%s, %d]: ", func, line); + int len = vsnprintf(s_ggmlqnn_log_internal_buf + len_prefix, GGML_QNN_LOGBUF_LEN - len_prefix, format, args); + if (len < (GGML_QNN_LOGBUF_LEN - len_prefix)) { +#if (defined __ANDROID__) || (defined ANDROID) + //for Android APK + __android_log_print(ANDROID_LOG_INFO, "ggml-qnn", "%s\n", s_ggmlqnn_log_internal_buf); +#endif +#if (defined __ANDROID__) || (defined ANDROID) + //do nothing when running on Android phone +#else + //for Windows on ARM + printf("%s\n", s_ggmlqnn_log_internal_buf); +#endif + } + va_end(args); + } +} + +// ================================================================================================= +// section-3: general helper macro / data structure / function +// ================================================================================================= +#define DISABLE_COPY(class_name) \ + class_name(const class_name &) = delete; \ + void operator=(const class_name &) = delete + +#define DISABLE_MOVE(class_name) \ + class_name(class_name &&) = delete; \ + void operator=(class_name &&) = delete + +#define GGMLQNN_MEM_ADD(alignment) (sizeof (size_t) + alignment) +#define GGMLQNN_MEM_MASK(alignment) ((uintptr_t)alignment - 1) + +static intptr_t ggmlqnn_align_to(size_t alignment, intptr_t offset) { + return offset % alignment == 0 ? offset + : offset + + (static_cast(alignment) - + offset % static_cast(alignment)); +} + +static void * ggmlqnn_mallocz_aligned(size_t size, size_t alignment) { + uint8_t * buffer = NULL; + size_t * sp = NULL; + buffer = static_cast(calloc(1, size + GGMLQNN_MEM_ADD(alignment))); + if (!buffer) + return NULL; + sp = (size_t *)buffer; + *sp = size; + buffer = (uint8_t *)(((uintptr_t) buffer + GGMLQNN_MEM_ADD(alignment)) & ~GGMLQNN_MEM_MASK(alignment)); + buffer[-1] = buffer - (uint8_t *)sp; + return buffer; +} + +static void * ggmlqnn_malloc_aligned(size_t size, size_t alignment) { + uint8_t * buffer = NULL; + size_t * sp = NULL; + buffer = static_cast(malloc(size + GGMLQNN_MEM_ADD(alignment))); + if (!buffer) + return NULL; + sp = (size_t *)buffer; + *sp = size; + buffer = (uint8_t *)(((uintptr_t) buffer + GGMLQNN_MEM_ADD(alignment)) & ~GGMLQNN_MEM_MASK(alignment)); + buffer[-1] = buffer - (uint8_t *)sp; + return buffer; +} + +static void ggmqnn_free_aligned(void * ptr) { + uint8_t * old = (uint8_t *)ptr; + if (!old) + return; + old -= old[-1]; + free(old); +} + +static size_t get_system_total_memory_in_bytes() { + struct sysinfo info = {}; + if (sysinfo(&info) == 0) { + return (info.totalram + info.totalswap) * info.mem_unit; + } + + auto pages = (size_t)sysconf(_SC_PHYS_PAGES); + auto page_size = (size_t)sysconf(_SC_PAGE_SIZE); + + return pages * page_size; +} + +static size_t get_system_free_memory_in_bytes() { + struct sysinfo info = {}; + if (sysinfo(&info) == 0) { + return (info.freeram + info.freeswap) * info.mem_unit; + } + + auto avail_pages = (size_t)sysconf(_SC_AVPHYS_PAGES); + auto page_size = (size_t)sysconf(_SC_PAGE_SIZE); + + return avail_pages * page_size; +} + +static size_t ggmlqnn_memscpy(void * dst, size_t dst_size, const void * src, size_t copy_size) { + if (!dst || !src || !dst_size || !copy_size) + return 0; + + size_t min_size = dst_size < copy_size ? dst_size : copy_size; + + memcpy(dst, src, min_size); + + return min_size; +} + +static char * ggmlqnn_strndup(const char * source, size_t maxlen) { + return ::strndup(source, maxlen); +} + +static void * ggmlqnn_host_malloc(size_t n) { + void * data = NULL; + int result = posix_memalign((void **) &data, sysconf(_SC_PAGESIZE), n); + if (result != 0) { + GGMLQNN_LOG_WARN("%s: error: posix_memalign failed\n", __func__); + return NULL; + } + + return data; +} + +// ================================================================================================= +// section-4: QNN helper macro / data structure / function +// ================================================================================================= +#define VALIDATE(value, status) \ + do { \ + status = value; \ + if (status != QNN_SUCCESS) { \ + GGMLQNN_LOG_WARN("%s expected QNN_SUCCESS\n", #value); \ + return status; \ + } \ + } while (0) + +#define VALIDATE_TENSOR_VERSION(tensor, err) VALIDATE(validate_tensor_version(tensor), err) + +#define VALIDATE_OP_CONFIG_VERSION(op, err) VALIDATE(validate_op_config_version(op), err) + +#define QNN_VER_PTR(x) (&((x).v1)) +#define QNN_OP_CFG_VALID(op_config) ((op_config).version == QNN_OPCONFIG_VERSION_1) + +#define QNN_OP_CFG_GET_NAME(op_config) get_qnn_oponfig_name(op_config) +#define QNN_OP_CFG_GET_PACKAGE_NAME(op_config) get_qnn_op_config_packagename(op_config) +#define QNN_OP_CFG_GET_TYPE_NAME(op_config) get_qnn_op_config_typename(op_config) +#define QNN_OP_CFG_GET_NUM_PARAMS(op_config) get_qnn_op_config_numparams(op_config) +#define QNN_OP_CFG_GET_PARAMS(op_config) get_qnn_op_config_params(op_config) +#define QNN_OP_CFG_GET_NUM_INPUTS(op_config) get_qnn_op_config_numinputs(op_config) +#define QNN_OP_CFG_GET_INPUTS(op_config) get_qnn_op_config_inputs(op_config) +#define QNN_OP_CFG_GET_NUM_OUTPUTS(op_config) get_qnn_op_config_numoutputs(op_config) +#define QNN_OP_CFG_GET_OUTPUTS(op_config) get_qnn_op_config_outputs(op_config) + +#define QNN_OP_CFG_SET_NAME(op_config, value) set_qnn_op_config_name(op_config, value) +#define QNN_OP_CFG_SET_PACKAGE_NAME(op_config, value) set_qnn_op_config_packagename(op_config, value) +#define QNN_OP_CFG_SET_TYPE_NAME(op_config, value) set_qnn_op_config_typename(op_config, value) + +#define QNN_OP_CFG_SET_PARAMS(op_config, num_of_params, params) \ + set_qnn_op_config_params(op_config, num_of_params, params) + +#define QNN_OP_CFG_SET_INPUTS(op_config, num_of_inputs, inputTensors) \ + set_qnn_op_config_inputs(op_config, num_of_inputs, inputTensors) + +#define QNN_OP_CFG_SET_OUTPUTS(op_config, num_of_outputs, output_tensors) \ + set_qnn_op_config_outputs(op_config, num_of_outputs, output_tensors) + +#define QNN_TENSOR_GET_ID(tensor) get_qnn_tensorid(tensor) +#define QNN_TENSOR_GET_NAME(tensor) get_qnn_tensorname(tensor) +#define QNN_TENSOR_GET_TYPE(tensor) get_qnn_tensortype(tensor) +#define QNN_TENSOR_GET_DATA_FORMAT(tensor) get_qnn_tensor_dataformat(tensor) +#define QNN_TENSOR_GET_DATA_TYPE(tensor) get_qnn_tensor_datatype(tensor) +#define QNN_TENSOR_GET_QUANT_PARAMS(tensor) get_qnn_tensor_quantparams(tensor) +#define QNN_TENSOR_GET_RANK(tensor) get_qnn_tensor_rank(tensor) +#define QNN_TENSOR_GET_DIMENSIONS(tensor) get_qnn_tensor_dimensions(tensor) +#define QNN_TENSOR_GET_MEM_TYPE(tensor) get_qnn_tensor_memtype(tensor) +#define QNN_TENSOR_GET_CLIENT_BUF(tensor) get_qnn_tensor_clientbuf(tensor) +#define QNN_TENSOR_GET_MEM_HANDLE(tensor) get_qnn_tensor_memhandle(tensor) + +#define QNN_TENSOR_SET_ID(tensor, value) set_qnn_tensor_id(tensor, value) +#define QNN_TENSOR_SET_NAME(tensor, value) set_qnn_tensor_name(tensor, value) +#define QNN_TENSOR_SET_TYPE(tensor, value) set_qnn_tensor_type(tensor, value) +#define QNN_TENSOR_SET_DATA_FORMAT(tensor, value) set_qnn_tensor_dataformat(tensor, value) +#define QNN_TENSOR_SET_DATA_TYPE(tensor, value) set_qnn_tensor_datatype(tensor, value) +#define QNN_TENSOR_SET_QUANT_PARAMS(tensor, value) set_qnn_tensor_quantparams(tensor, value) +#define QNN_TENSOR_SET_RANK(tensor, value) set_qnn_tensor_rank(tensor, value) +#define QNN_TENSOR_SET_DIMENSIONS(tensor, value) set_qnn_tensor_dimensions(tensor, value) +#define QNN_TENSOR_SET_MEM_TYPE(tensor, value) set_qnn_tensor_memtype(tensor, value) +#define QNN_TENSOR_SET_CLIENT_BUF(tensor, value) set_qnn_tensor_clientbuf(tensor, value) +#define QNN_TENSOR_SET_MEM_HANDLE(tensor, value) set_qnn_tensor_memhandle(tensor, value) + +static inline int validate_tensor_version(Qnn_Tensor_t tensor) { + if (tensor.version != QNN_TENSOR_VERSION_1) { + GGMLQNN_LOG_WARN("validate_tensor_version() tensor %s, got unsupported version %d\n", + tensor.v1.name, + tensor.version); + return 1; + } + return 0; +} + +[[maybe_unused]] static inline int validate_op_config_version(Qnn_OpConfig_t op_config) { + if (op_config.version != QNN_OPCONFIG_VERSION_1) { + GGMLQNN_LOG_WARN("validate_op_config_version() op %s, got unsupported version %d\n", + op_config.v1.name, + op_config.version); + return 1; + } + return 0; +} + +static inline const char * get_qnn_oponfig_name(const Qnn_OpConfig_t & op_config) { + if (op_config.version == QNN_OPCONFIG_VERSION_1) { + return op_config.v1.name; + } + return nullptr; +} + +[[maybe_unused]] static inline const char * get_qnn_oponfig_name(const Qnn_OpConfig_t * op_config) { + return get_qnn_oponfig_name(*op_config); +} + +static inline const char * get_qnn_op_config_packagename(const Qnn_OpConfig_t & op_config) { + if (op_config.version == QNN_OPCONFIG_VERSION_1) { + return op_config.v1.packageName; + } + return nullptr; +} + +[[maybe_unused]] static inline const char * get_qnn_op_config_packagename(const Qnn_OpConfig_t * op_config) { + return get_qnn_op_config_packagename(*op_config); +} + +static inline const char * get_qnn_op_config_typename(const Qnn_OpConfig_t & op_config) { + if (op_config.version == QNN_OPCONFIG_VERSION_1) { + return op_config.v1.typeName; + } + return nullptr; +} + +[[maybe_unused]] static inline const char * get_qnn_op_config_typename(const Qnn_OpConfig_t * op_config) { + return get_qnn_op_config_typename(*op_config); +} + +static inline uint32_t get_qnn_op_config_numparams(const Qnn_OpConfig_t & op_config) { + if (op_config.version == QNN_OPCONFIG_VERSION_1) { + return op_config.v1.numOfParams; + } + return 0u; +} + +[[maybe_unused]] static inline uint32_t get_qnn_op_config_numparams(const Qnn_OpConfig_t * op_config) { + return get_qnn_op_config_numparams(*op_config); +} + +static inline const Qnn_Param_t * get_qnn_op_config_params(const Qnn_OpConfig_t & op_config) { + if (op_config.version == QNN_OPCONFIG_VERSION_1) { + return op_config.v1.params; + } + return nullptr; +} + +[[maybe_unused]] static inline const Qnn_Param_t * get_qnn_op_config_params(const Qnn_OpConfig_t * op_config) { + return get_qnn_op_config_params(*op_config); +} + +static inline uint32_t get_qnn_op_config_numinputs(const Qnn_OpConfig_t & op_config) { + if (op_config.version == QNN_OPCONFIG_VERSION_1) { + return op_config.v1.numOfInputs; + } + return 0u; +} + +[[maybe_unused]] static inline uint32_t get_qnn_op_config_numinputs(const Qnn_OpConfig_t * op_config) { + return get_qnn_op_config_numinputs(*op_config); +} + +static inline const Qnn_Tensor_t * get_qnn_op_config_inputs(const Qnn_OpConfig_t & op_config) { + if (op_config.version == QNN_OPCONFIG_VERSION_1) { + return op_config.v1.inputTensors; + } + return nullptr; +} + +[[maybe_unused]] static inline const Qnn_Tensor_t * get_qnn_op_config_inputs(const Qnn_OpConfig_t * op_config) { + return get_qnn_op_config_inputs(*op_config); +} + +static inline uint32_t get_qnn_op_config_numoutputs(const Qnn_OpConfig_t & op_config) { + if (op_config.version == QNN_OPCONFIG_VERSION_1) { + return op_config.v1.numOfOutputs; + } + return 0u; +} + +[[maybe_unused]] static inline uint32_t get_qnn_op_config_numoutputs(const Qnn_OpConfig_t * op_config) { + return get_qnn_op_config_numoutputs(*op_config); +} + +static inline const Qnn_Tensor_t * get_qnn_op_config_outputs(const Qnn_OpConfig_t & op_config) { + if (op_config.version == QNN_OPCONFIG_VERSION_1) { + return op_config.v1.outputTensors; + } + return nullptr; +} + +[[maybe_unused]] static inline const Qnn_Tensor_t * get_qnn_op_config_outputs(const Qnn_OpConfig_t * op_config) { + return get_qnn_op_config_outputs(*op_config); +} + +static inline void set_qnn_op_config_name(Qnn_OpConfig_t & op_config, const char * name) { + if (op_config.version == QNN_OPCONFIG_VERSION_1) { + op_config.v1.name = name; + } +} + +[[maybe_unused]] static inline void set_qnn_op_config_name(Qnn_OpConfig_t * op_config, const char * name) { + set_qnn_op_config_name(*op_config, name); +} + +static inline void set_qnn_op_config_packagename(Qnn_OpConfig_t & op_config, const char * package_name) { + if (op_config.version == QNN_OPCONFIG_VERSION_1) { + op_config.v1.packageName = package_name; + } +} + +[[maybe_unused]] static inline void set_qnn_op_config_packagename(Qnn_OpConfig_t * op_config, const char * package_name) { + set_qnn_op_config_packagename(*op_config, package_name); +} + +static inline void set_qnn_op_config_typename(Qnn_OpConfig_t & op_config, const char * type_name) { + if (op_config.version == QNN_OPCONFIG_VERSION_1) { + op_config.v1.typeName = type_name; + } +} + +[[maybe_unused]] static inline void set_qnn_op_config_typename(Qnn_OpConfig_t * op_config, const char * type_name) { + set_qnn_op_config_typename(*op_config, type_name); +} + +static inline void set_qnn_op_config_params(Qnn_OpConfig_t & op_config, + uint32_t num_of_params, + Qnn_Param_t * params) { + if (op_config.version == QNN_OPCONFIG_VERSION_1) { + op_config.v1.numOfParams = num_of_params; + op_config.v1.params = params; + } +} + +[[maybe_unused]] static inline void set_qnn_op_config_params(Qnn_OpConfig_t * op_config, + uint32_t num_of_params, + Qnn_Param_t * params) { + set_qnn_op_config_params(*op_config, num_of_params, params); +} + +static inline void set_qnn_op_config_inputs(Qnn_OpConfig_t & op_config, + uint32_t num_of_inputs, + Qnn_Tensor_t * input_tensors) { + if (op_config.version == QNN_OPCONFIG_VERSION_1) { + op_config.v1.numOfInputs = num_of_inputs; + op_config.v1.inputTensors = input_tensors; + } +} + +[[maybe_unused]] static inline void set_qnn_op_config_inputs(Qnn_OpConfig_t * op_config, + uint32_t num_of_inputs, + Qnn_Tensor_t * input_tensors) { + set_qnn_op_config_inputs(*op_config, num_of_inputs, input_tensors); +} + +static inline void set_qnn_op_config_outputs(Qnn_OpConfig_t & op_config, + uint32_t num_of_outputs, + Qnn_Tensor_t * output_tensors) { + if (op_config.version == QNN_OPCONFIG_VERSION_1) { + op_config.v1.numOfOutputs = num_of_outputs; + op_config.v1.outputTensors = output_tensors; + } +} + +[[maybe_unused]] static inline void set_qnn_op_config_outputs(Qnn_OpConfig_t * op_config, + uint32_t num_of_outputs, + Qnn_Tensor_t * output_tensors) { + set_qnn_op_config_outputs(*op_config, num_of_outputs, output_tensors); +} + +static inline uint32_t get_qnn_tensorid(const Qnn_Tensor_t & tensor) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + return tensor.v1.id; + } + + return 0u; +} + +[[maybe_unused]] static inline uint32_t get_qnn_tensorid(const Qnn_Tensor_t * tensor) { + return get_qnn_tensorid(*tensor); +} + +static inline const char * get_qnn_tensorname(const Qnn_Tensor_t & tensor) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + return tensor.v1.name; + } + return nullptr; +} + +static inline const char * get_qnn_tensorname(const Qnn_Tensor_t * tensor) { + return get_qnn_tensorname(*tensor); +} + +static inline Qnn_TensorType_t get_qnn_tensortype(const Qnn_Tensor_t & tensor) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + return tensor.v1.type; + } + return QNN_TENSOR_TYPE_UNDEFINED; +} + +[[maybe_unused]] static inline Qnn_TensorType_t get_qnn_tensortype(const Qnn_Tensor_t * tensor) { + return get_qnn_tensortype(*tensor); +} + +static inline Qnn_TensorDataFormat_t get_qnn_tensor_dataformat(const Qnn_Tensor_t & tensor) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + return tensor.v1.dataFormat; + } + return QNN_TENSOR_DATA_FORMAT_FLAT_BUFFER; +} + +[[maybe_unused]] static inline Qnn_TensorDataFormat_t get_qnn_tensor_dataformat(const Qnn_Tensor_t * tensor) { + return get_qnn_tensor_dataformat(*tensor); +} + +static inline Qnn_DataType_t get_qnn_tensor_datatype(const Qnn_Tensor_t & tensor) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + return tensor.v1.dataType; + } + return QNN_DATATYPE_UNDEFINED; +} + +[[maybe_unused]] static inline Qnn_DataType_t get_qnn_tensor_datatype(const Qnn_Tensor_t * tensor) { + return get_qnn_tensor_datatype(*tensor); +} + +static inline Qnn_QuantizeParams_t get_qnn_tensor_quantparams(const Qnn_Tensor_t & tensor) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + return tensor.v1.quantizeParams; + } + return QNN_QUANTIZE_PARAMS_INIT; +} + +[[maybe_unused]] static inline Qnn_QuantizeParams_t get_qnn_tensor_quantparams(const Qnn_Tensor_t * tensor) { + return get_qnn_tensor_quantparams(*tensor); +} + +static inline uint32_t get_qnn_tensor_rank(const Qnn_Tensor_t & tensor) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + return tensor.v1.rank; + } + return 0u; +} + +[[maybe_unused]] static inline uint32_t get_qnn_tensor_rank(const Qnn_Tensor_t * tensor) { + return get_qnn_tensor_rank(*tensor); +} + +static inline uint32_t * get_qnn_tensor_dimensions(const Qnn_Tensor_t & tensor) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + return tensor.v1.dimensions; + } + return nullptr; +} + +[[maybe_unused]] static inline uint32_t * get_qnn_tensor_dimensions(const Qnn_Tensor_t * tensor) { + return get_qnn_tensor_dimensions(*tensor); +} + +static inline Qnn_TensorMemType_t get_qnn_tensor_memtype(const Qnn_Tensor_t & tensor) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + return tensor.v1.memType; + } + return QNN_TENSORMEMTYPE_UNDEFINED; +} + +[[maybe_unused]] static inline Qnn_TensorMemType_t get_qnn_tensor_memtype(const Qnn_Tensor_t * tensor) { + return get_qnn_tensor_memtype(*tensor); +} + +static inline Qnn_ClientBuffer_t get_qnn_tensor_clientbuf(const Qnn_Tensor_t & tensor) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + return tensor.v1.clientBuf; + } + return QNN_CLIENT_BUFFER_INIT; +} + +[[maybe_unused]] static inline Qnn_ClientBuffer_t get_qnn_tensor_clientbuf(const Qnn_Tensor_t * tensor) { + return get_qnn_tensor_clientbuf(*tensor); +} + +static inline Qnn_MemHandle_t get_qnn_tensor_memhandle(const Qnn_Tensor_t & tensor) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + return tensor.v1.memHandle; + } + return nullptr; +} + +[[maybe_unused]] static inline Qnn_MemHandle_t get_qnn_tensor_memhandle(const Qnn_Tensor_t * tensor) { + return get_qnn_tensor_memhandle(*tensor); +} + +static inline void set_qnn_tensor_id(Qnn_Tensor_t & tensor, uint32_t id) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + tensor.v1.id = id; + } +} + +[[maybe_unused]] static inline void set_qnn_tensor_id(Qnn_Tensor_t * tensor, uint32_t id) { + set_qnn_tensor_id(*tensor, id); +} + +static inline void set_qnn_tensor_name(Qnn_Tensor_t & tensor, const char * name) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + tensor.v1.name = name; + } +} + +[[maybe_unused]] static inline void set_qnn_tensor_name(Qnn_Tensor_t * tensor, const char * name) { + set_qnn_tensor_name(*tensor, name); +} + +static inline void set_qnn_tensor_type(Qnn_Tensor_t & tensor, Qnn_TensorType_t type) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + tensor.v1.type = type; + } +} + +[[maybe_unused]] static inline void set_qnn_tensor_type(Qnn_Tensor_t * tensor, Qnn_TensorType_t type) { + set_qnn_tensor_type(*tensor, type); +} + +static inline void set_qnn_tensor_dataformat(Qnn_Tensor_t & tensor, Qnn_TensorDataFormat_t format) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + tensor.v1.dataFormat = format; + } +} + +[[maybe_unused]] static inline void set_qnn_tensor_dataformat(Qnn_Tensor_t * tensor, Qnn_TensorDataFormat_t format) { + set_qnn_tensor_dataformat(*tensor, format); +} + +static inline void set_qnn_tensor_datatype(Qnn_Tensor_t & tensor, Qnn_DataType_t dataType) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + tensor.v1.dataType = dataType; + } +} + +[[maybe_unused]] static inline void set_qnn_tensor_datatype(Qnn_Tensor_t * tensor, Qnn_DataType_t dataType) { + set_qnn_tensor_datatype(*tensor, dataType); +} + +static inline void set_qnn_tensor_quantparams(Qnn_Tensor_t & tensor, Qnn_QuantizeParams_t params) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + tensor.v1.quantizeParams = params; + } +} + +[[maybe_unused]] static inline void set_qnn_tensor_quantparams(Qnn_Tensor_t * tensor, Qnn_QuantizeParams_t params) { + set_qnn_tensor_quantparams(*tensor, params); +} + +static inline void set_qnn_tensor_rank(Qnn_Tensor_t & tensor, uint32_t rank) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + tensor.v1.rank = rank; + } +} + +[[maybe_unused]] static inline void set_qnn_tensor_rank(Qnn_Tensor_t * tensor, uint32_t rank) { + set_qnn_tensor_rank(*tensor, rank); +} + +static inline void set_qnn_tensor_dimensions(Qnn_Tensor_t & tensor, uint32_t * dims) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + tensor.v1.dimensions = dims; + } +} + +[[maybe_unused]] static inline void set_qnn_tensor_dimensions(Qnn_Tensor_t * tensor, uint32_t * dims) { + set_qnn_tensor_dimensions(*tensor, dims); +} + +static inline void set_qnn_tensor_memtype(Qnn_Tensor_t & tensor, Qnn_TensorMemType_t memType) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + tensor.v1.memType = memType; + } +} + +[[maybe_unused]] static inline void set_qnn_tensor_memtype(Qnn_Tensor_t * tensor, Qnn_TensorMemType_t memType) { + set_qnn_tensor_memtype(*tensor, memType); +} + +static inline void set_qnn_tensor_clientbuf(Qnn_Tensor_t & tensor, Qnn_ClientBuffer_t clientBuf) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + tensor.v1.clientBuf = clientBuf; + } +} + +[[maybe_unused]] static inline void set_qnn_tensor_clientbuf(Qnn_Tensor_t * tensor, Qnn_ClientBuffer_t clientBuf) { + set_qnn_tensor_clientbuf(*tensor, clientBuf); +} + +static inline void set_qnn_tensor_memhandle(Qnn_Tensor_t & tensor, Qnn_MemHandle_t handle) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + tensor.v1.memHandle = handle; + } +} + +[[maybe_unused]] static inline void set_qnn_tensor_memhandle(Qnn_Tensor_t * tensor, Qnn_MemHandle_t handle) { + set_qnn_tensor_memhandle(*tensor, handle); +} + +inline static Qnn_Tensor_t qnn_tensor_init(Qnn_TensorVersion_t version) { + Qnn_Tensor_t tensor; + tensor.version = version; + if (version == QNN_TENSOR_VERSION_1) { + tensor.v1 = QNN_TENSOR_V1_INIT; + } else if (version == QNN_TENSOR_VERSION_2) { + tensor.v2 = QNN_TENSOR_V2_INIT; + } + return tensor; +} + +static int deep_copy_qnn_tensors(Qnn_Tensor_t & src, Qnn_Tensor_t & dst) { + int err = 0; + VALIDATE_TENSOR_VERSION(src, err); + + dst.version = src.version; + QNN_TENSOR_SET_NAME( + dst, ggmlqnn_strndup(QNN_TENSOR_GET_NAME(src), std::string(QNN_TENSOR_GET_NAME(src)).size())); + if (QNN_TENSOR_GET_NAME(dst) == nullptr) { + return 1; + } + QNN_TENSOR_SET_ID(dst, QNN_TENSOR_GET_ID(src)); + QNN_TENSOR_SET_TYPE(dst, QNN_TENSOR_GET_TYPE(src)); + QNN_TENSOR_SET_DATA_FORMAT(dst, QNN_TENSOR_GET_DATA_FORMAT(src)); + QNN_TENSOR_SET_DATA_TYPE(dst, QNN_TENSOR_GET_DATA_TYPE(src)); + QNN_TENSOR_SET_MEM_TYPE(dst, QNN_TENSOR_GET_MEM_TYPE(src)); + + if (QNN_TENSOR_GET_MEM_TYPE(src) == QNN_TENSORMEMTYPE_RAW) { + Qnn_ClientBuffer_t client_buf = {nullptr, 0}; + QNN_TENSOR_SET_CLIENT_BUF(dst, client_buf); + } else if (QNN_TENSOR_GET_MEM_TYPE(src) == QNN_TENSORMEMTYPE_MEMHANDLE) { + QNN_TENSOR_SET_MEM_HANDLE(dst, nullptr); + } else { + return 1; + } + + Qnn_QuantizeParams_t src_qparam = QNN_TENSOR_GET_QUANT_PARAMS(src); + Qnn_QuantizationEncoding_t encoding = src_qparam.quantizationEncoding; + if (encoding == QNN_QUANTIZATION_ENCODING_AXIS_SCALE_OFFSET) { + Qnn_QuantizeParams_t src_qparam_cpy = src_qparam; + Qnn_AxisScaleOffset_t & axis_scale_offset = src_qparam_cpy.axisScaleOffsetEncoding; + Qnn_ScaleOffset_t ** scale_offset = &axis_scale_offset.scaleOffset; + size_t scale_offset_size = axis_scale_offset.numScaleOffsets * sizeof(Qnn_ScaleOffset_t); + *scale_offset = (Qnn_ScaleOffset_t *)malloc(scale_offset_size); + ggmlqnn_memscpy(*scale_offset, + scale_offset_size, + src_qparam.axisScaleOffsetEncoding.scaleOffset, + scale_offset_size); + QNN_TENSOR_SET_QUANT_PARAMS(dst, src_qparam_cpy); + } else if (encoding == QNN_QUANTIZATION_ENCODING_BW_AXIS_SCALE_OFFSET) { + Qnn_QuantizeParams_t src_qparam_cpy = src_qparam; + Qnn_BwAxisScaleOffset_t & bwaxis_scale_offset = src_qparam_cpy.bwAxisScaleOffsetEncoding; + size_t scale_size = bwaxis_scale_offset.numElements * sizeof(float); + float ** scales = &bwaxis_scale_offset.scales; + int32_t ** offsets = &bwaxis_scale_offset.offsets; + *scales = (float *)malloc(scale_size); + ggmlqnn_memscpy(*scales, scale_size, src_qparam.bwAxisScaleOffsetEncoding.scales, scale_size); + + if (bwaxis_scale_offset.offsets != nullptr) { + size_t offset_size = bwaxis_scale_offset.numElements * sizeof(int32_t); + *offsets = (int32_t *)malloc(offset_size); + ggmlqnn_memscpy(*offsets, offset_size, src_qparam.bwAxisScaleOffsetEncoding.offsets, offset_size); + } + QNN_TENSOR_SET_QUANT_PARAMS(dst, src_qparam_cpy); + } else { + QNN_TENSOR_SET_QUANT_PARAMS(dst, src_qparam); + } + + uint32_t rank = QNN_TENSOR_GET_RANK(src); + QNN_TENSOR_SET_RANK(dst, rank); + size_t dim_size = rank * sizeof(uint32_t); + uint32_t * dimensions = (uint32_t *)malloc(dim_size); + GGMLQNN_LOG_DEBUG("tensor dims %p", dimensions); + if (dimensions == nullptr) { + GGMLQNN_LOG_WARN("deep_copy_qnn_tensors() allocation error while copying tensor %s\n", QNN_TENSOR_GET_NAME(src)); + return 1; + } + ggmlqnn_memscpy(dimensions, dim_size, QNN_TENSOR_GET_DIMENSIONS(src), dim_size); + QNN_TENSOR_SET_DIMENSIONS(dst, dimensions); + + return err; +} + +static int free_qnn_tensor(Qnn_Tensor_t * tensor) { + int err = 0; + VALIDATE_TENSOR_VERSION(*tensor, err); + free((void *) QNN_TENSOR_GET_NAME(*tensor)); + + Qnn_QuantizeParams_t src_qparam = QNN_TENSOR_GET_QUANT_PARAMS(*tensor); + Qnn_QuantizationEncoding_t encoding = src_qparam.quantizationEncoding; + if (encoding == QNN_QUANTIZATION_ENCODING_AXIS_SCALE_OFFSET) { + free(src_qparam.axisScaleOffsetEncoding.scaleOffset); + } else if (encoding == QNN_QUANTIZATION_ENCODING_BW_AXIS_SCALE_OFFSET) { + free(src_qparam.bwAxisScaleOffsetEncoding.scales); + if (src_qparam.bwAxisScaleOffsetEncoding.offsets != nullptr) { + free(src_qparam.bwAxisScaleOffsetEncoding.offsets); + } + } + //GGMLQNN_LOG_DEBUG("tensor dims %p", QNN_TENSOR_GET_DIMENSIONS(*tensor)); + free(QNN_TENSOR_GET_DIMENSIONS(*tensor)); + free(tensor); + + return err; +} + + +static size_t qnn_datatype_size(Qnn_DataType_t qnn_type) { + switch (qnn_type) { + case QNN_DATATYPE_FLOAT_32: + return sizeof(float); + case QNN_DATATYPE_FLOAT_16: + return sizeof(uint16_t); + case QNN_DATATYPE_UINT_32: + case QNN_DATATYPE_INT_32: + return sizeof(int32_t); + case QNN_DATATYPE_INT_16: + return sizeof(int16_t); + case QNN_DATATYPE_INT_8: + return sizeof(int8_t); + case QNN_DATATYPE_SFIXED_POINT_8: + return sizeof(int8_t); + case QNN_DATATYPE_SFIXED_POINT_4: + return sizeof(int8_t); + default: + break; + } + return 0; +} + +static const char * qnn_datatype_to_string(Qnn_DataType_t qnn_type) { + switch (qnn_type) { + case QNN_DATATYPE_FLOAT_32: + return "QNN_DATATYPE_FLOAT_32"; + case QNN_DATATYPE_FLOAT_16: + return "QNN_DATATYPE_FLOAT_16"; + case QNN_DATATYPE_UINT_32: + return "QNN_DATATYPE_UINT_32"; + case QNN_DATATYPE_INT_32: + return "QNN_DATATYPE_INT_32"; + case QNN_DATATYPE_INT_16: + return "QNN_DATATYPE_INT_16"; + case QNN_DATATYPE_INT_8: + return "QNN_DATATYPE_INT_8"; + case QNN_DATATYPE_SFIXED_POINT_8: + return "QNN_DATATYPE_SFIXED_POINT_8"; + case QNN_DATATYPE_SFIXED_POINT_4: + return "QNN_DATATYPE_SFIXED_POINT_4"; + default: + break; + } + return "QNN_DATATYPE_UNDEFINED"; +} + +static const char * qnn_get_error_string(Qnn_ErrorHandle_t qnn_error_code) { + // file:///opt/qcom/aistack/qairt/2.31.0.250130/docs/QNN/general/api_error_codes.html + switch (qnn_error_code) { + case QNN_SUCCESS: + return "QNN_SUCCESS"; + case QNN_COMMON_ERROR_GENERAL: + return "QNN_COMMON_ERROR_GENERAL"; + + // QnnGraph_Error_t + case QNN_GRAPH_ERROR_UNSUPPORTED_FEATURE: + return "QNN_GRAPH_ERROR_UNSUPPORTED_FEATURE"; + case QNN_GRAPH_ERROR_MEM_ALLOC: + return "QNN_GRAPH_ERROR_MEM_ALLOC"; + case QNN_GRAPH_ERROR_INVALID_ARGUMENT: + return "QNN_GRAPH_ERROR_INVALID_ARGUMENT"; + case QNN_GRAPH_ERROR_INVALID_HANDLE: + return "QNN_GRAPH_ERROR_INVALID_HANDLE"; + case QNN_GRAPH_ERROR_GRAPH_DOES_NOT_EXIST: + return "QNN_GRAPH_ERROR_GRAPH_DOES_NOT_EXIST"; + case QNN_GRAPH_ERROR_INVALID_NAME: + return "QNN_GRAPH_ERROR_INVALID_NAME"; + case QNN_GRAPH_ERROR_INVALID_TENSOR: + return "QNN_GRAPH_ERROR_INVALID_TENSOR"; + case QNN_GRAPH_ERROR_INVALID_OP_CONFIG: + return "QNN_GRAPH_ERROR_INVALID_OP_CONFIG"; + case QNN_GRAPH_ERROR_SET_PROFILE: + return "QNN_GRAPH_ERROR_SET_PROFILE"; + case QNN_GRAPH_ERROR_UNCONNECTED_NODE: + return "QNN_GRAPH_ERROR_UNCONNECTED_NODE"; + case QNN_GRAPH_ERROR_CREATE_FAILED: + return "QNN_GRAPH_ERROR_CREATE_FAILED"; + case QNN_GRAPH_ERROR_OPTIMIZATION_FAILED: + return "QNN_GRAPH_ERROR_OPTIMIZATION_FAILED"; + case QNN_GRAPH_ERROR_FINALIZE_FAILED: + return "QNN_GRAPH_ERROR_FINALIZE_FAILED"; + case QNN_GRAPH_ERROR_GRAPH_NOT_FINALIZED: + return "QNN_GRAPH_ERROR_GRAPH_NOT_FINALIZED"; + case QNN_GRAPH_ERROR_GRAPH_FINALIZED: + return "QNN_GRAPH_ERROR_GRAPH_FINALIZED"; + case QNN_GRAPH_ERROR_EXECUTION_ASYNC_FIFO_FULL: + return "QNN_GRAPH_ERROR_EXECUTION_ASYNC_FIFO_FULL"; + case QNN_GRAPH_ERROR_SIGNAL_IN_USE: + return "QNN_GRAPH_ERROR_SIGNAL_IN_USE"; + case QNN_GRAPH_ERROR_ABORTED: + return "QNN_GRAPH_ERROR_ABORTED"; + case QNN_GRAPH_ERROR_PROFILE_IN_USE: + return "QNN_GRAPH_ERROR_PROFILE_IN_USE"; + case QNN_GRAPH_ERROR_TIMED_OUT: + return "QNN_GRAPH_ERROR_TIMED_OUT"; + case QNN_GRAPH_ERROR_SUBGRAPH: + return "QNN_GRAPH_ERROR_SUBGRAPH"; + case QNN_GRAPH_ERROR_DISABLED: + return "QNN_GRAPH_ERROR_DISABLED"; + case QNN_GRAPH_ERROR_DYNAMIC_TENSOR_SHAPE: + return "QNN_GRAPH_ERROR_DYNAMIC_TENSOR_SHAPE"; + case QNN_GRAPH_ERROR_TENSOR_SPARSITY: + return "QNN_GRAPH_ERROR_TENSOR_SPARSITY"; + case QNN_GRAPH_ERROR_EARLY_TERMINATION: + return "QNN_GRAPH_ERROR_EARLY_TERMINATION"; + case QNN_GRAPH_ERROR_INVALID_CONTEXT: + return "QNN_GRAPH_ERROR_INVALID_CONTEXT"; + + //QQnnTensor_Error_t + //Invalid context/graph handle in creating tensor + case QNN_TENSOR_ERROR_INVALID_HANDLE: + return "QNN_TENSOR_ERROR_INVALID_HANDLE"; + //Tensor with specified credentials not registered with a context/graph + case QNN_TENSOR_ERROR_DOES_NOT_EXIST: + return "QNN_TENSOR_ERROR_DOES_NOT_EXIST"; + // (deprecated) Tensor has already been registered with backend + case QNN_TENSOR_ERROR_ALREADY_EXISTS: + return "QNN_TENSOR_ERROR_ALREADY_EXISTS"; + // Invalid tensor param. + case QNN_TENSOR_ERROR_INVALID_TENSOR_PARAM: + return "QNN_TENSOR_ERROR_INVALID_TENSOR_PARAM"; + // This tensor param is currently unsupported + case QNN_TENSOR_ERROR_UNSUPPORTED_TENSOR_PARAM: + return "QNN_TENSOR_ERROR_UNSUPPORTED_TENSOR_PARAM"; + // Tensor provided for update is invalid + case QNN_TENSOR_ERROR_INCOMPATIBLE_TENSOR_UPDATE: + return "QNN_TENSOR_ERROR_INCOMPATIBLE_TENSOR_UPDATE"; + + // QnnOpPackage_Error_t + case QNN_OP_PACKAGE_ERROR_LIBRARY_ALREADY_INITIALIZED: + return "QNN_OP_PACKAGE_ERROR_LIBRARY_ALREADY_INITIALIZED"; + case QNN_OP_PACKAGE_ERROR_LIBRARY_NOT_INITIALIZED: + return "QNN_OP_PACKAGE_ERROR_LIBRARY_NOT_INITIALIZED"; + case QNN_OP_PACKAGE_ERROR_INVALID_HANDLE: + return "QNN_OP_PACKAGE_ERROR_INVALID_HANDLE"; + case QNN_OP_PACKAGE_ERROR_INVALID_INFRASTRUCTURE: + return "QNN_OP_PACKAGE_ERROR_INVALID_INFRASTRUCTURE"; + case QNN_OP_PACKAGE_ERROR_INVALID_INFO: + return "QNN_OP_PACKAGE_ERROR_INVALID_INFO"; + case QNN_OP_PACKAGE_ERROR_VALIDATION_FAILURE: + return "QNN_OP_PACKAGE_ERROR_VALIDATION_FAILURE"; + case QNN_OP_PACKAGE_ERROR_INVALID_ARGUMENT: + return "QNN_OP_PACKAGE_ERROR_INVALID_ARGUMENT"; + + default: + return "unknown QNN error"; + } +} + +// ================================================================================================= +// section-5:ggml-qnn backend helper macro / data structure / function / class +// ================================================================================================= +#define RPCMEM_DEFAULT_FLAGS 1 +#define RPCMEM_HEAP_ID_SYSTEM 25 + +typedef void (* ggmlqnn_op_func_t)(ggml_backend_t backend, ggml_tensor * op); + +using pfn_rpc_mem_init = void (*)(void); +using pfn_rpc_mem_deinit = void (*)(void); +using pfn_rpc_mem_alloc = void *(*)(int, uint32_t, int); +using pfn_rpc_mem_free = void (*)(void *); +using pfn_rpc_mem_to_fd = int (*)(void *); +using _pfn_QnnSaver_initialize = decltype(QnnSaver_initialize); +using _pfn_QnnInterface_getProviders = decltype(QnnInterface_getProviders); +using _pfn_QnnSystemInterface_getProviders = decltype(QnnSystemInterface_getProviders); + +enum class ggml_qnn_profile_level { + profile_off = 0, + profile_basic = 1, + profile_detail = 2 +}; + +enum qcom_htp_arch { + NONE = 0, + V68 = 68, + V69 = 69, + V73 = 73, + V75 = 75, + V79 = 79, +}; + +enum qcom_chipset_soc_model { + UNKNOWN_SM = 0, + SM7450 = 41, // v69, 7 Gen1 + SM8350 = 30, // v68, 888 + SM8450 = 36, // v69, SD 8 Gen 1 + SM8475 = 42, // v69, SD 8+ Gen 1 + SM8550 = 43, // v73, SD 8 Gen 2 + SM8650 = 57, // v75, SD 8 Gen 3 + SM8750 = 69, // v79, SD 8 Gen 4 +}; + +struct qcom_socinfo { + uint32_t soc_model; + size_t htp_arch; + size_t vtcm_size_in_mb; + char soc_desc[GGML_MAX_NAME]; +}; + +//file:///opt/qcom/aistack/qairt/2.31.0.250130/docs/QNN/general/overview.html#tbl-supported-snapdragon-devices +static struct qcom_socinfo g_qnn_soc_info_table[] = { + /* Qualcomm SnapDragon 7 Gen 1 */ + [SM7450] = { + .soc_model = SM7450, + .htp_arch = V69, + .vtcm_size_in_mb = 8, + .soc_desc = "Qualcomm SnapDragon 7 Gen 1"}, + + /* Qualcomm SnapDragon 888 */ + [SM8350] = { + .soc_model = SM8350, + .htp_arch = V68, + .vtcm_size_in_mb = 8, + .soc_desc = "Qualcomm SnapDragon 888 "}, + + /* Qualcomm SnapDragon 8 Gen 1 */ + [SM8450] = { + .soc_model = SM8450, + .htp_arch = V69, + .vtcm_size_in_mb = 8, + .soc_desc = "Qualcomm SnapDragon 8 Gen 1"}, + + /* Qualcomm SnapDragon 8 Gen 1+ */ + [SM8475] = { + .soc_model = SM8475, + .htp_arch = V69, + .vtcm_size_in_mb = 8, + .soc_desc = "Qualcomm SnapDragon 8 Gen 1+"}, + + /* Qualcomm SnapDragon 8 Gen 2 */ + [SM8550] = { + .soc_model = SM8550, + .htp_arch = V73, + .vtcm_size_in_mb = 8, + .soc_desc = "Qualcomm SnapDragon 8 Gen 2"}, + + /* Qualcomm SnapDragon 8 Gen 3 */ + [SM8650] = { + .soc_model = SM8650, + .htp_arch = V75, + .vtcm_size_in_mb = 8, + .soc_desc = "Qualcomm SnapDragon 8 Gen 3 "}, + + /* Qualcomm SnapDragon 8 Gen 4 */ + [SM8750] = { + .soc_model = SM8750, + .htp_arch = V79, + .vtcm_size_in_mb = 8, + .soc_desc = "Qualcomm SnapDragon 8 Gen 4"}, + +}; + +struct ggml_backend_qnn_context { + int device; + int threads; + char name[GGML_MAX_NAME]; + char desc[GGML_MAX_NAME]; + char lib[GGML_MAX_NAME]; + qnn_instance * instance; + struct ggml_backend * backend; + QNN_INTERFACE_VER_TYPE raw_interface; + QNN_SYSTEM_INTERFACE_VER_TYPE raw_system_interface; + struct qcom_socinfo socinfo; + + //FIXME: should I move it from public member of class qnn_instance to here? + //std::map> _qnn_graph_map; +} ; + +//FIXME: the following global vars and three helper funcs should be removed in the future +static int32_t g_ggmltensor_idx = 0; +static void reset_idx() { + g_ggmltensor_idx = 0; +} + +static void inc_idx() { + g_ggmltensor_idx++; +} + +static int32_t get_idx() { + return g_ggmltensor_idx; +} + +// file:///opt/qcom/aistack/qairt/2.31.0.250130/docs/QNN/general/quantization.html +// CPU - Choose a non-quantized model.Quantized models are currently incompatible with the CPU backend +// GPU - Choose a non-quantized model.Quantized models are currently incompatible with the GPU backend +// HTP - Choose a quantized model. Quantized models are required when running on the HTP backend +// DSP - Choose a quantized model. Quantized models are required when running on the DSP backend +// HTA - Choose a quantized model. Quantized models are required when running on the HTA backend +static struct ggml_backend_qnn_context g_qnn_mgr[GGML_QNN_MAX_DEVICES] = { + [QNN_BACKEND_CPU] = {.device = 0, + .threads = 1, + .name = "qnn-cpu", + .desc = "Qualcomm Kryo CPU", + .lib = "libQnnCpu.so", + .instance = nullptr, + .backend = nullptr, + .raw_interface = {}, + .raw_system_interface = {}, + .socinfo = {}}, + + [QNN_BACKEND_GPU] = {.device = 1, + .threads = 1, + .name = "qnn-gpu", + .desc = "Qualcomm Adreno GPU", + .lib = "libQnnGpu.so", + .instance = nullptr, + .backend = nullptr, + .raw_interface = {}, + .raw_system_interface = {}, + .socinfo = {}}, + + [QNN_BACKEND_NPU] = {.device = 2, + .threads = 1, + .name = "qnn-npu", + .desc = "Qualcomm NPU(Hexagon Tensor Processor)", + .lib = "libQnnHtp.so", + .instance = nullptr, + .backend = nullptr, + .raw_interface = {}, + .raw_system_interface = {}, + .socinfo = {}}, +}; + +using ggml_dimension_array_t = int64_t[GGML_MAX_DIMS]; +using qnn_dimension_array_t = std::array; +using op_dims_calc_func_t = void (*)(const std::vector & input_dims, + ggml_dimension_array_t & output_dims); + +static void element_wise_op_dims(const std::vector & input_dims, + ggml_dimension_array_t &output_dims) { + for (size_t i = 1; i < std::size(output_dims); i++) { + output_dims[i] = input_dims.front()[i]; + } +} + +static void mat_mul_op_dims(const std::vector & input_dims, + ggml_dimension_array_t & output_dims) { + GGML_ASSERT(input_dims.size() == 2); + output_dims[0] = input_dims.front()[1]; + output_dims[1] = input_dims.back()[1]; +} + +struct qnn_op_caps_t { + const char * qnn_op_name = nullptr; + const size_t input_param_count = 0; + op_dims_calc_func_t calc_dims_func = nullptr; + const char * qnn_param_name = nullptr; +}; + +constexpr static const qnn_op_caps_t kOpCaps[] = { + {}, // GGML_OP_NONE + {}, // GGML_OP_DUP + { + // GGML_OP_ADD + QNN_OP_ELEMENT_WISE_ADD, // qnn_op_name + 2, // input_param_count + element_wise_op_dims, // calc_dims_func + }, + {}, // GGML_OP_ADD1 + {}, // GGML_OP_ACC + {}, // GGML_OP_SUB + {}, // GGML_OP_MUL + {}, // GGML_OP_DIV + {}, // GGML_OP_SQR + {}, // GGML_OP_SQRT + {}, // GGML_OP_LOG + {}, // GGML_OP_SIN + {}, // GGML_OP_COS + {}, // GGML_OP_SUM + {}, // GGML_OP_SUM_ROWS + {}, // GGML_OP_MEAN + {}, // GGML_OP_ARGMAX + {}, // GGML_OP_COUNT_EQUAL + {}, // GGML_OP_REPEAT + {}, // GGML_OP_REPEAT_BACK + {}, // GGML_OP_CONCAT + {}, // GGML_OP_SILU_BACK + {}, // GGML_OP_NORM + {}, // GGML_OP_RMS_NORM + {}, // GGML_OP_RMS_NORM_BACK + {}, // GGML_OP_GROUP_NORM + { + // GGML_OP_MUL_MAT + QNN_OP_MAT_MUL, // qnn_op_name + 2, // input_param_count + mat_mul_op_dims, // calc_dims_func + }, + {}, // GGML_OP_MUL_MAT_ID + {}, // GGML_OP_OUT_PROD + {}, // GGML_OP_SCALE + {}, // GGML_OP_SET + {}, // GGML_OP_CPY + {}, // GGML_OP_CONT + {}, // GGML_OP_RESHAPE + {}, // GGML_OP_VIEW + {}, // GGML_OP_PERMUTE + {}, // GGML_OP_TRANSPOSE + {}, // GGML_OP_GET_ROWS + {}, // GGML_OP_GET_ROWS_BACK + {}, // GGML_OP_DIAG + {}, // GGML_OP_DIAG_MASK_INF + {}, // GGML_OP_DIAG_MASK_ZERO + {}, // GGML_OP_SOFT_MAX + {}, // GGML_OP_SOFT_MAX_BACK + {}, // GGML_OP_ROPE + {}, // GGML_OP_ROPE_BACK + {}, // GGML_OP_CLAMP + {}, // GGML_OP_CONV_TRANSPOSE_1D + {}, // GGML_OP_IM2COL + {}, // GGML_OP_IM2COL_BACK + {}, // GGML_OP_CONV_TRANSPOSE_2D + {}, // GGML_OP_POOL_1D + {}, // GGML_OP_POOL_2D + {}, // GGML_OP_POOL_2D_BACK + {}, // GGML_OP_UPSCALE + {}, // GGML_OP_PAD + {}, // GGML_OP_PAD_REFLECT_1D + {}, // GGML_OP_ARANGE + {}, // GGML_OP_TIMESTEP_EMBEDDING + {}, // GGML_OP_ARGSORT + {}, // GGML_OP_LEAKY_RELU + {}, // GGML_OP_FLASH_ATTN_EXT + {}, // GGML_OP_FLASH_ATTN_BACK + {}, // GGML_OP_SSM_CONV + {}, // GGML_OP_SSM_SCAN + {}, // GGML_OP_WIN_PART + {}, // GGML_OP_WIN_UNPART + {}, // GGML_OP_GET_REL_POS + {}, // GGML_OP_ADD_REL_POS + {}, // GGML_OP_RWKV_WKV6 + {}, // GGML_OP_GATED_LINEAR_ATTN + {}, // GGML_OP_UNARY + {}, // GGML_OP_MAP_UNARY + {}, // GGML_OP_MAP_BINARY + {}, // GGML_OP_MAP_CUSTOM1_F32 + {}, // GGML_OP_MAP_CUSTOM2_F32 + {}, // GGML_OP_MAP_CUSTOM3_F32 + {}, // GGML_OP_MAP_CUSTOM1 + {}, // GGML_OP_MAP_CUSTOM2 + {}, // GGML_OP_MAP_CUSTOM3 + {}, // GGML_OP_CROSS_ENTROPY_LOSS + {}, // GGML_OP_CROSS_ENTROPY_LOSS_BACK + {}, // GGML_OP_OPT_STEP_ADAMW + {}, // GGML_UNARY_OP_ABS + {}, // GGML_UNARY_OP_SGN + {}, // GGML_UNARY_OP_NEG + {}, // GGML_UNARY_OP_STEP + {}, // GGML_UNARY_OP_TANH + {}, // GGML_UNARY_OP_ELU + {}, // GGML_UNARY_OP_RELU + {}, // GGML_UNARY_OP_SIGMOID + {}, // GGML_UNARY_OP_GELU + {}, // GGML_UNARY_OP_GELU_QUICK + {}, // GGML_UNARY_OP_SILU + {}, // GGML_UNARY_OP_HARDSWISH + {}, // GGML_UNARY_OP_HARDSIGMOID + {}, // GGML_UNARY_OP_EXP +}; + +static const char * qnn_get_socmodel_desc(uint32_t soc_model) { + switch (soc_model) { + case SM7450: + return "SM7450"; + case SM8350: + return "SM8350"; + case SM8450: + return "SM8450"; + case SM8475: + return "SM8475"; + case SM8550: + return "SM8550"; + case SM8650: + return "SM8650"; + case SM8750: + return "SM8750"; + default: + return "unknown"; + } +} + +static const char * qnn_get_htparch_desc(size_t htp_arch) { + switch (htp_arch) { + case V68: + return "QCOM_HTP_V68"; + case V69: + return "QCOM_HTP_V69"; + case V73: + return "QCOM_HTP_V73"; + case V75: + return "QCOM_HTP_V75"; + case V79: + return "QCOM_HTP_V79"; + default: + return "unknown"; + } +} + +static struct qcom_socinfo * qnn_get_socinfo_from_socmodel(uint32_t soc_model) { + size_t items = sizeof(g_qnn_soc_info_table) / sizeof(g_qnn_soc_info_table[0]); + for (size_t idx = 0; idx < items; idx++) { + if (soc_model == g_qnn_soc_info_table[idx].soc_model) { + return &g_qnn_soc_info_table[idx]; + } + } + return nullptr; +} + +static bool qnn_is_valid_params(ggml_backend_qnn_context * ctx, const ggml_tensor * src0, + const ggml_tensor * src1, ggml_tensor * dst) { + if ((nullptr == ctx) || (nullptr == src0) || (nullptr == src1) || (nullptr == dst)) { + GGMLQNN_LOG_WARN("invalid params\n"); + return false; + } + + qnn_instance * instance = ctx->instance; + if (nullptr == instance) { + GGMLQNN_LOG_WARN("invalid params\n"); + return false; + } + + return true; +} + +#define CHECK_PARAMS(ctx, src0, src1, dst) \ + do { \ + if (!qnn_is_valid_params((ctx), (src0), (src1), (dst))) { \ + return; \ + } \ + } while (0) + +static uint32_t ggml_get_tensor_rank(const ggml_tensor * tensor) { + /* + uint32_t rank = 0; + for (int i = 0; i < GGML_MAX_DIMS; i++) { + if ((0 != tensor->ne[i]) && (1 != tensor->ne[i])) { + rank++; + } + } + return rank; + */ + return ggml_n_dims(tensor); +} + +static uint32_t ggml_get_tensor_data_size(const ggml_tensor * tensor) { + /* + size_t data_size = ggml_row_size(tensor->type, tensor->ne[0]); + size_t n_dims = ggml_get_tensor_rank(tensor); + for (int i = 1; i < n_dims; i++) { + data_size *= tensor->ne[i]; + } + + return data_size; + */ + return ggml_nbytes(tensor); +} + +static const char * ggml_get_type_name(ggml_type type) { + const struct ggml_type_traits * traits = ggml_get_type_traits(type); + return traits->type_name; +} + +Qnn_Tensor_t * ggml_qnn_create_tensor(const ggml_tensor * tensor) { + Qnn_ErrorHandle_t error = QNN_SUCCESS; + char tensor_name[GGML_MAX_NAME] = {0}; + + //FIXME:remove get_idx() and inc_idx() in the future but ensure the tensor name is unique + snprintf(tensor_name, GGML_MAX_NAME, "tensor_%-8d", get_idx()); + GGMLQNN_LOG_DEBUG("init_tensor %d", get_idx()); + inc_idx(); + + uint32_t dimensions[] = {(uint32_t) tensor->ne[0], (uint32_t) tensor->ne[1], + (uint32_t) tensor->ne[2], (uint32_t) tensor->ne[3]}; + Qnn_DataType_t qnn_data_type = QNN_DATATYPE_FLOAT_32; + Qnn_TensorType_t qnn_tensor_type = QNN_TENSOR_TYPE_APP_WRITE; + + if (tensor->flags & GGML_TENSOR_FLAG_INPUT) { + qnn_tensor_type = QNN_TENSOR_TYPE_APP_WRITE; + } else if (tensor->flags & GGML_TENSOR_FLAG_OUTPUT) { + qnn_tensor_type = QNN_TENSOR_TYPE_APP_READ; + } + Qnn_Tensor_t qnn_tensor = { + .version= QNN_TENSOR_VERSION_1, + {.v1= { + .id = 0, + .name = tensor_name, + .type = qnn_tensor_type, + .dataFormat = QNN_TENSOR_DATA_FORMAT_FLAT_BUFFER, + .dataType = qnn_data_type, + .quantizeParams = {QNN_DEFINITION_UNDEFINED, + QNN_QUANTIZATION_ENCODING_UNDEFINED, + {.scaleOffsetEncoding = {.scale = 0.0000000000000000f, .offset = 0}}}, + .rank = ggml_get_tensor_rank(tensor), + .dimensions = dimensions, + .memType = QNN_TENSORMEMTYPE_RAW, + {.clientBuf = {.data = nullptr, + .dataSize = 0}}}} + }; + Qnn_Tensor_t * p_qnn_tensor = (Qnn_Tensor_t *)calloc(1, sizeof(Qnn_Tensor_t)); + if (nullptr == p_qnn_tensor) { + GGMLQNN_LOG_WARN("calloc failed"); + return nullptr; + } + error = deep_copy_qnn_tensors(qnn_tensor, * p_qnn_tensor); + if (error != QNN_SUCCESS) { + free(p_qnn_tensor); + GGMLQNN_LOG_WARN("init tensor failed"); + return nullptr; + } + + return p_qnn_tensor; +} + +//TODO: +// ref:explanation of k-quants, https://github.com/ggerganov/llama.cpp/pull/1684 +static Qnn_DataType_t qnn_datatype_from_ggml_datatype(enum ggml_type ggmltype) { + switch (ggmltype) { + case GGML_TYPE_F16: + return QNN_DATATYPE_FLOAT_16; + case GGML_TYPE_F32: + return QNN_DATATYPE_FLOAT_32; + case GGML_TYPE_I8: + return QNN_DATATYPE_INT_8; + case GGML_TYPE_Q8_0: + return QNN_DATATYPE_SFIXED_POINT_8; + case GGML_TYPE_Q4_0: + return QNN_DATATYPE_SFIXED_POINT_4; + default: + break; + } + return QNN_DATATYPE_UNDEFINED; +} + +//TODO: +static ggml_type ggml_datatype_from_qnn_datatype(Qnn_DataType_t qnn_type) { + switch (qnn_type) { + case QNN_DATATYPE_FLOAT_32: + return GGML_TYPE_F32; + case QNN_DATATYPE_FLOAT_16: + return GGML_TYPE_F16; + case QNN_DATATYPE_UINT_32: + case QNN_DATATYPE_INT_32: + return GGML_TYPE_I32; + case QNN_DATATYPE_INT_16: + return GGML_TYPE_I16; + case QNN_DATATYPE_INT_8: + return GGML_TYPE_I8; + case QNN_DATATYPE_SFIXED_POINT_8: + return GGML_TYPE_Q8_0; + case QNN_DATATYPE_SFIXED_POINT_4: + return GGML_TYPE_Q4_0; + default: + break; + } + return GGML_TYPE_COUNT; +} + +//TODO: +static const char * qnn_opname_from_ggmlop(enum ggml_op ggmlop) { + switch (ggmlop) { + case GGML_OP_ADD: + return QNN_OP_ELEMENT_WISE_ADD; + case GGML_OP_MUL_MAT: + return QNN_OP_MAT_MUL; + default: + break; + } + return nullptr; +} + +static const char * get_ggml_type_name(ggml_type type) { + const auto * traits = ggml_get_type_traits(type); + return traits->type_name; +} + +static void append_tensor_dimensions(const ggml_tensor * tensor, std::string & output) { + char buffer[256] = {}; + const char * type_name = get_ggml_type_name(tensor->type); + int len = 0; + switch (ggml_n_dims(tensor)) { + case 1: + len = snprintf(buffer, sizeof(buffer), "%ld%s", (long)tensor->ne[0], type_name); + break; + case 2: + len = snprintf(buffer, sizeof(buffer), "%ldx%ld%s", (long)tensor->ne[0], (long)tensor->ne[1], type_name); + break; + case 3: + len = snprintf(buffer, sizeof(buffer), "%ldx%ldx%ld%s", (long)tensor->ne[0], (long)tensor->ne[1], + (long)tensor->ne[2], type_name); + break; + case 4: + default: + len = snprintf(buffer, sizeof(buffer), "%ldx%ldx%ldx%ld%s", (long)tensor->ne[0], (long)tensor->ne[1], + (long)tensor->ne[2], (long)tensor->ne[3], type_name); + break; + } + GGML_ASSERT(len > 0 && len < (int)sizeof(buffer)); + output.append(buffer, len); +} + +constexpr const size_t kGgmlUnaryOpStart = GGML_OP_COUNT; + +static size_t get_qnn_op_index(const ggml_tensor * tensor) { + if (tensor->op == GGML_OP_UNARY) { + return kGgmlUnaryOpStart + ggml_get_unary_op(tensor); + } + + return tensor->op; +} + +static size_t get_qnn_op_input_param_count(const ggml_tensor * op) { + auto op_index = get_qnn_op_index(op); + GGML_ASSERT(op_index < std::size(kOpCaps)); + return kOpCaps[op_index].input_param_count; +} + +static void get_graph_key_from_op(const ggml_tensor * op, std::string & output) { + GGML_ASSERT(op->op != GGML_OP_NONE); + output += ggml_op_desc(op); + output += get_ggml_type_name(op->type); + size_t param_count = get_qnn_op_input_param_count(op); + for (size_t i = 0; i < param_count; ++i) { + auto * input = op->src[i]; + if (!input) { + break; + } + output += '_'; + append_tensor_dimensions(input, output); + } +} + +#if ENABLE_QNNBACKEND_PERF +class qnn_perf { +public: + qnn_perf(const std::string & perf_name) : _perf_name(std::move(perf_name)) {}; + qnn_perf() = delete; + qnn_perf(const qnn_perf & ) = delete; + qnn_perf & operator= (const qnn_perf & ) = delete; + + void start() { + _begin_time = ggml_time_us(); + } + + void info() { + _end_time = ggml_time_us(); + _duration = (_end_time - _begin_time); + GGMLQNN_LOG_DEBUG("duration of %s : %lld microseconds\n", _perf_name.c_str(), _duration); + } + +private: + int64_t _begin_time = 0LL; + int64_t _end_time = 0LL; + int64_t _duration = 0LL; + std::string _perf_name; +}; +#else +class qnn_perf { +public: + qnn_perf(const std::string & perf_name) {} + qnn_perf() = delete; + qnn_perf(const qnn_perf & ) = delete; + qnn_perf & operator= (const qnn_perf & ) = delete; + + void start() {} + void info() {} +}; +#endif + +template +Fn load_qnn_functionpointers(void * handle, const char * function_name) { + return reinterpret_cast(dlsym(handle, function_name)); +} + +class qnn_interface { + +#define DEFINE_SHIM_FUNCTION_INTERFACE(F, pointer_name) \ + template \ + inline auto qnn_##F(Args... args) const { \ + return (_qnn_interface->QNN_INTERFACE_VER_NAME.pointer_name)( \ + std::forward(args)...); \ + } + + +#define DEFINE_SHIM_FUNCTION_SYS_INTERFACE(F, pointer_name) \ + template \ + inline auto qnn_##F(Args... args) const { \ + return (_qnn_sys_interface->QNN_SYSTEM_INTERFACE_VER_NAME.pointer_name)( \ + std::forward(args)...); \ + } + + friend class qnn_instance; + +public: + qnn_interface() = default; + + // QnnBackend + DEFINE_SHIM_FUNCTION_INTERFACE(backend_create, backendCreate); + + DEFINE_SHIM_FUNCTION_INTERFACE(backend_free, backendFree); + + DEFINE_SHIM_FUNCTION_INTERFACE(backend_register_op_package, backendRegisterOpPackage); + + DEFINE_SHIM_FUNCTION_INTERFACE(backend_validate_op_config, backendValidateOpConfig); + + DEFINE_SHIM_FUNCTION_INTERFACE(backend_get_api_version, backendGetApiVersion); + + // QnnDevice + DEFINE_SHIM_FUNCTION_INTERFACE(device_create, deviceCreate); + + DEFINE_SHIM_FUNCTION_INTERFACE(device_free, deviceFree); + + DEFINE_SHIM_FUNCTION_INTERFACE(device_get_infrastructure, deviceGetInfrastructure); + + DEFINE_SHIM_FUNCTION_INTERFACE(device_get_platform_info, deviceGetPlatformInfo); + + DEFINE_SHIM_FUNCTION_INTERFACE(device_get_info, deviceGetInfo); + + // QnnContext + DEFINE_SHIM_FUNCTION_INTERFACE(context_create, contextCreate); + + DEFINE_SHIM_FUNCTION_INTERFACE(context_get_binary_size, contextGetBinarySize); + + DEFINE_SHIM_FUNCTION_INTERFACE(context_get_binary, contextGetBinary); + + DEFINE_SHIM_FUNCTION_INTERFACE(context_create_from_binary, contextCreateFromBinary); + + DEFINE_SHIM_FUNCTION_INTERFACE(context_free, contextFree); + + // QnnGraph + DEFINE_SHIM_FUNCTION_INTERFACE(graph_create, graphCreate); + + DEFINE_SHIM_FUNCTION_INTERFACE(graph_add_node, graphAddNode); + + DEFINE_SHIM_FUNCTION_INTERFACE(graph_finalize, graphFinalize); + + DEFINE_SHIM_FUNCTION_INTERFACE(graph_execute, graphExecute); + + DEFINE_SHIM_FUNCTION_INTERFACE(graph_retrieve, graphRetrieve); + + // QnnLog + DEFINE_SHIM_FUNCTION_INTERFACE(log_create, logCreate); + + DEFINE_SHIM_FUNCTION_INTERFACE(log_free, logFree); + + DEFINE_SHIM_FUNCTION_INTERFACE(log_set_log_level, logSetLogLevel); + + // QnnProfile + DEFINE_SHIM_FUNCTION_INTERFACE(profile_create, profileCreate); + + DEFINE_SHIM_FUNCTION_INTERFACE(profile_get_events, profileGetEvents); + + DEFINE_SHIM_FUNCTION_INTERFACE(profile_get_sub_events, profileGetSubEvents); + + DEFINE_SHIM_FUNCTION_INTERFACE(profile_get_event_data, profileGetEventData); + + DEFINE_SHIM_FUNCTION_INTERFACE(profile_free, profileFree); + + // QnnMem + DEFINE_SHIM_FUNCTION_INTERFACE(mem_register, memRegister); + + DEFINE_SHIM_FUNCTION_INTERFACE(mem_de_register, memDeRegister); + + // QnnProperty + DEFINE_SHIM_FUNCTION_INTERFACE(property_has_capability, propertyHasCapability); + + // QnnTensor + DEFINE_SHIM_FUNCTION_INTERFACE(tensor_create_context_tensor, tensorCreateContextTensor); + + DEFINE_SHIM_FUNCTION_INTERFACE(tensor_create_graph_tensor, tensorCreateGraphTensor); + + // QnnSystem + DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_create, systemContextCreate); + + DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_get_binary_info, systemContextGetBinaryInfo); + + DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_free, systemContextFree); + + void set_qnn_interface(const QnnInterface_t * qnn_interface) { + _qnn_interface = qnn_interface; + } + + void set_qnn_system_interface(const QnnSystemInterface_t * qnn_sys_interface) { + _qnn_sys_interface = qnn_sys_interface; + } + + uint32_t get_backend_id() const { + return _qnn_interface->backendId; + } + + bool is_loaded() const { + return ((_qnn_sys_interface != nullptr) && (_qnn_interface != nullptr)); + } + +private: + const QnnInterface_t *_qnn_interface = nullptr; + + const QnnSystemInterface_t *_qnn_sys_interface = nullptr; +}; + +class qnn_instance { +public: + using BackendIdType = decltype(QnnInterface_t{}.backendId); + + explicit qnn_instance(const std::string & lib_path, const std::string & backend_name, + const std::string & model_name) : + _lib_path(std::move(lib_path)), + _backend_name(std::move(backend_name)), + _model_name(std::move(model_name)) {}; + + ~qnn_instance() { + } + + int qnn_init(const QnnSaver_Config_t ** saver_config); + + int qnn_finalize(); + + const qnn_interface &get_qnn_interface() { + if (!_qnn_interface.is_loaded()) { + GGMLQNN_LOG_WARN("pls check why _qnn_interface is not loaded\n"); + } + return _qnn_interface; + } + + const QNN_INTERFACE_VER_TYPE &get_qnn_raw_interface() { + if (!_qnn_interface.is_loaded()) { + GGMLQNN_LOG_WARN("pls check why _qnn_interface is not loaded\n"); + } + return _qnn_raw_interface; + } + + const QNN_SYSTEM_INTERFACE_VER_TYPE &get_qnn_raw_system_interface() { + if (!_qnn_interface.is_loaded()) { + GGMLQNN_LOG_WARN("pls check why _qnn_interface is not loaded\n"); + } + return _qnn_raw_system_interface; + } + + const Qnn_LogHandle_t get_qnn_log_handle() { return _qnn_log_handle; } + + const Qnn_ProfileHandle_t get_qnn_profile_handle() { return _qnn_profile_handle; } + + const Qnn_DeviceHandle_t get_qnn_device_handle() { return _qnn_device_handle; } + + const Qnn_BackendHandle_t get_qnn_backend_handle() { return _qnn_backend_handle; } + + const Qnn_ContextHandle_t get_qnn_context_handle() { return _qnn_context_handle; } + + const QnnSystemContext_Handle_t get_qnn_system_handle() { return _qnn_system_handle; } + + const Qnn_GraphHandle_t get_qnn_graph_handle() { return _qnn_graph_handle; } + + int init_qnn_graph(const char * graph_name, + bool debug, + uint8_t do_node_validation = 1, + const QnnGraph_Config_t ** graph_configs = nullptr + ); + int init_qnn_graph(const std::string &graph_name, QNNBackend device, size_t vtcm_size_in_mb); + + int finalize_qnn_graph(); + + bool is_valid_graph() const { return _qnn_graph_handle != nullptr; } + + int init_htp_perfinfra() { + QnnDevice_Infrastructure_t device_infra = nullptr; + int error = _qnn_raw_interface.deviceGetInfrastructure(&device_infra); + if (error != QNN_SUCCESS) { + GGMLQNN_LOG_WARN("failed to get qnn device infra\n"); + return 1; + } + + QnnHtpDevice_Infrastructure_t *htp_infra = static_cast(device_infra); + QnnHtpDevice_PerfInfrastructure_t *htp_perfinfra = &htp_infra->perfInfra; + uint32_t power_configid = 1; + uint32_t device_id = 0; + uint32_t core_id = 0; + htp_perfinfra->createPowerConfigId(device_id, core_id, &power_configid); + _qnn_htp_perfinfra = htp_perfinfra; + _qnn_power_configid = power_configid; + + return 0; + } + + int set_rpc_polling() { + if (_qnn_rpc_pollingtime > 0) { + QnnHtpPerfInfrastructure_PowerConfig_t rpc_pollingtime; + memset(&rpc_pollingtime, 0, sizeof(rpc_pollingtime)); + rpc_pollingtime.option = + QNN_HTP_PERF_INFRASTRUCTURE_POWER_CONFIGOPTION_RPC_POLLING_TIME; + rpc_pollingtime.rpcPollingTimeConfig = _qnn_rpc_pollingtime; + const QnnHtpPerfInfrastructure_PowerConfig_t * power_configs[] = {&rpc_pollingtime, nullptr}; + if (_qnn_htp_perfinfra) { + _qnn_htp_perfinfra->setPowerConfig(_qnn_power_configid, power_configs); + } + } + return 0; + } + + int set_high_performance_mode() { + if (nullptr == _qnn_htp_perfinfra) { + GGMLQNN_LOG_DEBUG("perf intra is null\n"); + return 1; + } + + QnnHtpPerfInfrastructure_PowerConfig_t power_config; + memset(&power_config, 0, sizeof(power_config)); + power_config.option = QNN_HTP_PERF_INFRASTRUCTURE_POWER_CONFIGOPTION_DCVS_V3; + power_config.dcvsV3Config.dcvsEnable = 0; + power_config.dcvsV3Config.setDcvsEnable = 1; + power_config.dcvsV3Config.contextId = _qnn_power_configid; + power_config.dcvsV3Config.powerMode = QNN_HTP_PERF_INFRASTRUCTURE_POWERMODE_PERFORMANCE_MODE; + power_config.dcvsV3Config.setSleepLatency = 1; // True to consider Latency parameter otherwise False + power_config.dcvsV3Config.setBusParams = 1; // True to consider Bus parameter otherwise False + power_config.dcvsV3Config.setCoreParams = 1; // True to consider Core parameter otherwise False + power_config.dcvsV3Config.sleepDisable = 0; // True to consider sleep/LPM modes, False to enable + power_config.dcvsV3Config.setSleepDisable = 0; // True to consider sleep disable/enable parameter otherwise False + // set Sleep latency parameter + uint32_t latencyValue = 40; + power_config.dcvsV3Config.sleepLatency = latencyValue; // range 40-2000 micro sec + // set Bus Clock Parameters (refer QnnHtpPerfInfrastructure_VoltageCorner_t enum) + power_config.dcvsV3Config.busVoltageCornerMin = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; + power_config.dcvsV3Config.busVoltageCornerTarget = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; + power_config.dcvsV3Config.busVoltageCornerMax = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; + // set Core Clock Parameters (refer QnnHtpPerfInfrastructure_VoltageCorner_t enum) + power_config.dcvsV3Config.coreVoltageCornerMin = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; + power_config.dcvsV3Config.coreVoltageCornerTarget = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; + power_config.dcvsV3Config.coreVoltageCornerMax = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; + // set power config with different performance parameters + const QnnHtpPerfInfrastructure_PowerConfig_t * power_configs[] = {&power_config, nullptr}; + + _qnn_htp_perfinfra->setPowerConfig(_qnn_power_configid, power_configs); + + return 0; + } + + std::string &get_qnn_graph_name() { return _graph_name; } + + bool is_rpcmem_initialized() { + return _rpcmem_initialized; + } + + void set_rpcmem_initialized(bool initialized) { + _rpcmem_initialized = initialized; + } + + size_t get_rpcmem_capacity() { return _rpcmem_capacity; } + + int32_t rpcmem_to_fd(void * buf); + + int register_rpcmem(void * p_data, Qnn_Tensor_t * p_tensor); + Qnn_MemHandle_t register_rpcmem(void * p_data, const uint32_t rank, uint32_t * dimensions, Qnn_DataType_t data_type); + + void unregister_rpcmem(); + void unregister_rpcmem(Qnn_MemHandle_t mem_handle); + + void *alloc_rpcmem(size_t bytes, size_t alignment); + + void free_rpcmem(void * buf); + + bool is_rpcmem_allocated(void * buf); + + bool is_rpcmem_registered(Qnn_MemHandle_t handle) { + return _qnn_mem_set.count(handle) != 0U; + } + +public: + std::map> _qnn_graph_map; + +private: + int load_system(); + + int unload_system(); + + int load_backend(std::string & lib_path, const QnnSaver_Config_t ** saver_config); + + int unload_backend(); + + void set_qnn_raw_interface(QNN_INTERFACE_VER_TYPE & raw_interface) { + _qnn_raw_interface = raw_interface; + } + + void set_qnn_raw_system_interface(QNN_SYSTEM_INTERFACE_VER_TYPE & raw_interface) { + _qnn_raw_system_interface = raw_interface; + } + +private: + static constexpr const int _required_num_providers = 1; + +private: + std::string _lib_path; + std::string _backend_name; + std::string _model_name; // name of prebuilt QNN model, might be used in the future + BackendIdType _backend_id; + + bool _debug_tensor = false; // flag to indicate if requested graph is to be run in debug mode + bool _do_node_validations = true; // flag to indicate whether all add_node calls need to be validated + QnnLog_Level_t _qnn_log_level = QNN_LOG_LEVEL_DEBUG; + + ggml_qnn_profile_level _profile_level = ggml_qnn_profile_level::profile_detail; + + qnn_interface _qnn_interface; + + void *_system_lib_handle = nullptr; + + Qnn_GraphHandle_t _qnn_graph_handle = nullptr; + + Qnn_LogHandle_t _qnn_log_handle = nullptr; + + Qnn_ProfileHandle_t _qnn_profile_handle = nullptr; + + Qnn_DeviceHandle_t _qnn_device_handle = nullptr; + + Qnn_BackendHandle_t _qnn_backend_handle = nullptr; + + Qnn_ContextHandle_t _qnn_context_handle = nullptr; + + QnnSystemContext_Handle_t _qnn_system_handle = nullptr; + + QnnHtpDevice_PerfInfrastructure_t *_qnn_htp_perfinfra = nullptr; + uint32_t _qnn_power_configid = 1; + uint32_t _qnn_rpc_pollingtime = 9999; // 0-10000 us for high performing + + QNN_INTERFACE_VER_TYPE _qnn_raw_interface; + QNN_SYSTEM_INTERFACE_VER_TYPE _qnn_raw_system_interface; + + std::unordered_set _qnn_mem_set; + std::unordered_map _qnn_rpc_buffer_to_handles; + + static std::mutex _init_mutex; + static std::unordered_map _loaded_lib_handle; + static std::unordered_map _lib_path_to_backend_id; + static std::unordered_map _loaded_backend; + + void *_rpc_lib_handle = nullptr; + std::atomic_bool _rpcmem_initialized{false}; + pfn_rpc_mem_alloc _pfn_rpc_mem_alloc; + pfn_rpc_mem_free _pfn_rpc_mem_free; + pfn_rpc_mem_to_fd _pfn_rpc_mem_to_fd; + pfn_rpc_mem_init _pfn_rpc_mem_init; + pfn_rpc_mem_deinit _pfn_rpc_mem_deinit; + std::unordered_map _rpcmem_store_map; + size_t _rpcmem_capacity = 512; + + std::string _graph_name; + QNNBackend _device_id; +}; + +std::mutex qnn_instance::_init_mutex; +std::unordered_map qnn_instance::_loaded_lib_handle; +std::unordered_map qnn_instance::_lib_path_to_backend_id; +std::unordered_map qnn_instance::_loaded_backend; + +void * qnn_instance::alloc_rpcmem(size_t bytes, size_t alignment) { + if (!_rpcmem_initialized) { + GGMLQNN_LOG_WARN("rpc memory not initialized\n"); + return nullptr; + } + + auto allocate_bytes = static_cast(bytes + alignment); + void * buf = _pfn_rpc_mem_alloc(RPCMEM_HEAP_ID_SYSTEM, RPCMEM_DEFAULT_FLAGS, allocate_bytes); + if (buf == nullptr) { + GGMLQNN_LOG_WARN("failed to allocate rpc memory\n"); + return nullptr; + } + + auto aligned_buf = reinterpret_cast(ggmlqnn_align_to(alignment, + reinterpret_cast(buf))); + bool status = _rpcmem_store_map.insert(std::pair(aligned_buf, buf)).second; + if (!status) { + GGMLQNN_LOG_WARN("failed to allocate rpc memory\n"); + _pfn_rpc_mem_free(buf); + } + + return aligned_buf; +} + +void qnn_instance::free_rpcmem(void * buf) { + if (!_rpcmem_initialized) { + GGMLQNN_LOG_WARN("rpc memory not initialized\n"); + } else if (0 == _rpcmem_store_map.count(buf)) { + GGMLQNN_LOG_WARN("no allocated tensor\n"); + } else { + _pfn_rpc_mem_free(_rpcmem_store_map[buf]); + _rpcmem_store_map.erase(buf); + } +} + +int32_t qnn_instance::rpcmem_to_fd(void * buf) { + int32_t mem_fd = -1; + if (!is_rpcmem_initialized()) { + GGMLQNN_LOG_WARN("rpc memory not initialized\n"); + } else { + mem_fd = _pfn_rpc_mem_to_fd(buf); + } + + return mem_fd; +} + +int qnn_instance::register_rpcmem(void * p_data, Qnn_Tensor_t * p_tensor) { + if (nullptr == p_data || (nullptr == p_tensor)) { + GGMLQNN_LOG_WARN("invalid param\n"); + return 1; + } + + if (!is_rpcmem_initialized()) { + GGMLQNN_LOG_WARN("rpc memory not initialized\n"); + return 2; + } + + if (is_rpcmem_allocated(p_data)) { + GGMLQNN_LOG_WARN("rpc memory already allocated\n"); + //return 3; + } + if (is_rpcmem_registered((QNN_VER_PTR(*p_tensor)->memHandle))) { + GGMLQNN_LOG_WARN("tensor %s has been registered shared memory\n", (QNN_VER_PTR(*p_tensor)->name)); + return 4; + } + + int32_t mem_fd = rpcmem_to_fd(p_data); + if (-1 == mem_fd) { + GGMLQNN_LOG_WARN("failed to get file descriptor\n"); + return 5; + } + GGMLQNN_LOG_DEBUG("mem_fd %d\n", mem_fd); + Qnn_MemDescriptor_t descriptor = { + {QNN_VER_PTR(*p_tensor)->rank, QNN_VER_PTR(*p_tensor)->dimensions, nullptr}, + QNN_VER_PTR(*p_tensor)->dataType, + QNN_MEM_TYPE_ION, + {{mem_fd}}}; + Qnn_MemHandle_t handle = nullptr; + int error = QNN_SUCCESS; + error = _qnn_interface.qnn_mem_register( + _qnn_context_handle, + &descriptor, + /*numDescriptors=*/1, + &handle); + if (error != QNN_SUCCESS) { + GGMLQNN_LOG_WARN("failed to register shared memory, error %d, %s\n", QNN_GET_ERROR_CODE(error), + strerror(error)); + return 6; + } else { + GGMLQNN_LOG_INFO("tensor %s successfully register shared memory\n", (QNN_VER_PTR(*p_tensor)->name)); + } + QNN_VER_PTR(*p_tensor)->memHandle = handle; + _qnn_mem_set.insert(handle); + + return 0; +} + +Qnn_MemHandle_t qnn_instance::register_rpcmem(void * p_data, const uint32_t rank, uint32_t * dimensions, Qnn_DataType_t data_type) { + if (!p_data) { + GGMLQNN_LOG_WARN("invalid param"); + return nullptr; + } + + if (!is_rpcmem_initialized()) { + GGMLQNN_LOG_WARN("rpc memory not initialized"); + return nullptr; + } + + if (is_rpcmem_registered(p_data)) { + GGMLQNN_LOG_WARN("rpc memory already registered"); + return _qnn_rpc_buffer_to_handles[p_data]; + } + + auto mem_fd = rpcmem_to_fd(p_data); + if (mem_fd == -1) { + GGMLQNN_LOG_WARN("failed to get file descriptor"); + return nullptr; + } + + GGMLQNN_LOG_DEBUG("mem_fd %d", mem_fd); + Qnn_MemDescriptor_t descriptor = {{rank, dimensions, nullptr}, data_type, QNN_MEM_TYPE_ION, {{mem_fd}}}; + Qnn_MemHandle_t handle = nullptr; + auto error = _qnn_interface.qnn_mem_register(_qnn_context_handle, &descriptor, + /*numDescriptors=*/1, &handle); + if (error != QNN_SUCCESS) { + GGMLQNN_LOG_WARN("failed to register shared memory, error %d, %s", QNN_GET_ERROR_CODE(error), strerror(error)); + return nullptr; + } + + _qnn_rpc_buffer_to_handles.insert({p_data, handle}); + GGMLQNN_LOG_DEBUG("successfully register shared memory handler: %p", handle); + return handle; +} + +void qnn_instance::unregister_rpcmem() { + Qnn_ErrorHandle_t error = QNN_SUCCESS; + + if (_qnn_mem_set.empty()) { + GGMLQNN_LOG_WARN("no rpcmem registered\n"); + } + + for (auto &mem_handle : _qnn_mem_set) { + error = _qnn_interface.qnn_mem_de_register(&mem_handle, 1); + if (error != QNN_SUCCESS) { + GGMLQNN_LOG_WARN("failed to unregister shared memory, error %d\n", QNN_GET_ERROR_CODE(error)); + } + } + _qnn_mem_set.clear(); +} + +void qnn_instance::unregister_rpcmem(Qnn_MemHandle_t mem_handle) { + Qnn_ErrorHandle_t error = _qnn_interface.qnn_mem_de_register(&mem_handle, 1); + if (error != QNN_SUCCESS) { + GGMLQNN_LOG_WARN("failed to unregister shared memory, error %d", QNN_GET_ERROR_CODE(error)); + } + + auto it = std::find_if(_qnn_rpc_buffer_to_handles.begin(), _qnn_rpc_buffer_to_handles.end(), + [mem_handle](const auto &kv) { return kv.second == mem_handle; }); + if (it == _qnn_rpc_buffer_to_handles.end()) { + GGMLQNN_LOG_WARN("failed to find shared memory handler: %p", mem_handle); + return; + } + + _qnn_rpc_buffer_to_handles.erase(it); +} + +bool qnn_instance::is_rpcmem_allocated(void * buf) { + return _rpcmem_store_map.count(buf) != 0U; +} + +int qnn_instance::load_backend(std::string & lib_path, const QnnSaver_Config_t ** saver_config) { + Qnn_ErrorHandle_t error = QNN_SUCCESS; + GGMLQNN_LOG_DEBUG("lib_path:%s\n", lib_path.c_str()); + + void *lib_handle = dlopen(lib_path.c_str(), RTLD_NOW | RTLD_GLOBAL); + if (nullptr == lib_handle) { + GGMLQNN_LOG_WARN("can not open QNN library %s, with error: %s", lib_path.c_str(), dlerror()); + return 1; + } + + auto get_providers = + load_qnn_functionpointers<_pfn_QnnInterface_getProviders *>(lib_handle, + "QnnInterface_getProviders"); + if (nullptr == get_providers) { + GGMLQNN_LOG_WARN("can not load symbol QnnInterface_getProviders : %s", dlerror()); + return 2; + } + + // get QnnInterface Providers + std::uint32_t num_providers = 0; + const QnnInterface_t **provider_list = nullptr; + error = get_providers(&provider_list, &num_providers); + if (error != QNN_SUCCESS) { + GGMLQNN_LOG_WARN("failed to get providers, error %d", QNN_GET_ERROR_CODE(error)); + return 3; + } + GGMLQNN_LOG_DEBUG("num_providers=%d\n", num_providers); + if (num_providers != _required_num_providers) { + GGMLQNN_LOG_WARN("providers is %d instead of required %d", num_providers, _required_num_providers); + return 4; + } + + if (nullptr == provider_list) { + GGMLQNN_LOG_WARN("failed to get qnn interface providers\n"); + return 5; + } + bool found_valid_interface = false; + QNN_INTERFACE_VER_TYPE qnn_interface; + for (size_t idx = 0; idx < num_providers; idx++) { + if (QNN_API_VERSION_MAJOR == provider_list[idx]->apiVersion.coreApiVersion.major && + QNN_API_VERSION_MINOR <= provider_list[idx]->apiVersion.coreApiVersion.minor) { + found_valid_interface = true; + qnn_interface = provider_list[idx]->QNN_INTERFACE_VER_NAME; + break; + } + } + + if (!found_valid_interface) { + GGMLQNN_LOG_WARN("unable to find a valid qnn interface\n"); + return 6; + } else { + GGMLQNN_LOG_INFO("find a valid qnn interface\n"); + } + set_qnn_raw_interface(qnn_interface); + + BackendIdType backend_id = provider_list[0]->backendId; + _lib_path_to_backend_id[lib_path] = backend_id; + if (_loaded_backend.count(backend_id) > 0) { + GGMLQNN_LOG_WARN("lib_path %s is loaded, but backend %d already exists\n", + lib_path.c_str(), backend_id); + } + _loaded_backend[backend_id] = provider_list[0]; + if (_loaded_lib_handle.count(backend_id) > 0) { + GGMLQNN_LOG_WARN("closing %p\n", _loaded_lib_handle[backend_id]); + int dlclose_error = dlclose(_loaded_lib_handle[backend_id]); + if (dlclose_error != 0) { + GGMLQNN_LOG_WARN("fail to close %p with error %s\n", _loaded_lib_handle[backend_id], dlerror()); + } + } + _loaded_lib_handle[backend_id] = lib_handle; + _backend_id = backend_id; + +#if 0 //not used in PR, keep them here for further use + QnnSaver_Config_t outputdir_cfg; + outputdir_cfg.option = QNN_SAVER_CONFIG_OPTION_OUTPUT_DIRECTORY; + outputdir_cfg.outputDirectory = "/data/local/tmp/"; + QnnSaver_Config_t backendid_cfg; + backendid_cfg.option = QNN_SAVER_CONFIG_OPTION_BACKEND_ID; + backendid_cfg.backendId = _backend_id; + const QnnSaver_Config_t *saverCfg[] = {&outputdir_cfg, &backendid_cfg, nullptr}; + if (0 == QnnSaver_initialize(saverCfg)) { + GGMLQNN_LOG_INFO("QnnSaver_initialize successfully"); + } else { + GGMLQNN_LOG_WARN("QnnSaver_initialize failure"); + } +#endif + auto saver_initialize = + load_qnn_functionpointers<_pfn_QnnSaver_initialize *>( + _loaded_lib_handle[backend_id], "QnnSaver_initialize"); + if (nullptr != saver_initialize) { + error = saver_initialize(saver_config); + if (error != QNN_SUCCESS) { + GGMLQNN_LOG_WARN("failed to saver_initialize，error %d", QNN_GET_ERROR_CODE(error)); + return 7; + } + } else { + GGMLQNN_LOG_WARN("saver_initialize is null\n"); + } + + return 0; +} + +int qnn_instance::unload_backend() { + int dlclose_error = 0; + for (auto &it : _loaded_lib_handle) { + dlclose_error = dlclose(it.second); + if (dlclose_error != 0) { + GGMLQNN_LOG_WARN("failed to close QNN backend %d, error %s\n", it.first, dlerror()); + } + } + + _loaded_lib_handle.clear(); + _lib_path_to_backend_id.clear(); + _loaded_backend.clear(); + + return 0; +} + +int qnn_instance::load_system() { + Qnn_ErrorHandle_t error = QNN_SUCCESS; + + std::string system_lib_path = _lib_path + "libQnnSystem.so"; + GGMLQNN_LOG_DEBUG("system_lib_path:%s\n", system_lib_path.c_str()); + + _system_lib_handle = dlopen(system_lib_path.c_str(), RTLD_NOW | RTLD_LOCAL); + if (nullptr == _system_lib_handle) { + GGMLQNN_LOG_WARN("can not open QNN library %s, error: %s\n", system_lib_path.c_str(), dlerror()); + //re-try with Android APK's internal QNN runtime lib path + _lib_path = "/data/data/com.cdeos.kantv/qnnlib/"; + system_lib_path = _lib_path + "libQnnSystem.so"; + _system_lib_handle = dlopen(system_lib_path.c_str(), RTLD_NOW | RTLD_LOCAL); + if (nullptr == _system_lib_handle) { + GGMLQNN_LOG_WARN("can not open QNN library %s, error: %s\n", system_lib_path.c_str(), dlerror()); + return 1; + } + } + + auto * get_providers = reinterpret_cast<_pfn_QnnSystemInterface_getProviders *>(dlsym( + _system_lib_handle, "QnnSystemInterface_getProviders")); + if (nullptr == get_providers) { + GGMLQNN_LOG_WARN("can not load QNN symbol QnnSystemInterface_getProviders: %s\n", dlerror()); + return 2; + } + + uint32_t num_providers = 0; + const QnnSystemInterface_t ** provider_list = nullptr; + error = get_providers(&provider_list, &num_providers); + if (error != QNN_SUCCESS) { + GGMLQNN_LOG_WARN("failed to get providers, error %d\n", QNN_GET_ERROR_CODE(error)); + return 3; + } + + if (num_providers != _required_num_providers) { + GGMLQNN_LOG_WARN("providers is %d instead of required %d\n", num_providers, _required_num_providers); + return 4; + } + + if (nullptr == provider_list) { + GGMLQNN_LOG_WARN("can not get providers\n"); + return 5; + } + + QNN_SYSTEM_INTERFACE_VER_TYPE qnn_system_interface; + bool found_valid_system_interface = false; + for (size_t idx = 0; idx < num_providers; idx++) { + if (QNN_SYSTEM_API_VERSION_MAJOR == + provider_list[idx]->systemApiVersion.major && + QNN_SYSTEM_API_VERSION_MINOR <= + provider_list[idx]->systemApiVersion.minor) { + found_valid_system_interface = true; + qnn_system_interface = provider_list[idx]->QNN_SYSTEM_INTERFACE_VER_NAME; + break; + } + } + if (!found_valid_system_interface) { + GGMLQNN_LOG_WARN("unable to find a valid qnn system interface\n"); + return 6; + } else { + GGMLQNN_LOG_INFO("find a valid qnn system interface\n"); + } + set_qnn_raw_system_interface(qnn_system_interface); + + _qnn_interface.set_qnn_system_interface(provider_list[0]); + + _qnn_interface.qnn_system_context_create(&_qnn_system_handle); + if (nullptr == _qnn_system_handle) { + GGMLQNN_LOG_WARN("can not create QNN system contenxt\n"); + } else { + GGMLQNN_LOG_INFO("initialize qnn system successfully\n"); + } + + return 0; +} + +int qnn_instance::unload_system() { + int result = 0; + + if (nullptr == _system_lib_handle) { + GGMLQNN_LOG_DEBUG("system lib handle is null\n"); + return 1; + } + + if (nullptr != _qnn_system_handle) { + result = _qnn_interface.qnn_system_context_free(_qnn_system_handle); + if (result != QNN_SUCCESS) { + GGMLQNN_LOG_WARN("failed to free QNN system context\n"); + } + _qnn_system_handle = nullptr; + } + + int dlclose_error = dlclose(_system_lib_handle); + if (dlclose_error != 0) { + GGMLQNN_LOG_WARN("failed to close QnnSystem library, error %s\n", dlerror()); + return 2; + } + + _system_lib_handle = nullptr; + + return result; +} + +static void ggml_qnn_logcallback(const char * fmt, + QnnLog_Level_t level, + uint64_t timestamp, + va_list argp) { + + static std::mutex log_mutex; + static unsigned char s_ggml_qnn_logbuf[GGML_QNN_LOGBUF_LEN]; + + const char * log_level_desc = ""; + switch (level) { + case QNN_LOG_LEVEL_ERROR: + log_level_desc = " ERROR "; + break; + case QNN_LOG_LEVEL_WARN: + log_level_desc = "WARNING"; + break; + case QNN_LOG_LEVEL_INFO: + log_level_desc = " INFO "; + break; + case QNN_LOG_LEVEL_DEBUG: + log_level_desc = " DEBUG "; + break; + case QNN_LOG_LEVEL_VERBOSE: + log_level_desc = "VERBOSE"; + break; + case QNN_LOG_LEVEL_MAX: + log_level_desc = "UNKNOWN"; + break; + } + + double ms = (double) timestamp / 1000000.0; + + { + std::lock_guard lock(log_mutex); + + memset(s_ggml_qnn_logbuf, 0, GGML_QNN_LOGBUF_LEN); + vsnprintf(reinterpret_cast(s_ggml_qnn_logbuf), GGML_QNN_LOGBUF_LEN, fmt, argp); +#if GGMLQNN_PRINT_QNN_INTERNAL_LOG + GGMLQNN_LOG_INFO("%8.1fms [%-7s] %s\n", ms, log_level_desc, s_ggml_qnn_logbuf); +#endif + } +} + +int qnn_instance::qnn_init(const QnnSaver_Config_t ** saver_config) { + BackendIdType backend_id = QNN_BACKEND_ID_NULL; + GGMLQNN_LOG_DEBUG("enter qni_init\n"); + + const std::lock_guard lock(_init_mutex); + + if (0 != load_system()) { + GGMLQNN_LOG_WARN("can not load QNN system lib, pls check why?\n"); + return 1; + } else { + GGMLQNN_LOG_DEBUG("load QNN system lib successfully\n"); + } + + std::string bakend_lib_path = _lib_path + _backend_name; + if (0 == _lib_path_to_backend_id.count(bakend_lib_path)) { + int is_load_ok = load_backend(bakend_lib_path, saver_config); + if (0 != is_load_ok) { + GGMLQNN_LOG_WARN("failed to load QNN backend\n"); + return 2; + } + } + + backend_id = _lib_path_to_backend_id[bakend_lib_path]; + if (0 == _loaded_backend.count(backend_id) || + 0 == _loaded_lib_handle.count(backend_id)) { + GGMLQNN_LOG_WARN("library %s is loaded but loaded backend count=%zu, loaded lib_handle count=%zu\n", + bakend_lib_path.c_str(), + _loaded_backend.count(backend_id), + _loaded_lib_handle.count(backend_id)); + return 3; + } + + _qnn_interface.set_qnn_interface(_loaded_backend[backend_id]); + +#if 1 + _qnn_interface.qnn_log_create(ggml_qnn_logcallback, _qnn_log_level, &_qnn_log_handle); +#else + _qnn_raw_interface.logCreate(ggml_qnn_logcallback, _qnn_log_level, &_qnn_log_handle); +#endif + if (nullptr == _qnn_log_handle) { + GGMLQNN_LOG_WARN("why failed to initialize qnn log\n"); //NPU backend not work on Qualcomm SoC based low-end phone + return 4; + } else { + GGMLQNN_LOG_DEBUG("initialize qnn log successfully\n"); + } + + std::vector temp_backend_config; + _qnn_interface.qnn_backend_create(_qnn_log_handle, + temp_backend_config.empty() ? nullptr : temp_backend_config.data(), + &_qnn_backend_handle); + if (nullptr == _qnn_backend_handle) { + GGMLQNN_LOG_WARN("why failed to initialize qnn backend\n"); + return 5; + } else { + GGMLQNN_LOG_DEBUG("initialize qnn backend successfully\n"); + } + + if (nullptr != _qnn_raw_interface.propertyHasCapability) { + auto qnnstatus = _qnn_raw_interface.propertyHasCapability(QNN_PROPERTY_GROUP_DEVICE); + if (QNN_PROPERTY_NOT_SUPPORTED == qnnstatus) { + GGMLQNN_LOG_WARN("device property is not supported\n"); + } + if (QNN_PROPERTY_ERROR_UNKNOWN_KEY == qnnstatus) { + GGMLQNN_LOG_WARN("device property is not known to backend\n"); + } + } + + auto qnnstatus = _qnn_raw_interface.deviceCreate( + _qnn_log_handle, nullptr, &_qnn_device_handle); + if (QNN_SUCCESS != qnnstatus && QNN_DEVICE_ERROR_UNSUPPORTED_FEATURE != qnnstatus) { + GGMLQNN_LOG_WARN("failed to create QNN device\n"); + } else { + GGMLQNN_LOG_INFO("create device successfully\n"); + } + + if (ggml_qnn_profile_level::profile_off != _profile_level) { + GGMLQNN_LOG_INFO("profiling turned on; level = %d", _profile_level); + if (ggml_qnn_profile_level::profile_basic == _profile_level) { + GGMLQNN_LOG_INFO("basic profiling requested. creating Qnn Profile object\n"); + if (QNN_PROFILE_NO_ERROR != _qnn_raw_interface.profileCreate( + _qnn_backend_handle, QNN_PROFILE_LEVEL_BASIC, &_qnn_profile_handle)) { + GGMLQNN_LOG_WARN("unable to create profile handle in the backend\n"); + return 7; + } else { + GGMLQNN_LOG_DEBUG("initialize qnn profile successfully\n"); + } + } else if (ggml_qnn_profile_level::profile_detail == _profile_level) { + GGMLQNN_LOG_INFO("detailed profiling requested. Creating Qnn Profile object\n"); + if (QNN_PROFILE_NO_ERROR != _qnn_raw_interface.profileCreate( + _qnn_backend_handle, QNN_PROFILE_LEVEL_DETAILED, &_qnn_profile_handle)) { + GGMLQNN_LOG_WARN("unable to create profile handle in the backend\n"); + return 7; + } else { + GGMLQNN_LOG_DEBUG("initialize qnn profile successfully\n"); + } + } + } + + _rpc_lib_handle = dlopen("libcdsprpc.so", RTLD_NOW | RTLD_LOCAL); + if (nullptr == _rpc_lib_handle) { + GGMLQNN_LOG_WARN("failed to load qualcomm's rpc lib, error:%s\n", dlerror()); + return 9; + } else { + GGMLQNN_LOG_DEBUG("load rpcmem lib successfully\n"); + set_rpcmem_initialized(true); + } + _pfn_rpc_mem_init = reinterpret_cast(dlsym(_rpc_lib_handle, "rpcmem_init")); + _pfn_rpc_mem_deinit = reinterpret_cast(dlsym(_rpc_lib_handle, "rpcmem_deinit")); + _pfn_rpc_mem_alloc = reinterpret_cast(dlsym(_rpc_lib_handle,"rpcmem_alloc")); + _pfn_rpc_mem_free = reinterpret_cast(dlsym(_rpc_lib_handle, "rpcmem_free")); + _pfn_rpc_mem_to_fd = reinterpret_cast(dlsym(_rpc_lib_handle,"rpcmem_to_fd")); + if (nullptr == _pfn_rpc_mem_alloc || nullptr == _pfn_rpc_mem_free + || nullptr == _pfn_rpc_mem_to_fd) { + GGMLQNN_LOG_WARN("unable to access symbols in QNN RPC lib. dlerror(): %s", dlerror()); + dlclose(_rpc_lib_handle); + return 10; + } + + if (nullptr != _pfn_rpc_mem_init) // make Qualcomm's SoC based low-end phone happy + _pfn_rpc_mem_init(); + + std::vector temp_context_config; + _qnn_interface.qnn_context_create(_qnn_backend_handle, _qnn_device_handle, + temp_context_config.empty() ? nullptr : temp_context_config.data(), + &_qnn_context_handle); + if (nullptr == _qnn_context_handle) { + GGMLQNN_LOG_WARN("why failed to initialize qnn context\n"); + return 8; + } else { + GGMLQNN_LOG_DEBUG("initialize qnn context successfully\n"); + } + + if (_backend_name.find("Htp") != std::variant_npos) { + const QnnDevice_PlatformInfo_t * p_info = nullptr; + _qnn_raw_interface.deviceGetPlatformInfo(nullptr, &p_info); + GGMLQNN_LOG_INFO("device counts %d", p_info->v1.numHwDevices); + QnnDevice_HardwareDeviceInfo_t * infos = p_info->v1.hwDevices; + for (int i = 0; i < p_info->v1.numHwDevices; i++) { + GGMLQNN_LOG_INFO("deviceID:%d, deviceType:%d, numCores %d", infos[i].v1.deviceId, + infos[i].v1.deviceType, infos[i].v1.numCores); + QnnDevice_DeviceInfoExtension_t devinfo = infos[i].v1.deviceInfoExtension; + QnnHtpDevice_OnChipDeviceInfoExtension_t chipinfo = devinfo->onChipDevice; + QnnHtpDevice_Arch_t htp_arch = chipinfo.arch; + GGMLQNN_LOG_INFO("htp_type:%d(%s)", devinfo->devType, + (devinfo->devType == QNN_HTP_DEVICE_TYPE_ON_CHIP) ? "QNN_HTP_DEVICE_TYPE_ON_CHIP" : "QNN_HTP_DEVICE_TYPE_UNKNOWN"); + GGMLQNN_LOG_INFO("qualcomm soc_model:%d(%s), htp_arch:%d(%s), vtcm_size:%d MB", \ + chipinfo.socModel, qnn_get_socmodel_desc(chipinfo.socModel), \ + htp_arch, qnn_get_htparch_desc(htp_arch), chipinfo.vtcmSize); + struct qcom_socinfo * socinfo = qnn_get_socinfo_from_socmodel(chipinfo.socModel); + g_qnn_mgr[QNN_BACKEND_NPU].socinfo = { chipinfo.socModel, htp_arch, chipinfo.vtcmSize }; + if (nullptr != socinfo) { + memcpy(g_qnn_mgr[QNN_BACKEND_NPU].socinfo.soc_desc, socinfo->soc_desc, sizeof(socinfo->soc_desc)); + GGMLQNN_LOG_INFO("soc info:%s", socinfo->soc_desc); + } else { + memcpy(g_qnn_mgr[QNN_BACKEND_NPU].socinfo.soc_desc, "unknown", 7); + GGMLQNN_LOG_INFO("soc info:unknown"); + } + } + _qnn_raw_interface.deviceFreePlatformInfo(nullptr, p_info); + + //TODO: faster approach to probe the accurate capacity of QNN RPC ion memory + size_t candidate_size = 0; + uint8_t * rpc_buffer = nullptr; + const int SIZE_IN_MB = (1 << 20); + size_t probe_slots[] = {1024, 1536, 2048 - 48, 2048}; + size_t probe_counts = sizeof(probe_slots) / sizeof(size_t); + for (size_t idx = 0; idx < probe_counts; idx++) { + rpc_buffer = static_cast(alloc_rpcmem(probe_slots[idx] * SIZE_IN_MB, 4)); + if (nullptr == rpc_buffer) { + GGMLQNN_LOG_DEBUG("alloc rpcmem %d (MB) failure, %s\n", probe_slots[idx], strerror(errno)); + break; + } else { + candidate_size = probe_slots[idx]; + free_rpcmem(rpc_buffer); + rpc_buffer = nullptr; + } + } + if (candidate_size > _rpcmem_capacity) + _rpcmem_capacity = candidate_size; + GGMLQNN_LOG_INFO("capacity of rpc ion memory %d MB\n", _rpcmem_capacity); + + if (0 != init_htp_perfinfra()) { + GGMLQNN_LOG_WARN("initialize HTP performance failure"); + } + if (0 != set_rpc_polling()) { + GGMLQNN_LOG_WARN("set RPC polling failure"); + } + if (0 != set_high_performance_mode()) { + GGMLQNN_LOG_WARN("set HTP high performance mode failure"); + } + } + + GGMLQNN_LOG_DEBUG("leave qni_init\n"); + + return 0; +} + +int qnn_instance::qnn_finalize() { + int ret_status = 0; + Qnn_ErrorHandle_t error = QNN_SUCCESS; + + //FIXME:should be removed in the future + reset_idx(); + + if (nullptr != _pfn_rpc_mem_deinit) // make Qualcomm's mobile SoC equipped low-end phone happy + _pfn_rpc_mem_deinit(); + + if (dlclose(_rpc_lib_handle) != 0) { + GGMLQNN_LOG_WARN("failed to unload qualcomm's rpc lib, error:%s\n", dlerror()); + } else { + GGMLQNN_LOG_DEBUG("succeed to close rpcmem lib\n"); + } + + if (nullptr != _qnn_context_handle) { + error = _qnn_interface.qnn_context_free(_qnn_context_handle, _qnn_profile_handle); + if (error != QNN_SUCCESS) { + GGMLQNN_LOG_WARN("failed to free QNN context_handle: ID %u, error %d\n", + _qnn_interface.get_backend_id(), QNN_GET_ERROR_CODE(error)); + + } + _qnn_context_handle = nullptr; + } + + if (nullptr != _qnn_profile_handle) { + error = _qnn_interface.qnn_profile_free(_qnn_profile_handle); + if (error != QNN_SUCCESS) { + GGMLQNN_LOG_WARN("failed to free QNN profile_handle: ID %u, error %d\n", + _qnn_interface.get_backend_id(), QNN_GET_ERROR_CODE(error)); + + } + _qnn_profile_handle = nullptr; + } + + if (nullptr != _qnn_device_handle) { + error = _qnn_interface.qnn_device_free(_qnn_device_handle); + if (error != QNN_SUCCESS) { + GGMLQNN_LOG_WARN("failed to free QNN device_handle: ID %u, error %d\n", + _qnn_interface.get_backend_id(), QNN_GET_ERROR_CODE(error)); + + } + _qnn_device_handle = nullptr; + } + + if (nullptr != _qnn_backend_handle) { + error = _qnn_interface.qnn_backend_free(_qnn_backend_handle); + if (error != QNN_SUCCESS) { + GGMLQNN_LOG_WARN("failed to free QNN backend_handle: ID %u, error %d\n", + _qnn_interface.get_backend_id(), QNN_GET_ERROR_CODE(error)); + } + _qnn_backend_handle = nullptr; + + } + + if (nullptr != _qnn_log_handle) { + error = _qnn_interface.qnn_log_free(_qnn_log_handle); + if (error != QNN_SUCCESS) { + GGMLQNN_LOG_WARN("failed to free QNN log_handle: ID %u, error %d\n", + _qnn_interface.get_backend_id(), QNN_GET_ERROR_CODE(error)); + } + _qnn_log_handle = nullptr; + } + + unload_backend(); + + unload_system(); + + return ret_status; +} + +int qnn_instance::init_qnn_graph(const std::string & graph_name, QNNBackend device, size_t vtcm_size_in_mb) { + _graph_name = graph_name; + _device_id = device; + + GGMLQNN_LOG_DEBUG("[%s][%s]created", ggml_backend_qnn_get_devname(device), graph_name.c_str()); + + Qnn_ErrorHandle_t error = QNN_SUCCESS; + Qnn_GraphHandle_t graph_handle = nullptr; + if (device == QNN_BACKEND_NPU) { + QnnHtpGraph_CustomConfig_t hvx_config; + hvx_config.option = QNN_HTP_GRAPH_CONFIG_OPTION_NUM_HVX_THREADS; + hvx_config.numHvxThreads = 8; + QnnGraph_Config_t graph_hvx_config; + graph_hvx_config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM; + graph_hvx_config.customConfig = &hvx_config; + + QnnHtpGraph_CustomConfig_t dlbc_config; + dlbc_config.option = QNN_HTP_GRAPH_CONFIG_OPTION_OPTIMIZATION; + dlbc_config.optimizationOption.type = QNN_HTP_GRAPH_OPTIMIZATION_TYPE_ENABLE_DLBC; + dlbc_config.optimizationOption.floatValue = 1.0; // set to 0.0 to turn off DLBC + QnnGraph_Config_t graph_dlbc_config; + graph_dlbc_config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM; + graph_dlbc_config.customConfig = &dlbc_config; + + QnnHtpGraph_CustomConfig_t opt_config; + opt_config.optimizationOption.type = QNN_HTP_GRAPH_OPTIMIZATION_TYPE_FINALIZE_OPTIMIZATION_FLAG; + opt_config.optimizationOption.floatValue = 1; // 1 / 3 + QnnGraph_Config_t graph_opt_config; + graph_opt_config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM; + graph_opt_config.customConfig = &opt_config; + + QnnHtpGraph_CustomConfig_t vtcm_config; + vtcm_config.option = QNN_HTP_GRAPH_CONFIG_OPTION_VTCM_SIZE; + vtcm_config.vtcmSizeInMB = vtcm_size_in_mb; + QnnGraph_Config_t graph_vtcm_config; + graph_vtcm_config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM; + graph_vtcm_config.customConfig = &vtcm_config; + + const QnnGraph_Config_t * graph_configs[] = {&graph_hvx_config, &graph_dlbc_config, &graph_vtcm_config, + &graph_opt_config, nullptr}; + error = _qnn_interface.qnn_graph_create(_qnn_context_handle, graph_name.c_str(), graph_configs, &graph_handle); + } else { + error = _qnn_interface.qnn_graph_create(_qnn_context_handle, graph_name.c_str(), nullptr, &graph_handle); + } + + if (error != QNN_SUCCESS) { + GGMLQNN_LOG_ERROR("[%s][%s]failed to create qnn graph, error: %s", + ggml_backend_qnn_get_devname(device), graph_name.c_str(), + qnn_get_error_string(error)); + return error; + } + + GGMLQNN_LOG_INFO("[%s]create graph %s succeed", ggml_backend_qnn_get_devname(device), graph_name.c_str()); + _qnn_graph_handle = graph_handle; + return QNN_SUCCESS; +} + +int qnn_instance::init_qnn_graph(const char * graph_name, bool debug, uint8_t do_node_validation, + const QnnGraph_Config_t ** graph_configs) { + int result = 0; + + if (nullptr == graph_name) { + GGMLQNN_LOG_WARN("graph name is null\n"); + return 1; + } + + if (!_graph_name.empty()) { + GGMLQNN_LOG_WARN("qnn model for graph %s already initialized\n", graph_name); + return 2; + } + + if (!do_node_validation) { + GGMLQNN_LOG_WARN("node validation disabled, backend will not perform op validation prior to adding node\n"); + } + + _graph_name = graph_name; + _debug_tensor = debug; + _do_node_validations = do_node_validation; + + result = _qnn_raw_interface.graphCreate(_qnn_context_handle, + graph_name, + graph_configs, + &_qnn_graph_handle); + if (result != QNN_GRAPH_NO_ERROR || nullptr == _qnn_graph_handle) { + GGMLQNN_LOG_WARN("failed to create graph in qnn context\n"); + return 3; + } else { + GGMLQNN_LOG_INFO("succeed to create graph %s, %p\n", graph_name, _qnn_graph_handle); + } + + return 0; +} + +int qnn_instance::finalize_qnn_graph() { + if (nullptr != _qnn_graph_handle) { + if (_qnn_raw_interface.graphFinalize(_qnn_graph_handle, + _qnn_profile_handle, nullptr) + != QNN_GRAPH_NO_ERROR) { + GGMLQNN_LOG_WARN("finalizing graph failure\n"); + return 1; + } + } else { + GGMLQNN_LOG_DEBUG("qnn graph handle is null\n"); + } + + return 0; +} + +// ================================================================================================= +// section-6: implementation of ggml-qnn backend +// ================================================================================================= +static bool ggml_qnn_can_handle_op(const struct ggml_tensor * tensor, bool b_dump_tensor_info) { + if (tensor->op == GGML_OP_NONE) { + return true; + } + if (ggml_is_empty(tensor) || tensor->op == GGML_OP_RESHAPE + || tensor->op == GGML_OP_TRANSPOSE || tensor->op == GGML_OP_VIEW + || tensor->op == GGML_OP_PERMUTE) { + return false; + } + + bool supported_op = ((tensor->op == GGML_OP_ADD) || (tensor->op == GGML_OP_MUL_MAT)); + if (!supported_op) { + return false; + } + + struct ggml_tensor * src0 = tensor->src[0]; + struct ggml_tensor * src1 = tensor->src[1]; + + int64_t ne00 = tensor->src[0]->ne[0]; + int64_t ne01 = tensor->src[0]->ne[1]; + + int64_t ne10 = tensor->src[1]->ne[0]; + int64_t ne11 = tensor->src[1]->ne[1]; + + int64_t ne0 = tensor->ne[0]; + int64_t ne1 = tensor->ne[1]; + + if (tensor->op == GGML_OP_ADD) { + if (!ggml_are_same_shape(src0, src1)) { + return false; + } +#if GGMLQNN_PRINT_OP_ADD_LOG + if (b_dump_tensor_info) { + GGMLQNN_LOG_DEBUG("op name:%s, tensor type:%s", ggml_op_name(tensor->op), + ggml_type_name(tensor->type)); + GGMLQNN_LOG_DEBUG("src0 type:%s", ggml_type_name(tensor->src[0]->type)); + GGMLQNN_LOG_DEBUG("src1 type:%s", ggml_type_name(tensor->src[1]->type)); + GGMLQNN_LOG_DEBUG("GGML_OP_ADD"); + GGMLQNN_LOG_DEBUG( + "src0 %15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", + src0->name, + src0->type, ggml_type_name(src0->type), src0->ne[0], src0->ne[1], src0->ne[2], + src0->nb[0], src0->nb[1], src0->nb[2]); + GGMLQNN_LOG_DEBUG( + "src1 %15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", + src1->name, + src1->type, ggml_type_name(src1->type), src1->ne[0], src1->ne[1], src1->ne[2], + src1->nb[0], src1->nb[1], src1->nb[2]); + GGMLQNN_LOG_DEBUG( + " %15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", + tensor->name, + tensor->type, ggml_type_name(tensor->type), tensor->ne[0], tensor->ne[1], + tensor->ne[2], + tensor->nb[0], + tensor->nb[1], tensor->nb[2]); + + } +#endif + return (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16) + && (src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16); + + } + + if (tensor->op == GGML_OP_MUL_MAT) { +#if GGMLQNN_PRINT_OP_MUL_MAT_LOG + if (b_dump_tensor_info) { + GGMLQNN_LOG_DEBUG("op name:%s, tensor type:%s", ggml_op_name(tensor->op), + ggml_type_name(tensor->type)); + GGMLQNN_LOG_DEBUG("src0 type:%s", ggml_type_name(tensor->src[0]->type)); + GGMLQNN_LOG_DEBUG("src1 type:%s", ggml_type_name(tensor->src[1]->type)); + GGMLQNN_LOG_DEBUG("dst type:%s", ggml_type_name(tensor->type)); + GGMLQNN_LOG_DEBUG( + "src0 %15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", + src0->name, + src0->type, ggml_type_name(src0->type), src0->ne[0], src0->ne[1], src0->ne[2], + src0->nb[0], src0->nb[1], src0->nb[2]); + GGMLQNN_LOG_DEBUG( + "src1 %15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", + src1->name, + src1->type, ggml_type_name(src1->type), src1->ne[0], src1->ne[1], src1->ne[2], + src1->nb[0], src1->nb[1], src1->nb[2]); + GGMLQNN_LOG_DEBUG( + "dst %15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", + tensor->name, + tensor->type, ggml_type_name(tensor->type), tensor->ne[0], tensor->ne[1], + tensor->ne[2], + tensor->nb[0], + tensor->nb[1], tensor->nb[2]); + + } +#endif + //FIXME: 2048 is an experimental value between ASR inference and LLM inference because + // it's better only offload big matrix to QNN backend + if (ne01 <= 2048) { + return false; + } +#if 0 + //TODO: offload mul_mat to QNN backend + //we need to process type traint in func ggml_qnn_mul_mat(...) with following case: + //src0: q4_0, q6_k + //src1: f32 + //dst : f32 + return (src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16) + && (tensor->type == GGML_TYPE_F32 || tensor->type == GGML_TYPE_F16); +#else + //passthrough mul_mat + return (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16) + && (src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16) + && (src0->type == src1->type) && (src0->type == tensor->type); +#endif + } + + //TODO:for other op + return (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16) + && (src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16) + && (src0->type == src1->type) && (src0->type == tensor->type); +} + +static void ggml_qnn_add(ggml_backend_t backend, ggml_tensor * op) { + Qnn_ErrorHandle_t error = QNN_SUCCESS; + enum ggml_status result = GGML_STATUS_SUCCESS; + bool graph_initialized = false; + qnn_instance * instance = nullptr; + ggml_backend_qnn_context * ctx = (ggml_backend_qnn_context *)backend->context; + std::string graph_name = "ggml_op_qnn_add"; + qnn_perf op_perf = qnn_perf("ggml_qnn_add"); + Qnn_GraphHandle_t graph_handle = nullptr; + Qnn_Tensor_t * tensor_0 = nullptr; + Qnn_Tensor_t * tensor_1 = nullptr; + Qnn_Tensor_t * tensor_2 = nullptr; + Qnn_Param_t qnn_params[] = {}; + enum ggml_op ggmlop = GGML_OP_ADD; + Qnn_DataType_t src0_qnn_type = QNN_DATATYPE_FLOAT_32; + Qnn_DataType_t src1_qnn_type = QNN_DATATYPE_FLOAT_32; + Qnn_DataType_t dst_qnn_type = QNN_DATATYPE_FLOAT_32; + + instance = ctx->instance; + QNN_INTERFACE_VER_TYPE qnn_raw_interface = ctx->raw_interface; + + const ggml_tensor * src0 = op->src[0]; + const ggml_tensor * src1 = op->src[1]; + ggml_tensor * dst = op; + op_perf.start(); + + std::string map_entry; + get_graph_key_from_op(op, map_entry); + if (instance->_qnn_graph_map.find(map_entry) != instance->_qnn_graph_map.end()) { + graph_initialized = true; + auto & graph_item = instance->_qnn_graph_map[map_entry]; + graph_handle = std::get<0>(graph_item); + tensor_0 = std::get<1>(graph_item); + tensor_1 = std::get<2>(graph_item); + tensor_2 = std::get<3>(graph_item); + } else { + tensor_0 = ggml_qnn_create_tensor(src0); + tensor_1 = ggml_qnn_create_tensor(src1); + tensor_2 = ggml_qnn_create_tensor(dst); + } + +//#if GGMLQNN_DEBUG //uncomment this line and comment next line when troubleshooting mul_mat issue +#if GGMLQNN_PRINT_OP_ADD_LOG + GGMLQNN_LOG_DEBUG("call %s in dev %s\n", __func__, ctx->name); + GGMLQNN_LOG_DEBUG("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", + src0->name, + src0->type, ggml_type_name(src0->type), src0->ne[0], src0->ne[1], src0->ne[2], + src0->nb[0], src0->nb[1], src0->nb[2]); + GGMLQNN_LOG_DEBUG("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", + src1->name, + src1->type, ggml_type_name(src1->type), src1->ne[0], src1->ne[1], src1->ne[2], + src1->nb[0], src1->nb[1], src1->nb[2]); + GGMLQNN_LOG_DEBUG("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", + dst->name, + dst->type, ggml_type_name(dst->type), dst->ne[0], dst->ne[1], dst->ne[2], dst->nb[0], + dst->nb[1], dst->nb[2]); + GGMLQNN_LOG_DEBUG("%d, %d, %d, %d", src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3]); + GGMLQNN_LOG_DEBUG("tensor0 name %s", QNN_TENSOR_GET_NAME(tensor_0)); + GGMLQNN_LOG_DEBUG("tensor1 name %s", QNN_TENSOR_GET_NAME(tensor_1)); + GGMLQNN_LOG_DEBUG("tensor2 name %s", QNN_TENSOR_GET_NAME(tensor_2)); +#endif + + QNN_VER_PTR(*tensor_0)->type = QNN_TENSOR_TYPE_APP_WRITE; + QNN_VER_PTR(*tensor_1)->type = QNN_TENSOR_TYPE_APP_WRITE; + QNN_VER_PTR(*tensor_2)->type = QNN_TENSOR_TYPE_APP_READ; + + src0_qnn_type = qnn_datatype_from_ggml_datatype(src0->type); + src1_qnn_type = qnn_datatype_from_ggml_datatype(src1->type); + dst_qnn_type = qnn_datatype_from_ggml_datatype(dst->type); + + uint32_t * tensor_0_dimensions = QNN_VER_PTR(*tensor_0)->dimensions; + uint32_t * tensor_1_dimensions = QNN_VER_PTR(*tensor_1)->dimensions; + uint32_t * tensor_2_dimensions = QNN_VER_PTR(*tensor_2)->dimensions; + + if (!graph_initialized) { + graph_name = map_entry; + GGMLQNN_LOG_DEBUG("graph name %s", graph_name.c_str()); + if (ctx->device == QNN_BACKEND_NPU) { + QnnHtpGraph_CustomConfig_t hvx_config; + hvx_config.option = QNN_HTP_GRAPH_CONFIG_OPTION_NUM_HVX_THREADS; + hvx_config.numHvxThreads = 4; + QnnGraph_Config_t graph_hvx_config; + graph_hvx_config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM; + graph_hvx_config.customConfig = &hvx_config; + + QnnHtpGraph_CustomConfig_t dlbc_config; + dlbc_config.option = QNN_HTP_GRAPH_CONFIG_OPTION_OPTIMIZATION; + dlbc_config.optimizationOption.type = QNN_HTP_GRAPH_OPTIMIZATION_TYPE_ENABLE_DLBC; + dlbc_config.optimizationOption.floatValue = 1.0; // set to 0.0 to turn off DLBC + QnnGraph_Config_t graph_dlbc_config; + graph_dlbc_config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM; + graph_dlbc_config.customConfig = &dlbc_config; + + QnnHtpGraph_CustomConfig_t opt_config; + opt_config.optimizationOption.type = QNN_HTP_GRAPH_OPTIMIZATION_TYPE_FINALIZE_OPTIMIZATION_FLAG; + opt_config.optimizationOption.floatValue = 3; // 1 or 3 + QnnGraph_Config_t graph_opt_config; + graph_opt_config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM; + graph_opt_config.customConfig = &opt_config; + + QnnHtpGraph_CustomConfig_t vtcm_config; + vtcm_config.option = QNN_HTP_GRAPH_CONFIG_OPTION_VTCM_SIZE; + vtcm_config.vtcmSizeInMB = ctx->socinfo.vtcm_size_in_mb; + QnnGraph_Config_t graph_vtcm_config; + graph_vtcm_config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM; + graph_vtcm_config.customConfig = &vtcm_config; + + QnnHtpGraph_CustomConfig_t precision_config; + precision_config.option = QNN_HTP_GRAPH_CONFIG_OPTION_PRECISION; + precision_config.precision = QNN_PRECISION_FLOAT16; + QnnGraph_Config_t graph_precision_config; + graph_precision_config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM; + graph_precision_config.customConfig = &precision_config; + + const QnnGraph_Config_t * p_graphconfig[] = {&graph_hvx_config, + &graph_dlbc_config, + &graph_vtcm_config, + &graph_opt_config, + &graph_precision_config, + NULL}; + error = qnn_raw_interface.graphCreate(instance->get_qnn_context_handle(), + graph_name.c_str(), + p_graphconfig, &graph_handle); + } else { + error = qnn_raw_interface.graphCreate(instance->get_qnn_context_handle(), + graph_name.c_str(), + nullptr, &graph_handle); + } + if (QNN_SUCCESS != error) { + GGMLQNN_LOG_INFO("can't create qnn graph handle with graph name %s, error = %d\n", graph_name.c_str(), error); + return; + } + error = qnn_raw_interface.tensorCreateGraphTensor(graph_handle, tensor_0); + if (QNN_SUCCESS != error) { + GGMLQNN_LOG_INFO("error = %d\n", error); + } + error = qnn_raw_interface.tensorCreateGraphTensor(graph_handle, tensor_1); + if (QNN_SUCCESS != error) { + GGMLQNN_LOG_INFO("error = %d\n", error); + } + error = qnn_raw_interface.tensorCreateGraphTensor(graph_handle, tensor_2); + if (QNN_SUCCESS != error) { + GGMLQNN_LOG_INFO("error = %d\n", error); + } + + QNN_VER_PTR(*tensor_0)->clientBuf = {src0->data, ggml_get_tensor_data_size(src0)}; + QNN_VER_PTR(*tensor_1)->clientBuf = {src1->data, ggml_get_tensor_data_size(src1)}; + QNN_VER_PTR(*tensor_2)->clientBuf = {dst->data, ggml_get_tensor_data_size(dst)}; + + Qnn_Tensor_t tensor_inputs[] = { + *tensor_0, + *tensor_1 + }; + Qnn_Tensor_t tensor_outputs[] = { + *tensor_2 + }; + Qnn_OpConfig_t op_config = { + (Qnn_OpConfigVersion_t) 1, .v1 = { + "ggml_op_add", + QNN_OP_PACKAGE_NAME_QTI_AISW, + QNN_OP_ELEMENT_WISE_ADD, + 0, + qnn_params, + 2, + tensor_inputs, + 1, + tensor_outputs + } + }; + error = qnn_raw_interface.graphAddNode(graph_handle, op_config); + if (QNN_SUCCESS != error) { + GGMLQNN_LOG_INFO("error = %d\n", error); + } + error = qnn_raw_interface.graphFinalize(graph_handle, nullptr, nullptr); + if (QNN_SUCCESS != error) { + GGMLQNN_LOG_INFO("error = %d\n", error); + } + error = qnn_raw_interface.graphExecute(graph_handle, + tensor_inputs, 2, + tensor_outputs, 1, + nullptr, nullptr); + if (QNN_SUCCESS != error) { + GGMLQNN_LOG_INFO("error = %d\n", error); + } + auto graph_item = std::make_tuple(graph_handle, tensor_0, tensor_1, tensor_2); + instance->_qnn_graph_map[map_entry] = graph_item; + } else { + uint32_t dimensions_input_0[] = {(uint32_t) src0->ne[0], (uint32_t) src0->ne[1], + (uint32_t) src0->ne[2], (uint32_t) src0->ne[3]}; + uint32_t dimensions_input_1[] = {(uint32_t) src1->ne[0], (uint32_t) src1->ne[1], + (uint32_t) src1->ne[2], (uint32_t) src1->ne[3]}; + uint32_t dimensions_output[] = {(uint32_t) dst->ne[0], (uint32_t) dst->ne[1], + (uint32_t) dst->ne[2], (uint32_t) dst->ne[3]}; + QNN_VER_PTR(*tensor_0)->dimensions = dimensions_input_0; + QNN_VER_PTR(*tensor_0)->rank = ggml_get_tensor_rank(src0); + QNN_VER_PTR(*tensor_0)->dataType = src0_qnn_type; + QNN_VER_PTR(*tensor_1)->dimensions = dimensions_input_1; + QNN_VER_PTR(*tensor_1)->rank = ggml_get_tensor_rank(src1); + QNN_VER_PTR(*tensor_1)->dataType = src1_qnn_type; + QNN_VER_PTR(*tensor_2)->dimensions = dimensions_output; + QNN_VER_PTR(*tensor_2)->rank = ggml_get_tensor_rank(dst); + QNN_VER_PTR(*tensor_2)->dataType = dst_qnn_type; + + QNN_VER_PTR(*tensor_0)->clientBuf = {src0->data, ggml_get_tensor_data_size(src0)}; + QNN_VER_PTR(*tensor_1)->clientBuf = {src1->data, ggml_get_tensor_data_size(src1)}; + QNN_VER_PTR(*tensor_2)->clientBuf = {dst->data, ggml_get_tensor_data_size(dst)}; + + Qnn_Tensor_t tensor_inputs[] = { + *tensor_0, + *tensor_1 + }; + Qnn_Tensor_t tensor_outputs[] = { + *tensor_2 + }; + error = qnn_raw_interface.graphExecute(graph_handle, + tensor_inputs, 2, + tensor_outputs, 1, + nullptr, nullptr); + if (QNN_SUCCESS != error) { + GGMLQNN_LOG_INFO("error = %d\n", error); + } + } + + //avoid memory leak in func free_qnn_tensor + QNN_VER_PTR(*tensor_0)->dimensions = tensor_0_dimensions; + QNN_VER_PTR(*tensor_1)->dimensions = tensor_1_dimensions; + QNN_VER_PTR(*tensor_2)->dimensions = tensor_2_dimensions; +#if GGMLQNN_PRINT_OP_ADD_LOG + op_perf.info(); +#endif +} + +//TODO: type trait with op->src[0] +/* + * the procedure of ggml_qnn_mul_mat is similar to ggml_qnn_add,but there are type trait process + * for ggml_qnn_mul_mat, so it's a standalone function. + * + * MUL_MAT take most of the compute time (about 95%).so to speed up llama inference, we should focus on MUL_MAT. + * + * we have three kinds of MUL_MAT to compute: + * mul_mat_f32: both src0 and src1 are F32, this will be naturally handled in QNN backend + * mul_mat_f16_f32: src0 is F16 and src1 is F32, f16 in src0 -> f32 in src0', then src0' * src1 + * mul_mat_q_f32: src0 is quantized (Q4_0, Q4_1, ...) and src1 is F32, quantize in src0 -> f32 in src0', then src0' * src1 +*/ +static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { + Qnn_ErrorHandle_t error = QNN_SUCCESS; + bool graph_initialized = false; + qnn_perf op_perf = qnn_perf("ggml_qnn_mul_mat"); + qnn_instance * instance = nullptr; + ggml_backend_qnn_context * ctx = (ggml_backend_qnn_context *) backend->context; + + std::string graph_name = "ggml_op_qnn_mul_mat"; + Qnn_GraphHandle_t graph_handle = nullptr; + Qnn_Tensor_t * tensor_0 = nullptr; + Qnn_Tensor_t * tensor_1 = nullptr; + Qnn_Tensor_t * tensor_2 = nullptr; + + Qnn_Param_t qnn_params[] = {}; + + enum ggml_op ggmlop = GGML_OP_ADD; + Qnn_DataType_t src0_qnn_type = QNN_DATATYPE_FLOAT_32; + Qnn_DataType_t src1_qnn_type = QNN_DATATYPE_FLOAT_32; + Qnn_DataType_t dst_qnn_type = QNN_DATATYPE_FLOAT_32; + + instance = ctx->instance; + QNN_INTERFACE_VER_TYPE qnn_raw_interface = ctx->raw_interface; + + const ggml_tensor * src0 = op->src[0]; + const ggml_tensor * src1 = op->src[1]; + ggml_tensor * dst = op; + op_perf.start(); + + std::string map_entry; + get_graph_key_from_op(op, map_entry); + if (instance->_qnn_graph_map.find(map_entry) != instance->_qnn_graph_map.end()) { + graph_initialized = true; + auto & graph_item = instance->_qnn_graph_map[map_entry]; + graph_handle = std::get<0>(graph_item); + tensor_0 = std::get<1>(graph_item); + tensor_1 = std::get<2>(graph_item); + tensor_2 = std::get<3>(graph_item); + } else { + tensor_0 = ggml_qnn_create_tensor(src0); + tensor_1 = ggml_qnn_create_tensor(src1); + tensor_2 = ggml_qnn_create_tensor(dst); + } + +#if GGMLQNN_DEBUG + GGMLQNN_LOG_DEBUG("call %s in dev %s\n", __func__, ctx->name); + GGMLQNN_LOG_DEBUG("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", + src0->name, + src0->type, ggml_type_name(src0->type), src0->ne[0], src0->ne[1], src0->ne[2], + src0->nb[0], src0->nb[1], src0->nb[2]); + GGMLQNN_LOG_DEBUG("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", + src1->name, + src1->type, ggml_type_name(src1->type), src1->ne[0], src1->ne[1], src1->ne[2], + src1->nb[0], src1->nb[1], src1->nb[2]); + GGMLQNN_LOG_DEBUG("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", + dst->name, + dst->type, ggml_type_name(dst->type), dst->ne[0], dst->ne[1], dst->ne[2], dst->nb[0], + dst->nb[1], dst->nb[2]); + GGMLQNN_LOG_DEBUG("%d, %d, %d, %d", src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3]); + GGMLQNN_LOG_DEBUG("tensor0 name %s", QNN_TENSOR_GET_NAME(tensor_0)); + GGMLQNN_LOG_DEBUG("tensor1 name %s", QNN_TENSOR_GET_NAME(tensor_1)); + GGMLQNN_LOG_DEBUG("tensor2 name %s", QNN_TENSOR_GET_NAME(tensor_2)); +#endif + QNN_VER_PTR(*tensor_0)->type = QNN_TENSOR_TYPE_APP_WRITE; + QNN_VER_PTR(*tensor_1)->type = QNN_TENSOR_TYPE_APP_WRITE; + QNN_VER_PTR(*tensor_2)->type = QNN_TENSOR_TYPE_APP_READ; + + src0_qnn_type = qnn_datatype_from_ggml_datatype(src0->type); + src1_qnn_type = qnn_datatype_from_ggml_datatype(src1->type); + dst_qnn_type = qnn_datatype_from_ggml_datatype(dst->type); + + uint32_t * tensor_0_dimensions = QNN_VER_PTR(*tensor_0)->dimensions; + uint32_t * tensor_1_dimensions = QNN_VER_PTR(*tensor_1)->dimensions; + uint32_t * tensor_2_dimensions = QNN_VER_PTR(*tensor_2)->dimensions; + + if (!graph_initialized) { + graph_name = map_entry; + GGMLQNN_LOG_DEBUG("graph name %s", graph_name.c_str()); + error = qnn_raw_interface.graphCreate(instance->get_qnn_context_handle(), + graph_name.c_str(), nullptr, &graph_handle); + if (QNN_SUCCESS != error) { + GGMLQNN_LOG_INFO("can't create qnn graph handle with graph name %s, error = %d\n", graph_name.c_str(), error); + return; + } + error = qnn_raw_interface.tensorCreateGraphTensor(graph_handle, tensor_0); + if (QNN_SUCCESS != error) { + GGMLQNN_LOG_INFO("error = %d\n", error); + } + error = qnn_raw_interface.tensorCreateGraphTensor(graph_handle, tensor_1); + if (QNN_SUCCESS != error) { + GGMLQNN_LOG_INFO("error = %d\n", error); + } + error = qnn_raw_interface.tensorCreateGraphTensor(graph_handle, tensor_2); + if (QNN_SUCCESS != error) { + GGMLQNN_LOG_INFO("error = %d\n", error); + } + + QNN_VER_PTR(*tensor_0)->clientBuf = {src0->data, ggml_get_tensor_data_size(src0)}; + QNN_VER_PTR(*tensor_1)->clientBuf = {src1->data, ggml_get_tensor_data_size(src1)}; + QNN_VER_PTR(*tensor_2)->clientBuf = {dst->data, ggml_get_tensor_data_size(dst)}; + + Qnn_Tensor_t tensor_inputs[] = { + *tensor_0, + *tensor_1 + }; + Qnn_Tensor_t tensor_outputs[] = { + *tensor_2 + }; + Qnn_OpConfig_t op_config = { + (Qnn_OpConfigVersion_t) 1, .v1 = { + "ggml_op_mul_mat", + QNN_OP_PACKAGE_NAME_QTI_AISW, + QNN_OP_MAT_MUL, + 0, + qnn_params, + 2, + tensor_inputs, + 1, + tensor_outputs + } + }; + error = qnn_raw_interface.graphAddNode(graph_handle, op_config); + if (QNN_SUCCESS != error) { + GGMLQNN_LOG_INFO("error = %d\n", error); + } + error = qnn_raw_interface.graphFinalize(graph_handle, nullptr, nullptr); + if (QNN_SUCCESS != error) { + GGMLQNN_LOG_INFO("error = %d\n", error); + } + error = qnn_raw_interface.graphExecute(graph_handle, + tensor_inputs, 2, + tensor_outputs, 1, + nullptr, nullptr); + if (QNN_SUCCESS != error) { + GGMLQNN_LOG_INFO("error = %d\n", error); + } + auto graph_item = std::make_tuple(graph_handle, tensor_0, tensor_1, tensor_2); + instance->_qnn_graph_map[map_entry] = graph_item; + } else { + uint32_t dimensions_input_0[] = {(uint32_t) src0->ne[0], (uint32_t) src0->ne[1], + (uint32_t) src0->ne[2], (uint32_t) src0->ne[3]}; + uint32_t dimensions_input_1[] = {(uint32_t) src1->ne[0], (uint32_t) src1->ne[1], + (uint32_t) src1->ne[2], (uint32_t) src1->ne[3]}; + uint32_t dimensions_output[] = {(uint32_t) dst->ne[0], (uint32_t) dst->ne[1], + (uint32_t) dst->ne[2], (uint32_t) dst->ne[3]}; + QNN_VER_PTR(*tensor_0)->dimensions = dimensions_input_0; + QNN_VER_PTR(*tensor_0)->rank = ggml_get_tensor_rank(src0); + QNN_VER_PTR(*tensor_0)->dataType = src0_qnn_type; + QNN_VER_PTR(*tensor_1)->dimensions = dimensions_input_1; + QNN_VER_PTR(*tensor_1)->rank = ggml_get_tensor_rank(src1); + QNN_VER_PTR(*tensor_1)->dataType = src1_qnn_type; + QNN_VER_PTR(*tensor_2)->dimensions = dimensions_output; + QNN_VER_PTR(*tensor_2)->rank = ggml_get_tensor_rank(dst); + QNN_VER_PTR(*tensor_2)->dataType = dst_qnn_type; + + QNN_VER_PTR(*tensor_0)->clientBuf = {src0->data, ggml_get_tensor_data_size(src0)}; + QNN_VER_PTR(*tensor_1)->clientBuf = {src1->data, ggml_get_tensor_data_size(src1)}; + QNN_VER_PTR(*tensor_2)->clientBuf = {dst->data, ggml_get_tensor_data_size(dst)}; + + Qnn_Tensor_t tensor_inputs[] = { + *tensor_0, + *tensor_1 + }; + Qnn_Tensor_t tensor_outputs[] = { + *tensor_2 + }; + error = qnn_raw_interface.graphExecute(graph_handle, + tensor_inputs, 2, + tensor_outputs, 1, + nullptr, nullptr); + if (QNN_SUCCESS != error) { + GGMLQNN_LOG_INFO("error = %d\n", error); + } + } + + //avoid memory leak in func free_qnn_tensor + QNN_VER_PTR(*tensor_0)->dimensions = tensor_0_dimensions; + QNN_VER_PTR(*tensor_1)->dimensions = tensor_1_dimensions; + QNN_VER_PTR(*tensor_2)->dimensions = tensor_2_dimensions; + + op_perf.info(); +} + +static bool ggml_qnn_compute_forward(ggml_backend_t backend, struct ggml_tensor * tensor) { + ggmlqnn_op_func_t func = nullptr; + + switch (tensor->op) { + case GGML_OP_ADD: + func = ggml_qnn_add; + break; + + case GGML_OP_MUL_MAT: + func = ggml_qnn_mul_mat; + break; + + default: + return false; + } + + if (nullptr != func) + func(backend, tensor); + + return true; +} + +struct ggml_backend_qnn_buffer_context { + ~ggml_backend_qnn_buffer_context() { + if (buffer) { + free(buffer); + } + + for (auto * sub_buffer : sub_buffers) { + free(sub_buffer); + } + + for (auto * qnn_tensor : qnn_tensors) { + free_qnn_tensor(qnn_tensor); + } + + sub_buffers.clear(); + qnn_tensors.clear(); + } + void * buffer = nullptr; + + struct ggml_backend_qnn_context * backend_ctx = nullptr; + + size_t buffer_size = 0; + std::vector sub_buffers; + std::vector qnn_tensors; +}; + +static void ggml_backend_qnn_buffer_free_buffer(ggml_backend_buffer_t buffer) { + ggml_backend_qnn_buffer_context * ctx = (ggml_backend_qnn_buffer_context *)buffer->context; + delete ctx; +} + +static void * ggml_backend_qnn_buffer_get_base(ggml_backend_buffer_t buffer) { + ggml_backend_qnn_buffer_context * ctx = (ggml_backend_qnn_buffer_context *)buffer->context; + + return ctx->buffer; +} + +static void ggml_backend_qnn_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) { + Qnn_ErrorHandle_t error = QNN_SUCCESS; + ggml_backend_qnn_buffer_context * ctx = (ggml_backend_qnn_buffer_context *)buffer->context; + GGML_UNUSED(error); + GGML_UNUSED(ctx); + return; +} + +static void ggml_backend_qnn_buffer_set_tensor(ggml_backend_buffer_t buffer, + ggml_tensor * tensor, const void * data, + size_t offset, size_t size) { + GGML_UNUSED(buffer); + + memcpy((char *)tensor->data + offset, data, size); +} + +static void ggml_backend_qnn_buffer_memset_tensor(ggml_backend_buffer_t buffer, + struct ggml_tensor * tensor, + uint8_t value, size_t offset, size_t size) { + GGML_UNUSED(buffer); + memset((char *)tensor->data + offset, value, size); +} + +static void ggml_backend_qnn_buffer_get_tensor(ggml_backend_buffer_t buffer, + const ggml_tensor * tensor, + void * data, size_t offset, size_t size) { + GGML_UNUSED(buffer); + memcpy(data, (const char *)tensor->data + offset, size); +} + +static bool ggml_backend_qnn_buffer_cpy_tensor(ggml_backend_buffer_t buffer, + const struct ggml_tensor * src, + struct ggml_tensor * dst) { + GGML_UNUSED(buffer); + if (ggml_backend_buffer_is_host(src->buffer)) { + memcpy(dst->data, src->data, ggml_nbytes(src)); + return true; + } + + return false; +} + +static void ggml_backend_qnn_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) { + ggml_backend_qnn_buffer_context * ctx = (ggml_backend_qnn_buffer_context *)buffer->context; + memset(ctx->buffer, value, ctx->buffer_size); +} + +[[maybe_unused]]static void ggml_backend_qnn_buffer_reset(ggml_backend_buffer_t buffer) { + ggml_backend_qnn_buffer_context * ctx = (ggml_backend_qnn_buffer_context *)buffer->context; + for (auto * sub_buffer : ctx->sub_buffers) { + free(sub_buffer); + } + ctx->sub_buffers.clear(); +} + +static ggml_backend_buffer_i ggml_backend_qnn_buffer_interface = { + /* .free_buffer = */ ggml_backend_qnn_buffer_free_buffer, + /* .get_base = */ ggml_backend_qnn_buffer_get_base, + /* .init_tensor = */ ggml_backend_qnn_buffer_init_tensor, + /* .memset_tensor = */ ggml_backend_qnn_buffer_memset_tensor, + /* .set_tensor = */ ggml_backend_qnn_buffer_set_tensor, + /* .get_tensor = */ ggml_backend_qnn_buffer_get_tensor, + /* .cpy_tensor = */ ggml_backend_qnn_buffer_cpy_tensor, + /* .clear = */ ggml_backend_qnn_buffer_clear, + /* .reset = */ NULL, +}; + +static const char * ggml_backend_qnn_buffer_type_name(ggml_backend_buffer_type_t buft) { + return "qnn-buffer"; +} + +static ggml_backend_buffer_t ggml_backend_qnn_buffer_type_alloc_buffer( + ggml_backend_buffer_type_t buft, size_t size) { + ggml_backend_qnn_buffer_context * ctx = new ggml_backend_qnn_buffer_context; + + size_t size_page = sysconf(_SC_PAGESIZE); + size_t size_aligned = size; + if ((size_aligned % size_page) != 0) { + size_aligned += (size_page - (size_aligned % size_page)); + } + ctx->buffer = ggmlqnn_host_malloc(size_aligned); + ctx->buffer_size = size_aligned; + if (nullptr == ctx->buffer) { + GGMLQNN_LOG_WARN("%s: failed to allocate %.2f MiB\n", __func__, size / (1 << 20)); + return nullptr; + } + + return ggml_backend_buffer_init(buft, ggml_backend_qnn_buffer_interface, ctx, size); +} + +static size_t ggml_backend_qnn_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) { + GGML_UNUSED(buft); + return 32; +} + +//FIXME: this value is an experimental value on Xiaomi14 +static size_t ggml_backend_qnn_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) { + GGML_UNUSED(buft); + + return (2 * (1 << 30)); +} + +static bool ggml_backend_qnn_buffer_is_host(ggml_backend_buffer_type_t buft) { + GGML_UNUSED(buft); + return true; +} + +static const char * ggml_backend_qnn_name(ggml_backend_t backend) { + ggml_backend_qnn_context * ctx = (ggml_backend_qnn_context *) backend->context; + return g_qnn_mgr[ctx->device].name; +} + +static void ggml_backend_qnn_free(ggml_backend_t backend) { + GGMLQNN_LOG_DEBUG("enter %s", __func__ ); + ggml_backend_qnn_context * ctx = (ggml_backend_qnn_context *) backend->context; + GGMLQNN_LOG_DEBUG("idx %d, name:%s", ctx->device, g_qnn_mgr[ctx->device].name); + + qnn_instance * instance = (qnn_instance*)g_qnn_mgr[ctx->device].instance; + if (instance != nullptr) { + std::map>::iterator graph_it; + + for (graph_it = instance->_qnn_graph_map.begin(); + graph_it != instance->_qnn_graph_map.end(); graph_it++) { + auto & graph_item = graph_it->second; + Qnn_GraphHandle_t & graph_handle = std::get<0>(graph_item); + Qnn_Tensor_t * tensor_0 = std::get<1>(graph_item); + Qnn_Tensor_t * tensor_1 = std::get<2>(graph_item); + Qnn_Tensor_t * tensor_2 = std::get<3>(graph_item); + GGML_UNUSED(graph_handle); + GGMLQNN_LOG_DEBUG("graph type:%s", graph_it->first.c_str()); + free_qnn_tensor(tensor_0); + free_qnn_tensor(tensor_1); + free_qnn_tensor(tensor_2); + } + instance->_qnn_graph_map.clear(); + + instance->qnn_finalize(); + delete instance; + g_qnn_mgr[ctx->device].instance = nullptr; + } + + if (g_qnn_mgr[ctx->device].backend != nullptr) { + delete backend; + g_qnn_mgr[ctx->device].backend = nullptr; + } + GGMLQNN_LOG_DEBUG("leave %s", __func__ ); +} + +static enum ggml_status ggml_backend_qnn_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph) { + enum ggml_status result = GGML_STATUS_SUCCESS; + ggml_backend_qnn_context * ctx = (ggml_backend_qnn_context *) backend->context; + GGML_UNUSED(ctx); + + //GGMLQNN_LOG_DEBUG("cgraph->n_nodes %d", cgraph->n_nodes); + for (int i = 0; i < cgraph->n_nodes; i++) { + ggml_tensor * node = cgraph->nodes[i]; + if (ggml_is_empty(node) || node->op == GGML_OP_RESHAPE + || node->op == GGML_OP_TRANSPOSE || node->op == GGML_OP_VIEW + || node->op == GGML_OP_PERMUTE || node->op == GGML_OP_NONE) { + continue; + } + bool ok = ggml_qnn_compute_forward(backend, node); + if (!ok) { + GGMLQNN_LOG_DEBUG("%s: error: op not supported %s (%s)\n", + __func__, node->name, ggml_op_name(node->op)); + } + } + + return result; +} + +static const char * ggml_backend_qnn_device_get_name(ggml_backend_dev_t dev) { + struct ggml_backend_qnn_context *ctx = static_cast(dev->context); + if (nullptr == ctx) { + GGMLQNN_LOG_ERROR("pls check why ctx is null"); + return "unknown"; + } + return ctx->name; + + GGML_UNUSED(dev); +} + +static const char * ggml_backend_qnn_device_get_description(ggml_backend_dev_t dev) { + struct ggml_backend_qnn_context * ctx = static_cast(dev->context); + if (nullptr == ctx) { + GGMLQNN_LOG_ERROR("pls check why ctx is null"); + return "unknown"; + } + if (0 == strncmp(ctx->name, "qnn-npu", 7)) { + const char * soc_info = qnn_get_socmodel_desc(ctx->socinfo.soc_model); + const char * htp_arch = qnn_get_htparch_desc(ctx->socinfo.htp_arch); + std::string dev_desc = std::string(ctx->desc) + + std::string(soc_info) + "_" + std::string(htp_arch) + + "," + std::string(ctx->socinfo.soc_desc); + return dev_desc.c_str(); + } else { + return ctx->desc; + } +} + +static void ggml_backend_qnn_device_get_memory(ggml_backend_dev_t dev, size_t * free, size_t * total) { + //FIXME:this is NOT QNN device memory info + *free = get_system_free_memory_in_bytes(); + *total = get_system_total_memory_in_bytes(); + GGML_UNUSED(dev); +} + +static enum ggml_backend_dev_type ggml_backend_qnn_device_get_type(ggml_backend_dev_t dev) { + GGML_UNUSED(dev); + return GGML_BACKEND_DEVICE_TYPE_ACCEL; +} + +static void ggml_backend_qnn_device_get_props(ggml_backend_dev_t dev, + struct ggml_backend_dev_props * props) { + props->name = ggml_backend_qnn_device_get_name(dev); + props->description = ggml_backend_qnn_device_get_description(dev); + props->type = ggml_backend_qnn_device_get_type(dev); + ggml_backend_qnn_device_get_memory(dev, &props->memory_free, &props->memory_total); + props->caps = { + /* .async = */ false, + /* .host_buffer = */ false, + /* .buffer_from_host_ptr = */ true, + /* .events = */ false, + }; +} + +static ggml_backend_t ggml_backend_qnn_device_init_backend(ggml_backend_dev_t dev, const char * params) { + GGML_UNUSED(dev); + if (nullptr == params) { + params = 0; + } + ggml_backend_t qnn_backend = ggml_backend_qnn_init((int) (intptr_t) params, + "/data/local/tmp/"); + + return qnn_backend; + +} + +ggml_backend_buffer_type_t ggml_backend_qnn_buffer_type(size_t device_index) { + if (device_index >= GGML_QNN_MAX_DEVICES) { + GGMLQNN_LOG_DEBUG("ggml_backend_qnn_buffer_type error: device_index:%d is out of range [0, %d]\n", + device_index, GGML_QNN_MAX_DEVICES - 1); + return nullptr; + } + + static struct ggml_backend_buffer_type ggml_backend_buffer_type_qnn = { + /* .iface = */ { + /* .get_name = */ ggml_backend_qnn_buffer_type_name, + /* .alloc_buffer = */ ggml_backend_qnn_buffer_type_alloc_buffer, + /* .get_alignment = */ ggml_backend_qnn_buffer_type_get_alignment, + /* .get_max_size = */ ggml_backend_qnn_buffer_type_get_max_size, + /* .get_alloc_size = */ NULL,// defaults to ggml_nbytes + /* .is_host = */ ggml_backend_qnn_buffer_is_host + }, + /* .context = */ NULL, + }; + + return &ggml_backend_buffer_type_qnn; +} + +static ggml_backend_buffer_type_t ggml_backend_qnn_device_get_buffer_type(ggml_backend_dev_t dev) { + ggml_backend_qnn_context * ctx = (ggml_backend_qnn_context *) dev->context; + return ggml_backend_qnn_buffer_type(ctx->device); +} + +static ggml_backend_buffer_t ggml_backend_qnn_device_buffer_from_host_ptr(ggml_backend_dev_t dev, + void * ptr, size_t size, size_t max_tensor_size) { + return ggml_backend_cpu_buffer_from_ptr(ptr, size); + + GGML_UNUSED(dev); + GGML_UNUSED(max_tensor_size); +} + + +static bool ggml_backend_qnn_device_supports_op(ggml_backend_dev_t dev, const struct ggml_tensor * op) { + ggml_backend_qnn_context * ctx = (ggml_backend_qnn_context *) dev->context; + return (ggml_qnn_can_handle_op(op, true)); +} + +static bool ggml_backend_qnn_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) { + GGML_UNUSED(dev); + return ggml_backend_buft_is_host(buft); +} + +static struct ggml_backend_device_i ggml_backend_qnn_device_interface = { + /* .get_name = */ ggml_backend_qnn_device_get_name, + /* .get_description = */ ggml_backend_qnn_device_get_description, + /* .get_memory = */ ggml_backend_qnn_device_get_memory, + /* .get_type = */ ggml_backend_qnn_device_get_type, + /* .get_props = */ ggml_backend_qnn_device_get_props, + /* .init_backend = */ ggml_backend_qnn_device_init_backend, + /* .get_buffer_type = */ ggml_backend_qnn_device_get_buffer_type, + /* .get_host_buffer_type = */ NULL, + /* .buffer_from_host_ptr = */ ggml_backend_qnn_device_buffer_from_host_ptr, + /* .supports_op = */ ggml_backend_qnn_device_supports_op, + /* .supports_buft = */ ggml_backend_qnn_device_supports_buft, + /* .offload_op = */ NULL, + /* .event_new = */ NULL, + /* .event_free = */ NULL, + /* .event_synchronize = */ NULL, +}; + +static ggml_backend_i ggml_backend_qnn_interface = { + /* .get_name = */ ggml_backend_qnn_name, + /* .free = */ ggml_backend_qnn_free, + /* .set_tensor_async = */ nullptr, + /* .get_tensor_async = */ nullptr, + /* .cpy_tensor_async = */ nullptr, + /* .synchronize = */ nullptr, + /* .graph_plan_create = */ nullptr, + /* .graph_plan_free = */ nullptr, + /* .graph_plan_update = */ nullptr, + /* .graph_plan_compute = */ nullptr, + /* .graph_compute = */ ggml_backend_qnn_graph_compute, + /* .event_record = */ nullptr, + /* .event_wait = */ nullptr, +}; + +//FIXME: this guid is not make sense +static ggml_guid_t ggml_backend_qnn_guid() { + static ggml_guid guid = { + 0x1a, 0x2b, 0x3c, 0x4d, 0x5e, 0x6f, 0x70, 0x81, + 0x92, 0xa3, 0xb4, 0xc5, 0xd6, 0xe7, 0xf8, 0x09 + }; + return &guid; +} + +bool ggml_backend_is_qnn(ggml_backend_t backend) { + return backend != nullptr && ggml_guid_matches(backend->guid, ggml_backend_qnn_guid()); +} + +void ggml_backend_qnn_set_n_threads(ggml_backend_t backend, int n_threads) { + GGML_ASSERT(ggml_backend_is_qnn(backend)); + + struct ggml_backend_qnn_context * ctx = (struct ggml_backend_qnn_context *)backend->context; + ctx->threads = n_threads; +} + +int ggml_backend_qnn_get_device_count() { + return GGML_QNN_MAX_DEVICES; +} + +struct ggml_backend_qnn_reg_context { + std::vector devices; +}; + +static const char * ggml_backend_qnn_reg_get_name(ggml_backend_reg_t reg) { + return "ggml-qnn"; + + GGML_UNUSED(reg); +} + +static size_t ggml_backend_qnn_reg_get_device_count(ggml_backend_reg_t reg) { + GGML_UNUSED(reg); + return GGML_QNN_MAX_DEVICES; +} + +static ggml_backend_dev_t ggml_backend_qnn_reg_get_device(ggml_backend_reg_t reg, size_t index) { + GGML_UNUSED(reg); + GGML_UNUSED(index); + + GGMLQNN_LOG_DEBUG("index %d", index); + ggml_backend_qnn_reg_context * ctx = (ggml_backend_qnn_reg_context *)reg->context; + GGML_ASSERT(index < ctx->devices.size()); + return ctx->devices[index]; +} + +static void * ggml_backend_qnn_reg_get_proc_address(ggml_backend_reg_t reg, const char * name) { + GGML_UNUSED(reg); + + if (std::strcmp(name, "ggml_backend_set_n_threads") == 0) { + return (void *)ggml_backend_qnn_set_n_threads; + } + return NULL; +} + +static const ggml_backend_reg_i ggml_backend_qnn_reg_interface = { + /* .get_name = */ ggml_backend_qnn_reg_get_name, + /* .get_device_count = */ ggml_backend_qnn_reg_get_device_count, + /* .get_device = */ ggml_backend_qnn_reg_get_device, + /* .get_proc_address = */ ggml_backend_qnn_reg_get_proc_address, +}; + +ggml_backend_reg_t ggml_backend_qnn_reg() { + static ggml_backend_reg reg; + static bool initialized = false; + GGMLQNN_LOG_DEBUG("enter ggml_backend_qnn_reg"); + { + static std::mutex mutex; + std::lock_guard lock(mutex); + if (!initialized) { + ggml_backend_qnn_reg_context * ctx = new ggml_backend_qnn_reg_context; + + for (int i = 0; i < ggml_backend_qnn_get_device_count(); i++) { + ggml_backend_dev_t dev = new ggml_backend_device { + /* .iface = */ ggml_backend_qnn_device_interface, + /* .reg = */ ®, + /* .context = */ &g_qnn_mgr[i] + }; + ctx->devices.push_back(dev); + } + + reg = ggml_backend_reg { + /* .api_version = */ GGML_BACKEND_API_VERSION, + /* .iface = */ ggml_backend_qnn_reg_interface, + /* .context = */ ctx + }; + } + + initialized = true; + } + GGMLQNN_LOG_DEBUG("leave ggml_backend_qnn_reg"); + + return ® +} + +/** + * + * @param device 0: QNN_BACKEND_CPU 1: QNN_BACKEND_GPU 2: QNN_BACKEND_NPU + * @param qnn_lib_path QNN binrary runtime library path, such as "/data/local/tmp/" on Android or specified in JNI layer + * @return + */ +ggml_backend_t ggml_backend_qnn_init(size_t device, const char * qnn_lib_path) { + int result = 0; + + if (nullptr == qnn_lib_path) + return nullptr; + + GGMLQNN_LOG_DEBUG("device %d", device); + GGMLQNN_LOG_DEBUG("qnn_lib_path %s", qnn_lib_path); + if (device >= GGML_QNN_MAX_DEVICES) { + GGMLQNN_LOG_ERROR("invalid device %d", device); + return nullptr; + } + + if (nullptr != g_qnn_mgr[device].backend) { + GGMLQNN_LOG_WARN("qnn backend %d(%s) already loaded", device, ggml_backend_qnn_get_devname(device)); + return g_qnn_mgr[device].backend; + } + + std::string path = qnn_lib_path; + if (QNN_BACKEND_NPU == device) { + if (0 == setenv("LD_LIBRARY_PATH", + (path + + ":/vendor/dsp/cdsp:/vendor/lib64:/vendor/dsp/dsp:/vendor/dsp/images").c_str(), + 1)) { + GGMLQNN_LOG_INFO("QNN NPU backend setenv successfully"); + } else { + GGMLQNN_LOG_ERROR("QNN NPU backend setenv failure"); + } + if (0 == setenv("ADSP_LIBRARY_PATH", + (path + + ";/vendor/dsp/cdsp;/vendor/lib/rfsa/adsp;/system/lib/rfsa/adsp;/vendor/dsp/dsp;/vendor/dsp/images;/dsp").c_str(), + 1)) { + GGMLQNN_LOG_INFO("QNN NPU backend setenv successfully"); + } else { + GGMLQNN_LOG_ERROR("QNN NPU backend setenv failure"); + } + } else { + if (0 == setenv("LD_LIBRARY_PATH", + (path + + ":/vendor/dsp/cdsp:/vendor/lib64:/vendor/dsp/dsp:/vendor/dsp/images").c_str(), + 1)) { + GGMLQNN_LOG_INFO("%s backend setenv successfully\n", ggml_backend_qnn_get_devname(device)); + } else { + GGMLQNN_LOG_ERROR("%s backend setenv failure\n", ggml_backend_qnn_get_devname(device)); + } + } + + qnn_instance * instance = nullptr; + instance = new qnn_instance(qnn_lib_path, g_qnn_mgr[device].lib, ""); + result = instance->qnn_init(nullptr); + if (0 != result) { + GGMLQNN_LOG_WARN("init qnn subsystem failed with qnn backend %s, pls check why\n", ggml_backend_qnn_get_devname(device)); + delete instance; + return nullptr; + } + qnn_interface qnn_interface = instance->get_qnn_interface(); + if (!qnn_interface.is_loaded()) { + GGMLQNN_LOG_WARN("qnn subsystem failure\n"); + delete instance; + return nullptr; + } + + std::string device_name = ggml_backend_qnn_get_devname(device); + GGMLQNN_LOG_INFO("qnn device name %s", device_name.c_str()); + g_qnn_mgr[device].instance = instance; + g_qnn_mgr[device].raw_interface = instance->get_qnn_raw_interface(); + g_qnn_mgr[device].raw_system_interface = instance->get_qnn_raw_system_interface(); + + ggml_backend_t qnn_backend = new ggml_backend{ + /* .guid = */ ggml_backend_qnn_guid(), + /* .iface = */ ggml_backend_qnn_interface, + /* .device = */ ggml_backend_reg_dev_get(ggml_backend_qnn_reg(), device), + /* .context = */ &g_qnn_mgr[device] + }; + g_qnn_mgr[device].backend = qnn_backend; + + return qnn_backend; +} + +GGML_BACKEND_DL_IMPL(ggml_backend_qnn_reg) diff --git a/scripts/build-run-android.sh b/scripts/build-run-android.sh new file mode 100755 index 0000000000000..412ccadadaf6b --- /dev/null +++ b/scripts/build-run-android.sh @@ -0,0 +1,202 @@ +#!/bin/bash + +set -e + +PWD=`pwd` +ANDROID_PLATFORM=android-34 +ANDROID_NDK=${PWD}/android-ndk-r26c +REMOTE_PATH=/data/local/tmp/ +GGUF_MODEL_NAME=/sdcard/deepseek-r1-distill-qwen-1.5b-q4_0.gguf + +#QNN SDK could be found at: +#https://www.qualcomm.com/developer/software/qualcomm-ai-engine-direct-sdk +#https://developer.qualcomm.com/software/hexagon-dsp-sdk/tools +QNN_SDK_URL=https://www.qualcomm.com/developer/software/qualcomm-ai-engine-direct-sdk +QNN_SDK_PATH=/opt/qcom/aistack/qairt/2.31.0.250130/ + +function dump_vars() +{ + echo -e "ANDROID_NDK: ${ANDROID_NDK}" + echo -e "QNN_SDK_PATH: ${QNN_SDK_PATH}" +} + + +function show_pwd() +{ + echo -e "current working path:$(pwd)\n" +} + + +function check_qnn_sdk() +{ + if [ ! -d ${QNN_SDK_PATH} ]; then + echo -e "QNN_SDK_PATH ${QNN_SDK_PATH} not exist, pls check or download it from ${QNN_SDK_URL}...\n" + exit 1 + fi +} + + +function check_and_download_ndk() +{ + is_android_ndk_exist=1 + + if [ ! -d ${ANDROID_NDK} ]; then + is_android_ndk_exist=0 + fi + + if [ ! -f ${ANDROID_NDK}/build/cmake/android.toolchain.cmake ]; then + is_android_ndk_exist=0 + fi + + if [ ${is_android_ndk_exist} -eq 0 ]; then + + if [ ! -f android-ndk-r26c-linux.zip ]; then + wget --no-config --quiet --show-progress -O android-ndk-r26c-linux.zip https://dl.google.com/android/repository/android-ndk-r26c-linux.zip + fi + + unzip android-ndk-r26c-linux.zip + + if [ $? -ne 0 ]; then + printf "failed to download android ndk to %s \n" "${ANDROID_NDK}" + exit 1 + fi + + printf "android ndk saved to ${ANDROID_NDK} \n\n" + else + printf "android ndk already exist:${ANDROID_NDK} \n\n" + fi +} + + +function build_arm64 +{ + cmake -H. -B./out/android -DGGML_USE_QNN=ON -DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK}/build/cmake/android.toolchain.cmake -DANDROID_ABI=arm64-v8a -DANDROID_PLATFORM=latest -DCMAKE_C_FLAGS=-march=armv8.7-a -DGGML_QNN=ON -DGGML_QNN_SDK_PATH=${QNN_SDK_PATH} + cd out/android + make -j16 + show_pwd + + cd - +} + + +function remove_temp_dir() +{ + if [ -d out ]; then + echo "remove out directory in `pwd`" + rm -rf out + fi +} + + +function check_qnn_libs() +{ + #reuse the cached qnn libs on Android phone + adb shell ls ${REMOTE_PATH}/libQnnCpu.so + if [ $? -eq 0 ]; then + printf "QNN libs already exist on Android phone\n" + else + update_qnn_libs + fi +} + + +function update_qnn_libs() +{ + adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnSystem.so ${REMOTE_PATH}/ + adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnCpu.so ${REMOTE_PATH}/ + adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnGpu.so ${REMOTE_PATH}/ + + adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnHtp.so ${REMOTE_PATH}/ + adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnHtpNetRunExtensions.so ${REMOTE_PATH}/ + adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnHtpPrepare.so ${REMOTE_PATH}/ + adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnHtpV75Stub.so ${REMOTE_PATH}/ + adb push ${QNN_SDK_PATH}/lib/hexagon-v75/unsigned/libQnnHtpV75Skel.so ${REMOTE_PATH}/ +} + + +function build_ggml_qnn() +{ + show_pwd + check_and_download_ndk + check_qnn_sdk + dump_vars + remove_temp_dir + build_arm64 +} + + +function run_llamacli() +{ + check_qnn_libs + + if [ -f ./out/android/bin/libggml-qnn.so ]; then + adb push ./out/android/bin/*.so ${REMOTE_PATH}/ + fi + adb push ./out/android/bin/llama-cli ${REMOTE_PATH}/ + adb shell chmod +x ${REMOTE_PATH}/llama-cli + + adb shell "cd ${REMOTE_PATH} \ + && export LD_LIBRARY_PATH=${REMOTE_PATH} \ + && ${REMOTE_PATH}/llama-cli -mg 2 -m ${GGUF_MODEL_NAME} -p \"introduce the movie Once Upon a Time in America briefly.\n\"" + +} + +function run_test-backend-ops() +{ + check_qnn_libs + + if [ -f ./out/android/bin/libggml-qnn.so ]; then + adb push ./out/android/bin/*.so ${REMOTE_PATH}/ + fi + adb push ./out/android/bin/test-backend-ops ${REMOTE_PATH}/ + adb shell chmod +x ${REMOTE_PATH}/test-backend-ops + + adb shell "cd ${REMOTE_PATH} \ + && export LD_LIBRARY_PATH=${REMOTE_PATH} \ + && ${REMOTE_PATH}/test-backend-ops test" + +} + + +function show_usage() +{ + echo "Usage:" + echo " $0 build" + echo " $0 updateqnnlib" + echo " $0 run_llamacli" + echo " $0 run_testop" + echo -e "\n\n\n" +} + + +show_pwd + +check_qnn_sdk + +if [ $# == 0 ]; then + show_usage + exit 1 +elif [ $# == 1 ]; then + if [ "$1" == "-h" ]; then + show_usage + exit 1 + elif [ "$1" == "help" ]; then + show_usage + exit 1 + elif [ "$1" == "build" ]; then + build_ggml_qnn + exit 0 + elif [ "$1" == "run_llamacli" ]; then + run_llamacli + exit 0 + elif [ "$1" == "run_testop" ]; then + run_test-backend-ops + exit 0 + elif [ "$1" == "updateqnnlib" ]; then + update_qnn_libs + exit 0 + fi +else + show_usage + exit 1 +fi From ad4cc92bc0b1b5cc33de0013931519595238757c Mon Sep 17 00:00:00 2001 From: zhouwg Date: Sat, 15 Feb 2025 11:14:13 +0800 Subject: [PATCH 02/76] ggml-qnn: santiy check --- ggml/src/ggml-qnn/ggml-qnn.cpp | 67 ++++++++++++++++------------------ 1 file changed, 31 insertions(+), 36 deletions(-) diff --git a/ggml/src/ggml-qnn/ggml-qnn.cpp b/ggml/src/ggml-qnn/ggml-qnn.cpp index d29c6cb6f9222..780bc3553ab0f 100644 --- a/ggml/src/ggml-qnn/ggml-qnn.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn.cpp @@ -1,9 +1,6 @@ /* * Copyright (c) 2023-2024 The ggml authors * - * this is implementation of ggml-qnn(ggml-qnn backend of Qualcomm QNN(Qualcomm Neural Network, - * aka Qualcomm AI Engine Direct) - * * Qualcomm QNN SDK and reference tech guides could be found at: * https://www.qualcomm.com/developer/software/qualcomm-ai-engine-direct-sdk * https://developer.qualcomm.com/software/hexagon-dsp-sdk/tools @@ -17,7 +14,7 @@ * section-6 does implementation of ggml-qnn backend according to ggml's backend subsystem * * currently only provide GGML_OP_ADD's QNN backend implementation: - * - GGML_OP_ADD: this is skeleton, can expand other ggml ops as expertise + * - GGML_OP_ADD: this is skeleton, can expand other ggml ops according to expertise * * of course, can porting ggml-qnn to Windows on ARM as need. * @@ -105,10 +102,6 @@ class qnn_instance; struct ggml_backend_qnn_context; static int free_qnn_tensor(Qnn_Tensor_t * tensor); static enum ggml_status ggml_backend_qnn_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph); - -#if (defined __ANDROID__) || (defined ANDROID) -extern "C" int __android_log_print(int prio, const char * tag, const char * fmt, ...) __attribute__((__format__(printf, 3, 4))); -#endif static void ggmlqnn_log_internal(ggml_log_level level, const char * file, const char * func, int line, const char * format, ...); // ================================================================================================= @@ -142,13 +135,13 @@ static void ggmlqnn_log_internal(ggml_log_level level, const char * file, const int len = vsnprintf(s_ggmlqnn_log_internal_buf + len_prefix, GGML_QNN_LOGBUF_LEN - len_prefix, format, args); if (len < (GGML_QNN_LOGBUF_LEN - len_prefix)) { #if (defined __ANDROID__) || (defined ANDROID) - //for Android APK + //for Android application(standard APP or command line tool) __android_log_print(ANDROID_LOG_INFO, "ggml-qnn", "%s\n", s_ggmlqnn_log_internal_buf); #endif #if (defined __ANDROID__) || (defined ANDROID) - //do nothing when running on Android phone + //do nothing when running on Snapdragon based Android device #else - //for Windows on ARM + //for Snapdragon based WoA(Windows on ARM) device printf("%s\n", s_ggmlqnn_log_internal_buf); #endif } @@ -851,7 +844,6 @@ static int free_qnn_tensor(Qnn_Tensor_t * tensor) { free(src_qparam.bwAxisScaleOffsetEncoding.offsets); } } - //GGMLQNN_LOG_DEBUG("tensor dims %p", QNN_TENSOR_GET_DIMENSIONS(*tensor)); free(QNN_TENSOR_GET_DIMENSIONS(*tensor)); free(tensor); @@ -1367,8 +1359,8 @@ static struct qcom_socinfo * qnn_get_socinfo_from_socmodel(uint32_t soc_model) { return nullptr; } -static bool qnn_is_valid_params(ggml_backend_qnn_context * ctx, const ggml_tensor * src0, - const ggml_tensor * src1, ggml_tensor * dst) { +static bool ggmlqnn_is_valid_params(ggml_backend_qnn_context * ctx, const ggml_tensor * src0, + const ggml_tensor * src1, ggml_tensor * dst) { if ((nullptr == ctx) || (nullptr == src0) || (nullptr == src1) || (nullptr == dst)) { GGMLQNN_LOG_WARN("invalid params\n"); return false; @@ -1383,9 +1375,9 @@ static bool qnn_is_valid_params(ggml_backend_qnn_context * ctx, const ggml_tenso return true; } -#define CHECK_PARAMS(ctx, src0, src1, dst) \ +#define GGMLQNN_CHECK_PARAMS(ctx, src0, src1, dst) \ do { \ - if (!qnn_is_valid_params((ctx), (src0), (src1), (dst))) { \ + if (!ggmlqnn_is_valid_params((ctx), (src0), (src1), (dst))) { \ return; \ } \ } while (0) @@ -1516,7 +1508,7 @@ static ggml_type ggml_datatype_from_qnn_datatype(Qnn_DataType_t qnn_type) { return GGML_TYPE_COUNT; } -//TODO: +//TODO: add more ops static const char * qnn_opname_from_ggmlop(enum ggml_op ggmlop) { switch (ggmlop) { case GGML_OP_ADD: @@ -1540,7 +1532,7 @@ static void append_tensor_dimensions(const ggml_tensor * tensor, std::string & o int len = 0; switch (ggml_n_dims(tensor)) { case 1: - len = snprintf(buffer, sizeof(buffer), "%ld%s", (long)tensor->ne[0], type_name); + len = snprintf(buffer, sizeof(buffer), "%ldx1%s", (long)tensor->ne[0], type_name); break; case 2: len = snprintf(buffer, sizeof(buffer), "%ldx%ld%s", (long)tensor->ne[0], (long)tensor->ne[1], type_name); @@ -1913,7 +1905,7 @@ class qnn_instance { void unregister_rpcmem(); void unregister_rpcmem(Qnn_MemHandle_t mem_handle); - void *alloc_rpcmem(size_t bytes, size_t alignment); + void * alloc_rpcmem(size_t bytes, size_t alignment); void free_rpcmem(void * buf); @@ -2252,7 +2244,7 @@ int qnn_instance::load_backend(std::string & lib_path, const QnnSaver_Config_t * _loaded_lib_handle[backend_id] = lib_handle; _backend_id = backend_id; -#if 0 //not used in PR, keep them here for further use +#if 0 // keep them here for further use QnnSaver_Config_t outputdir_cfg; outputdir_cfg.option = QNN_SAVER_CONFIG_OPTION_OUTPUT_DIRECTORY; outputdir_cfg.outputDirectory = "/data/local/tmp/"; @@ -2307,8 +2299,8 @@ int qnn_instance::load_system() { _system_lib_handle = dlopen(system_lib_path.c_str(), RTLD_NOW | RTLD_LOCAL); if (nullptr == _system_lib_handle) { GGMLQNN_LOG_WARN("can not open QNN library %s, error: %s\n", system_lib_path.c_str(), dlerror()); - //re-try with Android APK's internal QNN runtime lib path - _lib_path = "/data/data/com.cdeos.kantv/qnnlib/"; + //re-try with default path of QNN binary runtime lib + _lib_path = "/data/local/tmp/"; system_lib_path = _lib_path + "libQnnSystem.so"; _system_lib_handle = dlopen(system_lib_path.c_str(), RTLD_NOW | RTLD_LOCAL); if (nullptr == _system_lib_handle) { @@ -2604,7 +2596,6 @@ int qnn_instance::qnn_init(const QnnSaver_Config_t ** saver_config) { } _qnn_raw_interface.deviceFreePlatformInfo(nullptr, p_info); - //TODO: faster approach to probe the accurate capacity of QNN RPC ion memory size_t candidate_size = 0; uint8_t * rpc_buffer = nullptr; const int SIZE_IN_MB = (1 << 20); @@ -2648,7 +2639,7 @@ int qnn_instance::qnn_finalize() { //FIXME:should be removed in the future reset_idx(); - if (nullptr != _pfn_rpc_mem_deinit) // make Qualcomm's mobile SoC equipped low-end phone happy + if (nullptr != _pfn_rpc_mem_deinit) _pfn_rpc_mem_deinit(); if (dlclose(_rpc_lib_handle) != 0) { @@ -2922,8 +2913,8 @@ static bool ggml_qnn_can_handle_op(const struct ggml_tensor * tensor, bool b_dum } #if 0 //TODO: offload mul_mat to QNN backend - //we need to process type traint in func ggml_qnn_mul_mat(...) with following case: - //src0: q4_0, q6_k + //need to process type trait in func ggml_qnn_mul_mat(...): + //src0: q4_0, q6_k, ... //src1: f32 //dst : f32 return (src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16) @@ -2959,13 +2950,15 @@ static void ggml_qnn_add(ggml_backend_t backend, ggml_tensor * op) { Qnn_DataType_t src0_qnn_type = QNN_DATATYPE_FLOAT_32; Qnn_DataType_t src1_qnn_type = QNN_DATATYPE_FLOAT_32; Qnn_DataType_t dst_qnn_type = QNN_DATATYPE_FLOAT_32; + const ggml_tensor * src0 = op->src[0]; + const ggml_tensor * src1 = op->src[1]; + ggml_tensor * dst = op; + + GGMLQNN_CHECK_PARAMS(ctx, src0, src1, dst); instance = ctx->instance; QNN_INTERFACE_VER_TYPE qnn_raw_interface = ctx->raw_interface; - const ggml_tensor * src0 = op->src[0]; - const ggml_tensor * src1 = op->src[1]; - ggml_tensor * dst = op; op_perf.start(); std::string map_entry; @@ -3174,17 +3167,17 @@ static void ggml_qnn_add(ggml_backend_t backend, ggml_tensor * op) { #endif } -//TODO: type trait with op->src[0] +//TODO: /* - * the procedure of ggml_qnn_mul_mat is similar to ggml_qnn_add,but there are type trait process - * for ggml_qnn_mul_mat, so it's a standalone function. + * the logic of ggml_qnn_mul_mat is similar to ggml_qnn_add,but type trait and matrix transpose are required + * for offload mulmat to QNN backend, so it's a standalone function. * * MUL_MAT take most of the compute time (about 95%).so to speed up llama inference, we should focus on MUL_MAT. * * we have three kinds of MUL_MAT to compute: * mul_mat_f32: both src0 and src1 are F32, this will be naturally handled in QNN backend * mul_mat_f16_f32: src0 is F16 and src1 is F32, f16 in src0 -> f32 in src0', then src0' * src1 - * mul_mat_q_f32: src0 is quantized (Q4_0, Q4_1, ...) and src1 is F32, quantize in src0 -> f32 in src0', then src0' * src1 + * mul_mat_q_f32: src0 is quantized (Q4_0, Q4_1, ...) and src1 is F32, src0 -> f32 in src0', then src0' * src1 */ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { Qnn_ErrorHandle_t error = QNN_SUCCESS; @@ -3205,13 +3198,15 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { Qnn_DataType_t src0_qnn_type = QNN_DATATYPE_FLOAT_32; Qnn_DataType_t src1_qnn_type = QNN_DATATYPE_FLOAT_32; Qnn_DataType_t dst_qnn_type = QNN_DATATYPE_FLOAT_32; + const ggml_tensor * src0 = op->src[0]; + const ggml_tensor * src1 = op->src[1]; + ggml_tensor * dst = op; + + GGMLQNN_CHECK_PARAMS(ctx, src0, src1, dst); instance = ctx->instance; QNN_INTERFACE_VER_TYPE qnn_raw_interface = ctx->raw_interface; - const ggml_tensor * src0 = op->src[0]; - const ggml_tensor * src1 = op->src[1]; - ggml_tensor * dst = op; op_perf.start(); std::string map_entry; From e4ddf3b8b63394329bb3e2f4d7f65bf7239af1c1 Mon Sep 17 00:00:00 2001 From: zhouwg Date: Sun, 16 Feb 2025 21:35:24 +0800 Subject: [PATCH 03/76] ggml-qnn: update script build-run-android.sh to compare peformance of ggml-qnn --- scripts/build-run-android.sh | 48 ++++++++++++++++++++++++++++++++---- 1 file changed, 43 insertions(+), 5 deletions(-) diff --git a/scripts/build-run-android.sh b/scripts/build-run-android.sh index 412ccadadaf6b..63614e6afe110 100755 --- a/scripts/build-run-android.sh +++ b/scripts/build-run-android.sh @@ -7,6 +7,7 @@ ANDROID_PLATFORM=android-34 ANDROID_NDK=${PWD}/android-ndk-r26c REMOTE_PATH=/data/local/tmp/ GGUF_MODEL_NAME=/sdcard/deepseek-r1-distill-qwen-1.5b-q4_0.gguf +GGUF_MODEL_NAME=/sdcard/qwen1_5-1_8b-chat-q4_0.gguf #QNN SDK could be found at: #https://www.qualcomm.com/developer/software/qualcomm-ai-engine-direct-sdk @@ -14,6 +15,9 @@ GGUF_MODEL_NAME=/sdcard/deepseek-r1-distill-qwen-1.5b-q4_0.gguf QNN_SDK_URL=https://www.qualcomm.com/developer/software/qualcomm-ai-engine-direct-sdk QNN_SDK_PATH=/opt/qcom/aistack/qairt/2.31.0.250130/ +#default is QNN NPU +qnnbackend=2 + function dump_vars() { echo -e "ANDROID_NDK: ${ANDROID_NDK}" @@ -137,10 +141,28 @@ function run_llamacli() adb shell "cd ${REMOTE_PATH} \ && export LD_LIBRARY_PATH=${REMOTE_PATH} \ - && ${REMOTE_PATH}/llama-cli -mg 2 -m ${GGUF_MODEL_NAME} -p \"introduce the movie Once Upon a Time in America briefly.\n\"" + && ${REMOTE_PATH}/llama-cli -mg ${qnnbackend} -m ${GGUF_MODEL_NAME} -p \"introduce the movie Once Upon a Time in America briefly.\n\"" } + +function run_llamabench() +{ + check_qnn_libs + + if [ -f ./out/android/bin/libggml-qnn.so ]; then + adb push ./out/android/bin/*.so ${REMOTE_PATH}/ + fi + adb push ./out/android/bin/llama-bench ${REMOTE_PATH}/ + adb shell chmod +x ${REMOTE_PATH}/llama-bench + + adb shell "cd ${REMOTE_PATH} \ + && export LD_LIBRARY_PATH=${REMOTE_PATH} \ + && ${REMOTE_PATH}/llama-bench -mg ${qnnbackend} -m ${GGUF_MODEL_NAME}" + +} + + function run_test-backend-ops() { check_qnn_libs @@ -163,8 +185,9 @@ function show_usage() echo "Usage:" echo " $0 build" echo " $0 updateqnnlib" - echo " $0 run_llamacli" echo " $0 run_testop" + echo " $0 run_llamacli 0 (QNN_CPU) / 1 (QNN_GPU) / 2 (QNN_NPU) / 3 (ggml)" + echo " $0 run_llamabench 0 (QNN_CPU) / 1 (QNN_GPU) / 2 (QNN_NPU) / 3 (ggml)" echo -e "\n\n\n" } @@ -186,15 +209,30 @@ elif [ $# == 1 ]; then elif [ "$1" == "build" ]; then build_ggml_qnn exit 0 - elif [ "$1" == "run_llamacli" ]; then - run_llamacli - exit 0 + elif [ "$1" == "run_testop" ]; then run_test-backend-ops exit 0 elif [ "$1" == "updateqnnlib" ]; then update_qnn_libs exit 0 + else + show_usage + exit 1 + fi +elif [ $# == 2 ]; then + qnnbackend=$2 + if [ ${qnnbackend} -gt 3 ]; then + show_usage + exit 1 + fi + + if [ "$1" == "run_llamacli" ]; then + run_llamacli + exit 0 + elif [ "$1" == "run_llamabench" ]; then + run_llamabench + exit 0 fi else show_usage From 56e73e5ba41ad7489b4905306b0431d93b163d4a Mon Sep 17 00:00:00 2001 From: zhouwg Date: Mon, 17 Feb 2025 19:01:06 +0800 Subject: [PATCH 04/76] ggml-qnn: fix minor issue in test-backend-ops.cpp --- tests/test-backend-ops.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 28f860a7f2969..20c66d10d6a6c 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -4651,7 +4651,11 @@ int main(int argc, char ** argv) { continue; } +#ifdef GGML_USE_QNN + ggml_backend_t backend = ggml_backend_dev_init(dev, reinterpret_cast(i)); +#else ggml_backend_t backend = ggml_backend_dev_init(dev, NULL); +#endif GGML_ASSERT(backend != NULL); ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(dev); From 5185cf778333b7bca00f3fd18fd699a0fd381260 Mon Sep 17 00:00:00 2001 From: zhouwg Date: Tue, 18 Feb 2025 09:53:57 +0800 Subject: [PATCH 05/76] ggml-qnn: merge QNN RPC feature from https://github.com/zhouwg/kantv/blob/ggml-qnn-quantize/core/ggml/llamacpp/ggml-qnn.cpp --- ggml/src/ggml-qnn/ggml-qnn.cpp | 534 +++++++++++++++++++-------------- 1 file changed, 307 insertions(+), 227 deletions(-) diff --git a/ggml/src/ggml-qnn/ggml-qnn.cpp b/ggml/src/ggml-qnn/ggml-qnn.cpp index 780bc3553ab0f..810711e41acc7 100644 --- a/ggml/src/ggml-qnn/ggml-qnn.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn.cpp @@ -266,6 +266,13 @@ static void * ggmlqnn_host_malloc(size_t n) { } \ } while (0) +#define CHECK_QNN_API(error) \ + do { \ + if (QNN_SUCCESS != (error)) { \ + GGMLQNN_LOG_INFO("error = %d\n", (error)); \ + } \ + } while (0) + #define VALIDATE_TENSOR_VERSION(tensor, err) VALIDATE(validate_tensor_version(tensor), err) #define VALIDATE_OP_CONFIG_VERSION(op, err) VALIDATE(validate_op_config_version(op), err) @@ -1175,40 +1182,20 @@ static struct ggml_backend_qnn_context g_qnn_mgr[GGML_QNN_MAX_DEVICES] = { .socinfo = {}}, }; -using ggml_dimension_array_t = int64_t[GGML_MAX_DIMS]; -using qnn_dimension_array_t = std::array; -using op_dims_calc_func_t = void (*)(const std::vector & input_dims, - ggml_dimension_array_t & output_dims); - -static void element_wise_op_dims(const std::vector & input_dims, - ggml_dimension_array_t &output_dims) { - for (size_t i = 1; i < std::size(output_dims); i++) { - output_dims[i] = input_dims.front()[i]; - } -} - -static void mat_mul_op_dims(const std::vector & input_dims, - ggml_dimension_array_t & output_dims) { - GGML_ASSERT(input_dims.size() == 2); - output_dims[0] = input_dims.front()[1]; - output_dims[1] = input_dims.back()[1]; -} struct qnn_op_caps_t { const char * qnn_op_name = nullptr; const size_t input_param_count = 0; - op_dims_calc_func_t calc_dims_func = nullptr; const char * qnn_param_name = nullptr; }; -constexpr static const qnn_op_caps_t kOpCaps[] = { +static const qnn_op_caps_t kOpCaps[] = { {}, // GGML_OP_NONE {}, // GGML_OP_DUP { // GGML_OP_ADD QNN_OP_ELEMENT_WISE_ADD, // qnn_op_name - 2, // input_param_count - element_wise_op_dims, // calc_dims_func + 2, // input_param_count }, {}, // GGML_OP_ADD1 {}, // GGML_OP_ACC @@ -1237,7 +1224,6 @@ constexpr static const qnn_op_caps_t kOpCaps[] = { // GGML_OP_MUL_MAT QNN_OP_MAT_MUL, // qnn_op_name 2, // input_param_count - mat_mul_op_dims, // calc_dims_func }, {}, // GGML_OP_MUL_MAT_ID {}, // GGML_OP_OUT_PROD @@ -1885,7 +1871,7 @@ class qnn_instance { return 0; } - std::string &get_qnn_graph_name() { return _graph_name; } + std::string & get_qnn_graph_name() { return _graph_name; } bool is_rpcmem_initialized() { return _rpcmem_initialized; @@ -1906,8 +1892,10 @@ class qnn_instance { void unregister_rpcmem(Qnn_MemHandle_t mem_handle); void * alloc_rpcmem(size_t bytes, size_t alignment); + void * get_rpcmem_from_memhandle(Qnn_MemHandle_t mem_handle); void free_rpcmem(void * buf); + void free_rpcmem(); bool is_rpcmem_allocated(void * buf); @@ -1915,6 +1903,10 @@ class qnn_instance { return _qnn_mem_set.count(handle) != 0U; } + bool enalbe_qnn_rpc() { + return _enable_qnn_rpc; + } + public: std::map> _qnn_graph_map; @@ -1975,15 +1967,16 @@ class qnn_instance { QNN_INTERFACE_VER_TYPE _qnn_raw_interface; QNN_SYSTEM_INTERFACE_VER_TYPE _qnn_raw_system_interface; - std::unordered_set _qnn_mem_set; + std::unordered_map _qnn_mem_set; std::unordered_map _qnn_rpc_buffer_to_handles; + static std::mutex _init_mutex; static std::unordered_map _loaded_lib_handle; static std::unordered_map _lib_path_to_backend_id; static std::unordered_map _loaded_backend; - void *_rpc_lib_handle = nullptr; + void * _rpc_lib_handle = nullptr; std::atomic_bool _rpcmem_initialized{false}; pfn_rpc_mem_alloc _pfn_rpc_mem_alloc; pfn_rpc_mem_free _pfn_rpc_mem_free; @@ -1995,6 +1988,7 @@ class qnn_instance { std::string _graph_name; QNNBackend _device_id; + bool _enable_qnn_rpc = false; //FIXME:unknown issue with QNN RPC feature }; std::mutex qnn_instance::_init_mutex; @@ -2032,11 +2026,30 @@ void qnn_instance::free_rpcmem(void * buf) { } else if (0 == _rpcmem_store_map.count(buf)) { GGMLQNN_LOG_WARN("no allocated tensor\n"); } else { + GGMLQNN_LOG_DEBUG("free rpc mem %p", _rpcmem_store_map[buf]); _pfn_rpc_mem_free(_rpcmem_store_map[buf]); _rpcmem_store_map.erase(buf); } } +void qnn_instance::free_rpcmem() { + Qnn_ErrorHandle_t error = QNN_SUCCESS; + + if (_rpcmem_store_map.empty()) { + GGMLQNN_LOG_WARN("no rpcmem allocated\n"); + return; + } + + for (std::unordered_map::iterator it = _rpcmem_store_map.begin(); + it != _qnn_mem_set.end(); + it++) { + void * rpcbuffer = it->second; + GGMLQNN_LOG_DEBUG("free rpc buffer %p", rpcbuffer); + _pfn_rpc_mem_free(rpcbuffer); + } + _rpcmem_store_map.clear(); +} + int32_t qnn_instance::rpcmem_to_fd(void * buf) { int32_t mem_fd = -1; if (!is_rpcmem_initialized()) { @@ -2059,10 +2072,6 @@ int qnn_instance::register_rpcmem(void * p_data, Qnn_Tensor_t * p_tensor) { return 2; } - if (is_rpcmem_allocated(p_data)) { - GGMLQNN_LOG_WARN("rpc memory already allocated\n"); - //return 3; - } if (is_rpcmem_registered((QNN_VER_PTR(*p_tensor)->memHandle))) { GGMLQNN_LOG_WARN("tensor %s has been registered shared memory\n", (QNN_VER_PTR(*p_tensor)->name)); return 4; @@ -2094,7 +2103,7 @@ int qnn_instance::register_rpcmem(void * p_data, Qnn_Tensor_t * p_tensor) { GGMLQNN_LOG_INFO("tensor %s successfully register shared memory\n", (QNN_VER_PTR(*p_tensor)->name)); } QNN_VER_PTR(*p_tensor)->memHandle = handle; - _qnn_mem_set.insert(handle); + _qnn_mem_set.insert((std::pair(p_data, handle))); return 0; } @@ -2136,6 +2145,19 @@ Qnn_MemHandle_t qnn_instance::register_rpcmem(void * p_data, const uint32_t ran return handle; } +void * qnn_instance::get_rpcmem_from_memhandle(Qnn_MemHandle_t mem_handle) { + for (std::unordered_map::iterator it = _qnn_mem_set.begin(); + it != _qnn_mem_set.end(); + it++) { + Qnn_MemHandle_t mem_handle = it->second; + if (it->second == mem_handle) { + return it->first; + } + } + GGMLQNN_LOG_WARN("can't find rpcmem from qnn mem handle %p", mem_handle); + return nullptr; +} + void qnn_instance::unregister_rpcmem() { Qnn_ErrorHandle_t error = QNN_SUCCESS; @@ -2143,10 +2165,16 @@ void qnn_instance::unregister_rpcmem() { GGMLQNN_LOG_WARN("no rpcmem registered\n"); } - for (auto &mem_handle : _qnn_mem_set) { + for (std::unordered_map::iterator it = _qnn_mem_set.begin(); + it != _qnn_mem_set.end(); + it++) { + Qnn_MemHandle_t mem_handle = it->second; error = _qnn_interface.qnn_mem_de_register(&mem_handle, 1); if (error != QNN_SUCCESS) { - GGMLQNN_LOG_WARN("failed to unregister shared memory, error %d\n", QNN_GET_ERROR_CODE(error)); + GGMLQNN_LOG_WARN("failed to unregister shared memory, error %d\n", + QNN_GET_ERROR_CODE(error)); + } else { + GGMLQNN_LOG_DEBUG("unregister shared memory ok"); } } _qnn_mem_set.clear(); @@ -2158,14 +2186,14 @@ void qnn_instance::unregister_rpcmem(Qnn_MemHandle_t mem_handle) { GGMLQNN_LOG_WARN("failed to unregister shared memory, error %d", QNN_GET_ERROR_CODE(error)); } - auto it = std::find_if(_qnn_rpc_buffer_to_handles.begin(), _qnn_rpc_buffer_to_handles.end(), + auto it = std::find_if(_qnn_mem_set.begin(), _qnn_mem_set.end(), [mem_handle](const auto &kv) { return kv.second == mem_handle; }); - if (it == _qnn_rpc_buffer_to_handles.end()) { + if (it == _qnn_mem_set.end()) { GGMLQNN_LOG_WARN("failed to find shared memory handler: %p", mem_handle); return; } - _qnn_rpc_buffer_to_handles.erase(it); + _qnn_mem_set.erase(it); } bool qnn_instance::is_rpcmem_allocated(void * buf) { @@ -2562,7 +2590,7 @@ int qnn_instance::qnn_init(const QnnSaver_Config_t ** saver_config) { temp_context_config.empty() ? nullptr : temp_context_config.data(), &_qnn_context_handle); if (nullptr == _qnn_context_handle) { - GGMLQNN_LOG_WARN("why failed to initialize qnn context\n"); + GGMLQNN_LOG_WARN("why failed to initialize qnn context, error:%s\n", strerror(errno)); return 8; } else { GGMLQNN_LOG_DEBUG("initialize qnn context successfully\n"); @@ -2636,9 +2664,13 @@ int qnn_instance::qnn_finalize() { int ret_status = 0; Qnn_ErrorHandle_t error = QNN_SUCCESS; + GGMLQNN_LOG_DEBUG("enter %s\n", __func__); //FIXME:should be removed in the future reset_idx(); + free_rpcmem(); + unregister_rpcmem(); + if (nullptr != _pfn_rpc_mem_deinit) _pfn_rpc_mem_deinit(); @@ -2700,6 +2732,7 @@ int qnn_instance::qnn_finalize() { unload_backend(); unload_system(); + GGMLQNN_LOG_DEBUG("leave %s\n", __func__); return ret_status; } @@ -2812,10 +2845,133 @@ int qnn_instance::finalize_qnn_graph() { return 0; } +static uint8_t * create_rpc_buffer(qnn_instance * instance, const ggml_tensor * ggml_tensor, Qnn_Tensor_t * qnn_tensor, bool b_copydata) { + if (nullptr == instance || nullptr == ggml_tensor || nullptr == qnn_tensor) { + GGMLQNN_LOG_WARN("invalid params\n"); + return nullptr; + } + + uint8_t * qnn_rpcbuffer = static_cast(instance->alloc_rpcmem(ggml_nbytes(ggml_tensor), 4)); + if (nullptr == qnn_rpcbuffer) { + GGMLQNN_LOG_WARN("alloc rpcmem failure, %s\n", strerror(errno)); + return nullptr; + } else { + GGMLQNN_LOG_DEBUG("alloc rpcmem %p successfully\n", qnn_rpcbuffer); + } + if (b_copydata) + memcpy(qnn_rpcbuffer, ggml_tensor->data, ggml_nbytes(ggml_tensor)); + instance->register_rpcmem(qnn_rpcbuffer, qnn_tensor); + return qnn_rpcbuffer; +} + +static Qnn_ErrorHandle_t create_htp_graph(ggml_backend_qnn_context * ctx, const std::string & graph_name, Qnn_GraphHandle_t * graph_handle) { + Qnn_ErrorHandle_t error = QNN_SUCCESS; + if (nullptr == ctx) + return QNN_MIN_ERROR_COMMON; + + qnn_instance * instance = ctx->instance; + QNN_INTERFACE_VER_TYPE qnn_raw_interface = ctx->raw_interface; + + QnnHtpGraph_CustomConfig_t hvx_config; + hvx_config.option = QNN_HTP_GRAPH_CONFIG_OPTION_NUM_HVX_THREADS; + hvx_config.numHvxThreads = 4; + QnnGraph_Config_t graph_hvx_config; + graph_hvx_config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM; + graph_hvx_config.customConfig = &hvx_config; + + QnnHtpGraph_CustomConfig_t dlbc_config; + dlbc_config.option = QNN_HTP_GRAPH_CONFIG_OPTION_OPTIMIZATION; + dlbc_config.optimizationOption.type = QNN_HTP_GRAPH_OPTIMIZATION_TYPE_ENABLE_DLBC; + dlbc_config.optimizationOption.floatValue = 1.0; // set to 0.0 to turn off DLBC + QnnGraph_Config_t graph_dlbc_config; + graph_dlbc_config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM; + graph_dlbc_config.customConfig = &dlbc_config; + + QnnHtpGraph_CustomConfig_t opt_config; + opt_config.optimizationOption.type = QNN_HTP_GRAPH_OPTIMIZATION_TYPE_FINALIZE_OPTIMIZATION_FLAG; + opt_config.optimizationOption.floatValue = 3; // 1 or 3 + QnnGraph_Config_t graph_opt_config; + graph_opt_config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM; + graph_opt_config.customConfig = &opt_config; + + QnnHtpGraph_CustomConfig_t vtcm_config; + vtcm_config.option = QNN_HTP_GRAPH_CONFIG_OPTION_VTCM_SIZE; + vtcm_config.vtcmSizeInMB = ctx->socinfo.vtcm_size_in_mb; + QnnGraph_Config_t graph_vtcm_config; + graph_vtcm_config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM; + graph_vtcm_config.customConfig = &vtcm_config; + + QnnHtpGraph_CustomConfig_t precision_config; + precision_config.option = QNN_HTP_GRAPH_CONFIG_OPTION_PRECISION; + precision_config.precision = QNN_PRECISION_FLOAT16; + QnnGraph_Config_t graph_precision_config; + graph_precision_config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM; + graph_precision_config.customConfig = &precision_config; + + const QnnGraph_Config_t * p_graphconfig[] = {&graph_hvx_config, + &graph_dlbc_config, + &graph_vtcm_config, + &graph_opt_config, + &graph_precision_config, + NULL}; + error = qnn_raw_interface.graphCreate(instance->get_qnn_context_handle(), + graph_name.c_str(), + p_graphconfig, graph_handle); + return error; +} + +static void print_tensors_info(const char * func_name, ggml_backend_qnn_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + //skip sanity check of params + GGMLQNN_LOG_DEBUG("call %s in dev %s\n", func_name, ctx->name); + GGMLQNN_LOG_DEBUG("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", + src0->name, + src0->type, ggml_type_name(src0->type), src0->ne[0], src0->ne[1], src0->ne[2], + src0->nb[0], src0->nb[1], src0->nb[2]); + GGMLQNN_LOG_DEBUG("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", + src1->name, + src1->type, ggml_type_name(src1->type), src1->ne[0], src1->ne[1], src1->ne[2], + src1->nb[0], src1->nb[1], src1->nb[2]); + GGMLQNN_LOG_DEBUG("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", + dst->name, + dst->type, ggml_type_name(dst->type), dst->ne[0], dst->ne[1], dst->ne[2], dst->nb[0], + dst->nb[1], dst->nb[2]); + GGMLQNN_LOG_DEBUG("%d, %d, %d, %d", src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3]); + GGMLQNN_LOG_DEBUG("tensor0 name %s", src0->name); + GGMLQNN_LOG_DEBUG("tensor1 name %s", src1->name); + GGMLQNN_LOG_DEBUG("tensor2 name %s", dst->name); +} + +static void dump_tensors_info(const struct ggml_tensor * tensor) { + //skip sanity check of params + struct ggml_tensor * src0 = tensor->src[0]; + struct ggml_tensor * src1 = tensor->src[1]; + GGMLQNN_LOG_DEBUG("op name:%s, tensor type:%s", ggml_op_name(tensor->op), + ggml_type_name(tensor->type)); + GGMLQNN_LOG_DEBUG("src0 type:%s", ggml_type_name(tensor->src[0]->type)); + GGMLQNN_LOG_DEBUG("src1 type:%s", ggml_type_name(tensor->src[1]->type)); + GGMLQNN_LOG_DEBUG( + "src0 %15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", + src0->name, + src0->type, ggml_type_name(src0->type), src0->ne[0], src0->ne[1], src0->ne[2], + src0->nb[0], src0->nb[1], src0->nb[2]); + GGMLQNN_LOG_DEBUG( + "src1 %15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", + src1->name, + src1->type, ggml_type_name(src1->type), src1->ne[0], src1->ne[1], src1->ne[2], + src1->nb[0], src1->nb[1], src1->nb[2]); + GGMLQNN_LOG_DEBUG( + " %15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", + tensor->name, + tensor->type, ggml_type_name(tensor->type), tensor->ne[0], tensor->ne[1], + tensor->ne[2], + tensor->nb[0], + tensor->nb[1], tensor->nb[2]); +} + // ================================================================================================= // section-6: implementation of ggml-qnn backend // ================================================================================================= -static bool ggml_qnn_can_handle_op(const struct ggml_tensor * tensor, bool b_dump_tensor_info) { +static bool ggml_qnn_can_handle_op(const struct ggml_tensor * tensor) { if (tensor->op == GGML_OP_NONE) { return true; } @@ -2846,32 +3002,12 @@ static bool ggml_qnn_can_handle_op(const struct ggml_tensor * tensor, bool b_dum if (!ggml_are_same_shape(src0, src1)) { return false; } -#if GGMLQNN_PRINT_OP_ADD_LOG - if (b_dump_tensor_info) { - GGMLQNN_LOG_DEBUG("op name:%s, tensor type:%s", ggml_op_name(tensor->op), - ggml_type_name(tensor->type)); - GGMLQNN_LOG_DEBUG("src0 type:%s", ggml_type_name(tensor->src[0]->type)); - GGMLQNN_LOG_DEBUG("src1 type:%s", ggml_type_name(tensor->src[1]->type)); - GGMLQNN_LOG_DEBUG("GGML_OP_ADD"); - GGMLQNN_LOG_DEBUG( - "src0 %15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", - src0->name, - src0->type, ggml_type_name(src0->type), src0->ne[0], src0->ne[1], src0->ne[2], - src0->nb[0], src0->nb[1], src0->nb[2]); - GGMLQNN_LOG_DEBUG( - "src1 %15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", - src1->name, - src1->type, ggml_type_name(src1->type), src1->ne[0], src1->ne[1], src1->ne[2], - src1->nb[0], src1->nb[1], src1->nb[2]); - GGMLQNN_LOG_DEBUG( - " %15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", - tensor->name, - tensor->type, ggml_type_name(tensor->type), tensor->ne[0], tensor->ne[1], - tensor->ne[2], - tensor->nb[0], - tensor->nb[1], tensor->nb[2]); - } + if (ne00 < 32) + return false; + +#if GGMLQNN_PRINT_OP_ADD_LOG + dump_tensors_info(tensor); #endif return (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16) && (src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16); @@ -2880,31 +3016,7 @@ static bool ggml_qnn_can_handle_op(const struct ggml_tensor * tensor, bool b_dum if (tensor->op == GGML_OP_MUL_MAT) { #if GGMLQNN_PRINT_OP_MUL_MAT_LOG - if (b_dump_tensor_info) { - GGMLQNN_LOG_DEBUG("op name:%s, tensor type:%s", ggml_op_name(tensor->op), - ggml_type_name(tensor->type)); - GGMLQNN_LOG_DEBUG("src0 type:%s", ggml_type_name(tensor->src[0]->type)); - GGMLQNN_LOG_DEBUG("src1 type:%s", ggml_type_name(tensor->src[1]->type)); - GGMLQNN_LOG_DEBUG("dst type:%s", ggml_type_name(tensor->type)); - GGMLQNN_LOG_DEBUG( - "src0 %15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", - src0->name, - src0->type, ggml_type_name(src0->type), src0->ne[0], src0->ne[1], src0->ne[2], - src0->nb[0], src0->nb[1], src0->nb[2]); - GGMLQNN_LOG_DEBUG( - "src1 %15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", - src1->name, - src1->type, ggml_type_name(src1->type), src1->ne[0], src1->ne[1], src1->ne[2], - src1->nb[0], src1->nb[1], src1->nb[2]); - GGMLQNN_LOG_DEBUG( - "dst %15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", - tensor->name, - tensor->type, ggml_type_name(tensor->type), tensor->ne[0], tensor->ne[1], - tensor->ne[2], - tensor->nb[0], - tensor->nb[1], tensor->nb[2]); - - } + dump_tensors_info(tensor); #endif //FIXME: 2048 is an experimental value between ASR inference and LLM inference because // it's better only offload big matrix to QNN backend @@ -2920,7 +3032,7 @@ static bool ggml_qnn_can_handle_op(const struct ggml_tensor * tensor, bool b_dum return (src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16) && (tensor->type == GGML_TYPE_F32 || tensor->type == GGML_TYPE_F16); #else - //passthrough mul_mat + //fall back to ggml cpu backend return (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16) && (src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16) && (src0->type == src1->type) && (src0->type == tensor->type); @@ -2954,6 +3066,10 @@ static void ggml_qnn_add(ggml_backend_t backend, ggml_tensor * op) { const ggml_tensor * src1 = op->src[1]; ggml_tensor * dst = op; + uint8_t * qnn_rpcbuffer_0 = nullptr; + uint8_t * qnn_rpcbuffer_1 = nullptr; + uint8_t * qnn_rpcbuffer_2 = nullptr; + GGMLQNN_CHECK_PARAMS(ctx, src0, src1, dst); instance = ctx->instance; @@ -2976,26 +3092,7 @@ static void ggml_qnn_add(ggml_backend_t backend, ggml_tensor * op) { tensor_2 = ggml_qnn_create_tensor(dst); } -//#if GGMLQNN_DEBUG //uncomment this line and comment next line when troubleshooting mul_mat issue -#if GGMLQNN_PRINT_OP_ADD_LOG - GGMLQNN_LOG_DEBUG("call %s in dev %s\n", __func__, ctx->name); - GGMLQNN_LOG_DEBUG("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", - src0->name, - src0->type, ggml_type_name(src0->type), src0->ne[0], src0->ne[1], src0->ne[2], - src0->nb[0], src0->nb[1], src0->nb[2]); - GGMLQNN_LOG_DEBUG("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", - src1->name, - src1->type, ggml_type_name(src1->type), src1->ne[0], src1->ne[1], src1->ne[2], - src1->nb[0], src1->nb[1], src1->nb[2]); - GGMLQNN_LOG_DEBUG("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", - dst->name, - dst->type, ggml_type_name(dst->type), dst->ne[0], dst->ne[1], dst->ne[2], dst->nb[0], - dst->nb[1], dst->nb[2]); - GGMLQNN_LOG_DEBUG("%d, %d, %d, %d", src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3]); - GGMLQNN_LOG_DEBUG("tensor0 name %s", QNN_TENSOR_GET_NAME(tensor_0)); - GGMLQNN_LOG_DEBUG("tensor1 name %s", QNN_TENSOR_GET_NAME(tensor_1)); - GGMLQNN_LOG_DEBUG("tensor2 name %s", QNN_TENSOR_GET_NAME(tensor_2)); -#endif + print_tensors_info(__func__, ctx, src0, src1, dst); QNN_VER_PTR(*tensor_0)->type = QNN_TENSOR_TYPE_APP_WRITE; QNN_VER_PTR(*tensor_1)->type = QNN_TENSOR_TYPE_APP_WRITE; @@ -3013,51 +3110,7 @@ static void ggml_qnn_add(ggml_backend_t backend, ggml_tensor * op) { graph_name = map_entry; GGMLQNN_LOG_DEBUG("graph name %s", graph_name.c_str()); if (ctx->device == QNN_BACKEND_NPU) { - QnnHtpGraph_CustomConfig_t hvx_config; - hvx_config.option = QNN_HTP_GRAPH_CONFIG_OPTION_NUM_HVX_THREADS; - hvx_config.numHvxThreads = 4; - QnnGraph_Config_t graph_hvx_config; - graph_hvx_config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM; - graph_hvx_config.customConfig = &hvx_config; - - QnnHtpGraph_CustomConfig_t dlbc_config; - dlbc_config.option = QNN_HTP_GRAPH_CONFIG_OPTION_OPTIMIZATION; - dlbc_config.optimizationOption.type = QNN_HTP_GRAPH_OPTIMIZATION_TYPE_ENABLE_DLBC; - dlbc_config.optimizationOption.floatValue = 1.0; // set to 0.0 to turn off DLBC - QnnGraph_Config_t graph_dlbc_config; - graph_dlbc_config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM; - graph_dlbc_config.customConfig = &dlbc_config; - - QnnHtpGraph_CustomConfig_t opt_config; - opt_config.optimizationOption.type = QNN_HTP_GRAPH_OPTIMIZATION_TYPE_FINALIZE_OPTIMIZATION_FLAG; - opt_config.optimizationOption.floatValue = 3; // 1 or 3 - QnnGraph_Config_t graph_opt_config; - graph_opt_config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM; - graph_opt_config.customConfig = &opt_config; - - QnnHtpGraph_CustomConfig_t vtcm_config; - vtcm_config.option = QNN_HTP_GRAPH_CONFIG_OPTION_VTCM_SIZE; - vtcm_config.vtcmSizeInMB = ctx->socinfo.vtcm_size_in_mb; - QnnGraph_Config_t graph_vtcm_config; - graph_vtcm_config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM; - graph_vtcm_config.customConfig = &vtcm_config; - - QnnHtpGraph_CustomConfig_t precision_config; - precision_config.option = QNN_HTP_GRAPH_CONFIG_OPTION_PRECISION; - precision_config.precision = QNN_PRECISION_FLOAT16; - QnnGraph_Config_t graph_precision_config; - graph_precision_config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM; - graph_precision_config.customConfig = &precision_config; - - const QnnGraph_Config_t * p_graphconfig[] = {&graph_hvx_config, - &graph_dlbc_config, - &graph_vtcm_config, - &graph_opt_config, - &graph_precision_config, - NULL}; - error = qnn_raw_interface.graphCreate(instance->get_qnn_context_handle(), - graph_name.c_str(), - p_graphconfig, &graph_handle); + error = create_htp_graph(ctx, graph_name, &graph_handle); } else { error = qnn_raw_interface.graphCreate(instance->get_qnn_context_handle(), graph_name.c_str(), @@ -3067,23 +3120,45 @@ static void ggml_qnn_add(ggml_backend_t backend, ggml_tensor * op) { GGMLQNN_LOG_INFO("can't create qnn graph handle with graph name %s, error = %d\n", graph_name.c_str(), error); return; } - error = qnn_raw_interface.tensorCreateGraphTensor(graph_handle, tensor_0); - if (QNN_SUCCESS != error) { - GGMLQNN_LOG_INFO("error = %d\n", error); + + if (instance->enalbe_qnn_rpc()) { + if (ctx->device == QNN_BACKEND_NPU) { // QNN RPC feature only available for NPU backend + QNN_VER_PTR(*tensor_0)->memType = QNN_TENSORMEMTYPE_MEMHANDLE; + QNN_VER_PTR(*tensor_0)->clientBuf = {.data=nullptr, .dataSize=0}; + + QNN_VER_PTR(*tensor_1)->memType = QNN_TENSORMEMTYPE_MEMHANDLE; + QNN_VER_PTR(*tensor_1)->clientBuf = {.data=nullptr, .dataSize=0}; + + QNN_VER_PTR(*tensor_2)->memType = QNN_TENSORMEMTYPE_MEMHANDLE; + QNN_VER_PTR(*tensor_2)->clientBuf = {.data=nullptr, .dataSize=0}; + } } + + error = qnn_raw_interface.tensorCreateGraphTensor(graph_handle, tensor_0); + CHECK_QNN_API(error); error = qnn_raw_interface.tensorCreateGraphTensor(graph_handle, tensor_1); - if (QNN_SUCCESS != error) { - GGMLQNN_LOG_INFO("error = %d\n", error); - } + CHECK_QNN_API(error); error = qnn_raw_interface.tensorCreateGraphTensor(graph_handle, tensor_2); - if (QNN_SUCCESS != error) { - GGMLQNN_LOG_INFO("error = %d\n", error); + CHECK_QNN_API(error); + + if (instance->enalbe_qnn_rpc()) { + if (ctx->device == QNN_BACKEND_NPU) { // QNN RPC feature only available for NPU backend + qnn_rpcbuffer_0 = create_rpc_buffer(instance, src0, tensor_0, true); + qnn_rpcbuffer_1 = create_rpc_buffer(instance, src1, tensor_1, true); + qnn_rpcbuffer_2 = create_rpc_buffer(instance, dst, tensor_2, false); + if (nullptr == qnn_rpcbuffer_0 || nullptr == qnn_rpcbuffer_1 || + nullptr == qnn_rpcbuffer_2) { + GGMLQNN_LOG_INFO("create rpc buffer failure\n"); + //FIXME: potential memory leak althought it shouldn't happen + return; + } + } + } else { + QNN_VER_PTR(*tensor_0)->clientBuf = {src0->data, ggml_get_tensor_data_size(src0)}; + QNN_VER_PTR(*tensor_1)->clientBuf = {src1->data, ggml_get_tensor_data_size(src1)}; + QNN_VER_PTR(*tensor_2)->clientBuf = {dst->data, ggml_get_tensor_data_size(dst)}; } - QNN_VER_PTR(*tensor_0)->clientBuf = {src0->data, ggml_get_tensor_data_size(src0)}; - QNN_VER_PTR(*tensor_1)->clientBuf = {src1->data, ggml_get_tensor_data_size(src1)}; - QNN_VER_PTR(*tensor_2)->clientBuf = {dst->data, ggml_get_tensor_data_size(dst)}; - Qnn_Tensor_t tensor_inputs[] = { *tensor_0, *tensor_1 @@ -3105,42 +3180,69 @@ static void ggml_qnn_add(ggml_backend_t backend, ggml_tensor * op) { } }; error = qnn_raw_interface.graphAddNode(graph_handle, op_config); - if (QNN_SUCCESS != error) { - GGMLQNN_LOG_INFO("error = %d\n", error); - } + CHECK_QNN_API(error); error = qnn_raw_interface.graphFinalize(graph_handle, nullptr, nullptr); - if (QNN_SUCCESS != error) { - GGMLQNN_LOG_INFO("error = %d\n", error); - } + CHECK_QNN_API(error); error = qnn_raw_interface.graphExecute(graph_handle, tensor_inputs, 2, tensor_outputs, 1, nullptr, nullptr); - if (QNN_SUCCESS != error) { - GGMLQNN_LOG_INFO("error = %d\n", error); + CHECK_QNN_API(error); + + if (instance->enalbe_qnn_rpc()) { + if (ctx->device == QNN_BACKEND_NPU) { // QNN RPC feature only available for NPU backend + uint8_t * qnn_rpcbuffer = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*tensor_2)->memHandle)); + GGMLQNN_LOG_INFO("qnn_rpcbuffer = %p\n", qnn_rpcbuffer); + if (nullptr != qnn_rpcbuffer) { + memcpy(dst->data, qnn_rpcbuffer, ggml_nbytes(dst)); + } + } } + auto graph_item = std::make_tuple(graph_handle, tensor_0, tensor_1, tensor_2); instance->_qnn_graph_map[map_entry] = graph_item; + } else { + uint32_t dimensions_input_0[] = {(uint32_t) src0->ne[0], (uint32_t) src0->ne[1], (uint32_t) src0->ne[2], (uint32_t) src0->ne[3]}; uint32_t dimensions_input_1[] = {(uint32_t) src1->ne[0], (uint32_t) src1->ne[1], (uint32_t) src1->ne[2], (uint32_t) src1->ne[3]}; uint32_t dimensions_output[] = {(uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->ne[2], (uint32_t) dst->ne[3]}; - QNN_VER_PTR(*tensor_0)->dimensions = dimensions_input_0; - QNN_VER_PTR(*tensor_0)->rank = ggml_get_tensor_rank(src0); - QNN_VER_PTR(*tensor_0)->dataType = src0_qnn_type; - QNN_VER_PTR(*tensor_1)->dimensions = dimensions_input_1; - QNN_VER_PTR(*tensor_1)->rank = ggml_get_tensor_rank(src1); - QNN_VER_PTR(*tensor_1)->dataType = src1_qnn_type; - QNN_VER_PTR(*tensor_2)->dimensions = dimensions_output; - QNN_VER_PTR(*tensor_2)->rank = ggml_get_tensor_rank(dst); - QNN_VER_PTR(*tensor_2)->dataType = dst_qnn_type; - QNN_VER_PTR(*tensor_0)->clientBuf = {src0->data, ggml_get_tensor_data_size(src0)}; - QNN_VER_PTR(*tensor_1)->clientBuf = {src1->data, ggml_get_tensor_data_size(src1)}; - QNN_VER_PTR(*tensor_2)->clientBuf = {dst->data, ggml_get_tensor_data_size(dst)}; + QNN_VER_PTR(*tensor_0)->dimensions = dimensions_input_0; + QNN_VER_PTR(*tensor_0)->rank = ggml_get_tensor_rank(src0); + QNN_VER_PTR(*tensor_0)->dataType = src0_qnn_type; + + QNN_VER_PTR(*tensor_1)->dimensions = dimensions_input_1; + QNN_VER_PTR(*tensor_1)->rank = ggml_get_tensor_rank(src1); + QNN_VER_PTR(*tensor_1)->dataType = src1_qnn_type; + + QNN_VER_PTR(*tensor_2)->dimensions = dimensions_output; + QNN_VER_PTR(*tensor_2)->rank = ggml_get_tensor_rank(dst); + QNN_VER_PTR(*tensor_2)->dataType = dst_qnn_type; + + if (instance->enalbe_qnn_rpc()) { + if (ctx->device == QNN_BACKEND_NPU) { // QNN RPC feature only available for NPU backend + //FIXME:why failure with test-backend-ops + uint8_t * qnn_buffer_0 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*tensor_0)->memHandle)); + GGMLQNN_LOG_INFO("qnn_rpcbuffer_0 = %p\n", qnn_rpcbuffer_0); + if (nullptr != qnn_buffer_0) { + memcpy(qnn_buffer_0, src0->data, ggml_nbytes(src0)); + } + + uint8_t * qnn_buffer_1 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*tensor_1)->memHandle)); + GGMLQNN_LOG_INFO("qnn_rpcbuffer_1 = %p\n", qnn_rpcbuffer_1); + if (nullptr != qnn_buffer_1) { + memcpy(qnn_buffer_1, src1->data, ggml_nbytes(src1)); + } + } + } else { + QNN_VER_PTR(*tensor_0)->clientBuf = {src0->data, ggml_get_tensor_data_size(src0)}; + QNN_VER_PTR(*tensor_1)->clientBuf = {src1->data, ggml_get_tensor_data_size(src1)}; + QNN_VER_PTR(*tensor_2)->clientBuf = {dst->data, ggml_get_tensor_data_size(dst)}; + } Qnn_Tensor_t tensor_inputs[] = { *tensor_0, @@ -3156,6 +3258,15 @@ static void ggml_qnn_add(ggml_backend_t backend, ggml_tensor * op) { if (QNN_SUCCESS != error) { GGMLQNN_LOG_INFO("error = %d\n", error); } + + if (instance->enalbe_qnn_rpc()) { + if (ctx->device == QNN_BACKEND_NPU) { // QNN RPC feature only available for NPU backend + //FIXME:why failure with test-backend-ops + uint8_t * qnn_buffer_2 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*tensor_2)->memHandle)); + if (nullptr != qnn_buffer_2) + memcpy(dst->data, qnn_buffer_2, ggml_nbytes(dst)); + } + } } //avoid memory leak in func free_qnn_tensor @@ -3224,25 +3335,8 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { tensor_2 = ggml_qnn_create_tensor(dst); } -#if GGMLQNN_DEBUG - GGMLQNN_LOG_DEBUG("call %s in dev %s\n", __func__, ctx->name); - GGMLQNN_LOG_DEBUG("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", - src0->name, - src0->type, ggml_type_name(src0->type), src0->ne[0], src0->ne[1], src0->ne[2], - src0->nb[0], src0->nb[1], src0->nb[2]); - GGMLQNN_LOG_DEBUG("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", - src1->name, - src1->type, ggml_type_name(src1->type), src1->ne[0], src1->ne[1], src1->ne[2], - src1->nb[0], src1->nb[1], src1->nb[2]); - GGMLQNN_LOG_DEBUG("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", - dst->name, - dst->type, ggml_type_name(dst->type), dst->ne[0], dst->ne[1], dst->ne[2], dst->nb[0], - dst->nb[1], dst->nb[2]); - GGMLQNN_LOG_DEBUG("%d, %d, %d, %d", src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3]); - GGMLQNN_LOG_DEBUG("tensor0 name %s", QNN_TENSOR_GET_NAME(tensor_0)); - GGMLQNN_LOG_DEBUG("tensor1 name %s", QNN_TENSOR_GET_NAME(tensor_1)); - GGMLQNN_LOG_DEBUG("tensor2 name %s", QNN_TENSOR_GET_NAME(tensor_2)); -#endif + print_tensors_info(__func__, ctx, src0, src1, dst); + QNN_VER_PTR(*tensor_0)->type = QNN_TENSOR_TYPE_APP_WRITE; QNN_VER_PTR(*tensor_1)->type = QNN_TENSOR_TYPE_APP_WRITE; QNN_VER_PTR(*tensor_2)->type = QNN_TENSOR_TYPE_APP_READ; @@ -3265,17 +3359,11 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { return; } error = qnn_raw_interface.tensorCreateGraphTensor(graph_handle, tensor_0); - if (QNN_SUCCESS != error) { - GGMLQNN_LOG_INFO("error = %d\n", error); - } + CHECK_QNN_API(error); error = qnn_raw_interface.tensorCreateGraphTensor(graph_handle, tensor_1); - if (QNN_SUCCESS != error) { - GGMLQNN_LOG_INFO("error = %d\n", error); - } + CHECK_QNN_API(error); error = qnn_raw_interface.tensorCreateGraphTensor(graph_handle, tensor_2); - if (QNN_SUCCESS != error) { - GGMLQNN_LOG_INFO("error = %d\n", error); - } + CHECK_QNN_API(error); QNN_VER_PTR(*tensor_0)->clientBuf = {src0->data, ggml_get_tensor_data_size(src0)}; QNN_VER_PTR(*tensor_1)->clientBuf = {src1->data, ggml_get_tensor_data_size(src1)}; @@ -3302,20 +3390,14 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { } }; error = qnn_raw_interface.graphAddNode(graph_handle, op_config); - if (QNN_SUCCESS != error) { - GGMLQNN_LOG_INFO("error = %d\n", error); - } + CHECK_QNN_API(error); error = qnn_raw_interface.graphFinalize(graph_handle, nullptr, nullptr); - if (QNN_SUCCESS != error) { - GGMLQNN_LOG_INFO("error = %d\n", error); - } + CHECK_QNN_API(error); error = qnn_raw_interface.graphExecute(graph_handle, tensor_inputs, 2, tensor_outputs, 1, nullptr, nullptr); - if (QNN_SUCCESS != error) { - GGMLQNN_LOG_INFO("error = %d\n", error); - } + CHECK_QNN_API(error); auto graph_item = std::make_tuple(graph_handle, tensor_0, tensor_1, tensor_2); instance->_qnn_graph_map[map_entry] = graph_item; } else { @@ -3350,9 +3432,7 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { tensor_inputs, 2, tensor_outputs, 1, nullptr, nullptr); - if (QNN_SUCCESS != error) { - GGMLQNN_LOG_INFO("error = %d\n", error); - } + CHECK_QNN_API(error); } //avoid memory leak in func free_qnn_tensor @@ -3699,7 +3779,7 @@ static ggml_backend_buffer_t ggml_backend_qnn_device_buffer_from_host_ptr(ggml_b static bool ggml_backend_qnn_device_supports_op(ggml_backend_dev_t dev, const struct ggml_tensor * op) { ggml_backend_qnn_context * ctx = (ggml_backend_qnn_context *) dev->context; - return (ggml_qnn_can_handle_op(op, true)); + return (ggml_qnn_can_handle_op(op)); } static bool ggml_backend_qnn_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) { From b77a867db0230bfd9b489c12a095806bc857ccbb Mon Sep 17 00:00:00 2001 From: zhouwg Date: Tue, 18 Feb 2025 17:35:04 +0800 Subject: [PATCH 06/76] ggml-qnn: sync from branch kantvai-ggmlqnn-npurpc --- ggml/src/ggml-qnn/ggml-qnn.cpp | 133 +++++++++++++++------------------ 1 file changed, 60 insertions(+), 73 deletions(-) diff --git a/ggml/src/ggml-qnn/ggml-qnn.cpp b/ggml/src/ggml-qnn/ggml-qnn.cpp index 810711e41acc7..6f2949333908e 100644 --- a/ggml/src/ggml-qnn/ggml-qnn.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn.cpp @@ -1903,7 +1903,7 @@ class qnn_instance { return _qnn_mem_set.count(handle) != 0U; } - bool enalbe_qnn_rpc() { + bool enable_qnn_rpc() { return _enable_qnn_rpc; } @@ -1989,6 +1989,9 @@ class qnn_instance { std::string _graph_name; QNNBackend _device_id; bool _enable_qnn_rpc = false; //FIXME:unknown issue with QNN RPC feature + + DISABLE_COPY(qnn_instance); + DISABLE_MOVE(qnn_instance); }; std::mutex qnn_instance::_init_mutex; @@ -3106,6 +3109,8 @@ static void ggml_qnn_add(ggml_backend_t backend, ggml_tensor * op) { uint32_t * tensor_1_dimensions = QNN_VER_PTR(*tensor_1)->dimensions; uint32_t * tensor_2_dimensions = QNN_VER_PTR(*tensor_2)->dimensions; + bool enable_npu_rpc = instance->enable_qnn_rpc() && ctx->device == QNN_BACKEND_NPU; + if (!graph_initialized) { graph_name = map_entry; GGMLQNN_LOG_DEBUG("graph name %s", graph_name.c_str()); @@ -3121,37 +3126,29 @@ static void ggml_qnn_add(ggml_backend_t backend, ggml_tensor * op) { return; } - if (instance->enalbe_qnn_rpc()) { - if (ctx->device == QNN_BACKEND_NPU) { // QNN RPC feature only available for NPU backend - QNN_VER_PTR(*tensor_0)->memType = QNN_TENSORMEMTYPE_MEMHANDLE; - QNN_VER_PTR(*tensor_0)->clientBuf = {.data=nullptr, .dataSize=0}; + if (enable_npu_rpc) { + QNN_VER_PTR(*tensor_0)->memType = QNN_TENSORMEMTYPE_MEMHANDLE; + QNN_VER_PTR(*tensor_0)->clientBuf = {.data=nullptr, .dataSize=0}; - QNN_VER_PTR(*tensor_1)->memType = QNN_TENSORMEMTYPE_MEMHANDLE; - QNN_VER_PTR(*tensor_1)->clientBuf = {.data=nullptr, .dataSize=0}; + QNN_VER_PTR(*tensor_1)->memType = QNN_TENSORMEMTYPE_MEMHANDLE; + QNN_VER_PTR(*tensor_1)->clientBuf = {.data=nullptr, .dataSize=0}; - QNN_VER_PTR(*tensor_2)->memType = QNN_TENSORMEMTYPE_MEMHANDLE; - QNN_VER_PTR(*tensor_2)->clientBuf = {.data=nullptr, .dataSize=0}; - } + QNN_VER_PTR(*tensor_2)->memType = QNN_TENSORMEMTYPE_MEMHANDLE; + QNN_VER_PTR(*tensor_2)->clientBuf = {.data=nullptr, .dataSize=0}; } - error = qnn_raw_interface.tensorCreateGraphTensor(graph_handle, tensor_0); - CHECK_QNN_API(error); - error = qnn_raw_interface.tensorCreateGraphTensor(graph_handle, tensor_1); - CHECK_QNN_API(error); - error = qnn_raw_interface.tensorCreateGraphTensor(graph_handle, tensor_2); - CHECK_QNN_API(error); - - if (instance->enalbe_qnn_rpc()) { - if (ctx->device == QNN_BACKEND_NPU) { // QNN RPC feature only available for NPU backend - qnn_rpcbuffer_0 = create_rpc_buffer(instance, src0, tensor_0, true); - qnn_rpcbuffer_1 = create_rpc_buffer(instance, src1, tensor_1, true); - qnn_rpcbuffer_2 = create_rpc_buffer(instance, dst, tensor_2, false); - if (nullptr == qnn_rpcbuffer_0 || nullptr == qnn_rpcbuffer_1 || - nullptr == qnn_rpcbuffer_2) { - GGMLQNN_LOG_INFO("create rpc buffer failure\n"); - //FIXME: potential memory leak althought it shouldn't happen - return; - } + CHECK_QNN_API(error = qnn_raw_interface.tensorCreateGraphTensor(graph_handle, tensor_0)); + CHECK_QNN_API(error = qnn_raw_interface.tensorCreateGraphTensor(graph_handle, tensor_1)); + CHECK_QNN_API(error = qnn_raw_interface.tensorCreateGraphTensor(graph_handle, tensor_2)); + + if (enable_npu_rpc) { + qnn_rpcbuffer_0 = create_rpc_buffer(instance, src0, tensor_0, true); + qnn_rpcbuffer_1 = create_rpc_buffer(instance, src1, tensor_1, true); + qnn_rpcbuffer_2 = create_rpc_buffer(instance, dst, tensor_2, false); + if (nullptr == qnn_rpcbuffer_0 || nullptr == qnn_rpcbuffer_1 || nullptr == qnn_rpcbuffer_2) { + GGMLQNN_LOG_INFO("create rpc buffer failure\n"); + //FIXME: potential memory leak althought it shouldn't happen + return; } } else { QNN_VER_PTR(*tensor_0)->clientBuf = {src0->data, ggml_get_tensor_data_size(src0)}; @@ -3179,23 +3176,19 @@ static void ggml_qnn_add(ggml_backend_t backend, ggml_tensor * op) { tensor_outputs } }; - error = qnn_raw_interface.graphAddNode(graph_handle, op_config); - CHECK_QNN_API(error); - error = qnn_raw_interface.graphFinalize(graph_handle, nullptr, nullptr); - CHECK_QNN_API(error); + CHECK_QNN_API(error = qnn_raw_interface.graphAddNode(graph_handle, op_config)); + CHECK_QNN_API(error = qnn_raw_interface.graphFinalize(graph_handle, nullptr, nullptr)); error = qnn_raw_interface.graphExecute(graph_handle, tensor_inputs, 2, tensor_outputs, 1, nullptr, nullptr); CHECK_QNN_API(error); - if (instance->enalbe_qnn_rpc()) { - if (ctx->device == QNN_BACKEND_NPU) { // QNN RPC feature only available for NPU backend - uint8_t * qnn_rpcbuffer = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*tensor_2)->memHandle)); - GGMLQNN_LOG_INFO("qnn_rpcbuffer = %p\n", qnn_rpcbuffer); - if (nullptr != qnn_rpcbuffer) { - memcpy(dst->data, qnn_rpcbuffer, ggml_nbytes(dst)); - } + if (enable_npu_rpc) { + uint8_t * qnn_rpcbuffer = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*tensor_2)->memHandle)); + GGMLQNN_LOG_INFO("qnn_rpcbuffer = %p\n", qnn_rpcbuffer); + if (nullptr != qnn_rpcbuffer) { + memcpy(dst->data, qnn_rpcbuffer, ggml_nbytes(dst)); } } @@ -3223,25 +3216,23 @@ static void ggml_qnn_add(ggml_backend_t backend, ggml_tensor * op) { QNN_VER_PTR(*tensor_2)->rank = ggml_get_tensor_rank(dst); QNN_VER_PTR(*tensor_2)->dataType = dst_qnn_type; - if (instance->enalbe_qnn_rpc()) { - if (ctx->device == QNN_BACKEND_NPU) { // QNN RPC feature only available for NPU backend - //FIXME:why failure with test-backend-ops - uint8_t * qnn_buffer_0 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*tensor_0)->memHandle)); - GGMLQNN_LOG_INFO("qnn_rpcbuffer_0 = %p\n", qnn_rpcbuffer_0); - if (nullptr != qnn_buffer_0) { - memcpy(qnn_buffer_0, src0->data, ggml_nbytes(src0)); - } + if (enable_npu_rpc) { + //FIXME:why failure with test-backend-ops + uint8_t * qnn_buffer_0 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*tensor_0)->memHandle)); + GGMLQNN_LOG_INFO("qnn_rpcbuffer_0 = %p\n", qnn_rpcbuffer_0); + if (nullptr != qnn_buffer_0) { + memcpy(qnn_buffer_0, src0->data, ggml_nbytes(src0)); + } - uint8_t * qnn_buffer_1 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*tensor_1)->memHandle)); - GGMLQNN_LOG_INFO("qnn_rpcbuffer_1 = %p\n", qnn_rpcbuffer_1); - if (nullptr != qnn_buffer_1) { - memcpy(qnn_buffer_1, src1->data, ggml_nbytes(src1)); - } + uint8_t * qnn_buffer_1 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*tensor_1)->memHandle)); + GGMLQNN_LOG_INFO("qnn_rpcbuffer_1 = %p\n", qnn_rpcbuffer_1); + if (nullptr != qnn_buffer_1) { + memcpy(qnn_buffer_1, src1->data, ggml_nbytes(src1)); } } else { QNN_VER_PTR(*tensor_0)->clientBuf = {src0->data, ggml_get_tensor_data_size(src0)}; QNN_VER_PTR(*tensor_1)->clientBuf = {src1->data, ggml_get_tensor_data_size(src1)}; - QNN_VER_PTR(*tensor_2)->clientBuf = {dst->data, ggml_get_tensor_data_size(dst)}; + QNN_VER_PTR(*tensor_2)->clientBuf = {dst->data, ggml_get_tensor_data_size(dst)}; } Qnn_Tensor_t tensor_inputs[] = { @@ -3255,16 +3246,13 @@ static void ggml_qnn_add(ggml_backend_t backend, ggml_tensor * op) { tensor_inputs, 2, tensor_outputs, 1, nullptr, nullptr); - if (QNN_SUCCESS != error) { - GGMLQNN_LOG_INFO("error = %d\n", error); - } + CHECK_QNN_API(error); - if (instance->enalbe_qnn_rpc()) { - if (ctx->device == QNN_BACKEND_NPU) { // QNN RPC feature only available for NPU backend - //FIXME:why failure with test-backend-ops - uint8_t * qnn_buffer_2 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*tensor_2)->memHandle)); - if (nullptr != qnn_buffer_2) - memcpy(dst->data, qnn_buffer_2, ggml_nbytes(dst)); + if (enable_npu_rpc) { + //FIXME:why failure with test-backend-ops + uint8_t * qnn_buffer_2 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*tensor_2)->memHandle)); + if (nullptr != qnn_buffer_2) { + memcpy(dst->data, qnn_buffer_2, ggml_nbytes(dst)); } } } @@ -3358,12 +3346,9 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { GGMLQNN_LOG_INFO("can't create qnn graph handle with graph name %s, error = %d\n", graph_name.c_str(), error); return; } - error = qnn_raw_interface.tensorCreateGraphTensor(graph_handle, tensor_0); - CHECK_QNN_API(error); - error = qnn_raw_interface.tensorCreateGraphTensor(graph_handle, tensor_1); - CHECK_QNN_API(error); - error = qnn_raw_interface.tensorCreateGraphTensor(graph_handle, tensor_2); - CHECK_QNN_API(error); + CHECK_QNN_API(error = qnn_raw_interface.tensorCreateGraphTensor(graph_handle, tensor_0)); + CHECK_QNN_API(error = qnn_raw_interface.tensorCreateGraphTensor(graph_handle, tensor_1)); + CHECK_QNN_API(error = qnn_raw_interface.tensorCreateGraphTensor(graph_handle, tensor_2)); QNN_VER_PTR(*tensor_0)->clientBuf = {src0->data, ggml_get_tensor_data_size(src0)}; QNN_VER_PTR(*tensor_1)->clientBuf = {src1->data, ggml_get_tensor_data_size(src1)}; @@ -3389,10 +3374,8 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { tensor_outputs } }; - error = qnn_raw_interface.graphAddNode(graph_handle, op_config); - CHECK_QNN_API(error); - error = qnn_raw_interface.graphFinalize(graph_handle, nullptr, nullptr); - CHECK_QNN_API(error); + CHECK_QNN_API(error = qnn_raw_interface.graphAddNode(graph_handle, op_config)); + CHECK_QNN_API(error = qnn_raw_interface.graphFinalize(graph_handle, nullptr, nullptr)); error = qnn_raw_interface.graphExecute(graph_handle, tensor_inputs, 2, tensor_outputs, 1, @@ -3400,7 +3383,9 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { CHECK_QNN_API(error); auto graph_item = std::make_tuple(graph_handle, tensor_0, tensor_1, tensor_2); instance->_qnn_graph_map[map_entry] = graph_item; + } else { + uint32_t dimensions_input_0[] = {(uint32_t) src0->ne[0], (uint32_t) src0->ne[1], (uint32_t) src0->ne[2], (uint32_t) src0->ne[3]}; uint32_t dimensions_input_1[] = {(uint32_t) src1->ne[0], (uint32_t) src1->ne[1], @@ -3410,9 +3395,11 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { QNN_VER_PTR(*tensor_0)->dimensions = dimensions_input_0; QNN_VER_PTR(*tensor_0)->rank = ggml_get_tensor_rank(src0); QNN_VER_PTR(*tensor_0)->dataType = src0_qnn_type; + QNN_VER_PTR(*tensor_1)->dimensions = dimensions_input_1; QNN_VER_PTR(*tensor_1)->rank = ggml_get_tensor_rank(src1); QNN_VER_PTR(*tensor_1)->dataType = src1_qnn_type; + QNN_VER_PTR(*tensor_2)->dimensions = dimensions_output; QNN_VER_PTR(*tensor_2)->rank = ggml_get_tensor_rank(dst); QNN_VER_PTR(*tensor_2)->dataType = dst_qnn_type; @@ -3656,7 +3643,7 @@ static enum ggml_status ggml_backend_qnn_graph_compute(ggml_backend_t backend, s ggml_backend_qnn_context * ctx = (ggml_backend_qnn_context *) backend->context; GGML_UNUSED(ctx); - //GGMLQNN_LOG_DEBUG("cgraph->n_nodes %d", cgraph->n_nodes); + GGMLQNN_LOG_DEBUG("cgraph->n_nodes %d", cgraph->n_nodes); for (int i = 0; i < cgraph->n_nodes; i++) { ggml_tensor * node = cgraph->nodes[i]; if (ggml_is_empty(node) || node->op == GGML_OP_RESHAPE From 7398aaa2d135fb984d61ac6c301333865bad37cf Mon Sep 17 00:00:00 2001 From: zhouwg Date: Wed, 19 Feb 2025 21:58:55 +0800 Subject: [PATCH 07/76] ggml-qnn: a concise approach to offload mulmat to QNN backend(sync from branch kantvai-ggmlqnn-npurpc, https://github.com/kantv-ai/llama.cpp/wiki/offloading-mulmat-to-QNN-backend) --- ggml/src/ggml-qnn/ggml-qnn.cpp | 626 ++++++++++++++++++++------------- 1 file changed, 377 insertions(+), 249 deletions(-) diff --git a/ggml/src/ggml-qnn/ggml-qnn.cpp b/ggml/src/ggml-qnn/ggml-qnn.cpp index 6f2949333908e..a1aca7940bf4f 100644 --- a/ggml/src/ggml-qnn/ggml-qnn.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn.cpp @@ -13,8 +13,9 @@ * section-5 does ggml-qnn backend helper macro / data structure / function / class * section-6 does implementation of ggml-qnn backend according to ggml's backend subsystem * - * currently only provide GGML_OP_ADD's QNN backend implementation: - * - GGML_OP_ADD: this is skeleton, can expand other ggml ops according to expertise + * currently only provide OPs' QNN backend implementation of GGML_OP_ADD & GGML_OP_MUL_MAT: + * - GGML_OP_ADD: this is a simple skeleton, can expand other ggml ops according to expertise + * - GGML_OP_MUL_MAT:this is a complicated skeleton, can expand other complex op accordingly * * of course, can porting ggml-qnn to Windows on ARM as need. * @@ -257,20 +258,25 @@ static void * ggmlqnn_host_malloc(size_t n) { // ================================================================================================= // section-4: QNN helper macro / data structure / function // ================================================================================================= -#define VALIDATE(value, status) \ - do { \ - status = value; \ - if (status != QNN_SUCCESS) { \ - GGMLQNN_LOG_WARN("%s expected QNN_SUCCESS\n", #value); \ - return status; \ - } \ +#define VALIDATE(value, status) \ + do { \ + status = value; \ + if (status != QNN_SUCCESS) { \ + GGMLQNN_LOG_WARN("%s expected QNN_SUCCESS\n", #value); \ + return status; \ + } \ } while (0) -#define CHECK_QNN_API(error) \ - do { \ - if (QNN_SUCCESS != (error)) { \ - GGMLQNN_LOG_INFO("error = %d\n", (error)); \ - } \ +#define CHECK_QNN_API(error, result) \ + do { \ + error = (result); \ + if (QNN_SUCCESS != error) { \ + if (error == QNN_COMMON_ERROR_NOT_SUPPORTED) { \ + GGMLQNN_LOG_WARN("WARNING: QNN feature/API not supported\n"); \ + } else { \ + GGMLQNN_LOG_INFO("QNN API error = %d(%s)\n", error, qnn_get_error_string(error)); \ + } \ + } \ } while (0) #define VALIDATE_TENSOR_VERSION(tensor, err) VALIDATE(validate_tensor_version(tensor), err) @@ -823,9 +829,8 @@ static int deep_copy_qnn_tensors(Qnn_Tensor_t & src, Qnn_Tensor_t & dst) { uint32_t rank = QNN_TENSOR_GET_RANK(src); QNN_TENSOR_SET_RANK(dst, rank); - size_t dim_size = rank * sizeof(uint32_t); + size_t dim_size = GGML_MAX_DIMS * sizeof(uint32_t); uint32_t * dimensions = (uint32_t *)malloc(dim_size); - GGMLQNN_LOG_DEBUG("tensor dims %p", dimensions); if (dimensions == nullptr) { GGMLQNN_LOG_WARN("deep_copy_qnn_tensors() allocation error while copying tensor %s\n", QNN_TENSOR_GET_NAME(src)); return 1; @@ -1025,6 +1030,9 @@ using _pfn_QnnSaver_initialize = decltype(QnnSaver_init using _pfn_QnnInterface_getProviders = decltype(QnnInterface_getProviders); using _pfn_QnnSystemInterface_getProviders = decltype(QnnSystemInterface_getProviders); +using qnn_res_t = std::tuple>; +using qnn_tensors_t = std::vector< Qnn_Tensor_t *>; + enum class ggml_qnn_profile_level { profile_off = 0, profile_basic = 1, @@ -1122,12 +1130,9 @@ struct ggml_backend_qnn_context { QNN_INTERFACE_VER_TYPE raw_interface; QNN_SYSTEM_INTERFACE_VER_TYPE raw_system_interface; struct qcom_socinfo socinfo; - - //FIXME: should I move it from public member of class qnn_instance to here? - //std::map> _qnn_graph_map; } ; -//FIXME: the following global vars and three helper funcs should be removed in the future +//TODO: the following global vars and three helper funcs should be removed in the future static int32_t g_ggmltensor_idx = 0; static void reset_idx() { g_ggmltensor_idx = 0; @@ -1399,11 +1404,11 @@ static const char * ggml_get_type_name(ggml_type type) { return traits->type_name; } -Qnn_Tensor_t * ggml_qnn_create_tensor(const ggml_tensor * tensor) { +static Qnn_Tensor_t * ggml_qnn_create_compute_tensor(const ggml_tensor * tensor) { Qnn_ErrorHandle_t error = QNN_SUCCESS; char tensor_name[GGML_MAX_NAME] = {0}; - //FIXME:remove get_idx() and inc_idx() in the future but ensure the tensor name is unique + //TODO:remove get_idx() and inc_idx() in the future but ensure the tensor name is unique snprintf(tensor_name, GGML_MAX_NAME, "tensor_%-8d", get_idx()); GGMLQNN_LOG_DEBUG("init_tensor %d", get_idx()); inc_idx(); @@ -1450,6 +1455,73 @@ Qnn_Tensor_t * ggml_qnn_create_tensor(const ggml_tensor * tensor) { return p_qnn_tensor; } +static Qnn_Tensor_t * ggml_qnn_create_mulmat_tensor(const ggml_tensor * tensor, const char * name, Qnn_TensorType_t qnn_tensor_type, + Qnn_DataType_t qnn_data_type, uint32_t rank, uint32_t * dims, void * data, uint32_t data_size) { + Qnn_ErrorHandle_t error = QNN_SUCCESS; + char tensor_name[GGML_MAX_NAME] = {0}; + + //TODO:remove get_idx() and inc_idx() in the future but ensure the tensor name is unique + if (nullptr != name) { + snprintf(tensor_name, GGML_MAX_NAME, "tensor_%-8d", get_idx()); + } else { + snprintf(tensor_name, GGML_MAX_NAME, "tensor_%s%-8d", name, get_idx()); + } + GGMLQNN_LOG_DEBUG("init_tensor %d", get_idx()); + inc_idx(); + + //there are different dimension order between ggml tensor and qnn tensor + uint32_t dimensions_transpose[GGML_MAX_DIMS] = {}; + uint32_t * tensor_dims = nullptr; + + if (nullptr != tensor) { + dimensions_transpose[0] = (uint32_t) tensor->ne[1]; + dimensions_transpose[1] = (uint32_t) tensor->ne[0]; + dimensions_transpose[2] = (uint32_t) tensor->ne[2]; + dimensions_transpose[3] = (uint32_t) tensor->ne[3]; + tensor_dims = dimensions_transpose; + } + if (nullptr != dims) { + tensor_dims = dims; + } + + Qnn_Tensor_t qnn_tensor = { + .version= QNN_TENSOR_VERSION_1, + {.v1= { + .id = 0, + .name = tensor_name, + .type = qnn_tensor_type, + .dataFormat = QNN_TENSOR_DATA_FORMAT_FLAT_BUFFER, + .dataType = qnn_data_type, + .quantizeParams = {QNN_DEFINITION_UNDEFINED, + QNN_QUANTIZATION_ENCODING_UNDEFINED, + {.scaleOffsetEncoding = {.scale = 0.0000000000000000f, .offset = 0}}}, + .rank = rank, + .dimensions = tensor_dims, + .memType = QNN_TENSORMEMTYPE_RAW, + {.clientBuf = {nullptr, 0} + } + } + } + }; + if (nullptr != name) { + QNN_VER_PTR(qnn_tensor)->name = name; + } + Qnn_Tensor_t * p_qnn_tensor = (Qnn_Tensor_t *)calloc(1, sizeof(Qnn_Tensor_t)); + if (nullptr == p_qnn_tensor) { + GGMLQNN_LOG_WARN("calloc failed"); + return nullptr; + } + error = deep_copy_qnn_tensors(qnn_tensor, * p_qnn_tensor); + if (error != QNN_SUCCESS) { + free(p_qnn_tensor); + GGMLQNN_LOG_WARN("init tensor failed"); + return nullptr; + } + QNN_VER_PTR(*p_qnn_tensor)->clientBuf = {data, data_size}; + + return p_qnn_tensor; +} + //TODO: // ref:explanation of k-quants, https://github.com/ggerganov/llama.cpp/pull/1684 static Qnn_DataType_t qnn_datatype_from_ggml_datatype(enum ggml_type ggmltype) { @@ -1908,7 +1980,7 @@ class qnn_instance { } public: - std::map> _qnn_graph_map; + std::map>> _qnn_graph_map; private: int load_system(); @@ -1988,7 +2060,7 @@ class qnn_instance { std::string _graph_name; QNNBackend _device_id; - bool _enable_qnn_rpc = false; //FIXME:unknown issue with QNN RPC feature + bool _enable_qnn_rpc = false; //TODO:unknown issue with QNN RPC feature DISABLE_COPY(qnn_instance); DISABLE_MOVE(qnn_instance); @@ -2207,7 +2279,7 @@ int qnn_instance::load_backend(std::string & lib_path, const QnnSaver_Config_t * Qnn_ErrorHandle_t error = QNN_SUCCESS; GGMLQNN_LOG_DEBUG("lib_path:%s\n", lib_path.c_str()); - void *lib_handle = dlopen(lib_path.c_str(), RTLD_NOW | RTLD_GLOBAL); + void * lib_handle = dlopen(lib_path.c_str(), RTLD_NOW | RTLD_GLOBAL); if (nullptr == lib_handle) { GGMLQNN_LOG_WARN("can not open QNN library %s, with error: %s", lib_path.c_str(), dlerror()); return 1; @@ -2223,7 +2295,7 @@ int qnn_instance::load_backend(std::string & lib_path, const QnnSaver_Config_t * // get QnnInterface Providers std::uint32_t num_providers = 0; - const QnnInterface_t **provider_list = nullptr; + const QnnInterface_t ** provider_list = nullptr; error = get_providers(&provider_list, &num_providers); if (error != QNN_SUCCESS) { GGMLQNN_LOG_WARN("failed to get providers, error %d", QNN_GET_ERROR_CODE(error)); @@ -2282,8 +2354,9 @@ int qnn_instance::load_backend(std::string & lib_path, const QnnSaver_Config_t * QnnSaver_Config_t backendid_cfg; backendid_cfg.option = QNN_SAVER_CONFIG_OPTION_BACKEND_ID; backendid_cfg.backendId = _backend_id; - const QnnSaver_Config_t *saverCfg[] = {&outputdir_cfg, &backendid_cfg, nullptr}; - if (0 == QnnSaver_initialize(saverCfg)) { + + const QnnSaver_Config_t * saver_cfg[] = {&outputdir_cfg, &backendid_cfg, nullptr}; + if (0 == QnnSaver_initialize(saver_cfg)) { GGMLQNN_LOG_INFO("QnnSaver_initialize successfully"); } else { GGMLQNN_LOG_WARN("QnnSaver_initialize failure"); @@ -2668,7 +2741,7 @@ int qnn_instance::qnn_finalize() { Qnn_ErrorHandle_t error = QNN_SUCCESS; GGMLQNN_LOG_DEBUG("enter %s\n", __func__); - //FIXME:should be removed in the future + //TODO:should be removed in the future reset_idx(); free_rpcmem(); @@ -2971,6 +3044,20 @@ static void dump_tensors_info(const struct ggml_tensor * tensor) { tensor->nb[1], tensor->nb[2]); } +//TODO: currently only support offloading 2D matrix to QNN backend +static void get_qnn_dimensions_from_ggml_dimensions(uint32_t * qnn_dimensions, uint32_t * ggml_dimensions, uint32_t rank) { + if (rank > GGML_MAX_DIMS) { + GGMLQNN_LOG_WARN("invalid params"); + return; + } + if (nullptr == qnn_dimensions || nullptr == ggml_dimensions) { + GGMLQNN_LOG_WARN("invalid params"); + return; + } + qnn_dimensions[0] = ggml_dimensions[1]; + qnn_dimensions[1] = ggml_dimensions[0]; +} + // ================================================================================================= // section-6: implementation of ggml-qnn backend // ================================================================================================= @@ -3010,7 +3097,7 @@ static bool ggml_qnn_can_handle_op(const struct ggml_tensor * tensor) { return false; #if GGMLQNN_PRINT_OP_ADD_LOG - dump_tensors_info(tensor); + //dump_tensors_info(tensor); #endif return (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16) && (src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16); @@ -3019,27 +3106,21 @@ static bool ggml_qnn_can_handle_op(const struct ggml_tensor * tensor) { if (tensor->op == GGML_OP_MUL_MAT) { #if GGMLQNN_PRINT_OP_MUL_MAT_LOG - dump_tensors_info(tensor); + //dump_tensors_info(tensor); #endif - //FIXME: 2048 is an experimental value between ASR inference and LLM inference because - // it's better only offload big matrix to QNN backend - if (ne01 <= 2048) { + uint32_t src0_rank = ggml_get_tensor_rank(src0); + uint32_t src1_rank = ggml_get_tensor_rank(src1); + + if ((src0_rank != 2) || (src1_rank != 2)) //TODO: only support offload 2D matrix mulmat to QNN backend return false; - } -#if 0 - //TODO: offload mul_mat to QNN backend - //need to process type trait in func ggml_qnn_mul_mat(...): + + //TODO: support more data type in func ggml_qnn_mul_mat(...): //src0: q4_0, q6_k, ... //src1: f32 //dst : f32 - return (src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16) - && (tensor->type == GGML_TYPE_F32 || tensor->type == GGML_TYPE_F16); -#else - //fall back to ggml cpu backend return (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16) && (src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16) && (src0->type == src1->type) && (src0->type == tensor->type); -#endif } //TODO:for other op @@ -3054,65 +3135,51 @@ static void ggml_qnn_add(ggml_backend_t backend, ggml_tensor * op) { bool graph_initialized = false; qnn_instance * instance = nullptr; ggml_backend_qnn_context * ctx = (ggml_backend_qnn_context *)backend->context; - std::string graph_name = "ggml_op_qnn_add"; qnn_perf op_perf = qnn_perf("ggml_qnn_add"); Qnn_GraphHandle_t graph_handle = nullptr; - Qnn_Tensor_t * tensor_0 = nullptr; - Qnn_Tensor_t * tensor_1 = nullptr; - Qnn_Tensor_t * tensor_2 = nullptr; + Qnn_Tensor_t * p_tensor0 = nullptr; + Qnn_Tensor_t * p_tensor1 = nullptr; + Qnn_Tensor_t * p_tensor2 = nullptr; Qnn_Param_t qnn_params[] = {}; - enum ggml_op ggmlop = GGML_OP_ADD; - Qnn_DataType_t src0_qnn_type = QNN_DATATYPE_FLOAT_32; - Qnn_DataType_t src1_qnn_type = QNN_DATATYPE_FLOAT_32; - Qnn_DataType_t dst_qnn_type = QNN_DATATYPE_FLOAT_32; const ggml_tensor * src0 = op->src[0]; const ggml_tensor * src1 = op->src[1]; ggml_tensor * dst = op; - uint8_t * qnn_rpcbuffer_0 = nullptr; - uint8_t * qnn_rpcbuffer_1 = nullptr; - uint8_t * qnn_rpcbuffer_2 = nullptr; - GGMLQNN_CHECK_PARAMS(ctx, src0, src1, dst); - instance = ctx->instance; QNN_INTERFACE_VER_TYPE qnn_raw_interface = ctx->raw_interface; - op_perf.start(); - std::string map_entry; - get_graph_key_from_op(op, map_entry); - if (instance->_qnn_graph_map.find(map_entry) != instance->_qnn_graph_map.end()) { + std::string graph_name; + get_graph_key_from_op(op, graph_name); + if (instance->_qnn_graph_map.find(graph_name) != instance->_qnn_graph_map.end()) { graph_initialized = true; - auto & graph_item = instance->_qnn_graph_map[map_entry]; + qnn_res_t & graph_item = instance->_qnn_graph_map[graph_name]; graph_handle = std::get<0>(graph_item); - tensor_0 = std::get<1>(graph_item); - tensor_1 = std::get<2>(graph_item); - tensor_2 = std::get<3>(graph_item); + qnn_tensors_t & tensor = std::get<1>(graph_item); + p_tensor0 = tensor[0]; + p_tensor1 = tensor[1]; + p_tensor2 = tensor[2]; } else { - tensor_0 = ggml_qnn_create_tensor(src0); - tensor_1 = ggml_qnn_create_tensor(src1); - tensor_2 = ggml_qnn_create_tensor(dst); + p_tensor0 = ggml_qnn_create_compute_tensor(src0); + p_tensor1 = ggml_qnn_create_compute_tensor(src1); + p_tensor2 = ggml_qnn_create_compute_tensor(dst); } - print_tensors_info(__func__, ctx, src0, src1, dst); - QNN_VER_PTR(*tensor_0)->type = QNN_TENSOR_TYPE_APP_WRITE; - QNN_VER_PTR(*tensor_1)->type = QNN_TENSOR_TYPE_APP_WRITE; - QNN_VER_PTR(*tensor_2)->type = QNN_TENSOR_TYPE_APP_READ; - - src0_qnn_type = qnn_datatype_from_ggml_datatype(src0->type); - src1_qnn_type = qnn_datatype_from_ggml_datatype(src1->type); - dst_qnn_type = qnn_datatype_from_ggml_datatype(dst->type); + //ensure QNN tensor has correct tensor type + QNN_VER_PTR(*p_tensor0)->type = QNN_TENSOR_TYPE_APP_WRITE; + QNN_VER_PTR(*p_tensor1)->type = QNN_TENSOR_TYPE_APP_WRITE; + QNN_VER_PTR(*p_tensor2)->type = QNN_TENSOR_TYPE_APP_READ; - uint32_t * tensor_0_dimensions = QNN_VER_PTR(*tensor_0)->dimensions; - uint32_t * tensor_1_dimensions = QNN_VER_PTR(*tensor_1)->dimensions; - uint32_t * tensor_2_dimensions = QNN_VER_PTR(*tensor_2)->dimensions; + //save the original dimensions of qnn tensors + uint32_t * tensor_0_dimensions = QNN_VER_PTR(*p_tensor0)->dimensions; + uint32_t * tensor_1_dimensions = QNN_VER_PTR(*p_tensor1)->dimensions; + uint32_t * tensor_2_dimensions = QNN_VER_PTR(*p_tensor2)->dimensions; bool enable_npu_rpc = instance->enable_qnn_rpc() && ctx->device == QNN_BACKEND_NPU; if (!graph_initialized) { - graph_name = map_entry; GGMLQNN_LOG_DEBUG("graph name %s", graph_name.c_str()); if (ctx->device == QNN_BACKEND_NPU) { error = create_htp_graph(ctx, graph_name, &graph_handle); @@ -3127,44 +3194,44 @@ static void ggml_qnn_add(ggml_backend_t backend, ggml_tensor * op) { } if (enable_npu_rpc) { - QNN_VER_PTR(*tensor_0)->memType = QNN_TENSORMEMTYPE_MEMHANDLE; - QNN_VER_PTR(*tensor_0)->clientBuf = {.data=nullptr, .dataSize=0}; + QNN_VER_PTR(*p_tensor0)->memType = QNN_TENSORMEMTYPE_MEMHANDLE; + QNN_VER_PTR(*p_tensor0)->clientBuf = {.data=nullptr, .dataSize=0}; - QNN_VER_PTR(*tensor_1)->memType = QNN_TENSORMEMTYPE_MEMHANDLE; - QNN_VER_PTR(*tensor_1)->clientBuf = {.data=nullptr, .dataSize=0}; + QNN_VER_PTR(*p_tensor1)->memType = QNN_TENSORMEMTYPE_MEMHANDLE; + QNN_VER_PTR(*p_tensor1)->clientBuf = {.data=nullptr, .dataSize=0}; - QNN_VER_PTR(*tensor_2)->memType = QNN_TENSORMEMTYPE_MEMHANDLE; - QNN_VER_PTR(*tensor_2)->clientBuf = {.data=nullptr, .dataSize=0}; + QNN_VER_PTR(*p_tensor2)->memType = QNN_TENSORMEMTYPE_MEMHANDLE; + QNN_VER_PTR(*p_tensor2)->clientBuf = {.data=nullptr, .dataSize=0}; } - CHECK_QNN_API(error = qnn_raw_interface.tensorCreateGraphTensor(graph_handle, tensor_0)); - CHECK_QNN_API(error = qnn_raw_interface.tensorCreateGraphTensor(graph_handle, tensor_1)); - CHECK_QNN_API(error = qnn_raw_interface.tensorCreateGraphTensor(graph_handle, tensor_2)); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor0)); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor1)); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor2)); if (enable_npu_rpc) { - qnn_rpcbuffer_0 = create_rpc_buffer(instance, src0, tensor_0, true); - qnn_rpcbuffer_1 = create_rpc_buffer(instance, src1, tensor_1, true); - qnn_rpcbuffer_2 = create_rpc_buffer(instance, dst, tensor_2, false); + uint8_t * qnn_rpcbuffer_0 = create_rpc_buffer(instance, src0, p_tensor0, true); + uint8_t * qnn_rpcbuffer_1 = create_rpc_buffer(instance, src1, p_tensor1, true); + uint8_t * qnn_rpcbuffer_2 = create_rpc_buffer(instance, dst, p_tensor2, false); if (nullptr == qnn_rpcbuffer_0 || nullptr == qnn_rpcbuffer_1 || nullptr == qnn_rpcbuffer_2) { GGMLQNN_LOG_INFO("create rpc buffer failure\n"); - //FIXME: potential memory leak althought it shouldn't happen + //TODO: potential memory leak although it shouldn't happen return; } } else { - QNN_VER_PTR(*tensor_0)->clientBuf = {src0->data, ggml_get_tensor_data_size(src0)}; - QNN_VER_PTR(*tensor_1)->clientBuf = {src1->data, ggml_get_tensor_data_size(src1)}; - QNN_VER_PTR(*tensor_2)->clientBuf = {dst->data, ggml_get_tensor_data_size(dst)}; + QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggml_get_tensor_data_size(src0)}; + QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggml_get_tensor_data_size(src1)}; + QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggml_get_tensor_data_size(dst)}; } Qnn_Tensor_t tensor_inputs[] = { - *tensor_0, - *tensor_1 + *p_tensor0, + *p_tensor1 }; Qnn_Tensor_t tensor_outputs[] = { - *tensor_2 + *p_tensor2 }; Qnn_OpConfig_t op_config = { - (Qnn_OpConfigVersion_t) 1, .v1 = { + QNN_OPCONFIG_VERSION_1, .v1 = { "ggml_op_add", QNN_OP_PACKAGE_NAME_QTI_AISW, QNN_OP_ELEMENT_WISE_ADD, @@ -3176,26 +3243,38 @@ static void ggml_qnn_add(ggml_backend_t backend, ggml_tensor * op) { tensor_outputs } }; - CHECK_QNN_API(error = qnn_raw_interface.graphAddNode(graph_handle, op_config)); - CHECK_QNN_API(error = qnn_raw_interface.graphFinalize(graph_handle, nullptr, nullptr)); - error = qnn_raw_interface.graphExecute(graph_handle, + CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, op_config)); + CHECK_QNN_API(error, qnn_raw_interface.graphFinalize(graph_handle, nullptr, nullptr)); + CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, tensor_inputs, 2, tensor_outputs, 1, - nullptr, nullptr); - CHECK_QNN_API(error); + nullptr, nullptr)); if (enable_npu_rpc) { - uint8_t * qnn_rpcbuffer = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*tensor_2)->memHandle)); + uint8_t * qnn_rpcbuffer = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*p_tensor2)->memHandle)); GGMLQNN_LOG_INFO("qnn_rpcbuffer = %p\n", qnn_rpcbuffer); if (nullptr != qnn_rpcbuffer) { memcpy(dst->data, qnn_rpcbuffer, ggml_nbytes(dst)); } } - auto graph_item = std::make_tuple(graph_handle, tensor_0, tensor_1, tensor_2); - instance->_qnn_graph_map[map_entry] = graph_item; + qnn_tensors_t ggml_op_add_tensors; + ggml_op_add_tensors.reserve(3); + ggml_op_add_tensors.push_back(p_tensor0); + ggml_op_add_tensors.push_back(p_tensor1); + ggml_op_add_tensors.push_back(p_tensor2); + + auto graph_item = std::make_tuple(graph_handle, ggml_op_add_tensors); + instance->_qnn_graph_map[graph_name] = graph_item; } else { + Qnn_DataType_t src0_qnn_type = QNN_DATATYPE_FLOAT_32; + Qnn_DataType_t src1_qnn_type = QNN_DATATYPE_FLOAT_32; + Qnn_DataType_t dst_qnn_type = QNN_DATATYPE_FLOAT_32; + + src0_qnn_type = qnn_datatype_from_ggml_datatype(src0->type); + src1_qnn_type = qnn_datatype_from_ggml_datatype(src1->type); + dst_qnn_type = qnn_datatype_from_ggml_datatype(dst->type); uint32_t dimensions_input_0[] = {(uint32_t) src0->ne[0], (uint32_t) src0->ne[1], (uint32_t) src0->ne[2], (uint32_t) src0->ne[3]}; @@ -3204,76 +3283,76 @@ static void ggml_qnn_add(ggml_backend_t backend, ggml_tensor * op) { uint32_t dimensions_output[] = {(uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->ne[2], (uint32_t) dst->ne[3]}; - QNN_VER_PTR(*tensor_0)->dimensions = dimensions_input_0; - QNN_VER_PTR(*tensor_0)->rank = ggml_get_tensor_rank(src0); - QNN_VER_PTR(*tensor_0)->dataType = src0_qnn_type; + QNN_VER_PTR(*p_tensor0)->dimensions = dimensions_input_0; + QNN_VER_PTR(*p_tensor0)->rank = ggml_get_tensor_rank(src0); + QNN_VER_PTR(*p_tensor0)->dataType = src0_qnn_type; - QNN_VER_PTR(*tensor_1)->dimensions = dimensions_input_1; - QNN_VER_PTR(*tensor_1)->rank = ggml_get_tensor_rank(src1); - QNN_VER_PTR(*tensor_1)->dataType = src1_qnn_type; + QNN_VER_PTR(*p_tensor1)->dimensions = dimensions_input_1; + QNN_VER_PTR(*p_tensor1)->rank = ggml_get_tensor_rank(src1); + QNN_VER_PTR(*p_tensor1)->dataType = src1_qnn_type; - QNN_VER_PTR(*tensor_2)->dimensions = dimensions_output; - QNN_VER_PTR(*tensor_2)->rank = ggml_get_tensor_rank(dst); - QNN_VER_PTR(*tensor_2)->dataType = dst_qnn_type; + QNN_VER_PTR(*p_tensor2)->dimensions = dimensions_output; + QNN_VER_PTR(*p_tensor2)->rank = ggml_get_tensor_rank(dst); + QNN_VER_PTR(*p_tensor2)->dataType = dst_qnn_type; if (enable_npu_rpc) { - //FIXME:why failure with test-backend-ops - uint8_t * qnn_buffer_0 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*tensor_0)->memHandle)); - GGMLQNN_LOG_INFO("qnn_rpcbuffer_0 = %p\n", qnn_rpcbuffer_0); + //TODO: NPU RPC feature will failed with test-backend-ops + uint8_t * qnn_buffer_0 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*p_tensor0)->memHandle)); + GGMLQNN_LOG_INFO("qnn_rpcbuffer_0 = %p\n", qnn_buffer_0); if (nullptr != qnn_buffer_0) { memcpy(qnn_buffer_0, src0->data, ggml_nbytes(src0)); } - uint8_t * qnn_buffer_1 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*tensor_1)->memHandle)); - GGMLQNN_LOG_INFO("qnn_rpcbuffer_1 = %p\n", qnn_rpcbuffer_1); + uint8_t * qnn_buffer_1 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*p_tensor1)->memHandle)); + GGMLQNN_LOG_INFO("qnn_rpcbuffer_1 = %p\n", qnn_buffer_1); if (nullptr != qnn_buffer_1) { memcpy(qnn_buffer_1, src1->data, ggml_nbytes(src1)); } } else { - QNN_VER_PTR(*tensor_0)->clientBuf = {src0->data, ggml_get_tensor_data_size(src0)}; - QNN_VER_PTR(*tensor_1)->clientBuf = {src1->data, ggml_get_tensor_data_size(src1)}; - QNN_VER_PTR(*tensor_2)->clientBuf = {dst->data, ggml_get_tensor_data_size(dst)}; + QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggml_get_tensor_data_size(src0)}; + QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggml_get_tensor_data_size(src1)}; + QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggml_get_tensor_data_size(dst)}; } Qnn_Tensor_t tensor_inputs[] = { - *tensor_0, - *tensor_1 + *p_tensor0, + *p_tensor1 }; Qnn_Tensor_t tensor_outputs[] = { - *tensor_2 + *p_tensor2 }; - error = qnn_raw_interface.graphExecute(graph_handle, + CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, tensor_inputs, 2, tensor_outputs, 1, - nullptr, nullptr); - CHECK_QNN_API(error); + nullptr, nullptr)); if (enable_npu_rpc) { - //FIXME:why failure with test-backend-ops - uint8_t * qnn_buffer_2 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*tensor_2)->memHandle)); + //TODO:NPU RPC feature will failed with test-backend-ops + uint8_t * qnn_buffer_2 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*p_tensor2)->memHandle)); if (nullptr != qnn_buffer_2) { memcpy(dst->data, qnn_buffer_2, ggml_nbytes(dst)); } } } - //avoid memory leak in func free_qnn_tensor - QNN_VER_PTR(*tensor_0)->dimensions = tensor_0_dimensions; - QNN_VER_PTR(*tensor_1)->dimensions = tensor_1_dimensions; - QNN_VER_PTR(*tensor_2)->dimensions = tensor_2_dimensions; + // restore the original dimensions of qnn tensors to avoid memory leak in func free_qnn_tensor + QNN_VER_PTR(*p_tensor0)->dimensions = tensor_0_dimensions; + QNN_VER_PTR(*p_tensor1)->dimensions = tensor_1_dimensions; + QNN_VER_PTR(*p_tensor2)->dimensions = tensor_2_dimensions; #if GGMLQNN_PRINT_OP_ADD_LOG op_perf.info(); #endif } -//TODO: /* - * the logic of ggml_qnn_mul_mat is similar to ggml_qnn_add,but type trait and matrix transpose are required - * for offload mulmat to QNN backend, so it's a standalone function. + * the logic of ggml_qnn_mul_mat is similar to ggml_qnn_add but much more complicated than ggml_qnn_add, + * matrix transpose and type trait are required for offload mulmat to QNN backend, + * so it's a standalone function. accordingly, this is another typical skeleton for offload other + * ggml ops to QNN backend * * MUL_MAT take most of the compute time (about 95%).so to speed up llama inference, we should focus on MUL_MAT. * - * we have three kinds of MUL_MAT to compute: + * have three kinds of MUL_MAT to compute: * mul_mat_f32: both src0 and src1 are F32, this will be naturally handled in QNN backend * mul_mat_f16_f32: src0 is F16 and src1 is F32, f16 in src0 -> f32 in src0', then src0' * src1 * mul_mat_q_f32: src0 is quantized (Q4_0, Q4_1, ...) and src1 is F32, src0 -> f32 in src0', then src0' * src1 @@ -3284,148 +3363,200 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { qnn_perf op_perf = qnn_perf("ggml_qnn_mul_mat"); qnn_instance * instance = nullptr; ggml_backend_qnn_context * ctx = (ggml_backend_qnn_context *) backend->context; - - std::string graph_name = "ggml_op_qnn_mul_mat"; Qnn_GraphHandle_t graph_handle = nullptr; - Qnn_Tensor_t * tensor_0 = nullptr; - Qnn_Tensor_t * tensor_1 = nullptr; - Qnn_Tensor_t * tensor_2 = nullptr; - - Qnn_Param_t qnn_params[] = {}; - - enum ggml_op ggmlop = GGML_OP_ADD; - Qnn_DataType_t src0_qnn_type = QNN_DATATYPE_FLOAT_32; - Qnn_DataType_t src1_qnn_type = QNN_DATATYPE_FLOAT_32; - Qnn_DataType_t dst_qnn_type = QNN_DATATYPE_FLOAT_32; + Qnn_Tensor_t * p_tensor0 = nullptr; + Qnn_Tensor_t * p_tensor1 = nullptr; + Qnn_Tensor_t * p_tensor2 = nullptr; + Qnn_Tensor_t * p_param_tensor = nullptr; + Qnn_Tensor_t * p_tensor2_transpose = nullptr; const ggml_tensor * src0 = op->src[0]; const ggml_tensor * src1 = op->src[1]; - ggml_tensor * dst = op; + ggml_tensor * dst = op; GGMLQNN_CHECK_PARAMS(ctx, src0, src1, dst); - instance = ctx->instance; QNN_INTERFACE_VER_TYPE qnn_raw_interface = ctx->raw_interface; - op_perf.start(); - std::string map_entry; - get_graph_key_from_op(op, map_entry); - if (instance->_qnn_graph_map.find(map_entry) != instance->_qnn_graph_map.end()) { - graph_initialized = true; - auto & graph_item = instance->_qnn_graph_map[map_entry]; - graph_handle = std::get<0>(graph_item); - tensor_0 = std::get<1>(graph_item); - tensor_1 = std::get<2>(graph_item); - tensor_2 = std::get<3>(graph_item); + std::string graph_name; + get_graph_key_from_op(op, graph_name); + if (instance->_qnn_graph_map.find(graph_name) != instance->_qnn_graph_map.end()) { + graph_initialized = true; + qnn_res_t & graph_item = instance->_qnn_graph_map[graph_name]; + graph_handle = std::get<0>(graph_item); + qnn_tensors_t & tensors = std::get<1>(graph_item); + p_tensor0 = tensors[0]; + p_tensor1 = tensors[1]; + p_tensor2 = tensors[2]; + p_param_tensor = tensors[3]; + p_tensor2_transpose = tensors[4]; } else { - tensor_0 = ggml_qnn_create_tensor(src0); - tensor_1 = ggml_qnn_create_tensor(src1); - tensor_2 = ggml_qnn_create_tensor(dst); + p_tensor0 = ggml_qnn_create_mulmat_tensor(src0, nullptr, QNN_TENSOR_TYPE_APP_WRITE,QNN_DATATYPE_FLOAT_32, 2, nullptr, nullptr, 0); + p_tensor1 = ggml_qnn_create_mulmat_tensor(src1, nullptr, QNN_TENSOR_TYPE_APP_WRITE,QNN_DATATYPE_FLOAT_32, 2, nullptr, nullptr, 0); + p_tensor2 = ggml_qnn_create_mulmat_tensor(dst, nullptr, QNN_TENSOR_TYPE_APP_READ,QNN_DATATYPE_FLOAT_32, 2, nullptr, nullptr, 0); } print_tensors_info(__func__, ctx, src0, src1, dst); - QNN_VER_PTR(*tensor_0)->type = QNN_TENSOR_TYPE_APP_WRITE; - QNN_VER_PTR(*tensor_1)->type = QNN_TENSOR_TYPE_APP_WRITE; - QNN_VER_PTR(*tensor_2)->type = QNN_TENSOR_TYPE_APP_READ; - - src0_qnn_type = qnn_datatype_from_ggml_datatype(src0->type); - src1_qnn_type = qnn_datatype_from_ggml_datatype(src1->type); - dst_qnn_type = qnn_datatype_from_ggml_datatype(dst->type); + //ensure QNN tensor has correct tensor type + QNN_VER_PTR(*p_tensor0)->type = QNN_TENSOR_TYPE_APP_WRITE; + QNN_VER_PTR(*p_tensor1)->type = QNN_TENSOR_TYPE_APP_WRITE; + QNN_VER_PTR(*p_tensor2)->type = QNN_TENSOR_TYPE_APP_READ; - uint32_t * tensor_0_dimensions = QNN_VER_PTR(*tensor_0)->dimensions; - uint32_t * tensor_1_dimensions = QNN_VER_PTR(*tensor_1)->dimensions; - uint32_t * tensor_2_dimensions = QNN_VER_PTR(*tensor_2)->dimensions; + //save the original dimensions of qnn tensors + uint32_t * tensor_0_dimensions = QNN_VER_PTR(*p_tensor0)->dimensions; + uint32_t * tensor_1_dimensions = QNN_VER_PTR(*p_tensor1)->dimensions; + uint32_t * tensor_2_dimensions = QNN_VER_PTR(*p_tensor2)->dimensions; if (!graph_initialized) { - graph_name = map_entry; GGMLQNN_LOG_DEBUG("graph name %s", graph_name.c_str()); + /* + there are two key-points in properly handling how to offload mulmat to the QNN backend in ggml-qnn + 1. transpose + a 3x2 f32 matrix which means 3 rows and 2 columns. in ggml, it could be created from: + struct ggml_tensor* matrix = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 2, 3); + which like this: + +---+---+ + | 0 | 1 | + +---+---+ + | 2 | 3 | + +---+---+ + | 4 | 5 | + +---+---+ + with + ne[0] = 2 + ne[1] = 3 + there are different dimension order between ggml tensor and qnn tensor + + 2. QNN's MatMul can only support input tensors with rank >= 2 + + there is gap between ggml mulmat and QNN mulmat,we need to perform a transpose operation when offloading mulmat to QNN backend. + */ + + //step-1: create qnn graph error = qnn_raw_interface.graphCreate(instance->get_qnn_context_handle(), graph_name.c_str(), nullptr, &graph_handle); if (QNN_SUCCESS != error) { GGMLQNN_LOG_INFO("can't create qnn graph handle with graph name %s, error = %d\n", graph_name.c_str(), error); return; } - CHECK_QNN_API(error = qnn_raw_interface.tensorCreateGraphTensor(graph_handle, tensor_0)); - CHECK_QNN_API(error = qnn_raw_interface.tensorCreateGraphTensor(graph_handle, tensor_1)); - CHECK_QNN_API(error = qnn_raw_interface.tensorCreateGraphTensor(graph_handle, tensor_2)); - - QNN_VER_PTR(*tensor_0)->clientBuf = {src0->data, ggml_get_tensor_data_size(src0)}; - QNN_VER_PTR(*tensor_1)->clientBuf = {src1->data, ggml_get_tensor_data_size(src1)}; - QNN_VER_PTR(*tensor_2)->clientBuf = {dst->data, ggml_get_tensor_data_size(dst)}; + //step-2: create param tensor for mulmat of 2d matrix + uint32_t param_tensor_dims[] = {2}; + uint32_t param_tensor_data[2] = {1, 0}; + p_param_tensor = ggml_qnn_create_mulmat_tensor(nullptr, "param", QNN_TENSOR_TYPE_STATIC,QNN_DATATYPE_UINT_32, 1, param_tensor_dims, param_tensor_data, 8); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_param_tensor)); + + //step-3: create compute tensor from ggml tensor + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor0)); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor1)); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor2)); + + QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggml_get_tensor_data_size(src0)}; + QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggml_get_tensor_data_size(src1)}; + QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggml_get_tensor_data_size(dst)}; + + //step-4: create a transpose tensor + uint32_t tensor2_transpose_dims[GGML_MAX_DIMS] = {}; + p_tensor2_transpose = ggml_qnn_create_mulmat_tensor(dst,"transpose",QNN_TENSOR_TYPE_NATIVE,QNN_DATATYPE_FLOAT_32, 2, nullptr, nullptr, 0); + get_qnn_dimensions_from_ggml_dimensions(tensor2_transpose_dims, tensor_2_dimensions,ggml_get_tensor_rank(dst)); + //save pointer because the dimensions of tensor p_tensor2_transpose will be changed later + uint32_t * tensor2_dimensions_transpose = QNN_VER_PTR(*p_tensor2_transpose)->dimensions; + //update dimensions of tensor p_tensor2_transpose to make QNN SDK happy + QNN_VER_PTR(*p_tensor2_transpose)->dimensions = tensor2_transpose_dims; + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor2_transpose)); + + //step-5: compose qnn graph: add mat_mul node + Qnn_Param_t out_0_params[] = { + {QNN_PARAMTYPE_SCALAR, + QNN_OP_MAT_MUL_PARAM_TRANSPOSE_IN1, + .scalarParam = {QNN_DATATYPE_BOOL_8, .bool8Value = 1} + } + }; - Qnn_Tensor_t tensor_inputs[] = { - *tensor_0, - *tensor_1 + Qnn_Tensor_t out_0_inputs[] = {*p_tensor0,*p_tensor1}; + Qnn_Tensor_t out_0_outputs[] = {*p_tensor2_transpose}; + Qnn_OpConfig_t out_0 = { + QNN_OPCONFIG_VERSION_1, .v1 = + {"ggmlqnn_mulmat_opconfig", QNN_OP_PACKAGE_NAME_QTI_AISW, QNN_OP_MAT_MUL, + 1, + out_0_params, + 2, + out_0_inputs, + 1, + out_0_outputs} }; - Qnn_Tensor_t tensor_outputs[] = { - *tensor_2 + CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle,out_0)); + + //step-5: compose qnn graph: add transpose node + Qnn_Param_t out_trans1_0_params[] = { + {(Qnn_ParamType_t) 1, + "perm", .tensorParam = *p_param_tensor + } }; - Qnn_OpConfig_t op_config = { - (Qnn_OpConfigVersion_t) 1, .v1 = { - "ggml_op_mul_mat", - QNN_OP_PACKAGE_NAME_QTI_AISW, - QNN_OP_MAT_MUL, - 0, - qnn_params, - 2, - tensor_inputs, + Qnn_Tensor_t out_trans1_0_inputs[] = {*p_tensor2_transpose}; + Qnn_Tensor_t out_trans1_0_outputs[] = {*p_tensor2}; + Qnn_OpConfig_t out_trans1_0 = { + QNN_OPCONFIG_VERSION_1, + .v1 = {"ggmlqnn_mulmat_transpose_opconfig", + "qti.aisw", + QNN_OP_TRANSPOSE, 1, + out_trans1_0_params, 1, - tensor_outputs - } + out_trans1_0_inputs, + 1, + out_trans1_0_outputs} }; - CHECK_QNN_API(error = qnn_raw_interface.graphAddNode(graph_handle, op_config)); - CHECK_QNN_API(error = qnn_raw_interface.graphFinalize(graph_handle, nullptr, nullptr)); - error = qnn_raw_interface.graphExecute(graph_handle, - tensor_inputs, 2, - tensor_outputs, 1, - nullptr, nullptr); - CHECK_QNN_API(error); - auto graph_item = std::make_tuple(graph_handle, tensor_0, tensor_1, tensor_2); - instance->_qnn_graph_map[map_entry] = graph_item; + CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle,out_trans1_0)); + + //step-6: finalize qnn graph and execute qnn graph + CHECK_QNN_API(error, qnn_raw_interface.graphFinalize(graph_handle, NULL, NULL)); + Qnn_Tensor_t input_tensors_0[] = {*p_tensor0,*p_tensor1}; + Qnn_Tensor_t output_tensors_0[] = {*p_tensor2}; + CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, + input_tensors_0, 2, + output_tensors_0, 1, + NULL, NULL)); + + qnn_tensors_t ggml_op_mulmat_tensors; + ggml_op_mulmat_tensors.reserve(5); + ggml_op_mulmat_tensors.push_back(p_tensor0); + ggml_op_mulmat_tensors.push_back(p_tensor1); + ggml_op_mulmat_tensors.push_back(p_tensor2); + ggml_op_mulmat_tensors.push_back(p_param_tensor); + ggml_op_mulmat_tensors.push_back(p_tensor2_transpose); + + auto graph_item = std::make_tuple(graph_handle, ggml_op_mulmat_tensors); + instance->_qnn_graph_map[graph_name] = graph_item; + + //avoid cleanup these resource to make test_backend_ops happy + //free_qnn_tensor(p_param_tensor); + //restore pointer to avoid memory leak + QNN_VER_PTR(*p_tensor2_transpose)->dimensions = tensor2_dimensions_transpose; + //free_qnn_tensor(p_tensor2_transpose); } else { - uint32_t dimensions_input_0[] = {(uint32_t) src0->ne[0], (uint32_t) src0->ne[1], - (uint32_t) src0->ne[2], (uint32_t) src0->ne[3]}; - uint32_t dimensions_input_1[] = {(uint32_t) src1->ne[0], (uint32_t) src1->ne[1], - (uint32_t) src1->ne[2], (uint32_t) src1->ne[3]}; - uint32_t dimensions_output[] = {(uint32_t) dst->ne[0], (uint32_t) dst->ne[1], - (uint32_t) dst->ne[2], (uint32_t) dst->ne[3]}; - QNN_VER_PTR(*tensor_0)->dimensions = dimensions_input_0; - QNN_VER_PTR(*tensor_0)->rank = ggml_get_tensor_rank(src0); - QNN_VER_PTR(*tensor_0)->dataType = src0_qnn_type; - - QNN_VER_PTR(*tensor_1)->dimensions = dimensions_input_1; - QNN_VER_PTR(*tensor_1)->rank = ggml_get_tensor_rank(src1); - QNN_VER_PTR(*tensor_1)->dataType = src1_qnn_type; - - QNN_VER_PTR(*tensor_2)->dimensions = dimensions_output; - QNN_VER_PTR(*tensor_2)->rank = ggml_get_tensor_rank(dst); - QNN_VER_PTR(*tensor_2)->dataType = dst_qnn_type; - - QNN_VER_PTR(*tensor_0)->clientBuf = {src0->data, ggml_get_tensor_data_size(src0)}; - QNN_VER_PTR(*tensor_1)->clientBuf = {src1->data, ggml_get_tensor_data_size(src1)}; - QNN_VER_PTR(*tensor_2)->clientBuf = {dst->data, ggml_get_tensor_data_size(dst)}; + QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggml_get_tensor_data_size(src0)}; + QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggml_get_tensor_data_size(src1)}; + QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggml_get_tensor_data_size(dst)}; Qnn_Tensor_t tensor_inputs[] = { - *tensor_0, - *tensor_1 + *p_tensor0, + *p_tensor1 }; Qnn_Tensor_t tensor_outputs[] = { - *tensor_2 + *p_tensor2 }; - error = qnn_raw_interface.graphExecute(graph_handle, + CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, tensor_inputs, 2, tensor_outputs, 1, - nullptr, nullptr); - CHECK_QNN_API(error); + nullptr, nullptr)); } - //avoid memory leak in func free_qnn_tensor - QNN_VER_PTR(*tensor_0)->dimensions = tensor_0_dimensions; - QNN_VER_PTR(*tensor_1)->dimensions = tensor_1_dimensions; - QNN_VER_PTR(*tensor_2)->dimensions = tensor_2_dimensions; + // restore the original dimensions of qnn tensors to avoid memory leak in func free_qnn_tensor + QNN_VER_PTR(*p_tensor0)->dimensions = tensor_0_dimensions; + QNN_VER_PTR(*p_tensor1)->dimensions = tensor_1_dimensions; + QNN_VER_PTR(*p_tensor2)->dimensions = tensor_2_dimensions; op_perf.info(); } @@ -3608,21 +3739,18 @@ static void ggml_backend_qnn_free(ggml_backend_t backend) { qnn_instance * instance = (qnn_instance*)g_qnn_mgr[ctx->device].instance; if (instance != nullptr) { - std::map>::iterator graph_it; + std::map>>::iterator graph_it; for (graph_it = instance->_qnn_graph_map.begin(); graph_it != instance->_qnn_graph_map.end(); graph_it++) { auto & graph_item = graph_it->second; Qnn_GraphHandle_t & graph_handle = std::get<0>(graph_item); - Qnn_Tensor_t * tensor_0 = std::get<1>(graph_item); - Qnn_Tensor_t * tensor_1 = std::get<2>(graph_item); - Qnn_Tensor_t * tensor_2 = std::get<3>(graph_item); + qnn_tensors_t & tensors = std::get<1>(graph_item); + for (auto tensor_it = tensors.begin(); tensor_it != tensors.end(); ++tensor_it) { + free_qnn_tensor(*tensor_it); + } GGML_UNUSED(graph_handle); GGMLQNN_LOG_DEBUG("graph type:%s", graph_it->first.c_str()); - free_qnn_tensor(tensor_0); - free_qnn_tensor(tensor_1); - free_qnn_tensor(tensor_2); } instance->_qnn_graph_map.clear(); From 7cd6648d8aca04c765a98ed654b29400d84f4f8e Mon Sep 17 00:00:00 2001 From: zhouwg Date: Thu, 20 Feb 2025 08:39:15 +0800 Subject: [PATCH 08/76] ggml-qnn: remove redundant codes --- ggml/src/ggml-qnn/ggml-qnn.cpp | 298 +++++++++++---------------------- 1 file changed, 97 insertions(+), 201 deletions(-) diff --git a/ggml/src/ggml-qnn/ggml-qnn.cpp b/ggml/src/ggml-qnn/ggml-qnn.cpp index a1aca7940bf4f..37c947f412f1f 100644 --- a/ggml/src/ggml-qnn/ggml-qnn.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn.cpp @@ -1404,58 +1404,69 @@ static const char * ggml_get_type_name(ggml_type type) { return traits->type_name; } -static Qnn_Tensor_t * ggml_qnn_create_compute_tensor(const ggml_tensor * tensor) { - Qnn_ErrorHandle_t error = QNN_SUCCESS; - char tensor_name[GGML_MAX_NAME] = {0}; - - //TODO:remove get_idx() and inc_idx() in the future but ensure the tensor name is unique - snprintf(tensor_name, GGML_MAX_NAME, "tensor_%-8d", get_idx()); - GGMLQNN_LOG_DEBUG("init_tensor %d", get_idx()); - inc_idx(); - - uint32_t dimensions[] = {(uint32_t) tensor->ne[0], (uint32_t) tensor->ne[1], - (uint32_t) tensor->ne[2], (uint32_t) tensor->ne[3]}; - Qnn_DataType_t qnn_data_type = QNN_DATATYPE_FLOAT_32; - Qnn_TensorType_t qnn_tensor_type = QNN_TENSOR_TYPE_APP_WRITE; +static const char * get_ggml_type_name(ggml_type type) { + const auto * traits = ggml_get_type_traits(type); + return traits->type_name; +} - if (tensor->flags & GGML_TENSOR_FLAG_INPUT) { - qnn_tensor_type = QNN_TENSOR_TYPE_APP_WRITE; - } else if (tensor->flags & GGML_TENSOR_FLAG_OUTPUT) { - qnn_tensor_type = QNN_TENSOR_TYPE_APP_READ; - } - Qnn_Tensor_t qnn_tensor = { - .version= QNN_TENSOR_VERSION_1, - {.v1= { - .id = 0, - .name = tensor_name, - .type = qnn_tensor_type, - .dataFormat = QNN_TENSOR_DATA_FORMAT_FLAT_BUFFER, - .dataType = qnn_data_type, - .quantizeParams = {QNN_DEFINITION_UNDEFINED, - QNN_QUANTIZATION_ENCODING_UNDEFINED, - {.scaleOffsetEncoding = {.scale = 0.0000000000000000f, .offset = 0}}}, - .rank = ggml_get_tensor_rank(tensor), - .dimensions = dimensions, - .memType = QNN_TENSORMEMTYPE_RAW, - {.clientBuf = {.data = nullptr, - .dataSize = 0}}}} - }; - Qnn_Tensor_t * p_qnn_tensor = (Qnn_Tensor_t *)calloc(1, sizeof(Qnn_Tensor_t)); - if (nullptr == p_qnn_tensor) { - GGMLQNN_LOG_WARN("calloc failed"); - return nullptr; +//TODO: +// ref:explanation of k-quants, https://github.com/ggerganov/llama.cpp/pull/1684 +static Qnn_DataType_t qnn_datatype_from_ggml_datatype(enum ggml_type ggmltype) { + switch (ggmltype) { + case GGML_TYPE_F16: + return QNN_DATATYPE_FLOAT_16; + case GGML_TYPE_F32: + return QNN_DATATYPE_FLOAT_32; + case GGML_TYPE_I8: + return QNN_DATATYPE_INT_8; + case GGML_TYPE_Q8_0: + return QNN_DATATYPE_SFIXED_POINT_8; + case GGML_TYPE_Q4_0: + return QNN_DATATYPE_SFIXED_POINT_4; + default: + break; } - error = deep_copy_qnn_tensors(qnn_tensor, * p_qnn_tensor); - if (error != QNN_SUCCESS) { - free(p_qnn_tensor); - GGMLQNN_LOG_WARN("init tensor failed"); - return nullptr; + return QNN_DATATYPE_UNDEFINED; +} + +//TODO: +static ggml_type ggml_datatype_from_qnn_datatype(Qnn_DataType_t qnn_type) { + switch (qnn_type) { + case QNN_DATATYPE_FLOAT_32: + return GGML_TYPE_F32; + case QNN_DATATYPE_FLOAT_16: + return GGML_TYPE_F16; + case QNN_DATATYPE_UINT_32: + case QNN_DATATYPE_INT_32: + return GGML_TYPE_I32; + case QNN_DATATYPE_INT_16: + return GGML_TYPE_I16; + case QNN_DATATYPE_INT_8: + return GGML_TYPE_I8; + case QNN_DATATYPE_SFIXED_POINT_8: + return GGML_TYPE_Q8_0; + case QNN_DATATYPE_SFIXED_POINT_4: + return GGML_TYPE_Q4_0; + default: + break; } + return GGML_TYPE_COUNT; +} - return p_qnn_tensor; +//TODO: add more ops +static const char * qnn_opname_from_ggmlop(enum ggml_op ggmlop) { + switch (ggmlop) { + case GGML_OP_ADD: + return QNN_OP_ELEMENT_WISE_ADD; + case GGML_OP_MUL_MAT: + return QNN_OP_MAT_MUL; + default: + break; + } + return nullptr; } -static Qnn_Tensor_t * ggml_qnn_create_mulmat_tensor(const ggml_tensor * tensor, const char * name, Qnn_TensorType_t qnn_tensor_type, +static Qnn_Tensor_t * ggml_qnn_create_general_tensor(const ggml_tensor * tensor, const char * name, Qnn_TensorType_t qnn_tensor_type, Qnn_DataType_t qnn_data_type, uint32_t rank, uint32_t * dims, void * data, uint32_t data_size) { Qnn_ErrorHandle_t error = QNN_SUCCESS; char tensor_name[GGML_MAX_NAME] = {0}; @@ -1480,6 +1491,7 @@ static Qnn_Tensor_t * ggml_qnn_create_mulmat_tensor(const ggml_tensor * tensor, dimensions_transpose[3] = (uint32_t) tensor->ne[3]; tensor_dims = dimensions_transpose; } + //re-assign tensor_dims if (nullptr != dims) { tensor_dims = dims; } @@ -1522,66 +1534,25 @@ static Qnn_Tensor_t * ggml_qnn_create_mulmat_tensor(const ggml_tensor * tensor, return p_qnn_tensor; } -//TODO: -// ref:explanation of k-quants, https://github.com/ggerganov/llama.cpp/pull/1684 -static Qnn_DataType_t qnn_datatype_from_ggml_datatype(enum ggml_type ggmltype) { - switch (ggmltype) { - case GGML_TYPE_F16: - return QNN_DATATYPE_FLOAT_16; - case GGML_TYPE_F32: - return QNN_DATATYPE_FLOAT_32; - case GGML_TYPE_I8: - return QNN_DATATYPE_INT_8; - case GGML_TYPE_Q8_0: - return QNN_DATATYPE_SFIXED_POINT_8; - case GGML_TYPE_Q4_0: - return QNN_DATATYPE_SFIXED_POINT_4; - default: - break; - } - return QNN_DATATYPE_UNDEFINED; -} +static Qnn_Tensor_t * ggml_qnn_create_compute_tensor(const ggml_tensor * tensor) { + uint32_t dimensions[] = {(uint32_t) tensor->ne[0], (uint32_t) tensor->ne[1], + (uint32_t) tensor->ne[2], (uint32_t) tensor->ne[3]}; + Qnn_DataType_t qnn_data_type = QNN_DATATYPE_FLOAT_32; + Qnn_TensorType_t qnn_tensor_type = QNN_TENSOR_TYPE_APP_WRITE; -//TODO: -static ggml_type ggml_datatype_from_qnn_datatype(Qnn_DataType_t qnn_type) { - switch (qnn_type) { - case QNN_DATATYPE_FLOAT_32: - return GGML_TYPE_F32; - case QNN_DATATYPE_FLOAT_16: - return GGML_TYPE_F16; - case QNN_DATATYPE_UINT_32: - case QNN_DATATYPE_INT_32: - return GGML_TYPE_I32; - case QNN_DATATYPE_INT_16: - return GGML_TYPE_I16; - case QNN_DATATYPE_INT_8: - return GGML_TYPE_I8; - case QNN_DATATYPE_SFIXED_POINT_8: - return GGML_TYPE_Q8_0; - case QNN_DATATYPE_SFIXED_POINT_4: - return GGML_TYPE_Q4_0; - default: - break; + if (tensor->flags & GGML_TENSOR_FLAG_INPUT) { + qnn_tensor_type = QNN_TENSOR_TYPE_APP_WRITE; + } else if (tensor->flags & GGML_TENSOR_FLAG_OUTPUT) { + qnn_tensor_type = QNN_TENSOR_TYPE_APP_READ; } - return GGML_TYPE_COUNT; -} -//TODO: add more ops -static const char * qnn_opname_from_ggmlop(enum ggml_op ggmlop) { - switch (ggmlop) { - case GGML_OP_ADD: - return QNN_OP_ELEMENT_WISE_ADD; - case GGML_OP_MUL_MAT: - return QNN_OP_MAT_MUL; - default: - break; - } - return nullptr; -} + qnn_data_type = qnn_datatype_from_ggml_datatype(tensor->type); + Qnn_Tensor_t * p_qnn_tensor = ggml_qnn_create_general_tensor(tensor, nullptr, + qnn_tensor_type, qnn_data_type, + ggml_n_dims(tensor), dimensions, + nullptr, 0); -static const char * get_ggml_type_name(ggml_type type) { - const auto * traits = ggml_get_type_traits(type); - return traits->type_name; + return p_qnn_tensor; } static void append_tensor_dimensions(const ggml_tensor * tensor, std::string & output) { @@ -1865,7 +1836,7 @@ class qnn_instance { uint8_t do_node_validation = 1, const QnnGraph_Config_t ** graph_configs = nullptr ); - int init_qnn_graph(const std::string &graph_name, QNNBackend device, size_t vtcm_size_in_mb); + int init_qnn_graph(const std::string &graph_name, QNNBackend device, size_t vtcm_size_in_mb = 8, size_t hvx_threads = 8); int finalize_qnn_graph(); @@ -2813,7 +2784,7 @@ int qnn_instance::qnn_finalize() { return ret_status; } -int qnn_instance::init_qnn_graph(const std::string & graph_name, QNNBackend device, size_t vtcm_size_in_mb) { +int qnn_instance::init_qnn_graph(const std::string & graph_name, QNNBackend device, size_t vtcm_size_in_mb, size_t hvx_threads) { _graph_name = graph_name; _device_id = device; @@ -2824,7 +2795,7 @@ int qnn_instance::init_qnn_graph(const std::string & graph_name, QNNBackend devi if (device == QNN_BACKEND_NPU) { QnnHtpGraph_CustomConfig_t hvx_config; hvx_config.option = QNN_HTP_GRAPH_CONFIG_OPTION_NUM_HVX_THREADS; - hvx_config.numHvxThreads = 8; + hvx_config.numHvxThreads = hvx_threads; QnnGraph_Config_t graph_hvx_config; graph_hvx_config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM; graph_hvx_config.customConfig = &hvx_config; @@ -2940,65 +2911,11 @@ static uint8_t * create_rpc_buffer(qnn_instance * instance, const ggml_tensor * return qnn_rpcbuffer; } -static Qnn_ErrorHandle_t create_htp_graph(ggml_backend_qnn_context * ctx, const std::string & graph_name, Qnn_GraphHandle_t * graph_handle) { - Qnn_ErrorHandle_t error = QNN_SUCCESS; - if (nullptr == ctx) - return QNN_MIN_ERROR_COMMON; - - qnn_instance * instance = ctx->instance; - QNN_INTERFACE_VER_TYPE qnn_raw_interface = ctx->raw_interface; - - QnnHtpGraph_CustomConfig_t hvx_config; - hvx_config.option = QNN_HTP_GRAPH_CONFIG_OPTION_NUM_HVX_THREADS; - hvx_config.numHvxThreads = 4; - QnnGraph_Config_t graph_hvx_config; - graph_hvx_config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM; - graph_hvx_config.customConfig = &hvx_config; - - QnnHtpGraph_CustomConfig_t dlbc_config; - dlbc_config.option = QNN_HTP_GRAPH_CONFIG_OPTION_OPTIMIZATION; - dlbc_config.optimizationOption.type = QNN_HTP_GRAPH_OPTIMIZATION_TYPE_ENABLE_DLBC; - dlbc_config.optimizationOption.floatValue = 1.0; // set to 0.0 to turn off DLBC - QnnGraph_Config_t graph_dlbc_config; - graph_dlbc_config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM; - graph_dlbc_config.customConfig = &dlbc_config; - - QnnHtpGraph_CustomConfig_t opt_config; - opt_config.optimizationOption.type = QNN_HTP_GRAPH_OPTIMIZATION_TYPE_FINALIZE_OPTIMIZATION_FLAG; - opt_config.optimizationOption.floatValue = 3; // 1 or 3 - QnnGraph_Config_t graph_opt_config; - graph_opt_config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM; - graph_opt_config.customConfig = &opt_config; - - QnnHtpGraph_CustomConfig_t vtcm_config; - vtcm_config.option = QNN_HTP_GRAPH_CONFIG_OPTION_VTCM_SIZE; - vtcm_config.vtcmSizeInMB = ctx->socinfo.vtcm_size_in_mb; - QnnGraph_Config_t graph_vtcm_config; - graph_vtcm_config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM; - graph_vtcm_config.customConfig = &vtcm_config; - - QnnHtpGraph_CustomConfig_t precision_config; - precision_config.option = QNN_HTP_GRAPH_CONFIG_OPTION_PRECISION; - precision_config.precision = QNN_PRECISION_FLOAT16; - QnnGraph_Config_t graph_precision_config; - graph_precision_config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM; - graph_precision_config.customConfig = &precision_config; - - const QnnGraph_Config_t * p_graphconfig[] = {&graph_hvx_config, - &graph_dlbc_config, - &graph_vtcm_config, - &graph_opt_config, - &graph_precision_config, - NULL}; - error = qnn_raw_interface.graphCreate(instance->get_qnn_context_handle(), - graph_name.c_str(), - p_graphconfig, graph_handle); - return error; -} - static void print_tensors_info(const char * func_name, ggml_backend_qnn_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { //skip sanity check of params - GGMLQNN_LOG_DEBUG("call %s in dev %s\n", func_name, ctx->name); + if (nullptr != func_name && nullptr != ctx) { + GGMLQNN_LOG_DEBUG("call %s in dev %s\n", func_name, ctx->name); + } GGMLQNN_LOG_DEBUG("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", src0->name, src0->type, ggml_type_name(src0->type), src0->ne[0], src0->ne[1], src0->ne[2], @@ -3019,29 +2936,14 @@ static void print_tensors_info(const char * func_name, ggml_backend_qnn_context static void dump_tensors_info(const struct ggml_tensor * tensor) { //skip sanity check of params - struct ggml_tensor * src0 = tensor->src[0]; + const struct ggml_tensor * src0 = tensor->src[0]; struct ggml_tensor * src1 = tensor->src[1]; + struct ggml_tensor * dst = const_cast(tensor); GGMLQNN_LOG_DEBUG("op name:%s, tensor type:%s", ggml_op_name(tensor->op), ggml_type_name(tensor->type)); GGMLQNN_LOG_DEBUG("src0 type:%s", ggml_type_name(tensor->src[0]->type)); GGMLQNN_LOG_DEBUG("src1 type:%s", ggml_type_name(tensor->src[1]->type)); - GGMLQNN_LOG_DEBUG( - "src0 %15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", - src0->name, - src0->type, ggml_type_name(src0->type), src0->ne[0], src0->ne[1], src0->ne[2], - src0->nb[0], src0->nb[1], src0->nb[2]); - GGMLQNN_LOG_DEBUG( - "src1 %15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", - src1->name, - src1->type, ggml_type_name(src1->type), src1->ne[0], src1->ne[1], src1->ne[2], - src1->nb[0], src1->nb[1], src1->nb[2]); - GGMLQNN_LOG_DEBUG( - " %15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", - tensor->name, - tensor->type, ggml_type_name(tensor->type), tensor->ne[0], tensor->ne[1], - tensor->ne[2], - tensor->nb[0], - tensor->nb[1], tensor->nb[2]); + print_tensors_info(nullptr, nullptr, src0, src1, dst); } //TODO: currently only support offloading 2D matrix to QNN backend @@ -3089,25 +2991,20 @@ static bool ggml_qnn_can_handle_op(const struct ggml_tensor * tensor) { int64_t ne1 = tensor->ne[1]; if (tensor->op == GGML_OP_ADD) { + //dump_tensors_info(tensor); if (!ggml_are_same_shape(src0, src1)) { return false; } if (ne00 < 32) return false; - -#if GGMLQNN_PRINT_OP_ADD_LOG - //dump_tensors_info(tensor); -#endif + return (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16) && (src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16); - } if (tensor->op == GGML_OP_MUL_MAT) { -#if GGMLQNN_PRINT_OP_MUL_MAT_LOG - //dump_tensors_info(tensor); -#endif + dump_tensors_info(tensor); uint32_t src0_rank = ggml_get_tensor_rank(src0); uint32_t src1_rank = ggml_get_tensor_rank(src1); @@ -3181,17 +3078,12 @@ static void ggml_qnn_add(ggml_backend_t backend, ggml_tensor * op) { if (!graph_initialized) { GGMLQNN_LOG_DEBUG("graph name %s", graph_name.c_str()); - if (ctx->device == QNN_BACKEND_NPU) { - error = create_htp_graph(ctx, graph_name, &graph_handle); - } else { - error = qnn_raw_interface.graphCreate(instance->get_qnn_context_handle(), - graph_name.c_str(), - nullptr, &graph_handle); - } + error = instance->init_qnn_graph(graph_name, static_cast(ctx->device), 8); if (QNN_SUCCESS != error) { GGMLQNN_LOG_INFO("can't create qnn graph handle with graph name %s, error = %d\n", graph_name.c_str(), error); return; } + graph_handle = instance->get_qnn_graph_handle(); if (enable_npu_rpc) { QNN_VER_PTR(*p_tensor0)->memType = QNN_TENSORMEMTYPE_MEMHANDLE; @@ -3391,9 +3283,9 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { p_param_tensor = tensors[3]; p_tensor2_transpose = tensors[4]; } else { - p_tensor0 = ggml_qnn_create_mulmat_tensor(src0, nullptr, QNN_TENSOR_TYPE_APP_WRITE,QNN_DATATYPE_FLOAT_32, 2, nullptr, nullptr, 0); - p_tensor1 = ggml_qnn_create_mulmat_tensor(src1, nullptr, QNN_TENSOR_TYPE_APP_WRITE,QNN_DATATYPE_FLOAT_32, 2, nullptr, nullptr, 0); - p_tensor2 = ggml_qnn_create_mulmat_tensor(dst, nullptr, QNN_TENSOR_TYPE_APP_READ,QNN_DATATYPE_FLOAT_32, 2, nullptr, nullptr, 0); + p_tensor0 = ggml_qnn_create_general_tensor(src0, nullptr, QNN_TENSOR_TYPE_APP_WRITE,QNN_DATATYPE_FLOAT_32, 2, nullptr, nullptr, 0); + p_tensor1 = ggml_qnn_create_general_tensor(src1, nullptr, QNN_TENSOR_TYPE_APP_WRITE,QNN_DATATYPE_FLOAT_32, 2, nullptr, nullptr, 0); + p_tensor2 = ggml_qnn_create_general_tensor(dst, nullptr, QNN_TENSOR_TYPE_APP_READ,QNN_DATATYPE_FLOAT_32, 2, nullptr, nullptr, 0); } print_tensors_info(__func__, ctx, src0, src1, dst); @@ -3443,7 +3335,7 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { //step-2: create param tensor for mulmat of 2d matrix uint32_t param_tensor_dims[] = {2}; uint32_t param_tensor_data[2] = {1, 0}; - p_param_tensor = ggml_qnn_create_mulmat_tensor(nullptr, "param", QNN_TENSOR_TYPE_STATIC,QNN_DATATYPE_UINT_32, 1, param_tensor_dims, param_tensor_data, 8); + p_param_tensor = ggml_qnn_create_general_tensor(nullptr, "param", QNN_TENSOR_TYPE_STATIC,QNN_DATATYPE_UINT_32, 1, param_tensor_dims, param_tensor_data, 8); CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_param_tensor)); //step-3: create compute tensor from ggml tensor @@ -3457,8 +3349,8 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { //step-4: create a transpose tensor uint32_t tensor2_transpose_dims[GGML_MAX_DIMS] = {}; - p_tensor2_transpose = ggml_qnn_create_mulmat_tensor(dst,"transpose",QNN_TENSOR_TYPE_NATIVE,QNN_DATATYPE_FLOAT_32, 2, nullptr, nullptr, 0); - get_qnn_dimensions_from_ggml_dimensions(tensor2_transpose_dims, tensor_2_dimensions,ggml_get_tensor_rank(dst)); + p_tensor2_transpose = ggml_qnn_create_general_tensor(dst, "transpose", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, 2, nullptr, nullptr, 0); + get_qnn_dimensions_from_ggml_dimensions(tensor2_transpose_dims, tensor_2_dimensions, ggml_get_tensor_rank(dst)); //save pointer because the dimensions of tensor p_tensor2_transpose will be changed later uint32_t * tensor2_dimensions_transpose = QNN_VER_PTR(*p_tensor2_transpose)->dimensions; //update dimensions of tensor p_tensor2_transpose to make QNN SDK happy @@ -3547,6 +3439,10 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { Qnn_Tensor_t tensor_outputs[] = { *p_tensor2 }; + //attention: + // this is the second technical approach of "how to utilize the Hexagon NPU maximally" through + // QNN SDK, details could be found at + // https://github.com/kantv-ai/llama.cpp/wiki/mapping-ggml-compute-graph-to-QNN-compute-graph CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, tensor_inputs, 2, tensor_outputs, 1, From 559bc1c746c1f57826b5a67d87d98182430c42d9 Mon Sep 17 00:00:00 2001 From: zhouwg Date: Thu, 20 Feb 2025 12:33:38 +0800 Subject: [PATCH 09/76] ggml-qnn: sync from branch kantvai-ggmlqnn-npurpc --- ggml/src/ggml-qnn/ggml-qnn.cpp | 282 ++++++++++++++++++++++----------- 1 file changed, 186 insertions(+), 96 deletions(-) diff --git a/ggml/src/ggml-qnn/ggml-qnn.cpp b/ggml/src/ggml-qnn/ggml-qnn.cpp index 37c947f412f1f..ee273503b9e8a 100644 --- a/ggml/src/ggml-qnn/ggml-qnn.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn.cpp @@ -13,9 +13,10 @@ * section-5 does ggml-qnn backend helper macro / data structure / function / class * section-6 does implementation of ggml-qnn backend according to ggml's backend subsystem * - * currently only provide OPs' QNN backend implementation of GGML_OP_ADD & GGML_OP_MUL_MAT: + * currently provide following ggml ops' QNN backend implementation: * - GGML_OP_ADD: this is a simple skeleton, can expand other ggml ops according to expertise - * - GGML_OP_MUL_MAT:this is a complicated skeleton, can expand other complex op accordingly + * - GGML_OP_MUL: this is a simple skeleton, can expand other ggml ops according to expertise + * - GGML_OP_MUL_MAT:this is a complicated skeleton, can expand other complex ggml ops accordingly * * of course, can porting ggml-qnn to Windows on ARM as need. * @@ -95,7 +96,6 @@ #include "ggml-qnn.h" #include "ggml-impl.h" #include "ggml-backend-impl.h" - // ================================================================================================= // section-1: forward/external declaration // ================================================================================================= @@ -110,9 +110,9 @@ static void ggmlqnn_log_internal(ggml_log_level level, const char * file, const // ================================================================================================= #define GGMLQNN_DEBUG 1 // for troubleshooting QNN backend #define GGML_QNN_LOGBUF_LEN 4096 -#define ENABLE_QNNBACKEND_PERF 1 // enable/disable op's perf info +#define ENABLE_QNNBACKEND_PERF 0 // enable/disable op's perf info #define GGMLQNN_PRINT_QNN_INTERNAL_LOG 0 // enable/disable QNN's internal log -#define GGMLQNN_PRINT_OP_ADD_LOG 1 // GGML_OP_ADD already verified with QNN-CPU / QNN-GPU / QNN-NPU +#define GGMLQNN_PRINT_OP_ADD_LOG 0 // GGML_OP_ADD already verified with QNN-CPU / QNN-GPU / QNN-NPU #define GGMLQNN_PRINT_OP_MUL_MAT_LOG 1 #define GGMLQNN_LOG_ERROR(...) ggmlqnn_log_internal(GGML_LOG_LEVEL_DEBUG, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) @@ -1187,25 +1187,28 @@ static struct ggml_backend_qnn_context g_qnn_mgr[GGML_QNN_MAX_DEVICES] = { .socinfo = {}}, }; - struct qnn_op_caps_t { const char * qnn_op_name = nullptr; const size_t input_param_count = 0; const char * qnn_param_name = nullptr; }; -static const qnn_op_caps_t kOpCaps[] = { +static const qnn_op_caps_t k_op_caps[] = { {}, // GGML_OP_NONE {}, // GGML_OP_DUP { // GGML_OP_ADD - QNN_OP_ELEMENT_WISE_ADD, // qnn_op_name - 2, // input_param_count + QNN_OP_ELEMENT_WISE_ADD, + 2, }, {}, // GGML_OP_ADD1 {}, // GGML_OP_ACC {}, // GGML_OP_SUB - {}, // GGML_OP_MUL + { + // GGML_OP_MUL + QNN_OP_ELEMENT_WISE_MULTIPLY, + 2, + }, {}, // GGML_OP_DIV {}, // GGML_OP_SQR {}, // GGML_OP_SQRT @@ -1227,8 +1230,8 @@ static const qnn_op_caps_t kOpCaps[] = { {}, // GGML_OP_GROUP_NORM { // GGML_OP_MUL_MAT - QNN_OP_MAT_MUL, // qnn_op_name - 2, // input_param_count + QNN_OP_MAT_MUL, + 2, }, {}, // GGML_OP_MUL_MAT_ID {}, // GGML_OP_OUT_PROD @@ -1580,11 +1583,9 @@ static void append_tensor_dimensions(const ggml_tensor * tensor, std::string & o output.append(buffer, len); } -constexpr const size_t kGgmlUnaryOpStart = GGML_OP_COUNT; - static size_t get_qnn_op_index(const ggml_tensor * tensor) { if (tensor->op == GGML_OP_UNARY) { - return kGgmlUnaryOpStart + ggml_get_unary_op(tensor); + return GGML_OP_COUNT + ggml_get_unary_op(tensor); } return tensor->op; @@ -1592,8 +1593,8 @@ static size_t get_qnn_op_index(const ggml_tensor * tensor) { static size_t get_qnn_op_input_param_count(const ggml_tensor * op) { auto op_index = get_qnn_op_index(op); - GGML_ASSERT(op_index < std::size(kOpCaps)); - return kOpCaps[op_index].input_param_count; + GGML_ASSERT(op_index < std::size(k_op_caps)); + return k_op_caps[op_index].input_param_count; } static void get_graph_key_from_op(const ggml_tensor * op, std::string & output) { @@ -1796,21 +1797,21 @@ class qnn_instance { int qnn_finalize(); - const qnn_interface &get_qnn_interface() { + const qnn_interface & get_qnn_interface() { if (!_qnn_interface.is_loaded()) { GGMLQNN_LOG_WARN("pls check why _qnn_interface is not loaded\n"); } return _qnn_interface; } - const QNN_INTERFACE_VER_TYPE &get_qnn_raw_interface() { + const QNN_INTERFACE_VER_TYPE & get_qnn_raw_interface() { if (!_qnn_interface.is_loaded()) { GGMLQNN_LOG_WARN("pls check why _qnn_interface is not loaded\n"); } return _qnn_raw_interface; } - const QNN_SYSTEM_INTERFACE_VER_TYPE &get_qnn_raw_system_interface() { + const QNN_SYSTEM_INTERFACE_VER_TYPE & get_qnn_raw_system_interface() { if (!_qnn_interface.is_loaded()) { GGMLQNN_LOG_WARN("pls check why _qnn_interface is not loaded\n"); } @@ -1836,7 +1837,7 @@ class qnn_instance { uint8_t do_node_validation = 1, const QnnGraph_Config_t ** graph_configs = nullptr ); - int init_qnn_graph(const std::string &graph_name, QNNBackend device, size_t vtcm_size_in_mb = 8, size_t hvx_threads = 8); + int init_qnn_graph(const std::string & graph_name, QNNBackend device, size_t vtcm_size_in_mb = 8, size_t hvx_threads = 8); int finalize_qnn_graph(); @@ -1850,8 +1851,8 @@ class qnn_instance { return 1; } - QnnHtpDevice_Infrastructure_t *htp_infra = static_cast(device_infra); - QnnHtpDevice_PerfInfrastructure_t *htp_perfinfra = &htp_infra->perfInfra; + QnnHtpDevice_Infrastructure_t * htp_infra = static_cast(device_infra); + QnnHtpDevice_PerfInfrastructure_t * htp_perfinfra = &htp_infra->perfInfra; uint32_t power_configid = 1; uint32_t device_id = 0; uint32_t core_id = 0; @@ -1925,6 +1926,7 @@ class qnn_instance { } size_t get_rpcmem_capacity() { return _rpcmem_capacity; } + size_t get_rpcmem_usage() { return _rpcmem_usage; } int32_t rpcmem_to_fd(void * buf); @@ -1950,6 +1952,32 @@ class qnn_instance { return _enable_qnn_rpc; } + void probe_device_meminfo() { + size_t candidate_size = 0; + uint8_t *rpc_buffer = nullptr; + const int SIZE_IN_MB = (1 << 20); + size_t probe_slots[] = {1024, 1536, 2048 - 48, 2048}; + size_t probe_counts = sizeof(probe_slots) / sizeof(size_t); + for (size_t idx = 0; idx < probe_counts; idx++) { + rpc_buffer = static_cast(alloc_rpcmem_internal(probe_slots[idx] * SIZE_IN_MB, 4)); + if (nullptr == rpc_buffer) { + GGMLQNN_LOG_DEBUG("alloc rpcmem %d (MB) failure, %s\n", probe_slots[idx], + strerror(errno)); + break; + } else { + candidate_size = probe_slots[idx]; + free_rpcmem(rpc_buffer); + rpc_buffer = nullptr; + } + } + if (candidate_size > _rpcmem_capacity) + _rpcmem_capacity = candidate_size; + + free_rpcmem(); + _rpcmem_usage = 0; + GGMLQNN_LOG_INFO("capacity of rpc ion memory %d MB\n", _rpcmem_capacity); + } + public: std::map>> _qnn_graph_map; @@ -1969,6 +1997,8 @@ class qnn_instance { void set_qnn_raw_system_interface(QNN_SYSTEM_INTERFACE_VER_TYPE & raw_interface) { _qnn_raw_system_interface = raw_interface; } + + void * alloc_rpcmem_internal(size_t bytes, size_t alignment); private: static constexpr const int _required_num_providers = 1; @@ -1987,7 +2017,7 @@ class qnn_instance { qnn_interface _qnn_interface; - void *_system_lib_handle = nullptr; + void * _system_lib_handle = nullptr; Qnn_GraphHandle_t _qnn_graph_handle = nullptr; @@ -2013,7 +2043,6 @@ class qnn_instance { std::unordered_map _qnn_mem_set; std::unordered_map _qnn_rpc_buffer_to_handles; - static std::mutex _init_mutex; static std::unordered_map _loaded_lib_handle; static std::unordered_map _lib_path_to_backend_id; @@ -2027,7 +2056,9 @@ class qnn_instance { pfn_rpc_mem_init _pfn_rpc_mem_init; pfn_rpc_mem_deinit _pfn_rpc_mem_deinit; std::unordered_map _rpcmem_store_map; - size_t _rpcmem_capacity = 512; + std::unordered_map _rpcmem_usage_map; + size_t _rpcmem_capacity = 512; // mempool size in Mbytes + size_t _rpcmem_usage = 0; // mempool usage in MBytes std::string _graph_name; QNNBackend _device_id; @@ -2042,7 +2073,7 @@ std::unordered_map qnn_instance::_loaded_li std::unordered_map qnn_instance::_lib_path_to_backend_id; std::unordered_map qnn_instance::_loaded_backend; -void * qnn_instance::alloc_rpcmem(size_t bytes, size_t alignment) { +void * qnn_instance::alloc_rpcmem_internal(size_t bytes, size_t alignment) { if (!_rpcmem_initialized) { GGMLQNN_LOG_WARN("rpc memory not initialized\n"); return nullptr; @@ -2062,17 +2093,50 @@ void * qnn_instance::alloc_rpcmem(size_t bytes, size_t alignment) { GGMLQNN_LOG_WARN("failed to allocate rpc memory\n"); _pfn_rpc_mem_free(buf); } + return aligned_buf; +} + +void * qnn_instance::alloc_rpcmem(size_t bytes, size_t alignment) { + if (_rpcmem_usage > (_rpcmem_capacity - 8)) { // reserve 8Mbytes in rpc mempool + GGMLQNN_LOG_WARN("rpc mempool capcaity: %d MB, usage: %d MB", _rpcmem_capacity, _rpcmem_usage); + return nullptr; + } + + auto aligned_buf = alloc_rpcmem_internal(bytes, alignment); + if (nullptr == aligned_buf) + return nullptr; + _rpcmem_usage_map.insert(std::pair(aligned_buf, bytes)); + size_t rpcmem_usage_in_bytes = _rpcmem_usage * (1 << 20); + rpcmem_usage_in_bytes += bytes; + _rpcmem_usage = rpcmem_usage_in_bytes / ( 1 << 20); return aligned_buf; } void qnn_instance::free_rpcmem(void * buf) { + size_t rpcbuffer_size = 0; if (!_rpcmem_initialized) { GGMLQNN_LOG_WARN("rpc memory not initialized\n"); } else if (0 == _rpcmem_store_map.count(buf)) { GGMLQNN_LOG_WARN("no allocated tensor\n"); } else { GGMLQNN_LOG_DEBUG("free rpc mem %p", _rpcmem_store_map[buf]); + for (std::unordered_map::iterator it = _rpcmem_usage_map.begin(); + it != _rpcmem_usage_map.end(); + it++) { + void * rpcbuffer = it->first; + if (buf == rpcbuffer) { + rpcbuffer_size = it->second; + size_t rpcmem_usage_in_bytes = _rpcmem_usage * (1 << 20); + rpcmem_usage_in_bytes -= rpcbuffer_size; + _rpcmem_usage = rpcmem_usage_in_bytes / ( 1 << 20); + } + } + if (rpcbuffer_size != 0) { + _rpcmem_usage_map.erase(buf); + } else { + GGMLQNN_LOG_WARN("it shouldn't happen, pls check why?"); + } _pfn_rpc_mem_free(_rpcmem_store_map[buf]); _rpcmem_store_map.erase(buf); } @@ -2094,6 +2158,8 @@ void qnn_instance::free_rpcmem() { _pfn_rpc_mem_free(rpcbuffer); } _rpcmem_store_map.clear(); + _rpcmem_usage_map.clear(); + _rpcmem_usage = 0; } int32_t qnn_instance::rpcmem_to_fd(void * buf) { @@ -2177,7 +2243,11 @@ Qnn_MemHandle_t qnn_instance::register_rpcmem(void * p_data, const uint32_t ran } GGMLQNN_LOG_DEBUG("mem_fd %d", mem_fd); - Qnn_MemDescriptor_t descriptor = {{rank, dimensions, nullptr}, data_type, QNN_MEM_TYPE_ION, {{mem_fd}}}; + Qnn_MemDescriptor_t descriptor = { + {rank, dimensions, nullptr}, + data_type, QNN_MEM_TYPE_ION, + {{mem_fd}} + }; Qnn_MemHandle_t handle = nullptr; auto error = _qnn_interface.qnn_mem_register(_qnn_context_handle, &descriptor, /*numDescriptors=*/1, &handle); @@ -2318,7 +2388,7 @@ int qnn_instance::load_backend(std::string & lib_path, const QnnSaver_Config_t * _loaded_lib_handle[backend_id] = lib_handle; _backend_id = backend_id; -#if 0 // keep them here for further use +#if 0 // leave them here for further use QnnSaver_Config_t outputdir_cfg; outputdir_cfg.option = QNN_SAVER_CONFIG_OPTION_OUTPUT_DIRECTORY; outputdir_cfg.outputDirectory = "/data/local/tmp/"; @@ -2468,6 +2538,7 @@ int qnn_instance::unload_system() { return result; } +#if GGMLQNN_PRINT_QNN_INTERNAL_LOG static void ggml_qnn_logcallback(const char * fmt, QnnLog_Level_t level, uint64_t timestamp, @@ -2499,24 +2570,25 @@ static void ggml_qnn_logcallback(const char * fmt, } double ms = (double) timestamp / 1000000.0; - { std::lock_guard lock(log_mutex); - memset(s_ggml_qnn_logbuf, 0, GGML_QNN_LOGBUF_LEN); vsnprintf(reinterpret_cast(s_ggml_qnn_logbuf), GGML_QNN_LOGBUF_LEN, fmt, argp); -#if GGMLQNN_PRINT_QNN_INTERNAL_LOG GGMLQNN_LOG_INFO("%8.1fms [%-7s] %s\n", ms, log_level_desc, s_ggml_qnn_logbuf); -#endif } } +#else +static void ggml_qnn_logcallback(const char * fmt, + QnnLog_Level_t level, + uint64_t timestamp, + va_list argp) { +} +#endif int qnn_instance::qnn_init(const QnnSaver_Config_t ** saver_config) { BackendIdType backend_id = QNN_BACKEND_ID_NULL; GGMLQNN_LOG_DEBUG("enter qni_init\n"); - const std::lock_guard lock(_init_mutex); - if (0 != load_system()) { GGMLQNN_LOG_WARN("can not load QNN system lib, pls check why?\n"); return 1; @@ -2542,9 +2614,7 @@ int qnn_instance::qnn_init(const QnnSaver_Config_t ** saver_config) { _loaded_lib_handle.count(backend_id)); return 3; } - _qnn_interface.set_qnn_interface(_loaded_backend[backend_id]); - #if 1 _qnn_interface.qnn_log_create(ggml_qnn_logcallback, _qnn_log_level, &_qnn_log_handle); #else @@ -2671,25 +2741,7 @@ int qnn_instance::qnn_init(const QnnSaver_Config_t ** saver_config) { } _qnn_raw_interface.deviceFreePlatformInfo(nullptr, p_info); - size_t candidate_size = 0; - uint8_t * rpc_buffer = nullptr; - const int SIZE_IN_MB = (1 << 20); - size_t probe_slots[] = {1024, 1536, 2048 - 48, 2048}; - size_t probe_counts = sizeof(probe_slots) / sizeof(size_t); - for (size_t idx = 0; idx < probe_counts; idx++) { - rpc_buffer = static_cast(alloc_rpcmem(probe_slots[idx] * SIZE_IN_MB, 4)); - if (nullptr == rpc_buffer) { - GGMLQNN_LOG_DEBUG("alloc rpcmem %d (MB) failure, %s\n", probe_slots[idx], strerror(errno)); - break; - } else { - candidate_size = probe_slots[idx]; - free_rpcmem(rpc_buffer); - rpc_buffer = nullptr; - } - } - if (candidate_size > _rpcmem_capacity) - _rpcmem_capacity = candidate_size; - GGMLQNN_LOG_INFO("capacity of rpc ion memory %d MB\n", _rpcmem_capacity); + probe_device_meminfo(); if (0 != init_htp_perfinfra()) { GGMLQNN_LOG_WARN("initialize HTP performance failure"); @@ -2963,6 +3015,7 @@ static void get_qnn_dimensions_from_ggml_dimensions(uint32_t * qnn_dimensions, u // ================================================================================================= // section-6: implementation of ggml-qnn backend // ================================================================================================= +//TODO: refine this function as it is a performance hotspot/bottleneck function static bool ggml_qnn_can_handle_op(const struct ggml_tensor * tensor) { if (tensor->op == GGML_OP_NONE) { return true; @@ -2973,7 +3026,9 @@ static bool ggml_qnn_can_handle_op(const struct ggml_tensor * tensor) { return false; } - bool supported_op = ((tensor->op == GGML_OP_ADD) || (tensor->op == GGML_OP_MUL_MAT)); + //TODO: support other op + bool supported_op = ((tensor->op == GGML_OP_ADD) || (tensor->op == GGML_OP_MUL_MAT) + || (tensor->op == GGML_OP_MUL)); if (!supported_op) { return false; } @@ -2981,37 +3036,34 @@ static bool ggml_qnn_can_handle_op(const struct ggml_tensor * tensor) { struct ggml_tensor * src0 = tensor->src[0]; struct ggml_tensor * src1 = tensor->src[1]; - int64_t ne00 = tensor->src[0]->ne[0]; - int64_t ne01 = tensor->src[0]->ne[1]; + const int64_t ne00 = tensor->src[0]->ne[0]; + const int64_t ne01 = tensor->src[0]->ne[1]; - int64_t ne10 = tensor->src[1]->ne[0]; - int64_t ne11 = tensor->src[1]->ne[1]; + const int64_t ne10 = tensor->src[1]->ne[0]; + const int64_t ne11 = tensor->src[1]->ne[1]; - int64_t ne0 = tensor->ne[0]; - int64_t ne1 = tensor->ne[1]; + const int64_t ne0 = tensor->ne[0]; + const int64_t ne1 = tensor->ne[1]; + + const uint32_t src0_rank = ggml_get_tensor_rank(src0); + const uint32_t src1_rank = ggml_get_tensor_rank(src1); if (tensor->op == GGML_OP_ADD) { //dump_tensors_info(tensor); if (!ggml_are_same_shape(src0, src1)) { return false; } - if (ne00 < 32) return false; - return (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16) && (src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16); } if (tensor->op == GGML_OP_MUL_MAT) { - dump_tensors_info(tensor); - uint32_t src0_rank = ggml_get_tensor_rank(src0); - uint32_t src1_rank = ggml_get_tensor_rank(src1); - + //dump_tensors_info(tensor); if ((src0_rank != 2) || (src1_rank != 2)) //TODO: only support offload 2D matrix mulmat to QNN backend return false; - - //TODO: support more data type in func ggml_qnn_mul_mat(...): + //TODO: support more data type in func ggml_qnn_mul_mat(...) //src0: q4_0, q6_k, ... //src1: f32 //dst : f32 @@ -3020,19 +3072,30 @@ static bool ggml_qnn_can_handle_op(const struct ggml_tensor * tensor) { && (src0->type == src1->type) && (src0->type == tensor->type); } - //TODO:for other op + if (tensor->op == GGML_OP_MUL) { + dump_tensors_info(tensor); + if ((src0_rank != 2) || (src1_rank != 2)) //TODO: only support offload 2D matrix mul to QNN backend + return false; + return (src0->type == GGML_TYPE_F32) + && (src1->type == GGML_TYPE_F32) + && (tensor->type == src1->type); + } + return (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16) && (src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16) && (src0->type == src1->type) && (src0->type == tensor->type); } -static void ggml_qnn_add(ggml_backend_t backend, ggml_tensor * op) { +/* + * provide a general skeleton to offload ggml op to QNN backend: a single node contains 2 input + * tensor and 1 output tensor +*/ +static void ggml_qnn_general_node(ggml_backend_t backend, ggml_tensor * op) { Qnn_ErrorHandle_t error = QNN_SUCCESS; enum ggml_status result = GGML_STATUS_SUCCESS; bool graph_initialized = false; qnn_instance * instance = nullptr; ggml_backend_qnn_context * ctx = (ggml_backend_qnn_context *)backend->context; - qnn_perf op_perf = qnn_perf("ggml_qnn_add"); Qnn_GraphHandle_t graph_handle = nullptr; Qnn_Tensor_t * p_tensor0 = nullptr; Qnn_Tensor_t * p_tensor1 = nullptr; @@ -3045,6 +3108,14 @@ static void ggml_qnn_add(ggml_backend_t backend, ggml_tensor * op) { GGMLQNN_CHECK_PARAMS(ctx, src0, src1, dst); instance = ctx->instance; QNN_INTERFACE_VER_TYPE qnn_raw_interface = ctx->raw_interface; + + size_t qnn_op_index = get_qnn_op_index(op); + GGML_ASSERT(qnn_op_index < std::size(k_op_caps)); + const char * qnn_op_name = k_op_caps[qnn_op_index].qnn_op_name; + std::string ggml_op_name_string = std::string("ggml_") + ggml_op_name(op->op); + const char * ggml_op_name = ggml_op_name_string.c_str(); + + qnn_perf op_perf = qnn_perf(ggml_op_name); op_perf.start(); std::string graph_name; @@ -3124,9 +3195,9 @@ static void ggml_qnn_add(ggml_backend_t backend, ggml_tensor * op) { }; Qnn_OpConfig_t op_config = { QNN_OPCONFIG_VERSION_1, .v1 = { - "ggml_op_add", + ggml_op_name, QNN_OP_PACKAGE_NAME_QTI_AISW, - QNN_OP_ELEMENT_WISE_ADD, + qnn_op_name, 0, qnn_params, 2, @@ -3138,9 +3209,9 @@ static void ggml_qnn_add(ggml_backend_t backend, ggml_tensor * op) { CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, op_config)); CHECK_QNN_API(error, qnn_raw_interface.graphFinalize(graph_handle, nullptr, nullptr)); CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, - tensor_inputs, 2, - tensor_outputs, 1, - nullptr, nullptr)); + tensor_inputs, 2, + tensor_outputs, 1, + nullptr, nullptr)); if (enable_npu_rpc) { uint8_t * qnn_rpcbuffer = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*p_tensor2)->memHandle)); @@ -3214,9 +3285,9 @@ static void ggml_qnn_add(ggml_backend_t backend, ggml_tensor * op) { *p_tensor2 }; CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, - tensor_inputs, 2, - tensor_outputs, 1, - nullptr, nullptr)); + tensor_inputs, 2, + tensor_outputs, 1, + nullptr, nullptr)); if (enable_npu_rpc) { //TODO:NPU RPC feature will failed with test-backend-ops @@ -3231,18 +3302,17 @@ static void ggml_qnn_add(ggml_backend_t backend, ggml_tensor * op) { QNN_VER_PTR(*p_tensor0)->dimensions = tensor_0_dimensions; QNN_VER_PTR(*p_tensor1)->dimensions = tensor_1_dimensions; QNN_VER_PTR(*p_tensor2)->dimensions = tensor_2_dimensions; -#if GGMLQNN_PRINT_OP_ADD_LOG op_perf.info(); -#endif } /* - * the logic of ggml_qnn_mul_mat is similar to ggml_qnn_add but much more complicated than ggml_qnn_add, + * the logic of ggml_qnn_mul_mat is similar to ggml_qnn_general_node but much more complicated + * than ggml_qnn_general_node. * matrix transpose and type trait are required for offload mulmat to QNN backend, * so it's a standalone function. accordingly, this is another typical skeleton for offload other * ggml ops to QNN backend * - * MUL_MAT take most of the compute time (about 95%).so to speed up llama inference, we should focus on MUL_MAT. + * MUL_MAT take most of the compute time (about 95%).so to speed up llama inference, should focus on MUL_MAT. * * have three kinds of MUL_MAT to compute: * mul_mat_f32: both src0 and src1 are F32, this will be naturally handled in QNN backend @@ -3288,7 +3358,7 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { p_tensor2 = ggml_qnn_create_general_tensor(dst, nullptr, QNN_TENSOR_TYPE_APP_READ,QNN_DATATYPE_FLOAT_32, 2, nullptr, nullptr, 0); } - print_tensors_info(__func__, ctx, src0, src1, dst); + //print_tensors_info(__func__, ctx, src0, src1, dst); //ensure QNN tensor has correct tensor type QNN_VER_PTR(*p_tensor0)->type = QNN_TENSOR_TYPE_APP_WRITE; @@ -3444,8 +3514,8 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { // QNN SDK, details could be found at // https://github.com/kantv-ai/llama.cpp/wiki/mapping-ggml-compute-graph-to-QNN-compute-graph CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, - tensor_inputs, 2, - tensor_outputs, 1, + tensor_inputs, 2, + tensor_outputs, 1, nullptr, nullptr)); } @@ -3453,7 +3523,6 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { QNN_VER_PTR(*p_tensor0)->dimensions = tensor_0_dimensions; QNN_VER_PTR(*p_tensor1)->dimensions = tensor_1_dimensions; QNN_VER_PTR(*p_tensor2)->dimensions = tensor_2_dimensions; - op_perf.info(); } @@ -3462,13 +3531,17 @@ static bool ggml_qnn_compute_forward(ggml_backend_t backend, struct ggml_tensor switch (tensor->op) { case GGML_OP_ADD: - func = ggml_qnn_add; + func = ggml_qnn_general_node; break; case GGML_OP_MUL_MAT: func = ggml_qnn_mul_mat; break; + case GGML_OP_MUL: + func = ggml_qnn_general_node; + break; + default: return false; } @@ -3667,7 +3740,6 @@ static enum ggml_status ggml_backend_qnn_graph_compute(ggml_backend_t backend, s ggml_backend_qnn_context * ctx = (ggml_backend_qnn_context *) backend->context; GGML_UNUSED(ctx); - GGMLQNN_LOG_DEBUG("cgraph->n_nodes %d", cgraph->n_nodes); for (int i = 0; i < cgraph->n_nodes; i++) { ggml_tensor * node = cgraph->nodes[i]; if (ggml_is_empty(node) || node->op == GGML_OP_RESHAPE @@ -3715,10 +3787,28 @@ static const char * ggml_backend_qnn_device_get_description(ggml_backend_dev_t d } static void ggml_backend_qnn_device_get_memory(ggml_backend_dev_t dev, size_t * free, size_t * total) { - //FIXME:this is NOT QNN device memory info - *free = get_system_free_memory_in_bytes(); - *total = get_system_total_memory_in_bytes(); - GGML_UNUSED(dev); + struct ggml_backend_qnn_context * ctx = static_cast(dev->context); + if ((nullptr == ctx) || (ctx->device > QNN_BACKEND_GGML)) { + GGMLQNN_LOG_ERROR("pls check params"); + *free = 0; + *total = 0; + } + + if (QNN_BACKEND_CPU == ctx->device || QNN_BACKEND_GGML == ctx->device) { + *total = get_system_total_memory_in_bytes(); + *free = get_system_free_memory_in_bytes(); + } else if (QNN_BACKEND_GPU == ctx->device) { + //TODO: probe GPU info in Qualcomm Adreno GPU + *total = get_system_total_memory_in_bytes(); + *free = get_system_free_memory_in_bytes(); + } else if (QNN_BACKEND_NPU == ctx->device) { + size_t rpc_ion_memsize = ctx->instance->get_rpcmem_capacity(); + size_t rpc_ion_usage = ctx->instance->get_rpcmem_usage(); + GGMLQNN_LOG_DEBUG("rpc memsize %d", rpc_ion_memsize); + GGMLQNN_LOG_DEBUG("rpc usage %d", rpc_ion_usage); + *total = rpc_ion_memsize * (1 << 20); + *free = (rpc_ion_memsize - rpc_ion_usage) * (1 << 20); + } } static enum ggml_backend_dev_type ggml_backend_qnn_device_get_type(ggml_backend_dev_t dev) { From 07d4cd44976b4cd86c8dfabbb5b81418c02e4d92 Mon Sep 17 00:00:00 2001 From: zhouwg Date: Thu, 20 Feb 2025 22:20:15 +0800 Subject: [PATCH 10/76] ggml-qnn: sync from branch kantvai-ggmlqnn-npurpc --- ggml/src/ggml-qnn/ggml-qnn.cpp | 106 +++++++++++++++++++-------------- 1 file changed, 60 insertions(+), 46 deletions(-) diff --git a/ggml/src/ggml-qnn/ggml-qnn.cpp b/ggml/src/ggml-qnn/ggml-qnn.cpp index ee273503b9e8a..9ef502421c051 100644 --- a/ggml/src/ggml-qnn/ggml-qnn.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn.cpp @@ -1483,15 +1483,13 @@ static Qnn_Tensor_t * ggml_qnn_create_general_tensor(const ggml_tensor * tensor, GGMLQNN_LOG_DEBUG("init_tensor %d", get_idx()); inc_idx(); - //there are different dimension order between ggml tensor and qnn tensor uint32_t dimensions_transpose[GGML_MAX_DIMS] = {}; uint32_t * tensor_dims = nullptr; - if (nullptr != tensor) { - dimensions_transpose[0] = (uint32_t) tensor->ne[1]; - dimensions_transpose[1] = (uint32_t) tensor->ne[0]; - dimensions_transpose[2] = (uint32_t) tensor->ne[2]; - dimensions_transpose[3] = (uint32_t) tensor->ne[3]; + //there are different dimension order between ggml tensor and qnn tensor + for (size_t idx = 0; idx < rank; idx++) { + dimensions_transpose[idx] = (uint32_t)tensor->ne[rank - 1 - idx]; + } tensor_dims = dimensions_transpose; } //re-assign tensor_dims @@ -2058,7 +2056,7 @@ class qnn_instance { std::unordered_map _rpcmem_store_map; std::unordered_map _rpcmem_usage_map; size_t _rpcmem_capacity = 512; // mempool size in Mbytes - size_t _rpcmem_usage = 0; // mempool usage in MBytes + size_t _rpcmem_usage = 0; // mempool usage in Mbytes std::string _graph_name; QNNBackend _device_id; @@ -2968,33 +2966,27 @@ static void print_tensors_info(const char * func_name, ggml_backend_qnn_context if (nullptr != func_name && nullptr != ctx) { GGMLQNN_LOG_DEBUG("call %s in dev %s\n", func_name, ctx->name); } - GGMLQNN_LOG_DEBUG("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", + GGMLQNN_LOG_DEBUG("%-6s: type = %i (%s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi, %5zi)", src0->name, - src0->type, ggml_type_name(src0->type), src0->ne[0], src0->ne[1], src0->ne[2], - src0->nb[0], src0->nb[1], src0->nb[2]); - GGMLQNN_LOG_DEBUG("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", + src0->type, ggml_type_name(src0->type), src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], + src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3]); + GGMLQNN_LOG_DEBUG("%-6s: type = %i (%s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi, %5zi)", src1->name, - src1->type, ggml_type_name(src1->type), src1->ne[0], src1->ne[1], src1->ne[2], - src1->nb[0], src1->nb[1], src1->nb[2]); - GGMLQNN_LOG_DEBUG("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", + src1->type, ggml_type_name(src1->type), src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3], + src1->nb[0], src1->nb[1], src1->nb[2], src1->nb[3]); + GGMLQNN_LOG_DEBUG("%-6s: type = %i (%s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi, %5zi)", dst->name, - dst->type, ggml_type_name(dst->type), dst->ne[0], dst->ne[1], dst->ne[2], dst->nb[0], - dst->nb[1], dst->nb[2]); - GGMLQNN_LOG_DEBUG("%d, %d, %d, %d", src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3]); - GGMLQNN_LOG_DEBUG("tensor0 name %s", src0->name); - GGMLQNN_LOG_DEBUG("tensor1 name %s", src1->name); - GGMLQNN_LOG_DEBUG("tensor2 name %s", dst->name); + dst->type, ggml_type_name(dst->type), dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], + dst->nb[0], dst->nb[1], dst->nb[2], dst->nb[3]); + GGMLQNN_LOG_DEBUG("\n"); } -static void dump_tensors_info(const struct ggml_tensor * tensor) { +static void dump_op_info(const struct ggml_tensor * tensor) { //skip sanity check of params const struct ggml_tensor * src0 = tensor->src[0]; - struct ggml_tensor * src1 = tensor->src[1]; - struct ggml_tensor * dst = const_cast(tensor); - GGMLQNN_LOG_DEBUG("op name:%s, tensor type:%s", ggml_op_name(tensor->op), - ggml_type_name(tensor->type)); - GGMLQNN_LOG_DEBUG("src0 type:%s", ggml_type_name(tensor->src[0]->type)); - GGMLQNN_LOG_DEBUG("src1 type:%s", ggml_type_name(tensor->src[1]->type)); + struct ggml_tensor * src1 = tensor->src[1]; + struct ggml_tensor * dst = const_cast(tensor); + GGMLQNN_LOG_DEBUG("op name:%s, tensor type:%s", ggml_op_name(tensor->op), ggml_type_name(tensor->type)); print_tensors_info(nullptr, nullptr, src0, src1, dst); } @@ -3008,8 +3000,13 @@ static void get_qnn_dimensions_from_ggml_dimensions(uint32_t * qnn_dimensions, u GGMLQNN_LOG_WARN("invalid params"); return; } - qnn_dimensions[0] = ggml_dimensions[1]; - qnn_dimensions[1] = ggml_dimensions[0]; + for (size_t idx = 0; idx < GGML_MAX_DIMS; idx++) + qnn_dimensions[idx] = ggml_dimensions[idx]; + + if (rank >= 2) { + qnn_dimensions[rank - 1] = ggml_dimensions[rank - 2]; + qnn_dimensions[rank - 2] = ggml_dimensions[rank - 1]; + } } // ================================================================================================= @@ -3060,9 +3057,16 @@ static bool ggml_qnn_can_handle_op(const struct ggml_tensor * tensor) { } if (tensor->op == GGML_OP_MUL_MAT) { - //dump_tensors_info(tensor); - if ((src0_rank != 2) || (src1_rank != 2)) //TODO: only support offload 2D matrix mulmat to QNN backend + dump_op_info(tensor); + if (src0_rank != src1_rank) // make QNN SDK happy + return false; + if (src0_rank < 2) // make QNN SDK happy + return false; + if (src0_rank > 3) //TODO: 4D matrix return false; + if ((src1->ne[2] != src0->ne[2]) || (src1->ne[3] != src0->ne[3])) // make QNN SDK happy + return false; + //TODO: support more data type in func ggml_qnn_mul_mat(...) //src0: q4_0, q6_k, ... //src1: f32 @@ -3073,8 +3077,8 @@ static bool ggml_qnn_can_handle_op(const struct ggml_tensor * tensor) { } if (tensor->op == GGML_OP_MUL) { - dump_tensors_info(tensor); - if ((src0_rank != 2) || (src1_rank != 2)) //TODO: only support offload 2D matrix mul to QNN backend + //dump_tensors_info(tensor); + if ((src0_rank != 2) || (src1_rank != 2)) //TODO: 3D and 4D matrix return false; return (src0->type == GGML_TYPE_F32) && (src1->type == GGML_TYPE_F32) @@ -3340,6 +3344,11 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { QNN_INTERFACE_VER_TYPE qnn_raw_interface = ctx->raw_interface; op_perf.start(); + uint32_t src0_rank = ggml_get_tensor_rank(src0); + uint32_t src1_rank = ggml_get_tensor_rank(src1); + GGML_ASSERT(src0_rank == src1_rank); + GGML_ASSERT(src0_rank >= 2); //QNN SDK's limitation + std::string graph_name; get_graph_key_from_op(op, graph_name); if (instance->_qnn_graph_map.find(graph_name) != instance->_qnn_graph_map.end()) { @@ -3353,12 +3362,12 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { p_param_tensor = tensors[3]; p_tensor2_transpose = tensors[4]; } else { - p_tensor0 = ggml_qnn_create_general_tensor(src0, nullptr, QNN_TENSOR_TYPE_APP_WRITE,QNN_DATATYPE_FLOAT_32, 2, nullptr, nullptr, 0); - p_tensor1 = ggml_qnn_create_general_tensor(src1, nullptr, QNN_TENSOR_TYPE_APP_WRITE,QNN_DATATYPE_FLOAT_32, 2, nullptr, nullptr, 0); - p_tensor2 = ggml_qnn_create_general_tensor(dst, nullptr, QNN_TENSOR_TYPE_APP_READ,QNN_DATATYPE_FLOAT_32, 2, nullptr, nullptr, 0); + p_tensor0 = ggml_qnn_create_general_tensor(src0, nullptr, QNN_TENSOR_TYPE_APP_WRITE,QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0); + p_tensor1 = ggml_qnn_create_general_tensor(src1, nullptr, QNN_TENSOR_TYPE_APP_WRITE,QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0); + p_tensor2 = ggml_qnn_create_general_tensor(dst, nullptr, QNN_TENSOR_TYPE_APP_READ,QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0); } - //print_tensors_info(__func__, ctx, src0, src1, dst); + print_tensors_info(__func__, ctx, src0, src1, dst); //ensure QNN tensor has correct tensor type QNN_VER_PTR(*p_tensor0)->type = QNN_TENSOR_TYPE_APP_WRITE; @@ -3403,9 +3412,16 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { return; } //step-2: create param tensor for mulmat of 2d matrix - uint32_t param_tensor_dims[] = {2}; - uint32_t param_tensor_data[2] = {1, 0}; - p_param_tensor = ggml_qnn_create_general_tensor(nullptr, "param", QNN_TENSOR_TYPE_STATIC,QNN_DATATYPE_UINT_32, 1, param_tensor_dims, param_tensor_data, 8); + const uint32_t param_tensor_data[GGML_MAX_DIMS][GGML_MAX_DIMS] = { + {0}, + {1, 0}, + {0, 2, 1}, + {0, 1, 3, 2}, + }; + uint32_t param_tensor_dims[1] = {src0_rank}; + p_param_tensor = ggml_qnn_create_general_tensor(nullptr, "param", QNN_TENSOR_TYPE_STATIC, QNN_DATATYPE_UINT_32, + 1, param_tensor_dims, + (void *) (param_tensor_data[src0_rank - 1]), src0_rank * sizeof(uint32_t)); CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_param_tensor)); //step-3: create compute tensor from ggml tensor @@ -3419,7 +3435,7 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { //step-4: create a transpose tensor uint32_t tensor2_transpose_dims[GGML_MAX_DIMS] = {}; - p_tensor2_transpose = ggml_qnn_create_general_tensor(dst, "transpose", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, 2, nullptr, nullptr, 0); + p_tensor2_transpose = ggml_qnn_create_general_tensor(dst, "transpose", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0); get_qnn_dimensions_from_ggml_dimensions(tensor2_transpose_dims, tensor_2_dimensions, ggml_get_tensor_rank(dst)); //save pointer because the dimensions of tensor p_tensor2_transpose will be changed later uint32_t * tensor2_dimensions_transpose = QNN_VER_PTR(*p_tensor2_transpose)->dimensions; @@ -3435,7 +3451,7 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { } }; - Qnn_Tensor_t out_0_inputs[] = {*p_tensor0,*p_tensor1}; + Qnn_Tensor_t out_0_inputs[] = {*p_tensor0,*p_tensor1}; Qnn_Tensor_t out_0_outputs[] = {*p_tensor2_transpose}; Qnn_OpConfig_t out_0 = { QNN_OPCONFIG_VERSION_1, .v1 = @@ -3455,7 +3471,7 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { "perm", .tensorParam = *p_param_tensor } }; - Qnn_Tensor_t out_trans1_0_inputs[] = {*p_tensor2_transpose}; + Qnn_Tensor_t out_trans1_0_inputs[] = {*p_tensor2_transpose}; Qnn_Tensor_t out_trans1_0_outputs[] = {*p_tensor2}; Qnn_OpConfig_t out_trans1_0 = { QNN_OPCONFIG_VERSION_1, @@ -3472,7 +3488,7 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { //step-6: finalize qnn graph and execute qnn graph CHECK_QNN_API(error, qnn_raw_interface.graphFinalize(graph_handle, NULL, NULL)); - Qnn_Tensor_t input_tensors_0[] = {*p_tensor0,*p_tensor1}; + Qnn_Tensor_t input_tensors_0[] = {*p_tensor0,*p_tensor1}; Qnn_Tensor_t output_tensors_0[] = {*p_tensor2}; CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, input_tensors_0, 2, @@ -3495,9 +3511,7 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { //restore pointer to avoid memory leak QNN_VER_PTR(*p_tensor2_transpose)->dimensions = tensor2_dimensions_transpose; //free_qnn_tensor(p_tensor2_transpose); - } else { - QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggml_get_tensor_data_size(src0)}; QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggml_get_tensor_data_size(src1)}; QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggml_get_tensor_data_size(dst)}; From 26c4383094b8246c414b43e7bc29299091f02e10 Mon Sep 17 00:00:00 2001 From: zhouwg Date: Fri, 21 Feb 2025 17:43:25 +0800 Subject: [PATCH 11/76] ggml-qnn: sync from branch kantvai-ggmlqnn-npurpc --- ggml/src/ggml-qnn/ggml-qnn.cpp | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/ggml/src/ggml-qnn/ggml-qnn.cpp b/ggml/src/ggml-qnn/ggml-qnn.cpp index 9ef502421c051..e862b07a234eb 100644 --- a/ggml/src/ggml-qnn/ggml-qnn.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn.cpp @@ -1132,8 +1132,8 @@ struct ggml_backend_qnn_context { struct qcom_socinfo socinfo; } ; -//TODO: the following global vars and three helper funcs should be removed in the future -static int32_t g_ggmltensor_idx = 0; +//the following helper funcs are used to ensure every QNN tensor name is unique +static std::atomic g_ggmltensor_idx(0); static void reset_idx() { g_ggmltensor_idx = 0; } @@ -1143,7 +1143,7 @@ static void inc_idx() { } static int32_t get_idx() { - return g_ggmltensor_idx; + return g_ggmltensor_idx.load(); } // file:///opt/qcom/aistack/qairt/2.31.0.250130/docs/QNN/general/quantization.html @@ -1474,7 +1474,7 @@ static Qnn_Tensor_t * ggml_qnn_create_general_tensor(const ggml_tensor * tensor, Qnn_ErrorHandle_t error = QNN_SUCCESS; char tensor_name[GGML_MAX_NAME] = {0}; - //TODO:remove get_idx() and inc_idx() in the future but ensure the tensor name is unique + //ensure the tensor name is unique if (nullptr != name) { snprintf(tensor_name, GGML_MAX_NAME, "tensor_%-8d", get_idx()); } else { @@ -2762,7 +2762,6 @@ int qnn_instance::qnn_finalize() { Qnn_ErrorHandle_t error = QNN_SUCCESS; GGMLQNN_LOG_DEBUG("enter %s\n", __func__); - //TODO:should be removed in the future reset_idx(); free_rpcmem(); @@ -3451,7 +3450,7 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { } }; - Qnn_Tensor_t out_0_inputs[] = {*p_tensor0,*p_tensor1}; + Qnn_Tensor_t out_0_inputs[] = {*p_tensor0, *p_tensor1}; Qnn_Tensor_t out_0_outputs[] = {*p_tensor2_transpose}; Qnn_OpConfig_t out_0 = { QNN_OPCONFIG_VERSION_1, .v1 = @@ -3488,7 +3487,7 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { //step-6: finalize qnn graph and execute qnn graph CHECK_QNN_API(error, qnn_raw_interface.graphFinalize(graph_handle, NULL, NULL)); - Qnn_Tensor_t input_tensors_0[] = {*p_tensor0,*p_tensor1}; + Qnn_Tensor_t input_tensors_0[] = {*p_tensor0, *p_tensor1}; Qnn_Tensor_t output_tensors_0[] = {*p_tensor2}; CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, input_tensors_0, 2, From d38e2a7169407291d7c6feec45daaf3f13e673a9 Mon Sep 17 00:00:00 2001 From: zhouwg Date: Fri, 14 Feb 2025 21:50:53 +0800 Subject: [PATCH 12/76] ggml-qnn: add Qualcomm QNN backend for GGML --- ggml/src/ggml-qnn/CMakeLists.txt | 8 +- ggml/src/ggml-qnn/ggml-qnn-impl.h | 611 +++++++ ggml/src/ggml-qnn/ggml-qnn-ops.cpp | 842 ++++++++++ ggml/src/ggml-qnn/ggml-qnn-ops.h | 52 + ggml/src/ggml-qnn/ggml-qnn.cpp | 2469 ++++++++-------------------- scripts/build-run-android.sh | 98 +- 6 files changed, 2241 insertions(+), 1839 deletions(-) create mode 100644 ggml/src/ggml-qnn/ggml-qnn-impl.h create mode 100644 ggml/src/ggml-qnn/ggml-qnn-ops.cpp create mode 100644 ggml/src/ggml-qnn/ggml-qnn-ops.h diff --git a/ggml/src/ggml-qnn/CMakeLists.txt b/ggml/src/ggml-qnn/CMakeLists.txt index 7bbb9be76b4f6..1156c98fbc9d7 100644 --- a/ggml/src/ggml-qnn/CMakeLists.txt +++ b/ggml/src/ggml-qnn/CMakeLists.txt @@ -4,12 +4,14 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Android") find_library(LOG_LIB log) set(QNN_LINK_LIBRARIES ${LOG_LIB}) set(QNN_DEFAULT_LIB_SEARCH_PATH "/data/local/tmp/" CACHE STRING "customized library search path for QNN backend") +elseif(CMAKE_SYSTEM_NAME STREQUAL "Windows") + set(QNN_DEFAULT_LIB_SEARCH_PATH "C:\\" CACHE STRING "customized library search path for QNN backend") else() - message(FATAL_ERROR "QNN now only available on Android") + message(FATAL_ERROR "QNN now only available on Android and Windows(Windows on ARM)") endif() if(NOT DEFINED GGML_QNN_SDK_PATH) - # try read from environment variable +# try read from environment variable if(DEFINED ENV{QNN_SDK_PATH}) set(GGML_QNN_SDK_PATH $ENV{QNN_SDK_PATH}) else() @@ -22,7 +24,7 @@ message("QNN_SDK_PATH: ${GGML_QNN_SDK_PATH}") set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3") file(GLOB QNN_SOURCES "${CMAKE_CURRENT_LIST_DIR}/*.cpp") -ggml_add_backend_library(ggml-qnn + ggml_add_backend_library(ggml-qnn ${QNN_SOURCES} ) diff --git a/ggml/src/ggml-qnn/ggml-qnn-impl.h b/ggml/src/ggml-qnn/ggml-qnn-impl.h new file mode 100644 index 0000000000000..5a2fe5752a097 --- /dev/null +++ b/ggml/src/ggml-qnn/ggml-qnn-impl.h @@ -0,0 +1,611 @@ +/* +* Copyright (c) 2023-2024 The ggml authors +* +* Permission is hereby granted, free of charge, to any person obtaining a copy +* of this software and associated documentation files (the "Software"), to +* deal in the Software without restriction, including without limitation the +* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +* sell copies of the Software, and to permit persons to whom the Software is +* furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +* IN THE SOFTWARE. +*/ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#if defined(__ANDROID__) || defined(__linux__) +#include +#include +#include +#include +#include +#include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if (defined __ANDROID__) || (defined ANDROID) +#include "android/log.h" +#endif + +#if defined(_WIN32) +#include +#include +#endif + +#include "QnnTypes.h" +#include "QnnCommon.h" +#include "QnnContext.h" +#include "QnnBackend.h" +#include "QnnGraph.h" +#include "QnnProperty.h" +#include "QnnTensor.h" +#include "QnnInterface.h" +#include "Saver/QnnSaver.h" +#include "System/QnnSystemInterface.h" +#include "HTP/QnnHtpDevice.h" +#include "HTP/QnnHtpGraph.h" + +#include "ggml-qnn.h" +#include "ggml-impl.h" +#include "ggml-backend-impl.h" + +class qnn_instance; +struct ggml_backend_qnn_context; +void ggmlqnn_log_internal(ggml_log_level level, const char * file, const char * func, int line, const char * format, ...); + +#if 0//def NDEBUG +#define GGMLQNN_DEBUG 0 +#define ENABLE_QNNBACKEND_PERF 0 // enable/disable op's perf info +#define GGMLQNN_PRINT_QNN_INTERNAL_LOG 0 // enable/disable QNN's internal log +#define GGMLQNN_PRINT_OP_ADD_LOG 0 // GGML_OP_ADD already verified with QNN-CPU / QNN-GPU / QNN-NPU +#define GGMLQNN_PRINT_OP_MUL_MAT_LOG 0 +#else +#define GGMLQNN_DEBUG 1 // for troubleshooting QNN backend +#define ENABLE_QNNBACKEND_PERF 0 // enable/disable op's perf info +#define GGMLQNN_PRINT_QNN_INTERNAL_LOG 0 // enable/disable QNN's internal log +#define GGMLQNN_PRINT_OP_ADD_LOG 0 // GGML_OP_ADD already verified with QNN-CPU / QNN-GPU / QNN-NPU +#define GGMLQNN_PRINT_OP_MUL_MAT_LOG 1 +#endif +#define GGML_QNN_LOGBUF_LEN 4096 + +#define GGMLQNN_LOG_ERROR(...) ggmlqnn_log_internal(GGML_LOG_LEVEL_ERROR, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) +#define GGMLQNN_LOG_WARN(...) ggmlqnn_log_internal(GGML_LOG_LEVEL_WARN , __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) +#define GGMLQNN_LOG_INFO(...) ggmlqnn_log_internal(GGML_LOG_LEVEL_INFO , __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) + +#if GGMLQNN_DEBUG +#define GGMLQNN_LOG_DEBUG(...) ggmlqnn_log_internal(GGML_LOG_LEVEL_DEBUG, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) +#else +#define GGMLQNN_LOG_DEBUG(...) +#endif + +#define CHECK_QNN_API(error, result) \ + do { \ + error = (result); \ + if (QNN_SUCCESS != error) { \ + if (error == QNN_COMMON_ERROR_NOT_SUPPORTED) { \ + GGMLQNN_LOG_WARN("WARNING: QNN feature/API not supported\n"); \ + } else { \ + GGMLQNN_LOG_INFO("QNN API error = %d(%s)\n", error, ggmlqnn_get_error_string(error)); \ + } \ + } \ + } while (0) + +#define QNN_VER_PTR(x) (&((x).v1)) +#define RPCMEM_DEFAULT_FLAGS 1 +#define RPCMEM_HEAP_ID_SYSTEM 25 + +#define DISABLE_COPY(class_name) \ + class_name(const class_name &) = delete; \ + void operator=(const class_name &) = delete + +#define DISABLE_MOVE(class_name) \ + class_name(class_name &&) = delete; \ + void operator=(class_name &&) = delete + +#define GQCGT ggmlqnn_create_general_tensor + +#if defined(_WIN32) +#define RTLD_GLOBAL 0x100 +#define RTLD_LOCAL 0x000 +#define RTLD_LAZY 0x000 +#define RTLD_NOW 0x001 +void * dlopen(const char * filename, int flag); +int dlclose(void * handle); +void * dlsym(void* handle, const char* name); +const char * dlerror(void); +#endif + +using pfn_rpc_mem_init = void (*)(void); +using pfn_rpc_mem_deinit = void (*)(void); +using pfn_rpc_mem_alloc = void *(*)(int, uint32_t, int); +using pfn_rpc_mem_free = void (*)(void *); +using pfn_rpc_mem_to_fd = int (*)(void *); +using _pfn_QnnSaver_initialize = decltype(QnnSaver_initialize); +using _pfn_QnnInterface_getProviders = decltype(QnnInterface_getProviders); +using _pfn_QnnSystemInterface_getProviders = decltype(QnnSystemInterface_getProviders); + +using qnn_res_t = std::tuple>; +using qnn_tensors_t = std::vector< Qnn_Tensor_t *>; + +enum class ggml_qnn_profile_level { + profile_off = 0, + profile_basic = 1, + profile_detail = 2 +}; + +enum qcom_htp_arch { + NONE = 0, + V68 = 68, + V69 = 69, + V73 = 73, + V75 = 75, + V79 = 79, +}; + +enum qcom_chipset_soc_model { + UNKNOWN_SM = 0, + SM7450 = 41, // v69, 7 Gen1 + SM8350 = 30, // v68, 888 + SM8450 = 36, // v69, SD 8 Gen 1 + SM8475 = 42, // v69, SD 8+ Gen 1 + SM8550 = 43, // v73, SD 8 Gen 2 + SM8650 = 57, // v75, SD 8 Gen 3 + SM8750 = 69, // v79, SD 8 Gen 4 +#if defined(_MSC_VER) + SC7280X = 44, + SC8280X = 37, + SC8380XP = 60, +#endif +}; + +struct qcom_socinfo { + uint32_t soc_model; + size_t htp_arch; + size_t vtcm_size_in_mb; + char soc_desc[GGML_MAX_NAME]; +}; + +struct ggml_backend_qnn_context { + int device; + int threads; + char name[GGML_MAX_NAME]; + char desc[GGML_MAX_NAME]; + char lib[GGML_MAX_NAME]; + qnn_instance * instance; + struct ggml_backend * backend; + QNN_INTERFACE_VER_TYPE raw_interface; + QNN_SYSTEM_INTERFACE_VER_TYPE raw_system_interface; + struct qcom_socinfo socinfo; + + std::unique_ptr work_data; + std::vector> tasks; + size_t work_size = 0; + size_t desired_size = 0; + int n_threads = GGML_DEFAULT_N_THREADS; +}; + +struct qnn_op_caps_t { + const char * qnn_op_name = nullptr; + const size_t input_param_count = 0; + const char * qnn_param_name = nullptr; +}; +extern const qnn_op_caps_t ggmlqnn_k_op_caps[]; + +#if ENABLE_QNNBACKEND_PERF +class qnn_perf { +public: + qnn_perf(const std::string & perf_name) : _perf_name(std::move(perf_name)) {}; + qnn_perf() = delete; + qnn_perf(const qnn_perf & ) = delete; + qnn_perf & operator= (const qnn_perf & ) = delete; + + void start() { + _begin_time = ggml_time_us(); + } + + void info() { + _end_time = ggml_time_us(); + _duration = (_end_time - _begin_time); + GGMLQNN_LOG_DEBUG("duration of %s : %lld microseconds\n", _perf_name.c_str(), _duration); + } + +private: + int64_t _begin_time = 0LL; + int64_t _end_time = 0LL; + int64_t _duration = 0LL; + std::string _perf_name; +}; +#else +class qnn_perf { +public: + qnn_perf(const std::string & perf_name) { + GGML_UNUSED(perf_name); + } + qnn_perf() = delete; + qnn_perf(const qnn_perf & ) = delete; + qnn_perf & operator= (const qnn_perf & ) = delete; + + void start() {} + void info() {} +}; +#endif + +class qnn_interface { +#define DEFINE_SHIM_FUNCTION_INTERFACE(F, pointer_name) \ + template \ + inline auto qnn_##F(Args... args) const { \ + return (_qnn_interface->QNN_INTERFACE_VER_NAME.pointer_name)( \ + std::forward(args)...); \ + } + + +#define DEFINE_SHIM_FUNCTION_SYS_INTERFACE(F, pointer_name) \ + template \ + inline auto qnn_##F(Args... args) const { \ + return (_qnn_sys_interface->QNN_SYSTEM_INTERFACE_VER_NAME.pointer_name)( \ + std::forward(args)...); \ + } + + friend class qnn_instance; + +public: + qnn_interface() = default; + + // QnnBackend + DEFINE_SHIM_FUNCTION_INTERFACE(backend_create, backendCreate) + + DEFINE_SHIM_FUNCTION_INTERFACE(backend_free, backendFree) + + DEFINE_SHIM_FUNCTION_INTERFACE(backend_register_op_package, backendRegisterOpPackage) + + DEFINE_SHIM_FUNCTION_INTERFACE(backend_validate_op_config, backendValidateOpConfig) + + DEFINE_SHIM_FUNCTION_INTERFACE(backend_get_api_version, backendGetApiVersion) + + // QnnDevice + DEFINE_SHIM_FUNCTION_INTERFACE(device_create, deviceCreate) + + DEFINE_SHIM_FUNCTION_INTERFACE(device_free, deviceFree) + + DEFINE_SHIM_FUNCTION_INTERFACE(device_get_infrastructure, deviceGetInfrastructure) + + DEFINE_SHIM_FUNCTION_INTERFACE(device_get_platform_info, deviceGetPlatformInfo) + + DEFINE_SHIM_FUNCTION_INTERFACE(device_get_info, deviceGetInfo) + + // QnnContext + DEFINE_SHIM_FUNCTION_INTERFACE(context_create, contextCreate) + + DEFINE_SHIM_FUNCTION_INTERFACE(context_get_binary_size, contextGetBinarySize) + + DEFINE_SHIM_FUNCTION_INTERFACE(context_get_binary, contextGetBinary) + + DEFINE_SHIM_FUNCTION_INTERFACE(context_create_from_binary, contextCreateFromBinary) + + DEFINE_SHIM_FUNCTION_INTERFACE(context_free, contextFree) + + // QnnGraph + DEFINE_SHIM_FUNCTION_INTERFACE(graph_create, graphCreate) + + DEFINE_SHIM_FUNCTION_INTERFACE(graph_add_node, graphAddNode) + + DEFINE_SHIM_FUNCTION_INTERFACE(graph_finalize, graphFinalize) + + DEFINE_SHIM_FUNCTION_INTERFACE(graph_execute, graphExecute) + + DEFINE_SHIM_FUNCTION_INTERFACE(graph_retrieve, graphRetrieve) + + // QnnLog + DEFINE_SHIM_FUNCTION_INTERFACE(log_create, logCreate) + + DEFINE_SHIM_FUNCTION_INTERFACE(log_free, logFree) + + DEFINE_SHIM_FUNCTION_INTERFACE(log_set_log_level, logSetLogLevel) + + // QnnProfile + DEFINE_SHIM_FUNCTION_INTERFACE(profile_create, profileCreate) + + DEFINE_SHIM_FUNCTION_INTERFACE(profile_get_events, profileGetEvents) + + DEFINE_SHIM_FUNCTION_INTERFACE(profile_get_sub_events, profileGetSubEvents) + + DEFINE_SHIM_FUNCTION_INTERFACE(profile_get_event_data, profileGetEventData) + + DEFINE_SHIM_FUNCTION_INTERFACE(profile_free, profileFree) + + // QnnMem + DEFINE_SHIM_FUNCTION_INTERFACE(mem_register, memRegister) + + DEFINE_SHIM_FUNCTION_INTERFACE(mem_de_register, memDeRegister) + + // QnnProperty + DEFINE_SHIM_FUNCTION_INTERFACE(property_has_capability, propertyHasCapability) + + // QnnTensor + DEFINE_SHIM_FUNCTION_INTERFACE(tensor_create_context_tensor, tensorCreateContextTensor) + + DEFINE_SHIM_FUNCTION_INTERFACE(tensor_create_graph_tensor, tensorCreateGraphTensor) + + // QnnSystem + DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_create, systemContextCreate) + + DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_get_binary_info, systemContextGetBinaryInfo) + + DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_free, systemContextFree) + + void set_qnn_interface(const QnnInterface_t * qnn_interface) { + _qnn_interface = qnn_interface; + } + + void set_qnn_system_interface(const QnnSystemInterface_t * qnn_sys_interface) { + _qnn_sys_interface = qnn_sys_interface; + } + + uint32_t get_backend_id() const { + return _qnn_interface->backendId; + } + + bool is_loaded() const { + return ((_qnn_sys_interface != nullptr) && (_qnn_interface != nullptr)); + } + +private: + const QnnInterface_t * _qnn_interface = nullptr; + + const QnnSystemInterface_t * _qnn_sys_interface = nullptr; +}; + +class qnn_instance { +public: + using BackendIdType = decltype(QnnInterface_t{}.backendId); + + explicit qnn_instance(const std::string & lib_path, const std::string & backend_name, + const std::string & model_name) : + _lib_path(std::move(lib_path)), + _backend_name(std::move(backend_name)), + _model_name(std::move(model_name)) {} + + ~qnn_instance() { + } + + int qnn_init(const QnnSaver_Config_t ** saver_config); + + int qnn_finalize(); + + const qnn_interface & get_qnn_interface() { + if (!_qnn_interface.is_loaded()) { + GGMLQNN_LOG_WARN("pls check why _qnn_interface is not loaded\n"); + } + return _qnn_interface; + } + + const QNN_INTERFACE_VER_TYPE & get_qnn_raw_interface() { + if (!_qnn_interface.is_loaded()) { + GGMLQNN_LOG_WARN("pls check why _qnn_interface is not loaded\n"); + } + return _qnn_raw_interface; + } + + const QNN_SYSTEM_INTERFACE_VER_TYPE & get_qnn_raw_system_interface() { + if (!_qnn_interface.is_loaded()) { + GGMLQNN_LOG_WARN("pls check why _qnn_interface is not loaded\n"); + } + return _qnn_raw_system_interface; + } + + Qnn_LogHandle_t get_qnn_log_handle() { return _qnn_log_handle; } + + Qnn_ProfileHandle_t get_qnn_profile_handle() { return _qnn_profile_handle; } + + Qnn_DeviceHandle_t get_qnn_device_handle() { return _qnn_device_handle; } + + Qnn_BackendHandle_t get_qnn_backend_handle() { return _qnn_backend_handle; } + + Qnn_ContextHandle_t get_qnn_context_handle() { return _qnn_context_handle; } + + QnnSystemContext_Handle_t get_qnn_system_handle() { return _qnn_system_handle; } + + Qnn_GraphHandle_t get_qnn_graph_handle() { return _qnn_graph_handle; } + + int init_qnn_graph(const char * graph_name, + bool debug, + uint8_t do_node_validation = 1, + const QnnGraph_Config_t ** graph_configs = nullptr + ); + int init_qnn_graph(const std::string & graph_name, QNNBackend device, size_t vtcm_size_in_mb = 8, size_t hvx_threads = 8); + + int finalize_qnn_graph(); + + bool is_valid_graph() const { return _qnn_graph_handle != nullptr; } + + int init_htp_perfinfra(); + + int set_rpc_polling(); + + int set_high_performance_mode(); + + std::string & get_qnn_graph_name() { return _graph_name; } + + bool is_rpcmem_initialized() { + return _rpcmem_initialized; + } + + void set_rpcmem_initialized(bool initialized) { + _rpcmem_initialized = initialized; + } + + size_t get_rpcmem_capacity() { return _rpcmem_capacity; } + size_t get_rpcmem_usage() { return _rpcmem_usage; } + + int32_t rpcmem_to_fd(void * buf); + + int register_rpcmem(void * p_data, Qnn_Tensor_t * p_tensor); + Qnn_MemHandle_t register_rpcmem(void * p_data, const uint32_t rank, uint32_t * dimensions, Qnn_DataType_t data_type); + + void unregister_rpcmem(); + void unregister_rpcmem(Qnn_MemHandle_t mem_handle); + + void * alloc_rpcmem(size_t bytes, size_t alignment); + void * get_rpcmem_from_memhandle(Qnn_MemHandle_t mem_handle); + + void free_rpcmem(void * buf); + void free_rpcmem(); + + bool is_rpcmem_allocated(void * buf); + + bool is_rpcmem_registered(Qnn_MemHandle_t handle) { + return _qnn_mem_set.count(handle) != 0U; + } + + bool enable_qnn_rpc() { + return _enable_qnn_rpc; + } + +public: + std::map>> _qnn_graph_map; + +private: + int load_system(); + + int unload_system(); + + int load_backend(std::string & lib_path, const QnnSaver_Config_t ** saver_config); + + int unload_backend(); + + void set_qnn_raw_interface(QNN_INTERFACE_VER_TYPE & raw_interface) { + _qnn_raw_interface = raw_interface; + } + + void set_qnn_raw_system_interface(QNN_SYSTEM_INTERFACE_VER_TYPE & raw_interface) { + _qnn_raw_system_interface = raw_interface; + } + + void * alloc_rpcmem_internal(size_t bytes, size_t alignment); + + void probe_device_meminfo(); + +private: + static constexpr const int _required_num_providers = 1; + +private: + std::string _lib_path; + std::string _backend_name; + std::string _model_name; // name of prebuilt QNN model, might be used in the future + BackendIdType _backend_id; + + bool _debug_tensor = false; // flag to indicate if requested graph is to be run in debug mode + bool _do_node_validations = true; // flag to indicate whether all add_node calls need to be validated + QnnLog_Level_t _qnn_log_level = QNN_LOG_LEVEL_DEBUG; + + ggml_qnn_profile_level _profile_level = ggml_qnn_profile_level::profile_detail; + + void * _system_lib_handle = nullptr; + + Qnn_GraphHandle_t _qnn_graph_handle = nullptr; + + Qnn_LogHandle_t _qnn_log_handle = nullptr; + + Qnn_ProfileHandle_t _qnn_profile_handle = nullptr; + + Qnn_DeviceHandle_t _qnn_device_handle = nullptr; + + Qnn_BackendHandle_t _qnn_backend_handle = nullptr; + + Qnn_ContextHandle_t _qnn_context_handle = nullptr; + + QnnSystemContext_Handle_t _qnn_system_handle = nullptr; + + QnnHtpDevice_PerfInfrastructure_t * _qnn_htp_perfinfra = nullptr; + uint32_t _qnn_power_configid = 1; + uint32_t _qnn_rpc_pollingtime = 9999; // 0-10000 us for high performing + + qnn_interface _qnn_interface; + QNN_INTERFACE_VER_TYPE _qnn_raw_interface; + QNN_SYSTEM_INTERFACE_VER_TYPE _qnn_raw_system_interface; + + std::unordered_map _qnn_mem_set; + std::unordered_map _qnn_rpc_buffer_to_handles; + + static std::mutex _init_mutex; + static std::unordered_map _loaded_lib_handle; + static std::unordered_map _lib_path_to_backend_id; + static std::unordered_map _loaded_backend; + + std::atomic_bool _rpcmem_initialized{false}; + pfn_rpc_mem_alloc _pfn_rpc_mem_alloc; + pfn_rpc_mem_free _pfn_rpc_mem_free; + pfn_rpc_mem_to_fd _pfn_rpc_mem_to_fd; + pfn_rpc_mem_init _pfn_rpc_mem_init; + pfn_rpc_mem_deinit _pfn_rpc_mem_deinit; + std::unordered_map _rpcmem_store_map; + std::unordered_map _rpcmem_usage_map; + size_t _rpcmem_usage = 0; // mempool usage in Mbytes + size_t _rpcmem_capacity = 512; // mempool size in Mbytes + + std::string _graph_name; + QNNBackend _device_id; + void * _rpc_lib_handle = nullptr; + bool _enable_qnn_rpc = false; //TODO:unknown issue with QNN RPC feature + + DISABLE_COPY(qnn_instance); + DISABLE_MOVE(qnn_instance); +}; + +size_t ggmlqnn_get_opcaps_size(void); +size_t ggmlqnn_get_op_index(const ggml_tensor * tensor); +Qnn_Tensor_t * ggmlqnn_create_compute_tensor(const ggml_tensor * tensor); +const char * ggmlqnn_get_error_string(Qnn_ErrorHandle_t qnn_error_code); +Qnn_DataType_t ggmlqnn_datatype_from_ggml_datatype(enum ggml_type ggmltype); +void * ggmlqnn_type_trait(ggml_backend_qnn_context * ctx, ggml_tensor * op); +void ggmlqnn_get_graphkey_from_op(const ggml_tensor * op, std::string & output); +uint8_t * ggmlqnn_create_rpc_buffer(qnn_instance * instance, const ggml_tensor * ggml_tensor, Qnn_Tensor_t * qnn_tensor, bool b_copydata); +void ggmlqnn_print_tensors_info(const char * func_name, ggml_backend_qnn_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); + +Qnn_OpConfig_t ggmlqnn_create_op_config(const char * name, const char * package, const char * type, + Qnn_Param_t * params, uint32_t num_params, + Qnn_Tensor_t * inputs, uint32_t num_inputs, + Qnn_Tensor_t * outputs, uint32_t num_outputs); +Qnn_Tensor_t * ggmlqnn_create_general_tensor(const ggml_tensor * tensor, const char * name, + Qnn_TensorType_t qnn_tensor_type, + Qnn_DataType_t qnn_data_type, + uint32_t rank, uint32_t * dims, + void * data, uint32_t data_size, + bool b_transpose = false); diff --git a/ggml/src/ggml-qnn/ggml-qnn-ops.cpp b/ggml/src/ggml-qnn/ggml-qnn-ops.cpp new file mode 100644 index 0000000000000..00cb7da32c183 --- /dev/null +++ b/ggml/src/ggml-qnn/ggml-qnn-ops.cpp @@ -0,0 +1,842 @@ +/* + * Copyright (c) 2023-2024 The ggml authors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ +#include "ggml-impl.h" +#include "ggml-common.h" +#include "ggml-qnn-ops.h" + +static inline uint32_t ggmlqnn_get_tensor_data_size(const ggml_tensor * tensor) { + /* + size_t data_size = ggml_row_size(tensor->type, tensor->ne[0]); + size_t n_dims = ggml_get_tensor_rank(tensor); + for (int i = 1; i < n_dims; i++) { + data_size *= tensor->ne[i]; + } + + return data_size; + */ + return ggml_nbytes(tensor); +} + +static inline bool ggmlqnn_is_valid_params(ggml_backend_qnn_context * ctx, const ggml_tensor * src0, + const ggml_tensor * src1, ggml_tensor * dst) { + if ((nullptr == ctx) || (nullptr == src0) || (nullptr == src1) || (nullptr == dst)) { + GGMLQNN_LOG_WARN("invalid params\n"); + return false; + } + + qnn_instance * instance = ctx->instance; + if (nullptr == instance) { + GGMLQNN_LOG_WARN("invalid params\n"); + return false; + } + + return true; +} + +#define GGMLQNN_CHECK_PARAMS(ctx, src0, src1, dst) \ + do { \ + if (!ggmlqnn_is_valid_params((ctx), (src0), (src1), (dst))) { \ + return; \ + } \ + } while (0) + +/* + * provide a general skeleton to offload ggml op to QNN backend: a single node contains 2 input + * tensor and 1 output tensor +*/ +void ggml_qnn_general_node(ggml_backend_qnn_context * ctx, ggml_tensor * op) { + Qnn_ErrorHandle_t error = QNN_SUCCESS; + enum ggml_status result = GGML_STATUS_SUCCESS; + bool graph_initialized = false; + qnn_instance * instance = nullptr; + Qnn_GraphHandle_t graph_handle = nullptr; + Qnn_Tensor_t * p_tensor0 = nullptr; + Qnn_Tensor_t * p_tensor1 = nullptr; + Qnn_Tensor_t * p_tensor2 = nullptr; + Qnn_Param_t qnn_params[] = {}; + const ggml_tensor * src0 = op->src[0]; + const ggml_tensor * src1 = op->src[1]; + ggml_tensor * dst = op; + + GGMLQNN_CHECK_PARAMS(ctx, src0, src1, dst); + instance = ctx->instance; + QNN_INTERFACE_VER_TYPE qnn_raw_interface = ctx->raw_interface; + size_t qnn_op_index = ggmlqnn_get_op_index(op); + GGML_ASSERT(qnn_op_index < ggmlqnn_get_opcaps_size()); + const char * qnn_op_name = ggmlqnn_k_op_caps[qnn_op_index].qnn_op_name; + std::string ggml_op_name_string = std::string("ggml_") + ggml_op_name(op->op); + const char * ggml_op_name = ggml_op_name_string.c_str(); + + qnn_perf op_perf = qnn_perf(ggml_op_name); + op_perf.start(); + + std::string graph_name; + ggmlqnn_get_graphkey_from_op(op, graph_name); + if (instance->_qnn_graph_map.find(graph_name) != instance->_qnn_graph_map.end()) { + graph_initialized = true; + qnn_res_t & graph_item = instance->_qnn_graph_map[graph_name]; + graph_handle = std::get<0>(graph_item); + qnn_tensors_t & tensor = std::get<1>(graph_item); + p_tensor0 = tensor[0]; + p_tensor1 = tensor[1]; + p_tensor2 = tensor[2]; + } else { + p_tensor0 = ggmlqnn_create_compute_tensor(src0); + p_tensor1 = ggmlqnn_create_compute_tensor(src1); + p_tensor2 = ggmlqnn_create_compute_tensor(dst); + } + //ggmlqnn_print_tensors_info(__func__, ctx, src0, src1, dst); + + //ensure QNN tensor has correct tensor type + QNN_VER_PTR(*p_tensor0)->type = QNN_TENSOR_TYPE_APP_WRITE; + QNN_VER_PTR(*p_tensor1)->type = QNN_TENSOR_TYPE_APP_WRITE; + QNN_VER_PTR(*p_tensor2)->type = QNN_TENSOR_TYPE_APP_READ; + + //save the original dimensions of qnn tensors + uint32_t * tensor_0_dimensions = QNN_VER_PTR(*p_tensor0)->dimensions; + uint32_t * tensor_1_dimensions = QNN_VER_PTR(*p_tensor1)->dimensions; + uint32_t * tensor_2_dimensions = QNN_VER_PTR(*p_tensor2)->dimensions; + + bool enable_npu_rpc = instance->enable_qnn_rpc() && ctx->device == QNN_BACKEND_NPU; + + if (!graph_initialized) { + GGMLQNN_LOG_DEBUG("graph name %s", graph_name.c_str()); + error = instance->init_qnn_graph(graph_name, static_cast(ctx->device), 8); + if (QNN_SUCCESS != error) { + GGMLQNN_LOG_INFO("can't create qnn graph handle with graph name %s, error = %d\n", graph_name.c_str(), error); + return; + } + graph_handle = instance->get_qnn_graph_handle(); + + if (enable_npu_rpc) { + QNN_VER_PTR(*p_tensor0)->memType = QNN_TENSORMEMTYPE_MEMHANDLE; + QNN_VER_PTR(*p_tensor0)->clientBuf = {.data=nullptr, .dataSize=0}; + + QNN_VER_PTR(*p_tensor1)->memType = QNN_TENSORMEMTYPE_MEMHANDLE; + QNN_VER_PTR(*p_tensor1)->clientBuf = {.data=nullptr, .dataSize=0}; + + QNN_VER_PTR(*p_tensor2)->memType = QNN_TENSORMEMTYPE_MEMHANDLE; + QNN_VER_PTR(*p_tensor2)->clientBuf = {.data=nullptr, .dataSize=0}; + } + + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor0)); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor1)); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor2)); + + if (enable_npu_rpc) { + uint8_t * qnn_rpcbuffer_0 = ggmlqnn_create_rpc_buffer(instance, src0, p_tensor0, true); + uint8_t * qnn_rpcbuffer_1 = ggmlqnn_create_rpc_buffer(instance, src1, p_tensor1, true); + uint8_t * qnn_rpcbuffer_2 = ggmlqnn_create_rpc_buffer(instance, dst, p_tensor2, false); + if (nullptr == qnn_rpcbuffer_0 || nullptr == qnn_rpcbuffer_1 || nullptr == qnn_rpcbuffer_2) { + GGMLQNN_LOG_INFO("create rpc buffer failure\n"); + //TODO: potential memory leak although it shouldn't happen + return; + } + } else { + QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggmlqnn_get_tensor_data_size(src0)}; + QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggmlqnn_get_tensor_data_size(src1)}; + QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggmlqnn_get_tensor_data_size(dst)}; + } + + Qnn_Tensor_t tensor_inputs[] = { + *p_tensor0, + *p_tensor1 + }; + Qnn_Tensor_t tensor_outputs[] = { + *p_tensor2 + }; + Qnn_OpConfig_t op_config = { + QNN_OPCONFIG_VERSION_1, .v1 = { + ggml_op_name, + QNN_OP_PACKAGE_NAME_QTI_AISW, + qnn_op_name, + 0, + qnn_params, + 2, + tensor_inputs, + 1, + tensor_outputs + } + }; + CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, op_config)); + CHECK_QNN_API(error, qnn_raw_interface.graphFinalize(graph_handle, nullptr, nullptr)); + CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, + tensor_inputs, 2, + tensor_outputs, 1, + nullptr, nullptr)); + + if (enable_npu_rpc) { + uint8_t * qnn_rpcbuffer = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*p_tensor2)->memHandle)); + GGMLQNN_LOG_INFO("qnn_rpcbuffer = %p\n", qnn_rpcbuffer); + if (nullptr != qnn_rpcbuffer) { + memcpy(dst->data, qnn_rpcbuffer, ggml_nbytes(dst)); + } + } + + qnn_tensors_t ggml_op_add_tensors; + ggml_op_add_tensors.reserve(3); + ggml_op_add_tensors.push_back(p_tensor0); + ggml_op_add_tensors.push_back(p_tensor1); + ggml_op_add_tensors.push_back(p_tensor2); + + auto graph_item = std::make_tuple(graph_handle, ggml_op_add_tensors); + instance->_qnn_graph_map[graph_name] = graph_item; + } else { + Qnn_DataType_t src0_qnn_type = QNN_DATATYPE_FLOAT_32; + Qnn_DataType_t src1_qnn_type = QNN_DATATYPE_FLOAT_32; + Qnn_DataType_t dst_qnn_type = QNN_DATATYPE_FLOAT_32; + + src0_qnn_type = ggmlqnn_datatype_from_ggml_datatype(src0->type); + src1_qnn_type = ggmlqnn_datatype_from_ggml_datatype(src1->type); + dst_qnn_type = ggmlqnn_datatype_from_ggml_datatype(dst->type); + + uint32_t dimensions_input_0[] = {(uint32_t) src0->ne[0], (uint32_t) src0->ne[1], + (uint32_t) src0->ne[2], (uint32_t) src0->ne[3]}; + uint32_t dimensions_input_1[] = {(uint32_t) src1->ne[0], (uint32_t) src1->ne[1], + (uint32_t) src1->ne[2], (uint32_t) src1->ne[3]}; + uint32_t dimensions_output[] = {(uint32_t) dst->ne[0], (uint32_t) dst->ne[1], + (uint32_t) dst->ne[2], (uint32_t) dst->ne[3]}; + + QNN_VER_PTR(*p_tensor0)->dimensions = dimensions_input_0; + QNN_VER_PTR(*p_tensor0)->rank = ggml_n_dims(src0); + QNN_VER_PTR(*p_tensor0)->dataType = src0_qnn_type; + + QNN_VER_PTR(*p_tensor1)->dimensions = dimensions_input_1; + QNN_VER_PTR(*p_tensor1)->rank = ggml_n_dims(src1); + QNN_VER_PTR(*p_tensor1)->dataType = src1_qnn_type; + + QNN_VER_PTR(*p_tensor2)->dimensions = dimensions_output; + QNN_VER_PTR(*p_tensor2)->rank = ggml_n_dims(dst); + QNN_VER_PTR(*p_tensor2)->dataType = dst_qnn_type; + + if (enable_npu_rpc) { + //TODO: NPU RPC feature will failed with test-backend-ops + uint8_t * qnn_buffer_0 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*p_tensor0)->memHandle)); + GGMLQNN_LOG_INFO("qnn_rpcbuffer_0 = %p\n", qnn_buffer_0); + if (nullptr != qnn_buffer_0) { + memcpy(qnn_buffer_0, src0->data, ggml_nbytes(src0)); + } + + uint8_t * qnn_buffer_1 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*p_tensor1)->memHandle)); + GGMLQNN_LOG_INFO("qnn_rpcbuffer_1 = %p\n", qnn_buffer_1); + if (nullptr != qnn_buffer_1) { + memcpy(qnn_buffer_1, src1->data, ggml_nbytes(src1)); + } + } else { + QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggmlqnn_get_tensor_data_size(src0)}; + QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggmlqnn_get_tensor_data_size(src1)}; + QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggmlqnn_get_tensor_data_size(dst)}; + } + + Qnn_Tensor_t tensor_inputs[] = { + *p_tensor0, + *p_tensor1 + }; + Qnn_Tensor_t tensor_outputs[] = { + *p_tensor2 + }; + CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, + tensor_inputs, 2, + tensor_outputs, 1, + nullptr, nullptr)); + + if (enable_npu_rpc) { + //TODO:NPU RPC feature will failed with test-backend-ops + uint8_t * qnn_buffer_2 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*p_tensor2)->memHandle)); + if (nullptr != qnn_buffer_2) { + memcpy(dst->data, qnn_buffer_2, ggml_nbytes(dst)); + } + } + } + + // restore the original dimensions of qnn tensors to avoid memory leak in func free_qnn_tensor + QNN_VER_PTR(*p_tensor0)->dimensions = tensor_0_dimensions; + QNN_VER_PTR(*p_tensor1)->dimensions = tensor_1_dimensions; + QNN_VER_PTR(*p_tensor2)->dimensions = tensor_2_dimensions; + +#if GGMLQNN_PRINT_OP_ADD_LOG + op_perf.info(); +#endif +} + +/* + * this function is AI-assisted code from Grok 3 for purpose of offload 4d matrix mulmat to QNN backend + * UT in ggml-qnn-ut.cpp passed: + * ./scripts/build-run-android.sh run_ut_mulmat 0 + * ./scripts/build-run-android.sh run_ut_mulmat 1 + * ./scripts/build-run-android.sh run_ut_mulmat 2 + * + * the logic of ggml_qnn_mul_mat_4d is similar to ggml_qnn_mul_mat but much more complicated + * than ggml_qnn_mul_mat, so it's a standalone function. + * it will be combined with ggml_qnn_mul_mat in the future + */ +static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) { + Qnn_ErrorHandle_t error = QNN_SUCCESS; + bool graph_initialized = false; + qnn_perf op_perf = qnn_perf("ggml_qnn_mul_mat_4d"); + qnn_instance *instance = ctx->instance; + QNN_INTERFACE_VER_TYPE qnn_raw_interface = ctx->raw_interface; + + const ggml_tensor *src0 = op->src[0]; + const ggml_tensor *src1 = op->src[1]; + ggml_tensor *dst = op; + + GGMLQNN_CHECK_PARAMS(ctx, src0, src1, dst); + GGML_ASSERT(ggml_n_dims(src0) == 4 && ggml_n_dims(src1) == 4); + op_perf.start(); + + std::string graph_name; + ggmlqnn_get_graphkey_from_op(op, graph_name); + GGMLQNN_LOG_DEBUG("graph name %s\n", graph_name.c_str()); + + ggmlqnn_print_tensors_info(__func__, ctx, src0, src1, dst); + + Qnn_GraphHandle_t graph_handle = nullptr; + Qnn_Tensor_t *p_tensor0 = nullptr; + Qnn_Tensor_t *p_reshape0_out = nullptr; + Qnn_Tensor_t *p_tile0_out = nullptr; + Qnn_Tensor_t *p_tensor1 = nullptr; + Qnn_Tensor_t *p_permute1_out = nullptr; + Qnn_Tensor_t *p_reshape1_out = nullptr; + Qnn_Tensor_t *p_matmul_out = nullptr; + Qnn_Tensor_t *p_reshape2_out = nullptr; + + if (instance->_qnn_graph_map.find(graph_name) != instance->_qnn_graph_map.end()) { + graph_initialized = true; + qnn_res_t &graph_item = instance->_qnn_graph_map[graph_name]; + graph_handle = std::get<0>(graph_item); + qnn_tensors_t &tensors = std::get<1>(graph_item); + p_tensor0 = tensors[0]; + p_reshape0_out = tensors[1]; + p_tile0_out = tensors[2]; + p_tensor1 = tensors[3]; + p_permute1_out = tensors[4]; + p_reshape1_out = tensors[5]; + p_matmul_out = tensors[6]; + p_reshape2_out = tensors[7]; + } else { + CHECK_QNN_API(error, qnn_raw_interface.graphCreate(instance->get_qnn_context_handle(), + graph_name.c_str(), NULL, &graph_handle)); + + // Define dimensions + uint32_t K = src0->ne[0]; // Inner dimension + uint32_t M = src0->ne[1]; // Rows of src0 + uint32_t N = src1->ne[1]; // Columns of src1 + uint32_t B0 = src0->ne[2] * src0->ne[3]; // src0 batch + uint32_t B1 = src1->ne[2] * src1->ne[3]; // src1 batch (drives output) + + // Validate K only + GGML_ASSERT(src0->ne[0] == src1->ne[0]); // K must match + + // src0: [K, M, H0, B0] -> QNN: [B0, H0, M, K] + uint32_t src0_dims[] = {static_cast(src0->ne[3]), static_cast(src0->ne[2]), static_cast(src0->ne[1]), static_cast(src0->ne[0])}; + p_tensor0 = GQCGT(src0, "input0", QNN_TENSOR_TYPE_APP_WRITE, QNN_DATATYPE_FLOAT_32, 4, + src0_dims, nullptr, 0); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor0)); + + // Reshape src0 to [B0, M, K] + uint32_t reshape0_out_dims[] = {B0, M, K}; + p_reshape0_out = GQCGT(nullptr, "reshape0_out", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, 3, + reshape0_out_dims, nullptr, 0); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_reshape0_out)); + Qnn_Tensor_t reshape0_inputs[] = {*p_tensor0}; + Qnn_Tensor_t reshape0_outputs[] = {*p_reshape0_out}; + Qnn_OpConfig_t reshape0_op = ggmlqnn_create_op_config("reshape0", QNN_OP_PACKAGE_NAME_QTI_AISW, + QNN_OP_RESHAPE, nullptr, 0, + reshape0_inputs, 1, reshape0_outputs, 1); + CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, reshape0_op)); + + // Tile src0 to match B1: [B0, M, K] -> [B1, M, K] + uint32_t tile0_out_dims[] = {B1, M, K}; + p_tile0_out = GQCGT(nullptr, "tile0_out", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, 3, + tile0_out_dims, nullptr, 0); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tile0_out)); + uint32_t tile_multiples[] = {B1 / B0, 1, 1}; + uint32_t tile_dims[] = {3}; + Qnn_Tensor_t *p_tile_multiples = GQCGT(nullptr, "tile_multiples", QNN_TENSOR_TYPE_STATIC, QNN_DATATYPE_UINT_32, 1, + tile_dims, tile_multiples, sizeof(tile_multiples)); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tile_multiples)); + Qnn_Param_t tile_params[] = {{QNN_PARAMTYPE_TENSOR, "multiples", .tensorParam = *p_tile_multiples}}; + Qnn_Tensor_t tile0_inputs[] = {*p_reshape0_out}; + Qnn_Tensor_t tile0_outputs[] = {*p_tile0_out}; + Qnn_OpConfig_t tile0_op = ggmlqnn_create_op_config("tile0", QNN_OP_PACKAGE_NAME_QTI_AISW, + QNN_OP_TILE, tile_params, 1, + tile0_inputs, 1, tile0_outputs, 1); + CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, tile0_op)); + + // src1: [N, K, H1, B1] -> QNN: [B1, H1, N, K] + uint32_t src1_dims[] = {static_cast(src1->ne[3]), static_cast(src1->ne[2]), static_cast(src1->ne[1]), static_cast(src1->ne[0])}; + p_tensor1 = GQCGT(src1, "input1", QNN_TENSOR_TYPE_APP_WRITE, QNN_DATATYPE_FLOAT_32, 4, + src1_dims, nullptr, 0); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor1)); + + // Permute src1 to [B1, H1, K, N] + uint32_t perm_data[] = {0, 1, 3, 2}; + uint32_t perm_dims[] = {4}; + Qnn_Tensor_t *p_perm = GQCGT(nullptr, "perm", QNN_TENSOR_TYPE_STATIC, QNN_DATATYPE_UINT_32, 1, + perm_dims, perm_data, sizeof(perm_data)); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_perm)); + uint32_t permute1_out_dims[] = {static_cast(src1->ne[3]), static_cast(src1->ne[2]), static_cast(src1->ne[0]), static_cast(src1->ne[1])}; + p_permute1_out = GQCGT(nullptr, "permute1_out", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, 4, + permute1_out_dims, nullptr, 0); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_permute1_out)); + Qnn_Param_t permute1_params[] = {{QNN_PARAMTYPE_TENSOR, "perm", .tensorParam = *p_perm}}; + Qnn_Tensor_t permute1_inputs[] = {*p_tensor1}; + Qnn_Tensor_t permute1_outputs[] = {*p_permute1_out}; + Qnn_OpConfig_t permute1_op = ggmlqnn_create_op_config("permute1", QNN_OP_PACKAGE_NAME_QTI_AISW, + QNN_OP_TRANSPOSE, permute1_params, 1, + permute1_inputs, 1, permute1_outputs, 1); + CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, permute1_op)); + + // Reshape src1 to [B1, K, N] + uint32_t reshape1_out_dims[] = {B1, K, N}; + p_reshape1_out = GQCGT(nullptr, "reshape1_out", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, 3, + reshape1_out_dims, nullptr, 0); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_reshape1_out)); + Qnn_Tensor_t reshape1_inputs[] = {*p_permute1_out}; + Qnn_Tensor_t reshape1_outputs[] = {*p_reshape1_out}; + Qnn_OpConfig_t reshape1_op = ggmlqnn_create_op_config("reshape1", QNN_OP_PACKAGE_NAME_QTI_AISW, + QNN_OP_RESHAPE, nullptr, 0, + reshape1_inputs, 1, reshape1_outputs, 1); + CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, reshape1_op)); + + // MatMul: [B1, M, K] x [B1, K, N] -> [B1, M, N] + uint32_t matmul_out_dims[] = {B1, M, N}; + p_matmul_out = GQCGT(nullptr, "matmul_out", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, 3, + matmul_out_dims, nullptr, 0); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_matmul_out)); + Qnn_Tensor_t matmul_inputs[] = {*p_tile0_out, *p_reshape1_out}; + Qnn_Tensor_t matmul_outputs[] = {*p_matmul_out}; + Qnn_OpConfig_t matmul_op = ggmlqnn_create_op_config("matmul", QNN_OP_PACKAGE_NAME_QTI_AISW, + QNN_OP_MAT_MUL, nullptr, 0, + matmul_inputs, 2, matmul_outputs, 1); + CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, matmul_op)); + + // Output: [N, M, H1, B1] -> QNN: [B1, H1, M, N] + uint32_t reshape2_out_dims[] = {static_cast(dst->ne[3]), static_cast(dst->ne[2]), static_cast(dst->ne[1]), static_cast(dst->ne[0])}; + p_reshape2_out = GQCGT(dst, "output", QNN_TENSOR_TYPE_APP_READ, QNN_DATATYPE_FLOAT_32, 4, + reshape2_out_dims, nullptr, 0); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_reshape2_out)); + Qnn_Tensor_t reshape2_inputs[] = {*p_matmul_out}; + Qnn_Tensor_t reshape2_outputs[] = {*p_reshape2_out}; + Qnn_OpConfig_t reshape2_op = ggmlqnn_create_op_config("reshape2", QNN_OP_PACKAGE_NAME_QTI_AISW, + QNN_OP_RESHAPE, nullptr, 0, + reshape2_inputs, 1, reshape2_outputs, 1); + CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, reshape2_op)); + + // Finalize + CHECK_QNN_API(error, qnn_raw_interface.graphFinalize(graph_handle, NULL, NULL)); + + // Cache + qnn_tensors_t ggml_op_mulmat_tensors = {p_tensor0, p_reshape0_out, p_tile0_out, p_tensor1, p_permute1_out, p_reshape1_out, p_matmul_out, p_reshape2_out}; + instance->_qnn_graph_map[graph_name] = std::make_tuple(graph_handle, ggml_op_mulmat_tensors); + } + + // Execute + QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, static_cast(ggml_nbytes(src0))}; + QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, static_cast(ggml_nbytes(src1))}; + QNN_VER_PTR(*p_reshape2_out)->clientBuf = {dst->data, static_cast(ggml_nbytes(dst))}; + + Qnn_Tensor_t input_tensors[] = {*p_tensor0, *p_tensor1}; + Qnn_Tensor_t output_tensors[] = {*p_reshape2_out}; + CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, input_tensors, 2, + output_tensors, 1, NULL, NULL)); + +#if 0 + // Log dst for debugging + float *dst_data = (float *)dst->data; + GGMLQNN_LOG_DEBUG("dst shape: [%d, %d, %d, %d]\n", dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]); + for (int i = 0; i < dst->ne[0] * dst->ne[1] * dst->ne[2] * dst->ne[3]; i++) { + GGMLQNN_LOG_DEBUG("dst[%d] = %f\n", i, dst_data[i]); + } +#endif + + op_perf.info(); +} + +/* + * @brief performs matrix multiplication with FP32 & quantized weights and floating-point inputs + * using the QNN backend. this function performs matrix multiplication of the input tensor + * `src1` and the weight tensor `src0`, handling transposing, and quantization as needed, + * and stores the result in the destination tensor `dst`. + * + * @param backend the context which got through (ggml_backend_qnn_context *)backend->context for the + * QNN backend operations. + * @param op the destination tensor where the result of the matrix multiplication will be stored. + * + * @note the logic of ggml_qnn_mul_mat is similar to ggml_qnn_general_node but much more complicated + * than ggml_qnn_general_node. so it's a standalone function. accordingly, this is another + * typical skeleton for offload other ggml ops to QNN backend. MUL_MAT take most of the compute + * time (about 95%).so to speed up llama inference, should focus on this func. there are three kinds + * of MUL_MAT to compute: + * mul_mat_f32: both src0 and src1 are F32, this will be naturally handled in QNN backend + * mul_mat_f16_f32: src0 is F16 and src1 is F32, f16 in src0 -> f32 in src0', then src0' * src1 + * mul_mat_q_f32: src0 is quantized (Q4_0, Q4_1, Q6_K...) + * and src1 is F32, src0 -> f32 in src0', then src0' * src1 +*/ +void ggml_qnn_mul_mat(ggml_backend_qnn_context * ctx, ggml_tensor * op) { + Qnn_ErrorHandle_t error = QNN_SUCCESS; + bool graph_initialized = false; + qnn_perf op_perf = qnn_perf("ggml_qnn_mul_mat"); + qnn_instance * instance = nullptr; + Qnn_GraphHandle_t graph_handle = nullptr; + Qnn_Tensor_t * p_tensor0 = nullptr; + Qnn_Tensor_t * p_tensor1 = nullptr; + Qnn_Tensor_t * p_tensor2 = nullptr; + Qnn_Tensor_t * p_param_tensor = nullptr; + Qnn_Tensor_t * p_tensor2_transpose = nullptr; + const ggml_tensor * src0 = op->src[0]; + const ggml_tensor * src1 = op->src[1]; + ggml_tensor * dst = op; + + GGMLQNN_CHECK_PARAMS(ctx, src0, src1, dst); + instance = ctx->instance; + QNN_INTERFACE_VER_TYPE qnn_raw_interface = ctx->raw_interface; + op_perf.start(); + + const enum ggml_type src0_type = src0->type; + const uint32_t src0_rank = ggml_n_dims(src0); + const uint32_t src1_rank = ggml_n_dims(src1); + GGML_ASSERT(src0_rank == src1_rank); + GGML_ASSERT(src0_rank >= 2); //QNN SDK's limitation, make QNN SDK happy + if (4 == src0_rank) { + return ggml_qnn_mul_mat_4d(ctx, op); + } + void * wdata = ggmlqnn_type_trait(ctx, op); + const size_t desired_size = ctx->desired_size; + + std::string graph_name; + ggmlqnn_get_graphkey_from_op(op, graph_name); + if (instance->_qnn_graph_map.find(graph_name) != instance->_qnn_graph_map.end()) { + graph_initialized = true; + qnn_res_t & graph_item = instance->_qnn_graph_map[graph_name]; + graph_handle = std::get<0>(graph_item); + qnn_tensors_t & tensors = std::get<1>(graph_item); + p_tensor0 = tensors[0]; + p_tensor1 = tensors[1]; + p_tensor2 = tensors[2]; + p_param_tensor = tensors[3]; + p_tensor2_transpose = tensors[4]; + } else { + p_tensor0 = GQCGT(src0, nullptr, QNN_TENSOR_TYPE_APP_WRITE,QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0); + p_tensor1 = GQCGT(src1, nullptr, QNN_TENSOR_TYPE_APP_WRITE,QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0); + p_tensor2 = GQCGT(dst, nullptr, QNN_TENSOR_TYPE_APP_READ,QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0); + } + ggmlqnn_print_tensors_info(__func__, ctx, src0, src1, dst); + + //ensure QNN tensor has correct tensor type + QNN_VER_PTR(*p_tensor0)->type = QNN_TENSOR_TYPE_APP_WRITE; + QNN_VER_PTR(*p_tensor1)->type = QNN_TENSOR_TYPE_APP_WRITE; + QNN_VER_PTR(*p_tensor2)->type = QNN_TENSOR_TYPE_APP_READ; + + //save the original dimensions of qnn tensors + uint32_t * tensor_0_dimensions = QNN_VER_PTR(*p_tensor0)->dimensions; + uint32_t * tensor_1_dimensions = QNN_VER_PTR(*p_tensor1)->dimensions; + uint32_t * tensor_2_dimensions = QNN_VER_PTR(*p_tensor2)->dimensions; + + if (!graph_initialized) { + GGMLQNN_LOG_DEBUG("graph name %s", graph_name.c_str()); + /* + there are two key-points in properly handling how to offload mulmat to the QNN backend in ggml-qnn + 1. transpose + a 3x2 f32 matrix which means 3 rows and 2 columns. in ggml, it could be created from: + struct ggml_tensor* matrix = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 2, 3); + which like this: + +---+---+ + | 0 | 1 | + +---+---+ + | 2 | 3 | + +---+---+ + | 4 | 5 | + +---+---+ + with + ne[0] = 2 + ne[1] = 3 + there are different dimension order between ggml tensor and qnn tensor + + 2. QNN's MatMul can only support input tensors with rank >= 2 + + in the all, there is gap between ggml mulmat and QNN mulmat,we need to perform a transpose + operation when offloading mulmat to QNN backend. this concise implementation will handle + transpose in func ggml_qnn_create_general_tensor() + */ + //step-1: create qnn graph + error = qnn_raw_interface.graphCreate(instance->get_qnn_context_handle(), + graph_name.c_str(), nullptr, &graph_handle); + if (QNN_SUCCESS != error) { + GGMLQNN_LOG_INFO("can't create qnn graph handle with graph name %s, error = %d\n", graph_name.c_str(), error); + return; + } + //step-2: create param tensor for mulmat of 2d/3d/4d matrix + const uint32_t param_tensor_data[GGML_MAX_DIMS][GGML_MAX_DIMS] = { + {0}, + {1, 0}, + {0, 2, 1}, + {0, 1, 3, 2}, + }; + uint32_t param_tensor_dims[1] = {src0_rank}; + p_param_tensor = GQCGT(nullptr, "param", QNN_TENSOR_TYPE_STATIC, QNN_DATATYPE_UINT_32, 1, param_tensor_dims, (void *)(param_tensor_data[src0_rank - 1]), src0_rank * sizeof(uint32_t)); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_param_tensor)); + + //step-3: create compute tensor from ggml tensor + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor0)); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor1)); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor2)); + if (src0_type != GGML_TYPE_F32) { + QNN_VER_PTR(*p_tensor0)->clientBuf = {wdata, static_cast(desired_size)}; + } else { + QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggmlqnn_get_tensor_data_size(src0)}; + } + QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggmlqnn_get_tensor_data_size(src1)}; + QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggmlqnn_get_tensor_data_size(dst)}; + + //step-4: create a transpose tensor + p_tensor2_transpose = GQCGT(dst, "transpose", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0, true); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor2_transpose)); + + //step-5: compose qnn graph: add mat_mul node + Qnn_Param_t out_0_params[] = { + {QNN_PARAMTYPE_SCALAR, + QNN_OP_MAT_MUL_PARAM_TRANSPOSE_IN1, + .scalarParam = {QNN_DATATYPE_BOOL_8, .bool8Value = 1} + } + }; + + Qnn_Tensor_t out_0_inputs[] = {*p_tensor0, *p_tensor1}; + Qnn_Tensor_t out_0_outputs[] = {*p_tensor2_transpose}; +#if 0 //leave here for easily understand code, can be removed in the future + Qnn_OpConfig_t out_0 = { + QNN_OPCONFIG_VERSION_1, .v1 = + {"ggmlqnn_mulmat_opconfig", QNN_OP_PACKAGE_NAME_QTI_AISW, QNN_OP_MAT_MUL, + 1, + out_0_params, + 2, + out_0_inputs, + 1, + out_0_outputs} + }; +#else + Qnn_OpConfig_t out_0 = ggmlqnn_create_op_config("ggmlqnn_mulmat_opconfig", QNN_OP_PACKAGE_NAME_QTI_AISW, QNN_OP_MAT_MUL, + out_0_params, 1, out_0_inputs, 2, out_0_outputs, 1); +#endif + CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle,out_0)); + + //step-5: compose qnn graph: add transpose node + Qnn_Param_t out_trans1_0_params[] = { + {QNN_PARAMTYPE_TENSOR, + "perm", .tensorParam = *p_param_tensor + } + }; + Qnn_Tensor_t out_trans1_0_inputs[] = {*p_tensor2_transpose}; + Qnn_Tensor_t out_trans1_0_outputs[] = {*p_tensor2}; +#if 0 //leave here for easily understand code, can be removed in the future + Qnn_OpConfig_t out_trans1_0 = { + QNN_OPCONFIG_VERSION_1, + .v1 = {"ggmlqnn_mulmat_transpose_opconfig", + QNN_OP_PACKAGE_NAME_QTI_AISW, + QNN_OP_TRANSPOSE, 1, + out_trans1_0_params, + 1, + out_trans1_0_inputs, + 1, + out_trans1_0_outputs} + }; +#else + Qnn_OpConfig_t out_trans1_0 = ggmlqnn_create_op_config("ggmlqnn_mulmat_transpose_opconfig", QNN_OP_PACKAGE_NAME_QTI_AISW, QNN_OP_TRANSPOSE, + out_trans1_0_params, 1, out_trans1_0_inputs, 1, out_trans1_0_outputs, 1); +#endif + CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle,out_trans1_0)); + + //step-6: finalize qnn graph and execute qnn graph + CHECK_QNN_API(error, qnn_raw_interface.graphFinalize(graph_handle, nullptr, nullptr)); + Qnn_Tensor_t input_tensors_0[] = {*p_tensor0, *p_tensor1}; + Qnn_Tensor_t output_tensors_0[] = {*p_tensor2}; + CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, + input_tensors_0, 2, + output_tensors_0, 1, + nullptr, nullptr)); + + qnn_tensors_t ggml_op_mulmat_tensors; + ggml_op_mulmat_tensors.reserve(5); + ggml_op_mulmat_tensors.push_back(p_tensor0); + ggml_op_mulmat_tensors.push_back(p_tensor1); + ggml_op_mulmat_tensors.push_back(p_tensor2); + ggml_op_mulmat_tensors.push_back(p_param_tensor); + ggml_op_mulmat_tensors.push_back(p_tensor2_transpose); + auto graph_item = std::make_tuple(graph_handle, ggml_op_mulmat_tensors); + instance->_qnn_graph_map[graph_name] = graph_item; + } else { + if (src0_type != GGML_TYPE_F32) { + QNN_VER_PTR(*p_tensor0)->clientBuf = {wdata, static_cast(desired_size)}; + } else { + QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggmlqnn_get_tensor_data_size(src0)}; + } + QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggmlqnn_get_tensor_data_size(src1)}; + QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggmlqnn_get_tensor_data_size(dst)}; + + Qnn_Tensor_t tensor_inputs[] = { + *p_tensor0, + *p_tensor1 + }; + Qnn_Tensor_t tensor_outputs[] = { + *p_tensor2 + }; + // this is the second technical approach or another pipeline of "how to utilize the Hexagon + // NPU maximally" through QNN SDK, details could be found at + // https://github.com/ggml-org/llama.cpp/pull/12049#issuecomment-2678308360 + CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, + tensor_inputs, 2, + tensor_outputs, 1, + nullptr, nullptr)); + } + + // restore the original dimensions of qnn tensors to avoid memory leak in func free_qnn_tensor + QNN_VER_PTR(*p_tensor0)->dimensions = tensor_0_dimensions; + QNN_VER_PTR(*p_tensor1)->dimensions = tensor_1_dimensions; + QNN_VER_PTR(*p_tensor2)->dimensions = tensor_2_dimensions; + op_perf.info(); +} + +void ggml_qnn_repeat(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); +} + +void ggml_qnn_div(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); +} + +void ggml_qnn_leaky_relu(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); +} + +void ggml_qnn_concat(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); +} + +void ggml_qnn_arange(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); +} + +void ggml_qnn_sqr(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); +} + +void ggml_qnn_clamp(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); +} + +void ggml_qnn_scale(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); +} + +void ggml_qnn_argsort(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); +} + +void ggml_qnn_norm(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); +} + +void ggml_qnn_group_norm(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); +} + +void ggml_qnn_acc(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); +} + +void ggml_qnn_sum_rows(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); +} + +void ggml_qnn_upsample_nearest2d(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); +} + +void ggml_qnn_pad(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); +} + +void ggml_qnn_pool2d(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); +} + +void ggml_qnn_dup(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); +} + +void ggml_qnn_rms_norm(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); +} + +void ggml_qnn_diag_mask(ggml_backend_qnn_context * ctx, ggml_tensor * dst, float value) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); + GGML_UNUSED(value); +} + +void ggml_qnn_im2col(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); +} + +void ggml_qnn_timestep_embedding(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); +} + +void ggml_qnn_cpy(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + ggml_qnn_dup(ctx, dst); +} + +void ggml_qnn_softmax(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); +} + +void ggml_qnn_get_rows(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); +} + +void ggml_qnn_rope(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); +} diff --git a/ggml/src/ggml-qnn/ggml-qnn-ops.h b/ggml/src/ggml-qnn/ggml-qnn-ops.h new file mode 100644 index 0000000000000..b1c388a32a87a --- /dev/null +++ b/ggml/src/ggml-qnn/ggml-qnn-ops.h @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2023-2024 The ggml authors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ +#pragma once + +#include "ggml-qnn-impl.h" +void ggml_qnn_general_node(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +void ggml_qnn_mul_mat(ggml_backend_qnn_context * ctx, ggml_tensor * dst); + +void ggml_qnn_repeat(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +void ggml_qnn_div(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +void ggml_qnn_leaky_relu(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +void ggml_qnn_concat(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +void ggml_qnn_arange(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +void ggml_qnn_sqr(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +void ggml_qnn_clamp(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +void ggml_qnn_scale(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +void ggml_qnn_argsort(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +void ggml_qnn_norm(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +void ggml_qnn_group_norm(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +void ggml_qnn_acc(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +void ggml_qnn_sum_rows(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +void ggml_qnn_upsample_nearest2d(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +void ggml_qnn_pad(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +void ggml_qnn_pool2d(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +void ggml_qnn_dup(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +void ggml_qnn_rms_norm(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +void ggml_qnn_diag_mask(ggml_backend_qnn_context * ctx, ggml_tensor * dst, float value); +void ggml_qnn_im2col(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +void ggml_qnn_timestep_embedding(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +void ggml_qnn_cpy(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +void ggml_qnn_softmax(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +void ggml_qnn_get_rows(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +void ggml_qnn_rope(ggml_backend_qnn_context * ctx, ggml_tensor * dst); diff --git a/ggml/src/ggml-qnn/ggml-qnn.cpp b/ggml/src/ggml-qnn/ggml-qnn.cpp index e862b07a234eb..c830128f750c8 100644 --- a/ggml/src/ggml-qnn/ggml-qnn.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn.cpp @@ -13,13 +13,11 @@ * section-5 does ggml-qnn backend helper macro / data structure / function / class * section-6 does implementation of ggml-qnn backend according to ggml's backend subsystem * - * currently provide following ggml ops' QNN backend implementation: + * currently provide following ggml ops' QNN backend implementation in ggml-qnn-ops.cpp: * - GGML_OP_ADD: this is a simple skeleton, can expand other ggml ops according to expertise * - GGML_OP_MUL: this is a simple skeleton, can expand other ggml ops according to expertise * - GGML_OP_MUL_MAT:this is a complicated skeleton, can expand other complex ggml ops accordingly * - * of course, can porting ggml-qnn to Windows on ARM as need. - * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to * deal in the Software without restriction, including without limitation the @@ -38,96 +36,23 @@ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS * IN THE SOFTWARE. */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#if (defined __ANDROID__) || (defined ANDROID) -#include "android/log.h" -#endif - -#include "QnnTypes.h" -#include "QnnCommon.h" -#include "QnnContext.h" -#include "QnnBackend.h" -#include "QnnGraph.h" -#include "QnnProperty.h" -#include "QnnTensor.h" -#include "QnnInterface.h" -#include "Saver/QnnSaver.h" -#include "System/QnnSystemInterface.h" -#include "HTP/QnnHtpDevice.h" -#include "HTP/QnnHtpGraph.h" - -#include "ggml-qnn.h" -#include "ggml-impl.h" -#include "ggml-backend-impl.h" +#include "ggml-qnn-impl.h" +#include "ggml-qnn-ops.h" // ================================================================================================= // section-1: forward/external declaration // ================================================================================================= -class qnn_instance; -struct ggml_backend_qnn_context; -static int free_qnn_tensor(Qnn_Tensor_t * tensor); +static int free_qnn_tensor(Qnn_Tensor_t * tensor); static enum ggml_status ggml_backend_qnn_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph); -static void ggmlqnn_log_internal(ggml_log_level level, const char * file, const char * func, int line, const char * format, ...); +typedef void (* ggmlqnn_op_func_t)(ggml_backend_qnn_context * ctx, ggml_tensor * op); // ================================================================================================= // section-2: ggml-qnn internal troubleshooting function // ================================================================================================= -#define GGMLQNN_DEBUG 1 // for troubleshooting QNN backend -#define GGML_QNN_LOGBUF_LEN 4096 -#define ENABLE_QNNBACKEND_PERF 0 // enable/disable op's perf info -#define GGMLQNN_PRINT_QNN_INTERNAL_LOG 0 // enable/disable QNN's internal log -#define GGMLQNN_PRINT_OP_ADD_LOG 0 // GGML_OP_ADD already verified with QNN-CPU / QNN-GPU / QNN-NPU -#define GGMLQNN_PRINT_OP_MUL_MAT_LOG 1 - -#define GGMLQNN_LOG_ERROR(...) ggmlqnn_log_internal(GGML_LOG_LEVEL_DEBUG, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) -#define GGMLQNN_LOG_WARN(...) ggmlqnn_log_internal(GGML_LOG_LEVEL_DEBUG , __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) -#define GGMLQNN_LOG_INFO(...) ggmlqnn_log_internal(GGML_LOG_LEVEL_DEBUG , __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) - -#if GGMLQNN_DEBUG -#define GGMLQNN_LOG_DEBUG(...) ggmlqnn_log_internal(GGML_LOG_LEVEL_DEBUG, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) -#else -#define GGMLQNN_LOG_DEBUG(...) -#endif -static void ggmlqnn_log_internal(ggml_log_level level, const char * file, const char * func, int line, const char * format, ...) { +void ggmlqnn_log_internal(ggml_log_level level, const char * file, const char * func, int line, const char * format, ...) { static std::mutex ggmlqnn_log_internal_mutex; static char s_ggmlqnn_log_internal_buf[GGML_QNN_LOGBUF_LEN]; + GGML_UNUSED(file); { std::lock_guard lock(ggmlqnn_log_internal_mutex); va_list args; @@ -138,11 +63,11 @@ static void ggmlqnn_log_internal(ggml_log_level level, const char * file, const #if (defined __ANDROID__) || (defined ANDROID) //for Android application(standard APP or command line tool) __android_log_print(ANDROID_LOG_INFO, "ggml-qnn", "%s\n", s_ggmlqnn_log_internal_buf); -#endif -#if (defined __ANDROID__) || (defined ANDROID) - //do nothing when running on Snapdragon based Android device + if (GGML_LOG_LEVEL_INFO == level) { + printf("%s\n", s_ggmlqnn_log_internal_buf); + } #else - //for Snapdragon based WoA(Windows on ARM) device + //for Snapdragon based WoA(Windows on ARM) device or Linux printf("%s\n", s_ggmlqnn_log_internal_buf); #endif } @@ -153,16 +78,48 @@ static void ggmlqnn_log_internal(ggml_log_level level, const char * file, const // ================================================================================================= // section-3: general helper macro / data structure / function // ================================================================================================= -#define DISABLE_COPY(class_name) \ - class_name(const class_name &) = delete; \ - void operator=(const class_name &) = delete +#if defined(_WIN32) +static const char * last_func = nullptr; +static long last_err; +void * dlopen(const char * dll, int flags) { + HINSTANCE h = LoadLibraryA(dll); + GGML_UNUSED(flags); + if (h == NULL) { + last_err = GetLastError(); + last_func = "dlopen"; + } + return h; +} -#define DISABLE_MOVE(class_name) \ - class_name(class_name &&) = delete; \ - void operator=(class_name &&) = delete +int dlclose(void * h) { + if (!FreeLibrary((HINSTANCE)h)) { + last_err = GetLastError(); + last_func = "dlclose"; + return -1; + } + return 0; +} -#define GGMLQNN_MEM_ADD(alignment) (sizeof (size_t) + alignment) -#define GGMLQNN_MEM_MASK(alignment) ((uintptr_t)alignment - 1) +void * dlsym(void * h, const char * name) { + FARPROC p = GetProcAddress((HINSTANCE)h, name); + if (!p) { + last_err = GetLastError(); + last_func = "dlsym"; + } + return (void*)(intptr_t)p; +} + +const char * dlerror(void) { + static char str[512]; + if (!last_err) return nullptr; + + snprintf(str, 512, "%s error #%ld", last_func, last_err); + last_err = 0; + last_func = NULL; + + return str; +} +#endif static intptr_t ggmlqnn_align_to(size_t alignment, intptr_t offset) { return offset % alignment == 0 ? offset @@ -171,62 +128,40 @@ static intptr_t ggmlqnn_align_to(size_t alignment, intptr_t offset) { offset % static_cast(alignment)); } -static void * ggmlqnn_mallocz_aligned(size_t size, size_t alignment) { - uint8_t * buffer = NULL; - size_t * sp = NULL; - buffer = static_cast(calloc(1, size + GGMLQNN_MEM_ADD(alignment))); - if (!buffer) - return NULL; - sp = (size_t *)buffer; - *sp = size; - buffer = (uint8_t *)(((uintptr_t) buffer + GGMLQNN_MEM_ADD(alignment)) & ~GGMLQNN_MEM_MASK(alignment)); - buffer[-1] = buffer - (uint8_t *)sp; - return buffer; -} - -static void * ggmlqnn_malloc_aligned(size_t size, size_t alignment) { - uint8_t * buffer = NULL; - size_t * sp = NULL; - buffer = static_cast(malloc(size + GGMLQNN_MEM_ADD(alignment))); - if (!buffer) - return NULL; - sp = (size_t *)buffer; - *sp = size; - buffer = (uint8_t *)(((uintptr_t) buffer + GGMLQNN_MEM_ADD(alignment)) & ~GGMLQNN_MEM_MASK(alignment)); - buffer[-1] = buffer - (uint8_t *)sp; - return buffer; -} - -static void ggmqnn_free_aligned(void * ptr) { - uint8_t * old = (uint8_t *)ptr; - if (!old) - return; - old -= old[-1]; - free(old); -} - static size_t get_system_total_memory_in_bytes() { +#if defined(__ANDROID__) || defined(__linux__) struct sysinfo info = {}; - if (sysinfo(&info) == 0) { + if (0 == sysinfo(&info)) { return (info.totalram + info.totalswap) * info.mem_unit; } - auto pages = (size_t)sysconf(_SC_PHYS_PAGES); auto page_size = (size_t)sysconf(_SC_PAGE_SIZE); return pages * page_size; +#elif defined(_WIN32) + //TODO: Snapdragon based WoA(Windows on ARM) + return 0; +#else +#error "ggml-qnn only support WoA, Android, Linux" +#endif } static size_t get_system_free_memory_in_bytes() { +#if defined(__ANDROID__) || defined(__linux__) struct sysinfo info = {}; - if (sysinfo(&info) == 0) { + if (0 == sysinfo(&info)) { return (info.freeram + info.freeswap) * info.mem_unit; } - auto avail_pages = (size_t)sysconf(_SC_AVPHYS_PAGES); auto page_size = (size_t)sysconf(_SC_PAGE_SIZE); return avail_pages * page_size; +#elif defined(_WIN32) + //TODO: Snapdragon based WoA(Windows on ARM) + return 0; +#else +#error "ggml-qnn only support WoA, Android, Linux" +#endif } static size_t ggmlqnn_memscpy(void * dst, size_t dst_size, const void * src, size_t copy_size) { @@ -241,16 +176,23 @@ static size_t ggmlqnn_memscpy(void * dst, size_t dst_size, const void * src, siz } static char * ggmlqnn_strndup(const char * source, size_t maxlen) { - return ::strndup(source, maxlen); + return strndup(source, maxlen); } static void * ggmlqnn_host_malloc(size_t n) { - void * data = NULL; - int result = posix_memalign((void **) &data, sysconf(_SC_PAGESIZE), n); +#if defined(__ANDROID__) || defined(__linux__) + void * data = nullptr; + int result = posix_memalign((void **)&data, sysconf(_SC_PAGESIZE), n); if (result != 0) { GGMLQNN_LOG_WARN("%s: error: posix_memalign failed\n", __func__); - return NULL; + return nullptr; } +#elif defined(_WIN32) + //TODO: Snapdragon based WoA(Windows on ARM) + return nullptr; +#else +#error "ggml-qnn only support WoA, Android, Linux" +#endif return data; } @@ -258,57 +200,6 @@ static void * ggmlqnn_host_malloc(size_t n) { // ================================================================================================= // section-4: QNN helper macro / data structure / function // ================================================================================================= -#define VALIDATE(value, status) \ - do { \ - status = value; \ - if (status != QNN_SUCCESS) { \ - GGMLQNN_LOG_WARN("%s expected QNN_SUCCESS\n", #value); \ - return status; \ - } \ - } while (0) - -#define CHECK_QNN_API(error, result) \ - do { \ - error = (result); \ - if (QNN_SUCCESS != error) { \ - if (error == QNN_COMMON_ERROR_NOT_SUPPORTED) { \ - GGMLQNN_LOG_WARN("WARNING: QNN feature/API not supported\n"); \ - } else { \ - GGMLQNN_LOG_INFO("QNN API error = %d(%s)\n", error, qnn_get_error_string(error)); \ - } \ - } \ - } while (0) - -#define VALIDATE_TENSOR_VERSION(tensor, err) VALIDATE(validate_tensor_version(tensor), err) - -#define VALIDATE_OP_CONFIG_VERSION(op, err) VALIDATE(validate_op_config_version(op), err) - -#define QNN_VER_PTR(x) (&((x).v1)) -#define QNN_OP_CFG_VALID(op_config) ((op_config).version == QNN_OPCONFIG_VERSION_1) - -#define QNN_OP_CFG_GET_NAME(op_config) get_qnn_oponfig_name(op_config) -#define QNN_OP_CFG_GET_PACKAGE_NAME(op_config) get_qnn_op_config_packagename(op_config) -#define QNN_OP_CFG_GET_TYPE_NAME(op_config) get_qnn_op_config_typename(op_config) -#define QNN_OP_CFG_GET_NUM_PARAMS(op_config) get_qnn_op_config_numparams(op_config) -#define QNN_OP_CFG_GET_PARAMS(op_config) get_qnn_op_config_params(op_config) -#define QNN_OP_CFG_GET_NUM_INPUTS(op_config) get_qnn_op_config_numinputs(op_config) -#define QNN_OP_CFG_GET_INPUTS(op_config) get_qnn_op_config_inputs(op_config) -#define QNN_OP_CFG_GET_NUM_OUTPUTS(op_config) get_qnn_op_config_numoutputs(op_config) -#define QNN_OP_CFG_GET_OUTPUTS(op_config) get_qnn_op_config_outputs(op_config) - -#define QNN_OP_CFG_SET_NAME(op_config, value) set_qnn_op_config_name(op_config, value) -#define QNN_OP_CFG_SET_PACKAGE_NAME(op_config, value) set_qnn_op_config_packagename(op_config, value) -#define QNN_OP_CFG_SET_TYPE_NAME(op_config, value) set_qnn_op_config_typename(op_config, value) - -#define QNN_OP_CFG_SET_PARAMS(op_config, num_of_params, params) \ - set_qnn_op_config_params(op_config, num_of_params, params) - -#define QNN_OP_CFG_SET_INPUTS(op_config, num_of_inputs, inputTensors) \ - set_qnn_op_config_inputs(op_config, num_of_inputs, inputTensors) - -#define QNN_OP_CFG_SET_OUTPUTS(op_config, num_of_outputs, output_tensors) \ - set_qnn_op_config_outputs(op_config, num_of_outputs, output_tensors) - #define QNN_TENSOR_GET_ID(tensor) get_qnn_tensorid(tensor) #define QNN_TENSOR_GET_NAME(tensor) get_qnn_tensorname(tensor) #define QNN_TENSOR_GET_TYPE(tensor) get_qnn_tensortype(tensor) @@ -333,200 +224,6 @@ static void * ggmlqnn_host_malloc(size_t n) { #define QNN_TENSOR_SET_CLIENT_BUF(tensor, value) set_qnn_tensor_clientbuf(tensor, value) #define QNN_TENSOR_SET_MEM_HANDLE(tensor, value) set_qnn_tensor_memhandle(tensor, value) -static inline int validate_tensor_version(Qnn_Tensor_t tensor) { - if (tensor.version != QNN_TENSOR_VERSION_1) { - GGMLQNN_LOG_WARN("validate_tensor_version() tensor %s, got unsupported version %d\n", - tensor.v1.name, - tensor.version); - return 1; - } - return 0; -} - -[[maybe_unused]] static inline int validate_op_config_version(Qnn_OpConfig_t op_config) { - if (op_config.version != QNN_OPCONFIG_VERSION_1) { - GGMLQNN_LOG_WARN("validate_op_config_version() op %s, got unsupported version %d\n", - op_config.v1.name, - op_config.version); - return 1; - } - return 0; -} - -static inline const char * get_qnn_oponfig_name(const Qnn_OpConfig_t & op_config) { - if (op_config.version == QNN_OPCONFIG_VERSION_1) { - return op_config.v1.name; - } - return nullptr; -} - -[[maybe_unused]] static inline const char * get_qnn_oponfig_name(const Qnn_OpConfig_t * op_config) { - return get_qnn_oponfig_name(*op_config); -} - -static inline const char * get_qnn_op_config_packagename(const Qnn_OpConfig_t & op_config) { - if (op_config.version == QNN_OPCONFIG_VERSION_1) { - return op_config.v1.packageName; - } - return nullptr; -} - -[[maybe_unused]] static inline const char * get_qnn_op_config_packagename(const Qnn_OpConfig_t * op_config) { - return get_qnn_op_config_packagename(*op_config); -} - -static inline const char * get_qnn_op_config_typename(const Qnn_OpConfig_t & op_config) { - if (op_config.version == QNN_OPCONFIG_VERSION_1) { - return op_config.v1.typeName; - } - return nullptr; -} - -[[maybe_unused]] static inline const char * get_qnn_op_config_typename(const Qnn_OpConfig_t * op_config) { - return get_qnn_op_config_typename(*op_config); -} - -static inline uint32_t get_qnn_op_config_numparams(const Qnn_OpConfig_t & op_config) { - if (op_config.version == QNN_OPCONFIG_VERSION_1) { - return op_config.v1.numOfParams; - } - return 0u; -} - -[[maybe_unused]] static inline uint32_t get_qnn_op_config_numparams(const Qnn_OpConfig_t * op_config) { - return get_qnn_op_config_numparams(*op_config); -} - -static inline const Qnn_Param_t * get_qnn_op_config_params(const Qnn_OpConfig_t & op_config) { - if (op_config.version == QNN_OPCONFIG_VERSION_1) { - return op_config.v1.params; - } - return nullptr; -} - -[[maybe_unused]] static inline const Qnn_Param_t * get_qnn_op_config_params(const Qnn_OpConfig_t * op_config) { - return get_qnn_op_config_params(*op_config); -} - -static inline uint32_t get_qnn_op_config_numinputs(const Qnn_OpConfig_t & op_config) { - if (op_config.version == QNN_OPCONFIG_VERSION_1) { - return op_config.v1.numOfInputs; - } - return 0u; -} - -[[maybe_unused]] static inline uint32_t get_qnn_op_config_numinputs(const Qnn_OpConfig_t * op_config) { - return get_qnn_op_config_numinputs(*op_config); -} - -static inline const Qnn_Tensor_t * get_qnn_op_config_inputs(const Qnn_OpConfig_t & op_config) { - if (op_config.version == QNN_OPCONFIG_VERSION_1) { - return op_config.v1.inputTensors; - } - return nullptr; -} - -[[maybe_unused]] static inline const Qnn_Tensor_t * get_qnn_op_config_inputs(const Qnn_OpConfig_t * op_config) { - return get_qnn_op_config_inputs(*op_config); -} - -static inline uint32_t get_qnn_op_config_numoutputs(const Qnn_OpConfig_t & op_config) { - if (op_config.version == QNN_OPCONFIG_VERSION_1) { - return op_config.v1.numOfOutputs; - } - return 0u; -} - -[[maybe_unused]] static inline uint32_t get_qnn_op_config_numoutputs(const Qnn_OpConfig_t * op_config) { - return get_qnn_op_config_numoutputs(*op_config); -} - -static inline const Qnn_Tensor_t * get_qnn_op_config_outputs(const Qnn_OpConfig_t & op_config) { - if (op_config.version == QNN_OPCONFIG_VERSION_1) { - return op_config.v1.outputTensors; - } - return nullptr; -} - -[[maybe_unused]] static inline const Qnn_Tensor_t * get_qnn_op_config_outputs(const Qnn_OpConfig_t * op_config) { - return get_qnn_op_config_outputs(*op_config); -} - -static inline void set_qnn_op_config_name(Qnn_OpConfig_t & op_config, const char * name) { - if (op_config.version == QNN_OPCONFIG_VERSION_1) { - op_config.v1.name = name; - } -} - -[[maybe_unused]] static inline void set_qnn_op_config_name(Qnn_OpConfig_t * op_config, const char * name) { - set_qnn_op_config_name(*op_config, name); -} - -static inline void set_qnn_op_config_packagename(Qnn_OpConfig_t & op_config, const char * package_name) { - if (op_config.version == QNN_OPCONFIG_VERSION_1) { - op_config.v1.packageName = package_name; - } -} - -[[maybe_unused]] static inline void set_qnn_op_config_packagename(Qnn_OpConfig_t * op_config, const char * package_name) { - set_qnn_op_config_packagename(*op_config, package_name); -} - -static inline void set_qnn_op_config_typename(Qnn_OpConfig_t & op_config, const char * type_name) { - if (op_config.version == QNN_OPCONFIG_VERSION_1) { - op_config.v1.typeName = type_name; - } -} - -[[maybe_unused]] static inline void set_qnn_op_config_typename(Qnn_OpConfig_t * op_config, const char * type_name) { - set_qnn_op_config_typename(*op_config, type_name); -} - -static inline void set_qnn_op_config_params(Qnn_OpConfig_t & op_config, - uint32_t num_of_params, - Qnn_Param_t * params) { - if (op_config.version == QNN_OPCONFIG_VERSION_1) { - op_config.v1.numOfParams = num_of_params; - op_config.v1.params = params; - } -} - -[[maybe_unused]] static inline void set_qnn_op_config_params(Qnn_OpConfig_t * op_config, - uint32_t num_of_params, - Qnn_Param_t * params) { - set_qnn_op_config_params(*op_config, num_of_params, params); -} - -static inline void set_qnn_op_config_inputs(Qnn_OpConfig_t & op_config, - uint32_t num_of_inputs, - Qnn_Tensor_t * input_tensors) { - if (op_config.version == QNN_OPCONFIG_VERSION_1) { - op_config.v1.numOfInputs = num_of_inputs; - op_config.v1.inputTensors = input_tensors; - } -} - -[[maybe_unused]] static inline void set_qnn_op_config_inputs(Qnn_OpConfig_t * op_config, - uint32_t num_of_inputs, - Qnn_Tensor_t * input_tensors) { - set_qnn_op_config_inputs(*op_config, num_of_inputs, input_tensors); -} - -static inline void set_qnn_op_config_outputs(Qnn_OpConfig_t & op_config, - uint32_t num_of_outputs, - Qnn_Tensor_t * output_tensors) { - if (op_config.version == QNN_OPCONFIG_VERSION_1) { - op_config.v1.numOfOutputs = num_of_outputs; - op_config.v1.outputTensors = output_tensors; - } -} - -[[maybe_unused]] static inline void set_qnn_op_config_outputs(Qnn_OpConfig_t * op_config, - uint32_t num_of_outputs, - Qnn_Tensor_t * output_tensors) { - set_qnn_op_config_outputs(*op_config, num_of_outputs, output_tensors); -} - static inline uint32_t get_qnn_tensorid(const Qnn_Tensor_t & tensor) { if (tensor.version == QNN_TENSOR_VERSION_1) { return tensor.v1.id; @@ -535,10 +232,6 @@ static inline uint32_t get_qnn_tensorid(const Qnn_Tensor_t & tensor) { return 0u; } -[[maybe_unused]] static inline uint32_t get_qnn_tensorid(const Qnn_Tensor_t * tensor) { - return get_qnn_tensorid(*tensor); -} - static inline const char * get_qnn_tensorname(const Qnn_Tensor_t & tensor) { if (tensor.version == QNN_TENSOR_VERSION_1) { return tensor.v1.name; @@ -546,10 +239,6 @@ static inline const char * get_qnn_tensorname(const Qnn_Tensor_t & tensor) { return nullptr; } -static inline const char * get_qnn_tensorname(const Qnn_Tensor_t * tensor) { - return get_qnn_tensorname(*tensor); -} - static inline Qnn_TensorType_t get_qnn_tensortype(const Qnn_Tensor_t & tensor) { if (tensor.version == QNN_TENSOR_VERSION_1) { return tensor.v1.type; @@ -557,10 +246,6 @@ static inline Qnn_TensorType_t get_qnn_tensortype(const Qnn_Tensor_t & tensor) { return QNN_TENSOR_TYPE_UNDEFINED; } -[[maybe_unused]] static inline Qnn_TensorType_t get_qnn_tensortype(const Qnn_Tensor_t * tensor) { - return get_qnn_tensortype(*tensor); -} - static inline Qnn_TensorDataFormat_t get_qnn_tensor_dataformat(const Qnn_Tensor_t & tensor) { if (tensor.version == QNN_TENSOR_VERSION_1) { return tensor.v1.dataFormat; @@ -568,10 +253,6 @@ static inline Qnn_TensorDataFormat_t get_qnn_tensor_dataformat(const Qnn_Tensor_ return QNN_TENSOR_DATA_FORMAT_FLAT_BUFFER; } -[[maybe_unused]] static inline Qnn_TensorDataFormat_t get_qnn_tensor_dataformat(const Qnn_Tensor_t * tensor) { - return get_qnn_tensor_dataformat(*tensor); -} - static inline Qnn_DataType_t get_qnn_tensor_datatype(const Qnn_Tensor_t & tensor) { if (tensor.version == QNN_TENSOR_VERSION_1) { return tensor.v1.dataType; @@ -579,10 +260,6 @@ static inline Qnn_DataType_t get_qnn_tensor_datatype(const Qnn_Tensor_t & tensor return QNN_DATATYPE_UNDEFINED; } -[[maybe_unused]] static inline Qnn_DataType_t get_qnn_tensor_datatype(const Qnn_Tensor_t * tensor) { - return get_qnn_tensor_datatype(*tensor); -} - static inline Qnn_QuantizeParams_t get_qnn_tensor_quantparams(const Qnn_Tensor_t & tensor) { if (tensor.version == QNN_TENSOR_VERSION_1) { return tensor.v1.quantizeParams; @@ -590,10 +267,6 @@ static inline Qnn_QuantizeParams_t get_qnn_tensor_quantparams(const Qnn_Tensor_t return QNN_QUANTIZE_PARAMS_INIT; } -[[maybe_unused]] static inline Qnn_QuantizeParams_t get_qnn_tensor_quantparams(const Qnn_Tensor_t * tensor) { - return get_qnn_tensor_quantparams(*tensor); -} - static inline uint32_t get_qnn_tensor_rank(const Qnn_Tensor_t & tensor) { if (tensor.version == QNN_TENSOR_VERSION_1) { return tensor.v1.rank; @@ -601,10 +274,6 @@ static inline uint32_t get_qnn_tensor_rank(const Qnn_Tensor_t & tensor) { return 0u; } -[[maybe_unused]] static inline uint32_t get_qnn_tensor_rank(const Qnn_Tensor_t * tensor) { - return get_qnn_tensor_rank(*tensor); -} - static inline uint32_t * get_qnn_tensor_dimensions(const Qnn_Tensor_t & tensor) { if (tensor.version == QNN_TENSOR_VERSION_1) { return tensor.v1.dimensions; @@ -612,10 +281,6 @@ static inline uint32_t * get_qnn_tensor_dimensions(const Qnn_Tensor_t & tensor) return nullptr; } -[[maybe_unused]] static inline uint32_t * get_qnn_tensor_dimensions(const Qnn_Tensor_t * tensor) { - return get_qnn_tensor_dimensions(*tensor); -} - static inline Qnn_TensorMemType_t get_qnn_tensor_memtype(const Qnn_Tensor_t & tensor) { if (tensor.version == QNN_TENSOR_VERSION_1) { return tensor.v1.memType; @@ -623,161 +288,78 @@ static inline Qnn_TensorMemType_t get_qnn_tensor_memtype(const Qnn_Tensor_t & te return QNN_TENSORMEMTYPE_UNDEFINED; } -[[maybe_unused]] static inline Qnn_TensorMemType_t get_qnn_tensor_memtype(const Qnn_Tensor_t * tensor) { - return get_qnn_tensor_memtype(*tensor); -} - -static inline Qnn_ClientBuffer_t get_qnn_tensor_clientbuf(const Qnn_Tensor_t & tensor) { - if (tensor.version == QNN_TENSOR_VERSION_1) { - return tensor.v1.clientBuf; - } - return QNN_CLIENT_BUFFER_INIT; -} - -[[maybe_unused]] static inline Qnn_ClientBuffer_t get_qnn_tensor_clientbuf(const Qnn_Tensor_t * tensor) { - return get_qnn_tensor_clientbuf(*tensor); -} - -static inline Qnn_MemHandle_t get_qnn_tensor_memhandle(const Qnn_Tensor_t & tensor) { - if (tensor.version == QNN_TENSOR_VERSION_1) { - return tensor.v1.memHandle; - } - return nullptr; -} - -[[maybe_unused]] static inline Qnn_MemHandle_t get_qnn_tensor_memhandle(const Qnn_Tensor_t * tensor) { - return get_qnn_tensor_memhandle(*tensor); -} - static inline void set_qnn_tensor_id(Qnn_Tensor_t & tensor, uint32_t id) { if (tensor.version == QNN_TENSOR_VERSION_1) { tensor.v1.id = id; } } -[[maybe_unused]] static inline void set_qnn_tensor_id(Qnn_Tensor_t * tensor, uint32_t id) { - set_qnn_tensor_id(*tensor, id); -} - static inline void set_qnn_tensor_name(Qnn_Tensor_t & tensor, const char * name) { if (tensor.version == QNN_TENSOR_VERSION_1) { tensor.v1.name = name; } } -[[maybe_unused]] static inline void set_qnn_tensor_name(Qnn_Tensor_t * tensor, const char * name) { - set_qnn_tensor_name(*tensor, name); -} - static inline void set_qnn_tensor_type(Qnn_Tensor_t & tensor, Qnn_TensorType_t type) { if (tensor.version == QNN_TENSOR_VERSION_1) { tensor.v1.type = type; } } -[[maybe_unused]] static inline void set_qnn_tensor_type(Qnn_Tensor_t * tensor, Qnn_TensorType_t type) { - set_qnn_tensor_type(*tensor, type); -} - static inline void set_qnn_tensor_dataformat(Qnn_Tensor_t & tensor, Qnn_TensorDataFormat_t format) { if (tensor.version == QNN_TENSOR_VERSION_1) { tensor.v1.dataFormat = format; } } -[[maybe_unused]] static inline void set_qnn_tensor_dataformat(Qnn_Tensor_t * tensor, Qnn_TensorDataFormat_t format) { - set_qnn_tensor_dataformat(*tensor, format); -} - static inline void set_qnn_tensor_datatype(Qnn_Tensor_t & tensor, Qnn_DataType_t dataType) { if (tensor.version == QNN_TENSOR_VERSION_1) { tensor.v1.dataType = dataType; } } -[[maybe_unused]] static inline void set_qnn_tensor_datatype(Qnn_Tensor_t * tensor, Qnn_DataType_t dataType) { - set_qnn_tensor_datatype(*tensor, dataType); -} - static inline void set_qnn_tensor_quantparams(Qnn_Tensor_t & tensor, Qnn_QuantizeParams_t params) { if (tensor.version == QNN_TENSOR_VERSION_1) { tensor.v1.quantizeParams = params; } } -[[maybe_unused]] static inline void set_qnn_tensor_quantparams(Qnn_Tensor_t * tensor, Qnn_QuantizeParams_t params) { - set_qnn_tensor_quantparams(*tensor, params); -} - static inline void set_qnn_tensor_rank(Qnn_Tensor_t & tensor, uint32_t rank) { if (tensor.version == QNN_TENSOR_VERSION_1) { tensor.v1.rank = rank; } } -[[maybe_unused]] static inline void set_qnn_tensor_rank(Qnn_Tensor_t * tensor, uint32_t rank) { - set_qnn_tensor_rank(*tensor, rank); -} - static inline void set_qnn_tensor_dimensions(Qnn_Tensor_t & tensor, uint32_t * dims) { if (tensor.version == QNN_TENSOR_VERSION_1) { tensor.v1.dimensions = dims; } } -[[maybe_unused]] static inline void set_qnn_tensor_dimensions(Qnn_Tensor_t * tensor, uint32_t * dims) { - set_qnn_tensor_dimensions(*tensor, dims); -} - static inline void set_qnn_tensor_memtype(Qnn_Tensor_t & tensor, Qnn_TensorMemType_t memType) { if (tensor.version == QNN_TENSOR_VERSION_1) { tensor.v1.memType = memType; } } -[[maybe_unused]] static inline void set_qnn_tensor_memtype(Qnn_Tensor_t * tensor, Qnn_TensorMemType_t memType) { - set_qnn_tensor_memtype(*tensor, memType); -} - static inline void set_qnn_tensor_clientbuf(Qnn_Tensor_t & tensor, Qnn_ClientBuffer_t clientBuf) { if (tensor.version == QNN_TENSOR_VERSION_1) { tensor.v1.clientBuf = clientBuf; } } -[[maybe_unused]] static inline void set_qnn_tensor_clientbuf(Qnn_Tensor_t * tensor, Qnn_ClientBuffer_t clientBuf) { - set_qnn_tensor_clientbuf(*tensor, clientBuf); -} - static inline void set_qnn_tensor_memhandle(Qnn_Tensor_t & tensor, Qnn_MemHandle_t handle) { if (tensor.version == QNN_TENSOR_VERSION_1) { tensor.v1.memHandle = handle; } } -[[maybe_unused]] static inline void set_qnn_tensor_memhandle(Qnn_Tensor_t * tensor, Qnn_MemHandle_t handle) { - set_qnn_tensor_memhandle(*tensor, handle); -} - -inline static Qnn_Tensor_t qnn_tensor_init(Qnn_TensorVersion_t version) { - Qnn_Tensor_t tensor; - tensor.version = version; - if (version == QNN_TENSOR_VERSION_1) { - tensor.v1 = QNN_TENSOR_V1_INIT; - } else if (version == QNN_TENSOR_VERSION_2) { - tensor.v2 = QNN_TENSOR_V2_INIT; - } - return tensor; -} - static int deep_copy_qnn_tensors(Qnn_Tensor_t & src, Qnn_Tensor_t & dst) { int err = 0; - VALIDATE_TENSOR_VERSION(src, err); dst.version = src.version; - QNN_TENSOR_SET_NAME( - dst, ggmlqnn_strndup(QNN_TENSOR_GET_NAME(src), std::string(QNN_TENSOR_GET_NAME(src)).size())); - if (QNN_TENSOR_GET_NAME(dst) == nullptr) { + QNN_TENSOR_SET_NAME(dst, ggmlqnn_strndup(QNN_TENSOR_GET_NAME(src), std::string(QNN_TENSOR_GET_NAME(src)).size())); + if (nullptr == QNN_TENSOR_GET_NAME(dst)) { return 1; } QNN_TENSOR_SET_ID(dst, QNN_TENSOR_GET_ID(src)); @@ -796,20 +378,20 @@ static int deep_copy_qnn_tensors(Qnn_Tensor_t & src, Qnn_Tensor_t & dst) { } Qnn_QuantizeParams_t src_qparam = QNN_TENSOR_GET_QUANT_PARAMS(src); - Qnn_QuantizationEncoding_t encoding = src_qparam.quantizationEncoding; + Qnn_QuantizationEncoding_t encoding = src_qparam.quantizationEncoding; if (encoding == QNN_QUANTIZATION_ENCODING_AXIS_SCALE_OFFSET) { - Qnn_QuantizeParams_t src_qparam_cpy = src_qparam; + Qnn_QuantizeParams_t src_qparam_cpy = src_qparam; Qnn_AxisScaleOffset_t & axis_scale_offset = src_qparam_cpy.axisScaleOffsetEncoding; - Qnn_ScaleOffset_t ** scale_offset = &axis_scale_offset.scaleOffset; + Qnn_ScaleOffset_t ** scale_offset = &axis_scale_offset.scaleOffset; size_t scale_offset_size = axis_scale_offset.numScaleOffsets * sizeof(Qnn_ScaleOffset_t); - *scale_offset = (Qnn_ScaleOffset_t *)malloc(scale_offset_size); + *scale_offset = (Qnn_ScaleOffset_t *)malloc(scale_offset_size); ggmlqnn_memscpy(*scale_offset, scale_offset_size, src_qparam.axisScaleOffsetEncoding.scaleOffset, scale_offset_size); QNN_TENSOR_SET_QUANT_PARAMS(dst, src_qparam_cpy); } else if (encoding == QNN_QUANTIZATION_ENCODING_BW_AXIS_SCALE_OFFSET) { - Qnn_QuantizeParams_t src_qparam_cpy = src_qparam; + Qnn_QuantizeParams_t src_qparam_cpy = src_qparam; Qnn_BwAxisScaleOffset_t & bwaxis_scale_offset = src_qparam_cpy.bwAxisScaleOffsetEncoding; size_t scale_size = bwaxis_scale_offset.numElements * sizeof(float); float ** scales = &bwaxis_scale_offset.scales; @@ -831,7 +413,7 @@ static int deep_copy_qnn_tensors(Qnn_Tensor_t & src, Qnn_Tensor_t & dst) { QNN_TENSOR_SET_RANK(dst, rank); size_t dim_size = GGML_MAX_DIMS * sizeof(uint32_t); uint32_t * dimensions = (uint32_t *)malloc(dim_size); - if (dimensions == nullptr) { + if (nullptr == dimensions) { GGMLQNN_LOG_WARN("deep_copy_qnn_tensors() allocation error while copying tensor %s\n", QNN_TENSOR_GET_NAME(src)); return 1; } @@ -843,10 +425,8 @@ static int deep_copy_qnn_tensors(Qnn_Tensor_t & src, Qnn_Tensor_t & dst) { static int free_qnn_tensor(Qnn_Tensor_t * tensor) { int err = 0; - VALIDATE_TENSOR_VERSION(*tensor, err); free((void *) QNN_TENSOR_GET_NAME(*tensor)); - - Qnn_QuantizeParams_t src_qparam = QNN_TENSOR_GET_QUANT_PARAMS(*tensor); + Qnn_QuantizeParams_t src_qparam = QNN_TENSOR_GET_QUANT_PARAMS(*tensor); Qnn_QuantizationEncoding_t encoding = src_qparam.quantizationEncoding; if (encoding == QNN_QUANTIZATION_ENCODING_AXIS_SCALE_OFFSET) { free(src_qparam.axisScaleOffsetEncoding.scaleOffset); @@ -862,55 +442,7 @@ static int free_qnn_tensor(Qnn_Tensor_t * tensor) { return err; } - -static size_t qnn_datatype_size(Qnn_DataType_t qnn_type) { - switch (qnn_type) { - case QNN_DATATYPE_FLOAT_32: - return sizeof(float); - case QNN_DATATYPE_FLOAT_16: - return sizeof(uint16_t); - case QNN_DATATYPE_UINT_32: - case QNN_DATATYPE_INT_32: - return sizeof(int32_t); - case QNN_DATATYPE_INT_16: - return sizeof(int16_t); - case QNN_DATATYPE_INT_8: - return sizeof(int8_t); - case QNN_DATATYPE_SFIXED_POINT_8: - return sizeof(int8_t); - case QNN_DATATYPE_SFIXED_POINT_4: - return sizeof(int8_t); - default: - break; - } - return 0; -} - -static const char * qnn_datatype_to_string(Qnn_DataType_t qnn_type) { - switch (qnn_type) { - case QNN_DATATYPE_FLOAT_32: - return "QNN_DATATYPE_FLOAT_32"; - case QNN_DATATYPE_FLOAT_16: - return "QNN_DATATYPE_FLOAT_16"; - case QNN_DATATYPE_UINT_32: - return "QNN_DATATYPE_UINT_32"; - case QNN_DATATYPE_INT_32: - return "QNN_DATATYPE_INT_32"; - case QNN_DATATYPE_INT_16: - return "QNN_DATATYPE_INT_16"; - case QNN_DATATYPE_INT_8: - return "QNN_DATATYPE_INT_8"; - case QNN_DATATYPE_SFIXED_POINT_8: - return "QNN_DATATYPE_SFIXED_POINT_8"; - case QNN_DATATYPE_SFIXED_POINT_4: - return "QNN_DATATYPE_SFIXED_POINT_4"; - default: - break; - } - return "QNN_DATATYPE_UNDEFINED"; -} - -static const char * qnn_get_error_string(Qnn_ErrorHandle_t qnn_error_code) { +const char * ggmlqnn_get_error_string(Qnn_ErrorHandle_t qnn_error_code) { // file:///opt/qcom/aistack/qairt/2.31.0.250130/docs/QNN/general/api_error_codes.html switch (qnn_error_code) { case QNN_SUCCESS: @@ -1013,59 +545,24 @@ static const char * qnn_get_error_string(Qnn_ErrorHandle_t qnn_error_code) { } } +// helper function to create an operation config +Qnn_OpConfig_t ggmlqnn_create_op_config(const char * name, const char * package, const char * type, + Qnn_Param_t * params, uint32_t num_params, + Qnn_Tensor_t * inputs, uint32_t num_inputs, + Qnn_Tensor_t * outputs, uint32_t num_outputs) { + Qnn_OpConfigV1_t v1 = {name, package, type, + num_params, params, + num_inputs, inputs, + num_outputs, outputs + }; + Qnn_OpConfig_t opcfg = {QNN_OPCONFIG_VERSION_1, {v1}}; + + return opcfg; +} + // ================================================================================================= // section-5:ggml-qnn backend helper macro / data structure / function / class // ================================================================================================= -#define RPCMEM_DEFAULT_FLAGS 1 -#define RPCMEM_HEAP_ID_SYSTEM 25 - -typedef void (* ggmlqnn_op_func_t)(ggml_backend_t backend, ggml_tensor * op); - -using pfn_rpc_mem_init = void (*)(void); -using pfn_rpc_mem_deinit = void (*)(void); -using pfn_rpc_mem_alloc = void *(*)(int, uint32_t, int); -using pfn_rpc_mem_free = void (*)(void *); -using pfn_rpc_mem_to_fd = int (*)(void *); -using _pfn_QnnSaver_initialize = decltype(QnnSaver_initialize); -using _pfn_QnnInterface_getProviders = decltype(QnnInterface_getProviders); -using _pfn_QnnSystemInterface_getProviders = decltype(QnnSystemInterface_getProviders); - -using qnn_res_t = std::tuple>; -using qnn_tensors_t = std::vector< Qnn_Tensor_t *>; - -enum class ggml_qnn_profile_level { - profile_off = 0, - profile_basic = 1, - profile_detail = 2 -}; - -enum qcom_htp_arch { - NONE = 0, - V68 = 68, - V69 = 69, - V73 = 73, - V75 = 75, - V79 = 79, -}; - -enum qcom_chipset_soc_model { - UNKNOWN_SM = 0, - SM7450 = 41, // v69, 7 Gen1 - SM8350 = 30, // v68, 888 - SM8450 = 36, // v69, SD 8 Gen 1 - SM8475 = 42, // v69, SD 8+ Gen 1 - SM8550 = 43, // v73, SD 8 Gen 2 - SM8650 = 57, // v75, SD 8 Gen 3 - SM8750 = 69, // v79, SD 8 Gen 4 -}; - -struct qcom_socinfo { - uint32_t soc_model; - size_t htp_arch; - size_t vtcm_size_in_mb; - char soc_desc[GGML_MAX_NAME]; -}; - //file:///opt/qcom/aistack/qairt/2.31.0.250130/docs/QNN/general/overview.html#tbl-supported-snapdragon-devices static struct qcom_socinfo g_qnn_soc_info_table[] = { /* Qualcomm SnapDragon 7 Gen 1 */ @@ -1117,20 +614,30 @@ static struct qcom_socinfo g_qnn_soc_info_table[] = { .vtcm_size_in_mb = 8, .soc_desc = "Qualcomm SnapDragon 8 Gen 4"}, -}; +#if defined(_WIN32) + /* Qualcomm SnapDragon 7c Gen 2 */ + [SC7280X] = { + .soc_model = SC7280X, + .htp_arch = V68, + .vtcm_size_in_mb = 8, + .soc_desc = "Qualcomm SnapDragon 7c Gen 2"}, -struct ggml_backend_qnn_context { - int device; - int threads; - char name[GGML_MAX_NAME]; - char desc[GGML_MAX_NAME]; - char lib[GGML_MAX_NAME]; - qnn_instance * instance; - struct ggml_backend * backend; - QNN_INTERFACE_VER_TYPE raw_interface; - QNN_SYSTEM_INTERFACE_VER_TYPE raw_system_interface; - struct qcom_socinfo socinfo; -} ; + /* Qualcomm SnapDragon 8cx Gen 3 */ + [SC8280X] = { + .soc_model = SC8280X, + .htp_arch = V68, + .vtcm_size_in_mb = 8, + .soc_desc = "Qualcomm SnapDragon 8cx Gen 3"}, + + /* Qualcomm SnapDragon 8cx Gen 4 */ + [SC8380XP] = { + .soc_model = SC8380XP, + .htp_arch = V73, + .vtcm_size_in_mb = 8, + .soc_desc = "Qualcomm SnapDragon 8cx Gen 4"}, +#endif + +}; //the following helper funcs are used to ensure every QNN tensor name is unique static std::atomic g_ggmltensor_idx(0); @@ -1157,7 +664,11 @@ static struct ggml_backend_qnn_context g_qnn_mgr[GGML_QNN_MAX_DEVICES] = { .threads = 1, .name = "qnn-cpu", .desc = "Qualcomm Kryo CPU", +#if defined(_WIN32) + .lib = "QnnCpu.dll", +#else .lib = "libQnnCpu.so", +#endif .instance = nullptr, .backend = nullptr, .raw_interface = {}, @@ -1168,7 +679,11 @@ static struct ggml_backend_qnn_context g_qnn_mgr[GGML_QNN_MAX_DEVICES] = { .threads = 1, .name = "qnn-gpu", .desc = "Qualcomm Adreno GPU", +#if defined(_WIN32) + .lib = "QnnGpu.dll", +#else .lib = "libQnnGpu.so", +#endif .instance = nullptr, .backend = nullptr, .raw_interface = {}, @@ -1179,7 +694,11 @@ static struct ggml_backend_qnn_context g_qnn_mgr[GGML_QNN_MAX_DEVICES] = { .threads = 1, .name = "qnn-npu", .desc = "Qualcomm NPU(Hexagon Tensor Processor)", +#if defined(_WIN32) + .lib = "QnnHtp.dll", +#else .lib = "libQnnHtp.so", +#endif .instance = nullptr, .backend = nullptr, .raw_interface = {}, @@ -1187,13 +706,7 @@ static struct ggml_backend_qnn_context g_qnn_mgr[GGML_QNN_MAX_DEVICES] = { .socinfo = {}}, }; -struct qnn_op_caps_t { - const char * qnn_op_name = nullptr; - const size_t input_param_count = 0; - const char * qnn_param_name = nullptr; -}; - -static const qnn_op_caps_t k_op_caps[] = { +const qnn_op_caps_t ggmlqnn_k_op_caps[] = { {}, // GGML_OP_NONE {}, // GGML_OP_DUP { @@ -1353,54 +866,6 @@ static struct qcom_socinfo * qnn_get_socinfo_from_socmodel(uint32_t soc_model) { return nullptr; } -static bool ggmlqnn_is_valid_params(ggml_backend_qnn_context * ctx, const ggml_tensor * src0, - const ggml_tensor * src1, ggml_tensor * dst) { - if ((nullptr == ctx) || (nullptr == src0) || (nullptr == src1) || (nullptr == dst)) { - GGMLQNN_LOG_WARN("invalid params\n"); - return false; - } - - qnn_instance * instance = ctx->instance; - if (nullptr == instance) { - GGMLQNN_LOG_WARN("invalid params\n"); - return false; - } - - return true; -} - -#define GGMLQNN_CHECK_PARAMS(ctx, src0, src1, dst) \ - do { \ - if (!ggmlqnn_is_valid_params((ctx), (src0), (src1), (dst))) { \ - return; \ - } \ - } while (0) - -static uint32_t ggml_get_tensor_rank(const ggml_tensor * tensor) { - /* - uint32_t rank = 0; - for (int i = 0; i < GGML_MAX_DIMS; i++) { - if ((0 != tensor->ne[i]) && (1 != tensor->ne[i])) { - rank++; - } - } - return rank; - */ - return ggml_n_dims(tensor); -} - -static uint32_t ggml_get_tensor_data_size(const ggml_tensor * tensor) { - /* - size_t data_size = ggml_row_size(tensor->type, tensor->ne[0]); - size_t n_dims = ggml_get_tensor_rank(tensor); - for (int i = 1; i < n_dims; i++) { - data_size *= tensor->ne[i]; - } - - return data_size; - */ - return ggml_nbytes(tensor); -} static const char * ggml_get_type_name(ggml_type type) { const struct ggml_type_traits * traits = ggml_get_type_traits(type); @@ -1412,9 +877,8 @@ static const char * get_ggml_type_name(ggml_type type) { return traits->type_name; } -//TODO: // ref:explanation of k-quants, https://github.com/ggerganov/llama.cpp/pull/1684 -static Qnn_DataType_t qnn_datatype_from_ggml_datatype(enum ggml_type ggmltype) { +Qnn_DataType_t ggmlqnn_datatype_from_ggml_datatype(enum ggml_type ggmltype) { switch (ggmltype) { case GGML_TYPE_F16: return QNN_DATATYPE_FLOAT_16; @@ -1432,7 +896,6 @@ static Qnn_DataType_t qnn_datatype_from_ggml_datatype(enum ggml_type ggmltype) { return QNN_DATATYPE_UNDEFINED; } -//TODO: static ggml_type ggml_datatype_from_qnn_datatype(Qnn_DataType_t qnn_type) { switch (qnn_type) { case QNN_DATATYPE_FLOAT_32: @@ -1456,23 +919,32 @@ static ggml_type ggml_datatype_from_qnn_datatype(Qnn_DataType_t qnn_type) { return GGML_TYPE_COUNT; } -//TODO: add more ops -static const char * qnn_opname_from_ggmlop(enum ggml_op ggmlop) { - switch (ggmlop) { - case GGML_OP_ADD: - return QNN_OP_ELEMENT_WISE_ADD; - case GGML_OP_MUL_MAT: - return QNN_OP_MAT_MUL; - default: - break; +static void get_qnn_dimensions_from_ggml_dimensions(uint32_t * qnn_dimensions, const uint32_t * ggml_dimensions, uint32_t rank) { + if (rank > GGML_MAX_DIMS) { + GGMLQNN_LOG_WARN("invalid params"); + return; + } + if (nullptr == qnn_dimensions || nullptr == ggml_dimensions) { + GGMLQNN_LOG_WARN("invalid params"); + return; + } + for (size_t idx = 0; idx < GGML_MAX_DIMS; idx++) + qnn_dimensions[idx] = ggml_dimensions[idx]; + + if (rank >= 2) { + qnn_dimensions[rank - 1] = ggml_dimensions[rank - 2]; + qnn_dimensions[rank - 2] = ggml_dimensions[rank - 1]; } - return nullptr; } -static Qnn_Tensor_t * ggml_qnn_create_general_tensor(const ggml_tensor * tensor, const char * name, Qnn_TensorType_t qnn_tensor_type, - Qnn_DataType_t qnn_data_type, uint32_t rank, uint32_t * dims, void * data, uint32_t data_size) { - Qnn_ErrorHandle_t error = QNN_SUCCESS; - char tensor_name[GGML_MAX_NAME] = {0}; +Qnn_Tensor_t * ggmlqnn_create_general_tensor(const ggml_tensor * tensor, const char * name, + Qnn_TensorType_t qnn_tensor_type, + Qnn_DataType_t qnn_data_type, + uint32_t rank, uint32_t * dims, + void * data, uint32_t data_size, + bool b_transpose) { + Qnn_ErrorHandle_t error = QNN_SUCCESS; + char tensor_name[GGML_MAX_NAME] = {}; //ensure the tensor name is unique if (nullptr != name) { @@ -1483,19 +955,36 @@ static Qnn_Tensor_t * ggml_qnn_create_general_tensor(const ggml_tensor * tensor, GGMLQNN_LOG_DEBUG("init_tensor %d", get_idx()); inc_idx(); - uint32_t dimensions_transpose[GGML_MAX_DIMS] = {}; - uint32_t * tensor_dims = nullptr; + uint32_t reverse_dims[GGML_MAX_DIMS] = {}; + uint32_t transpose_dims[GGML_MAX_DIMS] = {}; + uint32_t * tensor_dims = nullptr; + //case 1:use dims info from ggml tensor if (nullptr != tensor) { //there are different dimension order between ggml tensor and qnn tensor for (size_t idx = 0; idx < rank; idx++) { - dimensions_transpose[idx] = (uint32_t)tensor->ne[rank - 1 - idx]; + reverse_dims[idx] = (uint32_t)tensor->ne[rank - 1 - idx]; } - tensor_dims = dimensions_transpose; + tensor_dims = reverse_dims; } - //re-assign tensor_dims + //case 2: use user's specified tensor_dims if (nullptr != dims) { tensor_dims = dims; } + //case 3: transpose for dst tensor + if (b_transpose) { + GGML_ASSERT(tensor != nullptr); //ensure ggml_tensor is not nullptr for this special case + + get_qnn_dimensions_from_ggml_dimensions(transpose_dims, reverse_dims, ggml_n_dims(tensor)); + tensor_dims = transpose_dims; +#if 0 + for (size_t idx = 0; idx < 4; idx++) { + GGMLQNN_LOG_DEBUG("origin dim[%d]=%d\n", idx, reverse_dims[idx]); + } + for (size_t idx = 0; idx < 4; idx++) { + GGMLQNN_LOG_DEBUG("trans dim[%d]=%d\n", idx, transpose_dims[idx]); + } +#endif + } Qnn_Tensor_t qnn_tensor = { .version= QNN_TENSOR_VERSION_1, @@ -1505,14 +994,13 @@ static Qnn_Tensor_t * ggml_qnn_create_general_tensor(const ggml_tensor * tensor, .type = qnn_tensor_type, .dataFormat = QNN_TENSOR_DATA_FORMAT_FLAT_BUFFER, .dataType = qnn_data_type, - .quantizeParams = {QNN_DEFINITION_UNDEFINED, - QNN_QUANTIZATION_ENCODING_UNDEFINED, + .quantizeParams = {.encodingDefinition = QNN_DEFINITION_UNDEFINED, + .quantizationEncoding = QNN_QUANTIZATION_ENCODING_UNDEFINED, {.scaleOffsetEncoding = {.scale = 0.0000000000000000f, .offset = 0}}}, .rank = rank, .dimensions = tensor_dims, .memType = QNN_TENSORMEMTYPE_RAW, - {.clientBuf = {nullptr, 0} - } + .clientBuf = {.data = nullptr, .dataSize = 0} } } }; @@ -1526,545 +1014,166 @@ static Qnn_Tensor_t * ggml_qnn_create_general_tensor(const ggml_tensor * tensor, } error = deep_copy_qnn_tensors(qnn_tensor, * p_qnn_tensor); if (error != QNN_SUCCESS) { - free(p_qnn_tensor); - GGMLQNN_LOG_WARN("init tensor failed"); - return nullptr; - } - QNN_VER_PTR(*p_qnn_tensor)->clientBuf = {data, data_size}; - - return p_qnn_tensor; -} - -static Qnn_Tensor_t * ggml_qnn_create_compute_tensor(const ggml_tensor * tensor) { - uint32_t dimensions[] = {(uint32_t) tensor->ne[0], (uint32_t) tensor->ne[1], - (uint32_t) tensor->ne[2], (uint32_t) tensor->ne[3]}; - Qnn_DataType_t qnn_data_type = QNN_DATATYPE_FLOAT_32; - Qnn_TensorType_t qnn_tensor_type = QNN_TENSOR_TYPE_APP_WRITE; - - if (tensor->flags & GGML_TENSOR_FLAG_INPUT) { - qnn_tensor_type = QNN_TENSOR_TYPE_APP_WRITE; - } else if (tensor->flags & GGML_TENSOR_FLAG_OUTPUT) { - qnn_tensor_type = QNN_TENSOR_TYPE_APP_READ; - } - - qnn_data_type = qnn_datatype_from_ggml_datatype(tensor->type); - Qnn_Tensor_t * p_qnn_tensor = ggml_qnn_create_general_tensor(tensor, nullptr, - qnn_tensor_type, qnn_data_type, - ggml_n_dims(tensor), dimensions, - nullptr, 0); - - return p_qnn_tensor; -} - -static void append_tensor_dimensions(const ggml_tensor * tensor, std::string & output) { - char buffer[256] = {}; - const char * type_name = get_ggml_type_name(tensor->type); - int len = 0; - switch (ggml_n_dims(tensor)) { - case 1: - len = snprintf(buffer, sizeof(buffer), "%ldx1%s", (long)tensor->ne[0], type_name); - break; - case 2: - len = snprintf(buffer, sizeof(buffer), "%ldx%ld%s", (long)tensor->ne[0], (long)tensor->ne[1], type_name); - break; - case 3: - len = snprintf(buffer, sizeof(buffer), "%ldx%ldx%ld%s", (long)tensor->ne[0], (long)tensor->ne[1], - (long)tensor->ne[2], type_name); - break; - case 4: - default: - len = snprintf(buffer, sizeof(buffer), "%ldx%ldx%ldx%ld%s", (long)tensor->ne[0], (long)tensor->ne[1], - (long)tensor->ne[2], (long)tensor->ne[3], type_name); - break; - } - GGML_ASSERT(len > 0 && len < (int)sizeof(buffer)); - output.append(buffer, len); -} - -static size_t get_qnn_op_index(const ggml_tensor * tensor) { - if (tensor->op == GGML_OP_UNARY) { - return GGML_OP_COUNT + ggml_get_unary_op(tensor); - } - - return tensor->op; -} - -static size_t get_qnn_op_input_param_count(const ggml_tensor * op) { - auto op_index = get_qnn_op_index(op); - GGML_ASSERT(op_index < std::size(k_op_caps)); - return k_op_caps[op_index].input_param_count; -} - -static void get_graph_key_from_op(const ggml_tensor * op, std::string & output) { - GGML_ASSERT(op->op != GGML_OP_NONE); - output += ggml_op_desc(op); - output += get_ggml_type_name(op->type); - size_t param_count = get_qnn_op_input_param_count(op); - for (size_t i = 0; i < param_count; ++i) { - auto * input = op->src[i]; - if (!input) { - break; - } - output += '_'; - append_tensor_dimensions(input, output); - } -} - -#if ENABLE_QNNBACKEND_PERF -class qnn_perf { -public: - qnn_perf(const std::string & perf_name) : _perf_name(std::move(perf_name)) {}; - qnn_perf() = delete; - qnn_perf(const qnn_perf & ) = delete; - qnn_perf & operator= (const qnn_perf & ) = delete; - - void start() { - _begin_time = ggml_time_us(); - } - - void info() { - _end_time = ggml_time_us(); - _duration = (_end_time - _begin_time); - GGMLQNN_LOG_DEBUG("duration of %s : %lld microseconds\n", _perf_name.c_str(), _duration); - } - -private: - int64_t _begin_time = 0LL; - int64_t _end_time = 0LL; - int64_t _duration = 0LL; - std::string _perf_name; -}; -#else -class qnn_perf { -public: - qnn_perf(const std::string & perf_name) {} - qnn_perf() = delete; - qnn_perf(const qnn_perf & ) = delete; - qnn_perf & operator= (const qnn_perf & ) = delete; - - void start() {} - void info() {} -}; -#endif - -template -Fn load_qnn_functionpointers(void * handle, const char * function_name) { - return reinterpret_cast(dlsym(handle, function_name)); -} - -class qnn_interface { - -#define DEFINE_SHIM_FUNCTION_INTERFACE(F, pointer_name) \ - template \ - inline auto qnn_##F(Args... args) const { \ - return (_qnn_interface->QNN_INTERFACE_VER_NAME.pointer_name)( \ - std::forward(args)...); \ - } - - -#define DEFINE_SHIM_FUNCTION_SYS_INTERFACE(F, pointer_name) \ - template \ - inline auto qnn_##F(Args... args) const { \ - return (_qnn_sys_interface->QNN_SYSTEM_INTERFACE_VER_NAME.pointer_name)( \ - std::forward(args)...); \ - } - - friend class qnn_instance; - -public: - qnn_interface() = default; - - // QnnBackend - DEFINE_SHIM_FUNCTION_INTERFACE(backend_create, backendCreate); - - DEFINE_SHIM_FUNCTION_INTERFACE(backend_free, backendFree); - - DEFINE_SHIM_FUNCTION_INTERFACE(backend_register_op_package, backendRegisterOpPackage); - - DEFINE_SHIM_FUNCTION_INTERFACE(backend_validate_op_config, backendValidateOpConfig); - - DEFINE_SHIM_FUNCTION_INTERFACE(backend_get_api_version, backendGetApiVersion); - - // QnnDevice - DEFINE_SHIM_FUNCTION_INTERFACE(device_create, deviceCreate); - - DEFINE_SHIM_FUNCTION_INTERFACE(device_free, deviceFree); - - DEFINE_SHIM_FUNCTION_INTERFACE(device_get_infrastructure, deviceGetInfrastructure); - - DEFINE_SHIM_FUNCTION_INTERFACE(device_get_platform_info, deviceGetPlatformInfo); - - DEFINE_SHIM_FUNCTION_INTERFACE(device_get_info, deviceGetInfo); - - // QnnContext - DEFINE_SHIM_FUNCTION_INTERFACE(context_create, contextCreate); - - DEFINE_SHIM_FUNCTION_INTERFACE(context_get_binary_size, contextGetBinarySize); - - DEFINE_SHIM_FUNCTION_INTERFACE(context_get_binary, contextGetBinary); - - DEFINE_SHIM_FUNCTION_INTERFACE(context_create_from_binary, contextCreateFromBinary); - - DEFINE_SHIM_FUNCTION_INTERFACE(context_free, contextFree); - - // QnnGraph - DEFINE_SHIM_FUNCTION_INTERFACE(graph_create, graphCreate); - - DEFINE_SHIM_FUNCTION_INTERFACE(graph_add_node, graphAddNode); - - DEFINE_SHIM_FUNCTION_INTERFACE(graph_finalize, graphFinalize); - - DEFINE_SHIM_FUNCTION_INTERFACE(graph_execute, graphExecute); - - DEFINE_SHIM_FUNCTION_INTERFACE(graph_retrieve, graphRetrieve); - - // QnnLog - DEFINE_SHIM_FUNCTION_INTERFACE(log_create, logCreate); - - DEFINE_SHIM_FUNCTION_INTERFACE(log_free, logFree); - - DEFINE_SHIM_FUNCTION_INTERFACE(log_set_log_level, logSetLogLevel); - - // QnnProfile - DEFINE_SHIM_FUNCTION_INTERFACE(profile_create, profileCreate); - - DEFINE_SHIM_FUNCTION_INTERFACE(profile_get_events, profileGetEvents); - - DEFINE_SHIM_FUNCTION_INTERFACE(profile_get_sub_events, profileGetSubEvents); - - DEFINE_SHIM_FUNCTION_INTERFACE(profile_get_event_data, profileGetEventData); - - DEFINE_SHIM_FUNCTION_INTERFACE(profile_free, profileFree); - - // QnnMem - DEFINE_SHIM_FUNCTION_INTERFACE(mem_register, memRegister); - - DEFINE_SHIM_FUNCTION_INTERFACE(mem_de_register, memDeRegister); - - // QnnProperty - DEFINE_SHIM_FUNCTION_INTERFACE(property_has_capability, propertyHasCapability); - - // QnnTensor - DEFINE_SHIM_FUNCTION_INTERFACE(tensor_create_context_tensor, tensorCreateContextTensor); - - DEFINE_SHIM_FUNCTION_INTERFACE(tensor_create_graph_tensor, tensorCreateGraphTensor); - - // QnnSystem - DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_create, systemContextCreate); - - DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_get_binary_info, systemContextGetBinaryInfo); - - DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_free, systemContextFree); - - void set_qnn_interface(const QnnInterface_t * qnn_interface) { - _qnn_interface = qnn_interface; - } - - void set_qnn_system_interface(const QnnSystemInterface_t * qnn_sys_interface) { - _qnn_sys_interface = qnn_sys_interface; - } - - uint32_t get_backend_id() const { - return _qnn_interface->backendId; - } - - bool is_loaded() const { - return ((_qnn_sys_interface != nullptr) && (_qnn_interface != nullptr)); - } - -private: - const QnnInterface_t *_qnn_interface = nullptr; - - const QnnSystemInterface_t *_qnn_sys_interface = nullptr; -}; - -class qnn_instance { -public: - using BackendIdType = decltype(QnnInterface_t{}.backendId); - - explicit qnn_instance(const std::string & lib_path, const std::string & backend_name, - const std::string & model_name) : - _lib_path(std::move(lib_path)), - _backend_name(std::move(backend_name)), - _model_name(std::move(model_name)) {}; - - ~qnn_instance() { - } - - int qnn_init(const QnnSaver_Config_t ** saver_config); - - int qnn_finalize(); - - const qnn_interface & get_qnn_interface() { - if (!_qnn_interface.is_loaded()) { - GGMLQNN_LOG_WARN("pls check why _qnn_interface is not loaded\n"); - } - return _qnn_interface; - } - - const QNN_INTERFACE_VER_TYPE & get_qnn_raw_interface() { - if (!_qnn_interface.is_loaded()) { - GGMLQNN_LOG_WARN("pls check why _qnn_interface is not loaded\n"); - } - return _qnn_raw_interface; - } - - const QNN_SYSTEM_INTERFACE_VER_TYPE & get_qnn_raw_system_interface() { - if (!_qnn_interface.is_loaded()) { - GGMLQNN_LOG_WARN("pls check why _qnn_interface is not loaded\n"); - } - return _qnn_raw_system_interface; - } - - const Qnn_LogHandle_t get_qnn_log_handle() { return _qnn_log_handle; } - - const Qnn_ProfileHandle_t get_qnn_profile_handle() { return _qnn_profile_handle; } - - const Qnn_DeviceHandle_t get_qnn_device_handle() { return _qnn_device_handle; } - - const Qnn_BackendHandle_t get_qnn_backend_handle() { return _qnn_backend_handle; } - - const Qnn_ContextHandle_t get_qnn_context_handle() { return _qnn_context_handle; } - - const QnnSystemContext_Handle_t get_qnn_system_handle() { return _qnn_system_handle; } - - const Qnn_GraphHandle_t get_qnn_graph_handle() { return _qnn_graph_handle; } - - int init_qnn_graph(const char * graph_name, - bool debug, - uint8_t do_node_validation = 1, - const QnnGraph_Config_t ** graph_configs = nullptr - ); - int init_qnn_graph(const std::string & graph_name, QNNBackend device, size_t vtcm_size_in_mb = 8, size_t hvx_threads = 8); - - int finalize_qnn_graph(); - - bool is_valid_graph() const { return _qnn_graph_handle != nullptr; } - - int init_htp_perfinfra() { - QnnDevice_Infrastructure_t device_infra = nullptr; - int error = _qnn_raw_interface.deviceGetInfrastructure(&device_infra); - if (error != QNN_SUCCESS) { - GGMLQNN_LOG_WARN("failed to get qnn device infra\n"); - return 1; - } - - QnnHtpDevice_Infrastructure_t * htp_infra = static_cast(device_infra); - QnnHtpDevice_PerfInfrastructure_t * htp_perfinfra = &htp_infra->perfInfra; - uint32_t power_configid = 1; - uint32_t device_id = 0; - uint32_t core_id = 0; - htp_perfinfra->createPowerConfigId(device_id, core_id, &power_configid); - _qnn_htp_perfinfra = htp_perfinfra; - _qnn_power_configid = power_configid; - - return 0; - } - - int set_rpc_polling() { - if (_qnn_rpc_pollingtime > 0) { - QnnHtpPerfInfrastructure_PowerConfig_t rpc_pollingtime; - memset(&rpc_pollingtime, 0, sizeof(rpc_pollingtime)); - rpc_pollingtime.option = - QNN_HTP_PERF_INFRASTRUCTURE_POWER_CONFIGOPTION_RPC_POLLING_TIME; - rpc_pollingtime.rpcPollingTimeConfig = _qnn_rpc_pollingtime; - const QnnHtpPerfInfrastructure_PowerConfig_t * power_configs[] = {&rpc_pollingtime, nullptr}; - if (_qnn_htp_perfinfra) { - _qnn_htp_perfinfra->setPowerConfig(_qnn_power_configid, power_configs); - } - } - return 0; - } - - int set_high_performance_mode() { - if (nullptr == _qnn_htp_perfinfra) { - GGMLQNN_LOG_DEBUG("perf intra is null\n"); - return 1; - } - - QnnHtpPerfInfrastructure_PowerConfig_t power_config; - memset(&power_config, 0, sizeof(power_config)); - power_config.option = QNN_HTP_PERF_INFRASTRUCTURE_POWER_CONFIGOPTION_DCVS_V3; - power_config.dcvsV3Config.dcvsEnable = 0; - power_config.dcvsV3Config.setDcvsEnable = 1; - power_config.dcvsV3Config.contextId = _qnn_power_configid; - power_config.dcvsV3Config.powerMode = QNN_HTP_PERF_INFRASTRUCTURE_POWERMODE_PERFORMANCE_MODE; - power_config.dcvsV3Config.setSleepLatency = 1; // True to consider Latency parameter otherwise False - power_config.dcvsV3Config.setBusParams = 1; // True to consider Bus parameter otherwise False - power_config.dcvsV3Config.setCoreParams = 1; // True to consider Core parameter otherwise False - power_config.dcvsV3Config.sleepDisable = 0; // True to consider sleep/LPM modes, False to enable - power_config.dcvsV3Config.setSleepDisable = 0; // True to consider sleep disable/enable parameter otherwise False - // set Sleep latency parameter - uint32_t latencyValue = 40; - power_config.dcvsV3Config.sleepLatency = latencyValue; // range 40-2000 micro sec - // set Bus Clock Parameters (refer QnnHtpPerfInfrastructure_VoltageCorner_t enum) - power_config.dcvsV3Config.busVoltageCornerMin = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; - power_config.dcvsV3Config.busVoltageCornerTarget = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; - power_config.dcvsV3Config.busVoltageCornerMax = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; - // set Core Clock Parameters (refer QnnHtpPerfInfrastructure_VoltageCorner_t enum) - power_config.dcvsV3Config.coreVoltageCornerMin = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; - power_config.dcvsV3Config.coreVoltageCornerTarget = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; - power_config.dcvsV3Config.coreVoltageCornerMax = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; - // set power config with different performance parameters - const QnnHtpPerfInfrastructure_PowerConfig_t * power_configs[] = {&power_config, nullptr}; - - _qnn_htp_perfinfra->setPowerConfig(_qnn_power_configid, power_configs); - - return 0; - } - - std::string & get_qnn_graph_name() { return _graph_name; } - - bool is_rpcmem_initialized() { - return _rpcmem_initialized; - } - - void set_rpcmem_initialized(bool initialized) { - _rpcmem_initialized = initialized; - } - - size_t get_rpcmem_capacity() { return _rpcmem_capacity; } - size_t get_rpcmem_usage() { return _rpcmem_usage; } - - int32_t rpcmem_to_fd(void * buf); - - int register_rpcmem(void * p_data, Qnn_Tensor_t * p_tensor); - Qnn_MemHandle_t register_rpcmem(void * p_data, const uint32_t rank, uint32_t * dimensions, Qnn_DataType_t data_type); - - void unregister_rpcmem(); - void unregister_rpcmem(Qnn_MemHandle_t mem_handle); - - void * alloc_rpcmem(size_t bytes, size_t alignment); - void * get_rpcmem_from_memhandle(Qnn_MemHandle_t mem_handle); - - void free_rpcmem(void * buf); - void free_rpcmem(); - - bool is_rpcmem_allocated(void * buf); - - bool is_rpcmem_registered(Qnn_MemHandle_t handle) { - return _qnn_mem_set.count(handle) != 0U; + free(p_qnn_tensor); + GGMLQNN_LOG_WARN("init tensor failed"); + return nullptr; } + QNN_VER_PTR(*p_qnn_tensor)->clientBuf = {data, data_size}; - bool enable_qnn_rpc() { - return _enable_qnn_rpc; - } + return p_qnn_tensor; +} - void probe_device_meminfo() { - size_t candidate_size = 0; - uint8_t *rpc_buffer = nullptr; - const int SIZE_IN_MB = (1 << 20); - size_t probe_slots[] = {1024, 1536, 2048 - 48, 2048}; - size_t probe_counts = sizeof(probe_slots) / sizeof(size_t); - for (size_t idx = 0; idx < probe_counts; idx++) { - rpc_buffer = static_cast(alloc_rpcmem_internal(probe_slots[idx] * SIZE_IN_MB, 4)); - if (nullptr == rpc_buffer) { - GGMLQNN_LOG_DEBUG("alloc rpcmem %d (MB) failure, %s\n", probe_slots[idx], - strerror(errno)); - break; - } else { - candidate_size = probe_slots[idx]; - free_rpcmem(rpc_buffer); - rpc_buffer = nullptr; - } - } - if (candidate_size > _rpcmem_capacity) - _rpcmem_capacity = candidate_size; +Qnn_Tensor_t * ggmlqnn_create_compute_tensor(const ggml_tensor * tensor) { + uint32_t dimensions[] = {(uint32_t) tensor->ne[0], (uint32_t) tensor->ne[1], + (uint32_t) tensor->ne[2], (uint32_t) tensor->ne[3]}; + Qnn_DataType_t qnn_data_type = QNN_DATATYPE_FLOAT_32; + Qnn_TensorType_t qnn_tensor_type = QNN_TENSOR_TYPE_APP_WRITE; - free_rpcmem(); - _rpcmem_usage = 0; - GGMLQNN_LOG_INFO("capacity of rpc ion memory %d MB\n", _rpcmem_capacity); + if (tensor->flags & GGML_TENSOR_FLAG_INPUT) { + qnn_tensor_type = QNN_TENSOR_TYPE_APP_WRITE; + } else if (tensor->flags & GGML_TENSOR_FLAG_OUTPUT) { + qnn_tensor_type = QNN_TENSOR_TYPE_APP_READ; } -public: - std::map>> _qnn_graph_map; - -private: - int load_system(); - - int unload_system(); + qnn_data_type = ggmlqnn_datatype_from_ggml_datatype(tensor->type); + Qnn_Tensor_t * p_qnn_tensor = ggmlqnn_create_general_tensor(tensor, nullptr, + qnn_tensor_type, qnn_data_type, + ggml_n_dims(tensor), dimensions, + nullptr, 0); - int load_backend(std::string & lib_path, const QnnSaver_Config_t ** saver_config); + return p_qnn_tensor; +} - int unload_backend(); +void * ggmlqnn_type_trait(ggml_backend_qnn_context * ctx, ggml_tensor * op) { + const ggml_tensor * src0 = op->src[0]; + const ggml_tensor * src1 = op->src[1]; + ggml_tensor * dst = op; + const enum ggml_type src0_type = src0->type; + + GGML_TENSOR_BINARY_OP_LOCALS + GGML_ASSERT(ne0 == ne01); + GGML_ASSERT(ne1 == ne11); + GGML_ASSERT(ne2 == ne12); + GGML_ASSERT(ne3 == ne13); + GGML_ASSERT(nb00 == ggml_type_size(src0_type)); + GGML_ASSERT(nb10 == ggml_type_size(src1->type)); + + const int64_t ne_plane = ne01 * ne00; + const size_t desired_size = ((GGML_TYPE_F32 == src0_type) ? 0 : ne03 * ne02 * ne_plane * sizeof(float)); + ctx->desired_size = desired_size; + if (ctx->work_size < desired_size) { + ctx->work_data.reset(new char[desired_size]); + ctx->work_size = desired_size; + } + ctx->n_threads = std::thread::hardware_concurrency(); + void * wdata = ctx->work_data.get(); + // convert src0 to float + if (src0_type != GGML_TYPE_F32) { + const auto * type_traits = ggml_get_type_traits(src0_type); + ggml_to_float_t const to_float = type_traits->to_float; + + for (int64_t i03 = 0; i03 < ne03; i03++) { + for (int64_t i02 = 0; i02 < ne02; i02++) { + const void * x = (char *)src0->data + i02 * nb02 + i03 * nb03; + float * const wplane = (float *)wdata + i02 * ne_plane + i03 * ne02 * ne_plane; + + const int min_cols_per_thread = 4096; + const int min_rows_per_thread = std::max((int)(min_cols_per_thread / ne00), 1); + const int n_threads = std::max( + std::min(ctx->n_threads, (int)(ne01 / min_rows_per_thread)), 1); + for (int i = 1; i < n_threads; i++) { + const int64_t start = i * ne01 / n_threads; + const int64_t end = (i + 1) * ne01 / n_threads; + if (start < end) { + ctx->tasks.push_back(std::async(std::launch::async, [=]() { + for (int64_t i01 = start; i01 < end; i01++) { + to_float((const char *)x + i01 * nb01, wplane + i01 * ne00, ne00); + } + })); + } + } + { + // reuse the current thread for the first task + const int64_t start = 0; + const int64_t end = ne01 / n_threads; + for (int64_t i01 = start; i01 < end; i01++) { + to_float((const char *) x + i01 * nb01, wplane + i01 * ne00, ne00); + } + } + } + } - void set_qnn_raw_interface(QNN_INTERFACE_VER_TYPE & raw_interface) { - _qnn_raw_interface = raw_interface; + // wait for all tasks to finish + for (auto &task: ctx->tasks) { + task.get(); + } + ctx->tasks.clear(); } + return wdata; +} - void set_qnn_raw_system_interface(QNN_SYSTEM_INTERFACE_VER_TYPE & raw_interface) { - _qnn_raw_system_interface = raw_interface; +static void append_tensor_dimensions(const ggml_tensor * tensor, std::string & output) { + char buffer[256] = {}; + const char * type_name = get_ggml_type_name(tensor->type); + int len = 0; + switch (ggml_n_dims(tensor)) { + case 1: + len = snprintf(buffer, sizeof(buffer), "%ldx1%s", (long)tensor->ne[0], type_name); + break; + case 2: + len = snprintf(buffer, sizeof(buffer), "%ldx%ld%s", (long)tensor->ne[0], (long)tensor->ne[1], type_name); + break; + case 3: + len = snprintf(buffer, sizeof(buffer), "%ldx%ldx%ld%s", (long)tensor->ne[0], (long)tensor->ne[1], + (long)tensor->ne[2], type_name); + break; + case 4: + default: + len = snprintf(buffer, sizeof(buffer), "%ldx%ldx%ldx%ld%s", (long)tensor->ne[0], (long)tensor->ne[1], + (long)tensor->ne[2], (long)tensor->ne[3], type_name); + break; } - - void * alloc_rpcmem_internal(size_t bytes, size_t alignment); - -private: - static constexpr const int _required_num_providers = 1; - -private: - std::string _lib_path; - std::string _backend_name; - std::string _model_name; // name of prebuilt QNN model, might be used in the future - BackendIdType _backend_id; - - bool _debug_tensor = false; // flag to indicate if requested graph is to be run in debug mode - bool _do_node_validations = true; // flag to indicate whether all add_node calls need to be validated - QnnLog_Level_t _qnn_log_level = QNN_LOG_LEVEL_DEBUG; - - ggml_qnn_profile_level _profile_level = ggml_qnn_profile_level::profile_detail; - - qnn_interface _qnn_interface; - - void * _system_lib_handle = nullptr; - - Qnn_GraphHandle_t _qnn_graph_handle = nullptr; - - Qnn_LogHandle_t _qnn_log_handle = nullptr; - - Qnn_ProfileHandle_t _qnn_profile_handle = nullptr; - - Qnn_DeviceHandle_t _qnn_device_handle = nullptr; - - Qnn_BackendHandle_t _qnn_backend_handle = nullptr; - - Qnn_ContextHandle_t _qnn_context_handle = nullptr; - - QnnSystemContext_Handle_t _qnn_system_handle = nullptr; - - QnnHtpDevice_PerfInfrastructure_t *_qnn_htp_perfinfra = nullptr; - uint32_t _qnn_power_configid = 1; - uint32_t _qnn_rpc_pollingtime = 9999; // 0-10000 us for high performing + GGML_ASSERT(len > 0 && len < (int)sizeof(buffer)); + output.append(buffer, len); +} - QNN_INTERFACE_VER_TYPE _qnn_raw_interface; - QNN_SYSTEM_INTERFACE_VER_TYPE _qnn_raw_system_interface; +size_t ggmlqnn_get_opcaps_size() { + return std::size(ggmlqnn_k_op_caps); +} - std::unordered_map _qnn_mem_set; - std::unordered_map _qnn_rpc_buffer_to_handles; +size_t ggmlqnn_get_op_index(const ggml_tensor * tensor) { + if (tensor->op == GGML_OP_UNARY) { + return static_cast(GGML_OP_COUNT) + static_cast(ggml_get_unary_op(tensor)); + } - static std::mutex _init_mutex; - static std::unordered_map _loaded_lib_handle; - static std::unordered_map _lib_path_to_backend_id; - static std::unordered_map _loaded_backend; + return tensor->op; +} - void * _rpc_lib_handle = nullptr; - std::atomic_bool _rpcmem_initialized{false}; - pfn_rpc_mem_alloc _pfn_rpc_mem_alloc; - pfn_rpc_mem_free _pfn_rpc_mem_free; - pfn_rpc_mem_to_fd _pfn_rpc_mem_to_fd; - pfn_rpc_mem_init _pfn_rpc_mem_init; - pfn_rpc_mem_deinit _pfn_rpc_mem_deinit; - std::unordered_map _rpcmem_store_map; - std::unordered_map _rpcmem_usage_map; - size_t _rpcmem_capacity = 512; // mempool size in Mbytes - size_t _rpcmem_usage = 0; // mempool usage in Mbytes +static size_t ggmlqnn_get_op_input_param_count(const ggml_tensor * op) { + auto op_index = ggmlqnn_get_op_index(op); + GGML_ASSERT(op_index < std::size(ggmlqnn_k_op_caps)); + return ggmlqnn_k_op_caps[op_index].input_param_count; +} - std::string _graph_name; - QNNBackend _device_id; - bool _enable_qnn_rpc = false; //TODO:unknown issue with QNN RPC feature +void ggmlqnn_get_graphkey_from_op(const ggml_tensor * op, std::string & output) { + GGML_ASSERT(op->op != GGML_OP_NONE); + output += ggml_op_desc(op); + output += get_ggml_type_name(op->type); + size_t param_count = ggmlqnn_get_op_input_param_count(op); + for (size_t i = 0; i < param_count; ++i) { + auto * input = op->src[i]; + if (!input) { + break; + } + output += '_'; + append_tensor_dimensions(input, output); + } +} - DISABLE_COPY(qnn_instance); - DISABLE_MOVE(qnn_instance); -}; +template +Fn load_qnn_functionpointers(void * handle, const char * function_name) { + return reinterpret_cast(dlsym(handle, function_name)); +} std::mutex qnn_instance::_init_mutex; std::unordered_map qnn_instance::_loaded_lib_handle; @@ -2079,13 +1188,13 @@ void * qnn_instance::alloc_rpcmem_internal(size_t bytes, size_t alignment) { auto allocate_bytes = static_cast(bytes + alignment); void * buf = _pfn_rpc_mem_alloc(RPCMEM_HEAP_ID_SYSTEM, RPCMEM_DEFAULT_FLAGS, allocate_bytes); - if (buf == nullptr) { + if (nullptr == buf) { GGMLQNN_LOG_WARN("failed to allocate rpc memory\n"); return nullptr; } auto aligned_buf = reinterpret_cast(ggmlqnn_align_to(alignment, - reinterpret_cast(buf))); + reinterpret_cast(buf))); bool status = _rpcmem_store_map.insert(std::pair(aligned_buf, buf)).second; if (!status) { GGMLQNN_LOG_WARN("failed to allocate rpc memory\n"); @@ -2141,8 +1250,6 @@ void qnn_instance::free_rpcmem(void * buf) { } void qnn_instance::free_rpcmem() { - Qnn_ErrorHandle_t error = QNN_SUCCESS; - if (_rpcmem_store_map.empty()) { GGMLQNN_LOG_WARN("no rpcmem allocated\n"); return; @@ -2184,13 +1291,13 @@ int qnn_instance::register_rpcmem(void * p_data, Qnn_Tensor_t * p_tensor) { if (is_rpcmem_registered((QNN_VER_PTR(*p_tensor)->memHandle))) { GGMLQNN_LOG_WARN("tensor %s has been registered shared memory\n", (QNN_VER_PTR(*p_tensor)->name)); - return 4; + return 3; } int32_t mem_fd = rpcmem_to_fd(p_data); if (-1 == mem_fd) { GGMLQNN_LOG_WARN("failed to get file descriptor\n"); - return 5; + return 4; } GGMLQNN_LOG_DEBUG("mem_fd %d\n", mem_fd); Qnn_MemDescriptor_t descriptor = { @@ -2206,9 +1313,8 @@ int qnn_instance::register_rpcmem(void * p_data, Qnn_Tensor_t * p_tensor) { /*numDescriptors=*/1, &handle); if (error != QNN_SUCCESS) { - GGMLQNN_LOG_WARN("failed to register shared memory, error %d, %s\n", QNN_GET_ERROR_CODE(error), - strerror(error)); - return 6; + GGMLQNN_LOG_WARN("failed to register shared memory, error %d, %s\n", QNN_GET_ERROR_CODE(error), strerror(error)); + return 5; } else { GGMLQNN_LOG_INFO("tensor %s successfully register shared memory\n", (QNN_VER_PTR(*p_tensor)->name)); } @@ -2247,8 +1353,7 @@ Qnn_MemHandle_t qnn_instance::register_rpcmem(void * p_data, const uint32_t ran {{mem_fd}} }; Qnn_MemHandle_t handle = nullptr; - auto error = _qnn_interface.qnn_mem_register(_qnn_context_handle, &descriptor, - /*numDescriptors=*/1, &handle); + auto error = _qnn_interface.qnn_mem_register(_qnn_context_handle, &descriptor, /*numDescriptors=*/1, &handle); if (error != QNN_SUCCESS) { GGMLQNN_LOG_WARN("failed to register shared memory, error %d, %s", QNN_GET_ERROR_CODE(error), strerror(error)); return nullptr; @@ -2285,8 +1390,7 @@ void qnn_instance::unregister_rpcmem() { Qnn_MemHandle_t mem_handle = it->second; error = _qnn_interface.qnn_mem_de_register(&mem_handle, 1); if (error != QNN_SUCCESS) { - GGMLQNN_LOG_WARN("failed to unregister shared memory, error %d\n", - QNN_GET_ERROR_CODE(error)); + GGMLQNN_LOG_WARN("failed to unregister shared memory, error %d\n", QNN_GET_ERROR_CODE(error)); } else { GGMLQNN_LOG_DEBUG("unregister shared memory ok"); } @@ -2324,9 +1428,9 @@ int qnn_instance::load_backend(std::string & lib_path, const QnnSaver_Config_t * return 1; } - auto get_providers = - load_qnn_functionpointers<_pfn_QnnInterface_getProviders *>(lib_handle, - "QnnInterface_getProviders"); + auto get_providers = load_qnn_functionpointers<_pfn_QnnInterface_getProviders *>( + lib_handle, + "QnnInterface_getProviders"); if (nullptr == get_providers) { GGMLQNN_LOG_WARN("can not load symbol QnnInterface_getProviders : %s", dlerror()); return 2; @@ -2386,21 +1490,6 @@ int qnn_instance::load_backend(std::string & lib_path, const QnnSaver_Config_t * _loaded_lib_handle[backend_id] = lib_handle; _backend_id = backend_id; -#if 0 // leave them here for further use - QnnSaver_Config_t outputdir_cfg; - outputdir_cfg.option = QNN_SAVER_CONFIG_OPTION_OUTPUT_DIRECTORY; - outputdir_cfg.outputDirectory = "/data/local/tmp/"; - QnnSaver_Config_t backendid_cfg; - backendid_cfg.option = QNN_SAVER_CONFIG_OPTION_BACKEND_ID; - backendid_cfg.backendId = _backend_id; - - const QnnSaver_Config_t * saver_cfg[] = {&outputdir_cfg, &backendid_cfg, nullptr}; - if (0 == QnnSaver_initialize(saver_cfg)) { - GGMLQNN_LOG_INFO("QnnSaver_initialize successfully"); - } else { - GGMLQNN_LOG_WARN("QnnSaver_initialize failure"); - } -#endif auto saver_initialize = load_qnn_functionpointers<_pfn_QnnSaver_initialize *>( _loaded_lib_handle[backend_id], "QnnSaver_initialize"); @@ -2419,7 +1508,7 @@ int qnn_instance::load_backend(std::string & lib_path, const QnnSaver_Config_t * int qnn_instance::unload_backend() { int dlclose_error = 0; - for (auto &it : _loaded_lib_handle) { + for (auto & it : _loaded_lib_handle) { dlclose_error = dlclose(it.second); if (dlclose_error != 0) { GGMLQNN_LOG_WARN("failed to close QNN backend %d, error %s\n", it.first, dlerror()); @@ -2436,7 +1525,11 @@ int qnn_instance::unload_backend() { int qnn_instance::load_system() { Qnn_ErrorHandle_t error = QNN_SUCCESS; +#ifdef _WIN32 + std::string system_lib_path = _lib_path + "QnnSystem.dll"; +#else std::string system_lib_path = _lib_path + "libQnnSystem.so"; +#endif GGMLQNN_LOG_DEBUG("system_lib_path:%s\n", system_lib_path.c_str()); _system_lib_handle = dlopen(system_lib_path.c_str(), RTLD_NOW | RTLD_LOCAL); @@ -2444,7 +1537,11 @@ int qnn_instance::load_system() { GGMLQNN_LOG_WARN("can not open QNN library %s, error: %s\n", system_lib_path.c_str(), dlerror()); //re-try with default path of QNN binary runtime lib _lib_path = "/data/local/tmp/"; +#ifdef _WIN32 + system_lib_path = _lib_path + "QnnSystem.dll"; +#else system_lib_path = _lib_path + "libQnnSystem.so"; +#endif _system_lib_handle = dlopen(system_lib_path.c_str(), RTLD_NOW | RTLD_LOCAL); if (nullptr == _system_lib_handle) { GGMLQNN_LOG_WARN("can not open QNN library %s, error: %s\n", system_lib_path.c_str(), dlerror()); @@ -2572,7 +1669,7 @@ static void ggml_qnn_logcallback(const char * fmt, std::lock_guard lock(log_mutex); memset(s_ggml_qnn_logbuf, 0, GGML_QNN_LOGBUF_LEN); vsnprintf(reinterpret_cast(s_ggml_qnn_logbuf), GGML_QNN_LOGBUF_LEN, fmt, argp); - GGMLQNN_LOG_INFO("%8.1fms [%-7s] %s\n", ms, log_level_desc, s_ggml_qnn_logbuf); + GGMLQNN_LOG_DEBUG("%8.1fms [%-7s] %s\n", ms, log_level_desc, s_ggml_qnn_logbuf); } } #else @@ -2580,6 +1677,10 @@ static void ggml_qnn_logcallback(const char * fmt, QnnLog_Level_t level, uint64_t timestamp, va_list argp) { + GGML_UNUSED(fmt); + GGML_UNUSED(level); + GGML_UNUSED(timestamp); + GGML_UNUSED(argp); } #endif @@ -2594,20 +1695,20 @@ int qnn_instance::qnn_init(const QnnSaver_Config_t ** saver_config) { GGMLQNN_LOG_DEBUG("load QNN system lib successfully\n"); } - std::string bakend_lib_path = _lib_path + _backend_name; - if (0 == _lib_path_to_backend_id.count(bakend_lib_path)) { - int is_load_ok = load_backend(bakend_lib_path, saver_config); + std::string backend_lib_path = _lib_path + _backend_name; + if (0 == _lib_path_to_backend_id.count(backend_lib_path)) { + int is_load_ok = load_backend(backend_lib_path, saver_config); if (0 != is_load_ok) { GGMLQNN_LOG_WARN("failed to load QNN backend\n"); return 2; } } - backend_id = _lib_path_to_backend_id[bakend_lib_path]; + backend_id = _lib_path_to_backend_id[backend_lib_path]; if (0 == _loaded_backend.count(backend_id) || 0 == _loaded_lib_handle.count(backend_id)) { GGMLQNN_LOG_WARN("library %s is loaded but loaded backend count=%zu, loaded lib_handle count=%zu\n", - bakend_lib_path.c_str(), + backend_lib_path.c_str(), _loaded_backend.count(backend_id), _loaded_lib_handle.count(backend_id)); return 3; @@ -2661,7 +1762,7 @@ int qnn_instance::qnn_init(const QnnSaver_Config_t ** saver_config) { if (QNN_PROFILE_NO_ERROR != _qnn_raw_interface.profileCreate( _qnn_backend_handle, QNN_PROFILE_LEVEL_BASIC, &_qnn_profile_handle)) { GGMLQNN_LOG_WARN("unable to create profile handle in the backend\n"); - return 7; + return 6; } else { GGMLQNN_LOG_DEBUG("initialize qnn profile successfully\n"); } @@ -2677,10 +1778,16 @@ int qnn_instance::qnn_init(const QnnSaver_Config_t ** saver_config) { } } +#if defined(__ANDROID__) || defined(__linux__) _rpc_lib_handle = dlopen("libcdsprpc.so", RTLD_NOW | RTLD_LOCAL); +#elif defined(_WIN32) + _rpc_lib_handle = dlopen("libcdsprpc.dll", RTLD_NOW | RTLD_LOCAL); +#else +#error "ggml-qnn only support WoA, Android, Linux" +#endif if (nullptr == _rpc_lib_handle) { GGMLQNN_LOG_WARN("failed to load qualcomm's rpc lib, error:%s\n", dlerror()); - return 9; + return 8; } else { GGMLQNN_LOG_DEBUG("load rpcmem lib successfully\n"); set_rpcmem_initialized(true); @@ -2694,7 +1801,7 @@ int qnn_instance::qnn_init(const QnnSaver_Config_t ** saver_config) { || nullptr == _pfn_rpc_mem_to_fd) { GGMLQNN_LOG_WARN("unable to access symbols in QNN RPC lib. dlerror(): %s", dlerror()); dlclose(_rpc_lib_handle); - return 10; + return 9; } if (nullptr != _pfn_rpc_mem_init) // make Qualcomm's SoC based low-end phone happy @@ -2706,7 +1813,7 @@ int qnn_instance::qnn_init(const QnnSaver_Config_t ** saver_config) { &_qnn_context_handle); if (nullptr == _qnn_context_handle) { GGMLQNN_LOG_WARN("why failed to initialize qnn context, error:%s\n", strerror(errno)); - return 8; + return 10; } else { GGMLQNN_LOG_DEBUG("initialize qnn context successfully\n"); } @@ -2716,7 +1823,7 @@ int qnn_instance::qnn_init(const QnnSaver_Config_t ** saver_config) { _qnn_raw_interface.deviceGetPlatformInfo(nullptr, &p_info); GGMLQNN_LOG_INFO("device counts %d", p_info->v1.numHwDevices); QnnDevice_HardwareDeviceInfo_t * infos = p_info->v1.hwDevices; - for (int i = 0; i < p_info->v1.numHwDevices; i++) { + for (size_t i = 0; i < p_info->v1.numHwDevices; i++) { GGMLQNN_LOG_INFO("deviceID:%d, deviceType:%d, numCores %d", infos[i].v1.deviceId, infos[i].v1.deviceType, infos[i].v1.numCores); QnnDevice_DeviceInfoExtension_t devinfo = infos[i].v1.deviceInfoExtension; @@ -2728,7 +1835,7 @@ int qnn_instance::qnn_init(const QnnSaver_Config_t ** saver_config) { chipinfo.socModel, qnn_get_socmodel_desc(chipinfo.socModel), \ htp_arch, qnn_get_htparch_desc(htp_arch), chipinfo.vtcmSize); struct qcom_socinfo * socinfo = qnn_get_socinfo_from_socmodel(chipinfo.socModel); - g_qnn_mgr[QNN_BACKEND_NPU].socinfo = { chipinfo.socModel, htp_arch, chipinfo.vtcmSize }; + g_qnn_mgr[QNN_BACKEND_NPU].socinfo = { chipinfo.socModel, htp_arch, chipinfo.vtcmSize, {}}; if (nullptr != socinfo) { memcpy(g_qnn_mgr[QNN_BACKEND_NPU].socinfo.soc_desc, socinfo->soc_desc, sizeof(socinfo->soc_desc)); GGMLQNN_LOG_INFO("soc info:%s", socinfo->soc_desc); @@ -2881,17 +1988,17 @@ int qnn_instance::init_qnn_graph(const std::string & graph_name, QNNBackend devi if (error != QNN_SUCCESS) { GGMLQNN_LOG_ERROR("[%s][%s]failed to create qnn graph, error: %s", ggml_backend_qnn_get_devname(device), graph_name.c_str(), - qnn_get_error_string(error)); + ggmlqnn_get_error_string(error)); return error; } - GGMLQNN_LOG_INFO("[%s]create graph %s succeed", ggml_backend_qnn_get_devname(device), graph_name.c_str()); + GGMLQNN_LOG_DEBUG("[%s]create graph %s succeed", ggml_backend_qnn_get_devname(device), graph_name.c_str()); _qnn_graph_handle = graph_handle; return QNN_SUCCESS; } int qnn_instance::init_qnn_graph(const char * graph_name, bool debug, uint8_t do_node_validation, - const QnnGraph_Config_t ** graph_configs) { + const QnnGraph_Config_t ** graph_configs) { int result = 0; if (nullptr == graph_name) { @@ -2941,7 +2048,103 @@ int qnn_instance::finalize_qnn_graph() { return 0; } -static uint8_t * create_rpc_buffer(qnn_instance * instance, const ggml_tensor * ggml_tensor, Qnn_Tensor_t * qnn_tensor, bool b_copydata) { +int qnn_instance::init_htp_perfinfra() { + QnnDevice_Infrastructure_t device_infra = nullptr; + int error = _qnn_raw_interface.deviceGetInfrastructure(&device_infra); + if (error != QNN_SUCCESS) { + GGMLQNN_LOG_WARN("failed to get qnn device infra\n"); + return 1; + } + + QnnHtpDevice_Infrastructure_t * htp_infra = static_cast(device_infra); + QnnHtpDevice_PerfInfrastructure_t * htp_perfinfra = &htp_infra->perfInfra; + uint32_t power_configid = 1; + uint32_t device_id = 0; + uint32_t core_id = 0; + htp_perfinfra->createPowerConfigId(device_id, core_id, &power_configid); + _qnn_htp_perfinfra = htp_perfinfra; + _qnn_power_configid = power_configid; + + return 0; +} + +int qnn_instance::set_rpc_polling() { + if (_qnn_rpc_pollingtime > 0) { + QnnHtpPerfInfrastructure_PowerConfig_t rpc_pollingtime; + memset(&rpc_pollingtime, 0, sizeof(rpc_pollingtime)); + rpc_pollingtime.option = QNN_HTP_PERF_INFRASTRUCTURE_POWER_CONFIGOPTION_RPC_POLLING_TIME; + rpc_pollingtime.rpcPollingTimeConfig = _qnn_rpc_pollingtime; + const QnnHtpPerfInfrastructure_PowerConfig_t * power_configs[] = {&rpc_pollingtime, nullptr}; + if (_qnn_htp_perfinfra) { + _qnn_htp_perfinfra->setPowerConfig(_qnn_power_configid, power_configs); + } + } + return 0; +} + +int qnn_instance::set_high_performance_mode() { + if (nullptr == _qnn_htp_perfinfra) { + GGMLQNN_LOG_DEBUG("perf intra is null\n"); + return 1; + } + + QnnHtpPerfInfrastructure_PowerConfig_t power_config; + memset(&power_config, 0, sizeof(power_config)); + power_config.option = QNN_HTP_PERF_INFRASTRUCTURE_POWER_CONFIGOPTION_DCVS_V3; + power_config.dcvsV3Config.dcvsEnable = 0; + power_config.dcvsV3Config.setDcvsEnable = 1; + power_config.dcvsV3Config.contextId = _qnn_power_configid; + power_config.dcvsV3Config.powerMode = QNN_HTP_PERF_INFRASTRUCTURE_POWERMODE_PERFORMANCE_MODE; + power_config.dcvsV3Config.setSleepLatency = 1; // True to consider Latency parameter otherwise False + power_config.dcvsV3Config.setBusParams = 1; // True to consider Bus parameter otherwise False + power_config.dcvsV3Config.setCoreParams = 1; // True to consider Core parameter otherwise False + power_config.dcvsV3Config.sleepDisable = 0; // True to consider sleep/LPM modes, False to enable + power_config.dcvsV3Config.setSleepDisable = 0; // True to consider sleep disable/enable parameter otherwise False + // set Sleep latency parameter + uint32_t latencyValue = 40; + power_config.dcvsV3Config.sleepLatency = latencyValue; // range 40-2000 micro sec + // set Bus Clock Parameters (refer QnnHtpPerfInfrastructure_VoltageCorner_t enum) + power_config.dcvsV3Config.busVoltageCornerMin = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; + power_config.dcvsV3Config.busVoltageCornerTarget = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; + power_config.dcvsV3Config.busVoltageCornerMax = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; + // set Core Clock Parameters (refer QnnHtpPerfInfrastructure_VoltageCorner_t enum) + power_config.dcvsV3Config.coreVoltageCornerMin = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; + power_config.dcvsV3Config.coreVoltageCornerTarget = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; + power_config.dcvsV3Config.coreVoltageCornerMax = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; + // set power config with different performance parameters + const QnnHtpPerfInfrastructure_PowerConfig_t * power_configs[] = {&power_config, nullptr}; + + _qnn_htp_perfinfra->setPowerConfig(_qnn_power_configid, power_configs); + + return 0; +} + +void qnn_instance::probe_device_meminfo() { + size_t candidate_size = 0; + uint8_t * rpc_buffer = nullptr; + const int SIZE_IN_MB = (1 << 20); + size_t probe_slots[] = {1024, 1536, 2048 - 48, 2048}; + size_t probe_counts = sizeof(probe_slots) / sizeof(size_t); + for (size_t idx = 0; idx < probe_counts; idx++) { + rpc_buffer = static_cast(alloc_rpcmem_internal(probe_slots[idx] * SIZE_IN_MB, 4)); + if (nullptr == rpc_buffer) { + GGMLQNN_LOG_DEBUG("alloc rpcmem %d (MB) failure, %s\n", probe_slots[idx], strerror(errno)); + break; + } else { + candidate_size = probe_slots[idx]; + free_rpcmem(rpc_buffer); + rpc_buffer = nullptr; + } + } + if (candidate_size > _rpcmem_capacity) + _rpcmem_capacity = candidate_size; + + free_rpcmem(); + _rpcmem_usage = 0; + GGMLQNN_LOG_INFO("capacity of rpc ion memory %d MB\n", _rpcmem_capacity); +} + +uint8_t * ggmlqnn_create_rpc_buffer(qnn_instance * instance, const ggml_tensor * ggml_tensor, Qnn_Tensor_t * qnn_tensor, bool b_copydata) { if (nullptr == instance || nullptr == ggml_tensor || nullptr == qnn_tensor) { GGMLQNN_LOG_WARN("invalid params\n"); return nullptr; @@ -2960,7 +2163,7 @@ static uint8_t * create_rpc_buffer(qnn_instance * instance, const ggml_tensor * return qnn_rpcbuffer; } -static void print_tensors_info(const char * func_name, ggml_backend_qnn_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +void ggmlqnn_print_tensors_info(const char * func_name, ggml_backend_qnn_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { //skip sanity check of params if (nullptr != func_name && nullptr != ctx) { GGMLQNN_LOG_DEBUG("call %s in dev %s\n", func_name, ctx->name); @@ -2986,45 +2189,30 @@ static void dump_op_info(const struct ggml_tensor * tensor) { struct ggml_tensor * src1 = tensor->src[1]; struct ggml_tensor * dst = const_cast(tensor); GGMLQNN_LOG_DEBUG("op name:%s, tensor type:%s", ggml_op_name(tensor->op), ggml_type_name(tensor->type)); - print_tensors_info(nullptr, nullptr, src0, src1, dst); -} - -//TODO: currently only support offloading 2D matrix to QNN backend -static void get_qnn_dimensions_from_ggml_dimensions(uint32_t * qnn_dimensions, uint32_t * ggml_dimensions, uint32_t rank) { - if (rank > GGML_MAX_DIMS) { - GGMLQNN_LOG_WARN("invalid params"); - return; - } - if (nullptr == qnn_dimensions || nullptr == ggml_dimensions) { - GGMLQNN_LOG_WARN("invalid params"); - return; - } - for (size_t idx = 0; idx < GGML_MAX_DIMS; idx++) - qnn_dimensions[idx] = ggml_dimensions[idx]; - - if (rank >= 2) { - qnn_dimensions[rank - 1] = ggml_dimensions[rank - 2]; - qnn_dimensions[rank - 2] = ggml_dimensions[rank - 1]; - } + ggmlqnn_print_tensors_info(nullptr, nullptr, src0, src1, dst); } // ================================================================================================= // section-6: implementation of ggml-qnn backend // ================================================================================================= //TODO: refine this function as it is a performance hotspot/bottleneck function -static bool ggml_qnn_can_handle_op(const struct ggml_tensor * tensor) { +static bool ggml_qnn_can_handle_op(const ggml_backend_qnn_context * ctx, const struct ggml_tensor * tensor) { if (tensor->op == GGML_OP_NONE) { return true; } if (ggml_is_empty(tensor) || tensor->op == GGML_OP_RESHAPE - || tensor->op == GGML_OP_TRANSPOSE || tensor->op == GGML_OP_VIEW - || tensor->op == GGML_OP_PERMUTE) { + || tensor->op == GGML_OP_TRANSPOSE + || tensor->op == GGML_OP_VIEW + || tensor->op == GGML_OP_PERMUTE + ) { return false; } - //TODO: support other op - bool supported_op = ((tensor->op == GGML_OP_ADD) || (tensor->op == GGML_OP_MUL_MAT) - || (tensor->op == GGML_OP_MUL)); + //TODO: add other op here + bool supported_op = ((tensor->op == GGML_OP_ADD) + || (tensor->op == GGML_OP_MUL_MAT) + || (tensor->op == GGML_OP_MUL) + ); if (!supported_op) { return false; } @@ -3032,20 +2220,25 @@ static bool ggml_qnn_can_handle_op(const struct ggml_tensor * tensor) { struct ggml_tensor * src0 = tensor->src[0]; struct ggml_tensor * src1 = tensor->src[1]; - const int64_t ne00 = tensor->src[0]->ne[0]; - const int64_t ne01 = tensor->src[0]->ne[1]; + const int64_t ne00 = tensor->src[0]->ne[0]; + const int64_t ne01 = tensor->src[0]->ne[1]; - const int64_t ne10 = tensor->src[1]->ne[0]; - const int64_t ne11 = tensor->src[1]->ne[1]; + const int64_t ne10 = tensor->src[1]->ne[0]; + const int64_t ne11 = tensor->src[1]->ne[1]; - const int64_t ne0 = tensor->ne[0]; - const int64_t ne1 = tensor->ne[1]; + const int64_t ne0 = tensor->ne[0]; + const int64_t ne1 = tensor->ne[1]; - const uint32_t src0_rank = ggml_get_tensor_rank(src0); - const uint32_t src1_rank = ggml_get_tensor_rank(src1); + const uint32_t src0_rank = ggml_n_dims(src0); + const uint32_t src1_rank = ggml_n_dims(src1); + GGML_UNUSED(ne01); + GGML_UNUSED(ne10); + GGML_UNUSED(ne11); + GGML_UNUSED(ne0); + GGML_UNUSED(ne1); if (tensor->op == GGML_OP_ADD) { - //dump_tensors_info(tensor); + //dump_op_info(tensor); if (!ggml_are_same_shape(src0, src1)) { return false; } @@ -3056,27 +2249,31 @@ static bool ggml_qnn_can_handle_op(const struct ggml_tensor * tensor) { } if (tensor->op == GGML_OP_MUL_MAT) { - dump_op_info(tensor); + //dump_op_info(tensor); if (src0_rank != src1_rank) // make QNN SDK happy return false; - if (src0_rank < 2) // make QNN SDK happy + if (src0_rank < 2) // QNN's limitation, make QNN SDK happy return false; - if (src0_rank > 3) //TODO: 4D matrix + if (4 == src0_rank) //TODO: 4D matrix mulmat in CT return false; if ((src1->ne[2] != src0->ne[2]) || (src1->ne[3] != src0->ne[3])) // make QNN SDK happy return false; - //TODO: support more data type in func ggml_qnn_mul_mat(...) - //src0: q4_0, q6_k, ... - //src1: f32 - //dst : f32 - return (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16) - && (src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16) - && (src0->type == src1->type) && (src0->type == tensor->type); + if (ctx->device == QNN_BACKEND_NPU) + if (2 == src0_rank) + return (src0->type == GGML_TYPE_F32 + || src0->type == GGML_TYPE_Q4_0 || src0->type == GGML_TYPE_Q8_0 + || src0->type == GGML_TYPE_Q6_K || src0->type == GGML_TYPE_Q8_K + ) && (src1->type == GGML_TYPE_F32) && (tensor->type == GGML_TYPE_F32); + else + return (src0->type == GGML_TYPE_F32) && (src1->type == GGML_TYPE_F32) && (tensor->type == GGML_TYPE_F32); + else + return (src0->type == GGML_TYPE_F32 || ggml_is_quantized(src0->type)) + && (src1->type == GGML_TYPE_F32) && (tensor->type == GGML_TYPE_F32); } if (tensor->op == GGML_OP_MUL) { - //dump_tensors_info(tensor); + //dump_op_info(tensor); if ((src0_rank != 2) || (src1_rank != 2)) //TODO: 3D and 4D matrix return false; return (src0->type == GGML_TYPE_F32) @@ -3084,483 +2281,135 @@ static bool ggml_qnn_can_handle_op(const struct ggml_tensor * tensor) { && (tensor->type == src1->type); } - return (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16) - && (src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16) - && (src0->type == src1->type) && (src0->type == tensor->type); -} - -/* - * provide a general skeleton to offload ggml op to QNN backend: a single node contains 2 input - * tensor and 1 output tensor -*/ -static void ggml_qnn_general_node(ggml_backend_t backend, ggml_tensor * op) { - Qnn_ErrorHandle_t error = QNN_SUCCESS; - enum ggml_status result = GGML_STATUS_SUCCESS; - bool graph_initialized = false; - qnn_instance * instance = nullptr; - ggml_backend_qnn_context * ctx = (ggml_backend_qnn_context *)backend->context; - Qnn_GraphHandle_t graph_handle = nullptr; - Qnn_Tensor_t * p_tensor0 = nullptr; - Qnn_Tensor_t * p_tensor1 = nullptr; - Qnn_Tensor_t * p_tensor2 = nullptr; - Qnn_Param_t qnn_params[] = {}; - const ggml_tensor * src0 = op->src[0]; - const ggml_tensor * src1 = op->src[1]; - ggml_tensor * dst = op; - - GGMLQNN_CHECK_PARAMS(ctx, src0, src1, dst); - instance = ctx->instance; - QNN_INTERFACE_VER_TYPE qnn_raw_interface = ctx->raw_interface; - - size_t qnn_op_index = get_qnn_op_index(op); - GGML_ASSERT(qnn_op_index < std::size(k_op_caps)); - const char * qnn_op_name = k_op_caps[qnn_op_index].qnn_op_name; - std::string ggml_op_name_string = std::string("ggml_") + ggml_op_name(op->op); - const char * ggml_op_name = ggml_op_name_string.c_str(); - - qnn_perf op_perf = qnn_perf(ggml_op_name); - op_perf.start(); - - std::string graph_name; - get_graph_key_from_op(op, graph_name); - if (instance->_qnn_graph_map.find(graph_name) != instance->_qnn_graph_map.end()) { - graph_initialized = true; - qnn_res_t & graph_item = instance->_qnn_graph_map[graph_name]; - graph_handle = std::get<0>(graph_item); - qnn_tensors_t & tensor = std::get<1>(graph_item); - p_tensor0 = tensor[0]; - p_tensor1 = tensor[1]; - p_tensor2 = tensor[2]; - } else { - p_tensor0 = ggml_qnn_create_compute_tensor(src0); - p_tensor1 = ggml_qnn_create_compute_tensor(src1); - p_tensor2 = ggml_qnn_create_compute_tensor(dst); - } - print_tensors_info(__func__, ctx, src0, src1, dst); - - //ensure QNN tensor has correct tensor type - QNN_VER_PTR(*p_tensor0)->type = QNN_TENSOR_TYPE_APP_WRITE; - QNN_VER_PTR(*p_tensor1)->type = QNN_TENSOR_TYPE_APP_WRITE; - QNN_VER_PTR(*p_tensor2)->type = QNN_TENSOR_TYPE_APP_READ; - - //save the original dimensions of qnn tensors - uint32_t * tensor_0_dimensions = QNN_VER_PTR(*p_tensor0)->dimensions; - uint32_t * tensor_1_dimensions = QNN_VER_PTR(*p_tensor1)->dimensions; - uint32_t * tensor_2_dimensions = QNN_VER_PTR(*p_tensor2)->dimensions; - - bool enable_npu_rpc = instance->enable_qnn_rpc() && ctx->device == QNN_BACKEND_NPU; - - if (!graph_initialized) { - GGMLQNN_LOG_DEBUG("graph name %s", graph_name.c_str()); - error = instance->init_qnn_graph(graph_name, static_cast(ctx->device), 8); - if (QNN_SUCCESS != error) { - GGMLQNN_LOG_INFO("can't create qnn graph handle with graph name %s, error = %d\n", graph_name.c_str(), error); - return; - } - graph_handle = instance->get_qnn_graph_handle(); - - if (enable_npu_rpc) { - QNN_VER_PTR(*p_tensor0)->memType = QNN_TENSORMEMTYPE_MEMHANDLE; - QNN_VER_PTR(*p_tensor0)->clientBuf = {.data=nullptr, .dataSize=0}; - - QNN_VER_PTR(*p_tensor1)->memType = QNN_TENSORMEMTYPE_MEMHANDLE; - QNN_VER_PTR(*p_tensor1)->clientBuf = {.data=nullptr, .dataSize=0}; - - QNN_VER_PTR(*p_tensor2)->memType = QNN_TENSORMEMTYPE_MEMHANDLE; - QNN_VER_PTR(*p_tensor2)->clientBuf = {.data=nullptr, .dataSize=0}; - } - - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor0)); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor1)); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor2)); - - if (enable_npu_rpc) { - uint8_t * qnn_rpcbuffer_0 = create_rpc_buffer(instance, src0, p_tensor0, true); - uint8_t * qnn_rpcbuffer_1 = create_rpc_buffer(instance, src1, p_tensor1, true); - uint8_t * qnn_rpcbuffer_2 = create_rpc_buffer(instance, dst, p_tensor2, false); - if (nullptr == qnn_rpcbuffer_0 || nullptr == qnn_rpcbuffer_1 || nullptr == qnn_rpcbuffer_2) { - GGMLQNN_LOG_INFO("create rpc buffer failure\n"); - //TODO: potential memory leak although it shouldn't happen - return; - } - } else { - QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggml_get_tensor_data_size(src0)}; - QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggml_get_tensor_data_size(src1)}; - QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggml_get_tensor_data_size(dst)}; - } - - Qnn_Tensor_t tensor_inputs[] = { - *p_tensor0, - *p_tensor1 - }; - Qnn_Tensor_t tensor_outputs[] = { - *p_tensor2 - }; - Qnn_OpConfig_t op_config = { - QNN_OPCONFIG_VERSION_1, .v1 = { - ggml_op_name, - QNN_OP_PACKAGE_NAME_QTI_AISW, - qnn_op_name, - 0, - qnn_params, - 2, - tensor_inputs, - 1, - tensor_outputs - } - }; - CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, op_config)); - CHECK_QNN_API(error, qnn_raw_interface.graphFinalize(graph_handle, nullptr, nullptr)); - CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, - tensor_inputs, 2, - tensor_outputs, 1, - nullptr, nullptr)); - - if (enable_npu_rpc) { - uint8_t * qnn_rpcbuffer = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*p_tensor2)->memHandle)); - GGMLQNN_LOG_INFO("qnn_rpcbuffer = %p\n", qnn_rpcbuffer); - if (nullptr != qnn_rpcbuffer) { - memcpy(dst->data, qnn_rpcbuffer, ggml_nbytes(dst)); - } - } - - qnn_tensors_t ggml_op_add_tensors; - ggml_op_add_tensors.reserve(3); - ggml_op_add_tensors.push_back(p_tensor0); - ggml_op_add_tensors.push_back(p_tensor1); - ggml_op_add_tensors.push_back(p_tensor2); - - auto graph_item = std::make_tuple(graph_handle, ggml_op_add_tensors); - instance->_qnn_graph_map[graph_name] = graph_item; - - } else { - Qnn_DataType_t src0_qnn_type = QNN_DATATYPE_FLOAT_32; - Qnn_DataType_t src1_qnn_type = QNN_DATATYPE_FLOAT_32; - Qnn_DataType_t dst_qnn_type = QNN_DATATYPE_FLOAT_32; - - src0_qnn_type = qnn_datatype_from_ggml_datatype(src0->type); - src1_qnn_type = qnn_datatype_from_ggml_datatype(src1->type); - dst_qnn_type = qnn_datatype_from_ggml_datatype(dst->type); - - uint32_t dimensions_input_0[] = {(uint32_t) src0->ne[0], (uint32_t) src0->ne[1], - (uint32_t) src0->ne[2], (uint32_t) src0->ne[3]}; - uint32_t dimensions_input_1[] = {(uint32_t) src1->ne[0], (uint32_t) src1->ne[1], - (uint32_t) src1->ne[2], (uint32_t) src1->ne[3]}; - uint32_t dimensions_output[] = {(uint32_t) dst->ne[0], (uint32_t) dst->ne[1], - (uint32_t) dst->ne[2], (uint32_t) dst->ne[3]}; - - QNN_VER_PTR(*p_tensor0)->dimensions = dimensions_input_0; - QNN_VER_PTR(*p_tensor0)->rank = ggml_get_tensor_rank(src0); - QNN_VER_PTR(*p_tensor0)->dataType = src0_qnn_type; - - QNN_VER_PTR(*p_tensor1)->dimensions = dimensions_input_1; - QNN_VER_PTR(*p_tensor1)->rank = ggml_get_tensor_rank(src1); - QNN_VER_PTR(*p_tensor1)->dataType = src1_qnn_type; - - QNN_VER_PTR(*p_tensor2)->dimensions = dimensions_output; - QNN_VER_PTR(*p_tensor2)->rank = ggml_get_tensor_rank(dst); - QNN_VER_PTR(*p_tensor2)->dataType = dst_qnn_type; - - if (enable_npu_rpc) { - //TODO: NPU RPC feature will failed with test-backend-ops - uint8_t * qnn_buffer_0 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*p_tensor0)->memHandle)); - GGMLQNN_LOG_INFO("qnn_rpcbuffer_0 = %p\n", qnn_buffer_0); - if (nullptr != qnn_buffer_0) { - memcpy(qnn_buffer_0, src0->data, ggml_nbytes(src0)); - } - - uint8_t * qnn_buffer_1 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*p_tensor1)->memHandle)); - GGMLQNN_LOG_INFO("qnn_rpcbuffer_1 = %p\n", qnn_buffer_1); - if (nullptr != qnn_buffer_1) { - memcpy(qnn_buffer_1, src1->data, ggml_nbytes(src1)); - } - } else { - QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggml_get_tensor_data_size(src0)}; - QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggml_get_tensor_data_size(src1)}; - QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggml_get_tensor_data_size(dst)}; - } - - Qnn_Tensor_t tensor_inputs[] = { - *p_tensor0, - *p_tensor1 - }; - Qnn_Tensor_t tensor_outputs[] = { - *p_tensor2 - }; - CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, - tensor_inputs, 2, - tensor_outputs, 1, - nullptr, nullptr)); - - if (enable_npu_rpc) { - //TODO:NPU RPC feature will failed with test-backend-ops - uint8_t * qnn_buffer_2 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*p_tensor2)->memHandle)); - if (nullptr != qnn_buffer_2) { - memcpy(dst->data, qnn_buffer_2, ggml_nbytes(dst)); - } - } - } - - // restore the original dimensions of qnn tensors to avoid memory leak in func free_qnn_tensor - QNN_VER_PTR(*p_tensor0)->dimensions = tensor_0_dimensions; - QNN_VER_PTR(*p_tensor1)->dimensions = tensor_1_dimensions; - QNN_VER_PTR(*p_tensor2)->dimensions = tensor_2_dimensions; - op_perf.info(); -} - -/* - * the logic of ggml_qnn_mul_mat is similar to ggml_qnn_general_node but much more complicated - * than ggml_qnn_general_node. - * matrix transpose and type trait are required for offload mulmat to QNN backend, - * so it's a standalone function. accordingly, this is another typical skeleton for offload other - * ggml ops to QNN backend - * - * MUL_MAT take most of the compute time (about 95%).so to speed up llama inference, should focus on MUL_MAT. - * - * have three kinds of MUL_MAT to compute: - * mul_mat_f32: both src0 and src1 are F32, this will be naturally handled in QNN backend - * mul_mat_f16_f32: src0 is F16 and src1 is F32, f16 in src0 -> f32 in src0', then src0' * src1 - * mul_mat_q_f32: src0 is quantized (Q4_0, Q4_1, ...) and src1 is F32, src0 -> f32 in src0', then src0' * src1 -*/ -static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { - Qnn_ErrorHandle_t error = QNN_SUCCESS; - bool graph_initialized = false; - qnn_perf op_perf = qnn_perf("ggml_qnn_mul_mat"); - qnn_instance * instance = nullptr; - ggml_backend_qnn_context * ctx = (ggml_backend_qnn_context *) backend->context; - Qnn_GraphHandle_t graph_handle = nullptr; - Qnn_Tensor_t * p_tensor0 = nullptr; - Qnn_Tensor_t * p_tensor1 = nullptr; - Qnn_Tensor_t * p_tensor2 = nullptr; - Qnn_Tensor_t * p_param_tensor = nullptr; - Qnn_Tensor_t * p_tensor2_transpose = nullptr; - const ggml_tensor * src0 = op->src[0]; - const ggml_tensor * src1 = op->src[1]; - ggml_tensor * dst = op; - - GGMLQNN_CHECK_PARAMS(ctx, src0, src1, dst); - instance = ctx->instance; - QNN_INTERFACE_VER_TYPE qnn_raw_interface = ctx->raw_interface; - op_perf.start(); - - uint32_t src0_rank = ggml_get_tensor_rank(src0); - uint32_t src1_rank = ggml_get_tensor_rank(src1); - GGML_ASSERT(src0_rank == src1_rank); - GGML_ASSERT(src0_rank >= 2); //QNN SDK's limitation - - std::string graph_name; - get_graph_key_from_op(op, graph_name); - if (instance->_qnn_graph_map.find(graph_name) != instance->_qnn_graph_map.end()) { - graph_initialized = true; - qnn_res_t & graph_item = instance->_qnn_graph_map[graph_name]; - graph_handle = std::get<0>(graph_item); - qnn_tensors_t & tensors = std::get<1>(graph_item); - p_tensor0 = tensors[0]; - p_tensor1 = tensors[1]; - p_tensor2 = tensors[2]; - p_param_tensor = tensors[3]; - p_tensor2_transpose = tensors[4]; - } else { - p_tensor0 = ggml_qnn_create_general_tensor(src0, nullptr, QNN_TENSOR_TYPE_APP_WRITE,QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0); - p_tensor1 = ggml_qnn_create_general_tensor(src1, nullptr, QNN_TENSOR_TYPE_APP_WRITE,QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0); - p_tensor2 = ggml_qnn_create_general_tensor(dst, nullptr, QNN_TENSOR_TYPE_APP_READ,QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0); - } - - print_tensors_info(__func__, ctx, src0, src1, dst); - - //ensure QNN tensor has correct tensor type - QNN_VER_PTR(*p_tensor0)->type = QNN_TENSOR_TYPE_APP_WRITE; - QNN_VER_PTR(*p_tensor1)->type = QNN_TENSOR_TYPE_APP_WRITE; - QNN_VER_PTR(*p_tensor2)->type = QNN_TENSOR_TYPE_APP_READ; - - //save the original dimensions of qnn tensors - uint32_t * tensor_0_dimensions = QNN_VER_PTR(*p_tensor0)->dimensions; - uint32_t * tensor_1_dimensions = QNN_VER_PTR(*p_tensor1)->dimensions; - uint32_t * tensor_2_dimensions = QNN_VER_PTR(*p_tensor2)->dimensions; - - if (!graph_initialized) { - GGMLQNN_LOG_DEBUG("graph name %s", graph_name.c_str()); - /* - there are two key-points in properly handling how to offload mulmat to the QNN backend in ggml-qnn - 1. transpose - a 3x2 f32 matrix which means 3 rows and 2 columns. in ggml, it could be created from: - struct ggml_tensor* matrix = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 2, 3); - which like this: - +---+---+ - | 0 | 1 | - +---+---+ - | 2 | 3 | - +---+---+ - | 4 | 5 | - +---+---+ - with - ne[0] = 2 - ne[1] = 3 - there are different dimension order between ggml tensor and qnn tensor - - 2. QNN's MatMul can only support input tensors with rank >= 2 - - there is gap between ggml mulmat and QNN mulmat,we need to perform a transpose operation when offloading mulmat to QNN backend. - */ - - //step-1: create qnn graph - error = qnn_raw_interface.graphCreate(instance->get_qnn_context_handle(), - graph_name.c_str(), nullptr, &graph_handle); - if (QNN_SUCCESS != error) { - GGMLQNN_LOG_INFO("can't create qnn graph handle with graph name %s, error = %d\n", graph_name.c_str(), error); - return; - } - //step-2: create param tensor for mulmat of 2d matrix - const uint32_t param_tensor_data[GGML_MAX_DIMS][GGML_MAX_DIMS] = { - {0}, - {1, 0}, - {0, 2, 1}, - {0, 1, 3, 2}, - }; - uint32_t param_tensor_dims[1] = {src0_rank}; - p_param_tensor = ggml_qnn_create_general_tensor(nullptr, "param", QNN_TENSOR_TYPE_STATIC, QNN_DATATYPE_UINT_32, - 1, param_tensor_dims, - (void *) (param_tensor_data[src0_rank - 1]), src0_rank * sizeof(uint32_t)); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_param_tensor)); - - //step-3: create compute tensor from ggml tensor - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor0)); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor1)); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor2)); - - QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggml_get_tensor_data_size(src0)}; - QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggml_get_tensor_data_size(src1)}; - QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggml_get_tensor_data_size(dst)}; - - //step-4: create a transpose tensor - uint32_t tensor2_transpose_dims[GGML_MAX_DIMS] = {}; - p_tensor2_transpose = ggml_qnn_create_general_tensor(dst, "transpose", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0); - get_qnn_dimensions_from_ggml_dimensions(tensor2_transpose_dims, tensor_2_dimensions, ggml_get_tensor_rank(dst)); - //save pointer because the dimensions of tensor p_tensor2_transpose will be changed later - uint32_t * tensor2_dimensions_transpose = QNN_VER_PTR(*p_tensor2_transpose)->dimensions; - //update dimensions of tensor p_tensor2_transpose to make QNN SDK happy - QNN_VER_PTR(*p_tensor2_transpose)->dimensions = tensor2_transpose_dims; - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor2_transpose)); - - //step-5: compose qnn graph: add mat_mul node - Qnn_Param_t out_0_params[] = { - {QNN_PARAMTYPE_SCALAR, - QNN_OP_MAT_MUL_PARAM_TRANSPOSE_IN1, - .scalarParam = {QNN_DATATYPE_BOOL_8, .bool8Value = 1} - } - }; - - Qnn_Tensor_t out_0_inputs[] = {*p_tensor0, *p_tensor1}; - Qnn_Tensor_t out_0_outputs[] = {*p_tensor2_transpose}; - Qnn_OpConfig_t out_0 = { - QNN_OPCONFIG_VERSION_1, .v1 = - {"ggmlqnn_mulmat_opconfig", QNN_OP_PACKAGE_NAME_QTI_AISW, QNN_OP_MAT_MUL, - 1, - out_0_params, - 2, - out_0_inputs, - 1, - out_0_outputs} - }; - CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle,out_0)); - - //step-5: compose qnn graph: add transpose node - Qnn_Param_t out_trans1_0_params[] = { - {(Qnn_ParamType_t) 1, - "perm", .tensorParam = *p_param_tensor - } - }; - Qnn_Tensor_t out_trans1_0_inputs[] = {*p_tensor2_transpose}; - Qnn_Tensor_t out_trans1_0_outputs[] = {*p_tensor2}; - Qnn_OpConfig_t out_trans1_0 = { - QNN_OPCONFIG_VERSION_1, - .v1 = {"ggmlqnn_mulmat_transpose_opconfig", - "qti.aisw", - QNN_OP_TRANSPOSE, 1, - out_trans1_0_params, - 1, - out_trans1_0_inputs, - 1, - out_trans1_0_outputs} - }; - CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle,out_trans1_0)); - - //step-6: finalize qnn graph and execute qnn graph - CHECK_QNN_API(error, qnn_raw_interface.graphFinalize(graph_handle, NULL, NULL)); - Qnn_Tensor_t input_tensors_0[] = {*p_tensor0, *p_tensor1}; - Qnn_Tensor_t output_tensors_0[] = {*p_tensor2}; - CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, - input_tensors_0, 2, - output_tensors_0, 1, - NULL, NULL)); - - qnn_tensors_t ggml_op_mulmat_tensors; - ggml_op_mulmat_tensors.reserve(5); - ggml_op_mulmat_tensors.push_back(p_tensor0); - ggml_op_mulmat_tensors.push_back(p_tensor1); - ggml_op_mulmat_tensors.push_back(p_tensor2); - ggml_op_mulmat_tensors.push_back(p_param_tensor); - ggml_op_mulmat_tensors.push_back(p_tensor2_transpose); - - auto graph_item = std::make_tuple(graph_handle, ggml_op_mulmat_tensors); - instance->_qnn_graph_map[graph_name] = graph_item; - - //avoid cleanup these resource to make test_backend_ops happy - //free_qnn_tensor(p_param_tensor); - //restore pointer to avoid memory leak - QNN_VER_PTR(*p_tensor2_transpose)->dimensions = tensor2_dimensions_transpose; - //free_qnn_tensor(p_tensor2_transpose); - } else { - QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggml_get_tensor_data_size(src0)}; - QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggml_get_tensor_data_size(src1)}; - QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggml_get_tensor_data_size(dst)}; - - Qnn_Tensor_t tensor_inputs[] = { - *p_tensor0, - *p_tensor1 - }; - Qnn_Tensor_t tensor_outputs[] = { - *p_tensor2 - }; - //attention: - // this is the second technical approach of "how to utilize the Hexagon NPU maximally" through - // QNN SDK, details could be found at - // https://github.com/kantv-ai/llama.cpp/wiki/mapping-ggml-compute-graph-to-QNN-compute-graph - CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, - tensor_inputs, 2, - tensor_outputs, 1, - nullptr, nullptr)); - } - - // restore the original dimensions of qnn tensors to avoid memory leak in func free_qnn_tensor - QNN_VER_PTR(*p_tensor0)->dimensions = tensor_0_dimensions; - QNN_VER_PTR(*p_tensor1)->dimensions = tensor_1_dimensions; - QNN_VER_PTR(*p_tensor2)->dimensions = tensor_2_dimensions; - op_perf.info(); + return false; } -static bool ggml_qnn_compute_forward(ggml_backend_t backend, struct ggml_tensor * tensor) { +static bool ggml_qnn_compute_forward(ggml_backend_t backend, struct ggml_tensor * dst) { ggmlqnn_op_func_t func = nullptr; + ggml_backend_qnn_context * ctx = (ggml_backend_qnn_context *)backend->context; - switch (tensor->op) { + switch (dst->op) { + case GGML_OP_REPEAT: + ggml_qnn_repeat(ctx, dst); + break; + case GGML_OP_GET_ROWS: + ggml_qnn_get_rows(ctx, dst); + break; + case GGML_OP_DUP: + ggml_qnn_dup(ctx, dst); + break; case GGML_OP_ADD: func = ggml_qnn_general_node; break; - - case GGML_OP_MUL_MAT: - func = ggml_qnn_mul_mat; + case GGML_OP_ACC: + ggml_qnn_acc(ctx, dst); break; - case GGML_OP_MUL: func = ggml_qnn_general_node; break; - + case GGML_OP_DIV: + ggml_qnn_div(ctx, dst); + break; + case GGML_OP_UNARY: + switch (ggml_get_unary_op(dst)) { + case GGML_UNARY_OP_GELU: + break; + case GGML_UNARY_OP_SILU: + break; + case GGML_UNARY_OP_GELU_QUICK: + break; + case GGML_UNARY_OP_TANH: + break; + case GGML_UNARY_OP_RELU: + break; + case GGML_UNARY_OP_HARDSIGMOID: + break; + case GGML_UNARY_OP_HARDSWISH: + break; + default: + return false; + } + break; + case GGML_OP_NORM: + ggml_qnn_norm(ctx, dst); + break; + case GGML_OP_GROUP_NORM: + ggml_qnn_group_norm(ctx, dst); + break; + case GGML_OP_CONCAT: + ggml_qnn_concat(ctx, dst); + break; + case GGML_OP_UPSCALE: + ggml_qnn_upsample_nearest2d(ctx, dst); + break; + case GGML_OP_PAD: + ggml_qnn_pad(ctx, dst); + break; + case GGML_OP_ARANGE: + ggml_qnn_arange(ctx, dst); + break; + case GGML_OP_TIMESTEP_EMBEDDING: + ggml_qnn_timestep_embedding(ctx, dst); + break; + case GGML_OP_LEAKY_RELU: + ggml_qnn_leaky_relu(ctx, dst); + break; + case GGML_OP_RMS_NORM: + ggml_qnn_rms_norm(ctx, dst); + break; + case GGML_OP_MUL_MAT: + ggml_qnn_mul_mat(ctx, dst); + break; + case GGML_OP_MUL_MAT_ID: + return false; + case GGML_OP_SCALE: + ggml_qnn_scale(ctx, dst); + break; + case GGML_OP_SQR: + ggml_qnn_sqr(ctx, dst); + break; + case GGML_OP_CLAMP: + ggml_qnn_clamp(ctx, dst); + break; + case GGML_OP_CPY: + ggml_qnn_cpy(ctx, dst); + break; + case GGML_OP_CONT: + ggml_qnn_dup(ctx, dst); + break; + case GGML_OP_NONE: + case GGML_OP_RESHAPE: + case GGML_OP_VIEW: + case GGML_OP_PERMUTE: + case GGML_OP_TRANSPOSE: + break; + case GGML_OP_DIAG_MASK_INF: + ggml_qnn_diag_mask(ctx, dst, -INFINITY); + break; + case GGML_OP_SOFT_MAX: + ggml_qnn_softmax(ctx, dst); + break; + case GGML_OP_ROPE: + ggml_qnn_rope(ctx, dst); + break; + case GGML_OP_IM2COL: + ggml_qnn_im2col(ctx, dst); + break; + case GGML_OP_POOL_2D: + ggml_qnn_pool2d(ctx, dst); + break; + case GGML_OP_SUM_ROWS: + ggml_qnn_sum_rows(ctx, dst); + break; + case GGML_OP_ARGSORT: + ggml_qnn_argsort(ctx, dst); + break; default: return false; } if (nullptr != func) - func(backend, tensor); + func(ctx, dst); return true; } @@ -3598,14 +2447,12 @@ static void ggml_backend_qnn_buffer_free_buffer(ggml_backend_buffer_t buffer) { static void * ggml_backend_qnn_buffer_get_base(ggml_backend_buffer_t buffer) { ggml_backend_qnn_buffer_context * ctx = (ggml_backend_qnn_buffer_context *)buffer->context; - return ctx->buffer; } static void ggml_backend_qnn_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) { - Qnn_ErrorHandle_t error = QNN_SUCCESS; ggml_backend_qnn_buffer_context * ctx = (ggml_backend_qnn_buffer_context *)buffer->context; - GGML_UNUSED(error); + GGML_UNUSED(tensor); GGML_UNUSED(ctx); return; } @@ -3649,14 +2496,6 @@ static void ggml_backend_qnn_buffer_clear(ggml_backend_buffer_t buffer, uint8_t memset(ctx->buffer, value, ctx->buffer_size); } -[[maybe_unused]]static void ggml_backend_qnn_buffer_reset(ggml_backend_buffer_t buffer) { - ggml_backend_qnn_buffer_context * ctx = (ggml_backend_qnn_buffer_context *)buffer->context; - for (auto * sub_buffer : ctx->sub_buffers) { - free(sub_buffer); - } - ctx->sub_buffers.clear(); -} - static ggml_backend_buffer_i ggml_backend_qnn_buffer_interface = { /* .free_buffer = */ ggml_backend_qnn_buffer_free_buffer, /* .get_base = */ ggml_backend_qnn_buffer_get_base, @@ -3666,10 +2505,11 @@ static ggml_backend_buffer_i ggml_backend_qnn_buffer_interface = { /* .get_tensor = */ ggml_backend_qnn_buffer_get_tensor, /* .cpy_tensor = */ ggml_backend_qnn_buffer_cpy_tensor, /* .clear = */ ggml_backend_qnn_buffer_clear, - /* .reset = */ NULL, + /* .reset = */ nullptr, }; static const char * ggml_backend_qnn_buffer_type_name(ggml_backend_buffer_type_t buft) { + GGML_UNUSED(buft); return "qnn-buffer"; } @@ -3677,7 +2517,13 @@ static ggml_backend_buffer_t ggml_backend_qnn_buffer_type_alloc_buffer( ggml_backend_buffer_type_t buft, size_t size) { ggml_backend_qnn_buffer_context * ctx = new ggml_backend_qnn_buffer_context; +#if defined(__ANDROID__) || defined(__linux__) size_t size_page = sysconf(_SC_PAGESIZE); +#elif defined(_WIN32) + SYSTEM_INFO systeminfo; + GetSystemInfo(&systeminfo); + size_t size_page = systeminfo.dwPageSize; +#endif size_t size_aligned = size; if ((size_aligned % size_page) != 0) { size_aligned += (size_page - (size_aligned % size_page)); @@ -3697,11 +2543,11 @@ static size_t ggml_backend_qnn_buffer_type_get_alignment(ggml_backend_buffer_typ return 32; } -//FIXME: this value is an experimental value on Xiaomi14 +//TODO:not used currently static size_t ggml_backend_qnn_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) { GGML_UNUSED(buft); - return (2 * (1 << 30)); + return (2 * (1 << 20)); } static bool ggml_backend_qnn_buffer_is_host(ggml_backend_buffer_type_t buft) { @@ -3777,12 +2623,11 @@ static const char * ggml_backend_qnn_device_get_name(ggml_backend_dev_t dev) { return "unknown"; } return ctx->name; - - GGML_UNUSED(dev); } static const char * ggml_backend_qnn_device_get_description(ggml_backend_dev_t dev) { struct ggml_backend_qnn_context * ctx = static_cast(dev->context); + static char qnn_device_desc[256]; if (nullptr == ctx) { GGMLQNN_LOG_ERROR("pls check why ctx is null"); return "unknown"; @@ -3793,7 +2638,9 @@ static const char * ggml_backend_qnn_device_get_description(ggml_backend_dev_t d std::string dev_desc = std::string(ctx->desc) + std::string(soc_info) + "_" + std::string(htp_arch) + "," + std::string(ctx->socinfo.soc_desc); - return dev_desc.c_str(); + memset(qnn_device_desc, 0, 256); + memcpy(qnn_device_desc, dev_desc.c_str(), strlen(dev_desc.c_str())); + return qnn_device_desc; } else { return ctx->desc; } @@ -3855,7 +2702,7 @@ static ggml_backend_t ggml_backend_qnn_device_init_backend(ggml_backend_dev_t de } -ggml_backend_buffer_type_t ggml_backend_qnn_buffer_type(size_t device_index) { +static ggml_backend_buffer_type_t ggml_backend_qnn_buffer_type(size_t device_index) { if (device_index >= GGML_QNN_MAX_DEVICES) { GGMLQNN_LOG_DEBUG("ggml_backend_qnn_buffer_type error: device_index:%d is out of range [0, %d]\n", device_index, GGML_QNN_MAX_DEVICES - 1); @@ -3868,10 +2715,11 @@ ggml_backend_buffer_type_t ggml_backend_qnn_buffer_type(size_t device_index) { /* .alloc_buffer = */ ggml_backend_qnn_buffer_type_alloc_buffer, /* .get_alignment = */ ggml_backend_qnn_buffer_type_get_alignment, /* .get_max_size = */ ggml_backend_qnn_buffer_type_get_max_size, - /* .get_alloc_size = */ NULL,// defaults to ggml_nbytes + /* .get_alloc_size = */ nullptr,// defaults to ggml_nbytes /* .is_host = */ ggml_backend_qnn_buffer_is_host }, - /* .context = */ NULL, + /* .device = */ nullptr, + /* .context = */ nullptr, }; return &ggml_backend_buffer_type_qnn; @@ -3890,10 +2738,9 @@ static ggml_backend_buffer_t ggml_backend_qnn_device_buffer_from_host_ptr(ggml_b GGML_UNUSED(max_tensor_size); } - static bool ggml_backend_qnn_device_supports_op(ggml_backend_dev_t dev, const struct ggml_tensor * op) { ggml_backend_qnn_context * ctx = (ggml_backend_qnn_context *) dev->context; - return (ggml_qnn_can_handle_op(op)); + return (ggml_qnn_can_handle_op(ctx,op)); } static bool ggml_backend_qnn_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) { @@ -3909,14 +2756,14 @@ static struct ggml_backend_device_i ggml_backend_qnn_device_interface = { /* .get_props = */ ggml_backend_qnn_device_get_props, /* .init_backend = */ ggml_backend_qnn_device_init_backend, /* .get_buffer_type = */ ggml_backend_qnn_device_get_buffer_type, - /* .get_host_buffer_type = */ NULL, + /* .get_host_buffer_type = */ nullptr, /* .buffer_from_host_ptr = */ ggml_backend_qnn_device_buffer_from_host_ptr, /* .supports_op = */ ggml_backend_qnn_device_supports_op, /* .supports_buft = */ ggml_backend_qnn_device_supports_buft, - /* .offload_op = */ NULL, - /* .event_new = */ NULL, - /* .event_free = */ NULL, - /* .event_synchronize = */ NULL, + /* .offload_op = */ nullptr, + /* .event_new = */ nullptr, + /* .event_free = */ nullptr, + /* .event_synchronize = */ nullptr, }; static ggml_backend_i ggml_backend_qnn_interface = { @@ -3964,9 +2811,8 @@ struct ggml_backend_qnn_reg_context { }; static const char * ggml_backend_qnn_reg_get_name(ggml_backend_reg_t reg) { - return "ggml-qnn"; - GGML_UNUSED(reg); + return "ggml-qnn"; } static size_t ggml_backend_qnn_reg_get_device_count(ggml_backend_reg_t reg) { @@ -3987,10 +2833,15 @@ static ggml_backend_dev_t ggml_backend_qnn_reg_get_device(ggml_backend_reg_t reg static void * ggml_backend_qnn_reg_get_proc_address(ggml_backend_reg_t reg, const char * name) { GGML_UNUSED(reg); - if (std::strcmp(name, "ggml_backend_set_n_threads") == 0) { + if (nullptr == name) + return nullptr; + + const char * slot_name = "ggml_backend_set_n_threads"; + //avoid buffer attack rather than strcmp + if (0 == std::memcmp(name, slot_name, strlen(slot_name))) { return (void *)ggml_backend_qnn_set_n_threads; } - return NULL; + return nullptr; } static const ggml_backend_reg_i ggml_backend_qnn_reg_interface = { @@ -4057,6 +2908,7 @@ ggml_backend_t ggml_backend_qnn_init(size_t device, const char * qnn_lib_path) { return g_qnn_mgr[device].backend; } +#if defined(__ANDROID__) std::string path = qnn_lib_path; if (QNN_BACKEND_NPU == device) { if (0 == setenv("LD_LIBRARY_PATH", @@ -4085,6 +2937,7 @@ ggml_backend_t ggml_backend_qnn_init(size_t device, const char * qnn_lib_path) { GGMLQNN_LOG_ERROR("%s backend setenv failure\n", ggml_backend_qnn_get_devname(device)); } } +#endif qnn_instance * instance = nullptr; instance = new qnn_instance(qnn_lib_path, g_qnn_mgr[device].lib, ""); diff --git a/scripts/build-run-android.sh b/scripts/build-run-android.sh index 63614e6afe110..3d239510b8d63 100755 --- a/scripts/build-run-android.sh +++ b/scripts/build-run-android.sh @@ -74,7 +74,7 @@ function check_and_download_ndk() function build_arm64 { - cmake -H. -B./out/android -DGGML_USE_QNN=ON -DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK}/build/cmake/android.toolchain.cmake -DANDROID_ABI=arm64-v8a -DANDROID_PLATFORM=latest -DCMAKE_C_FLAGS=-march=armv8.7-a -DGGML_QNN=ON -DGGML_QNN_SDK_PATH=${QNN_SDK_PATH} + cmake -H. -B./out/android -DCMAKE_BUILD_TYPE=Release -DGGML_USE_QNN=ON -DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK}/build/cmake/android.toolchain.cmake -DANDROID_ABI=arm64-v8a -DANDROID_PLATFORM=latest -DCMAKE_C_FLAGS=-march=armv8.7-a -DGGML_QNN=ON -DGGML_QNN_SDK_PATH=${QNN_SDK_PATH} cd out/android make -j16 show_pwd @@ -106,15 +106,15 @@ function check_qnn_libs() function update_qnn_libs() { - adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnSystem.so ${REMOTE_PATH}/ - adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnCpu.so ${REMOTE_PATH}/ - adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnGpu.so ${REMOTE_PATH}/ - - adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnHtp.so ${REMOTE_PATH}/ - adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnHtpNetRunExtensions.so ${REMOTE_PATH}/ - adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnHtpPrepare.so ${REMOTE_PATH}/ - adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnHtpV75Stub.so ${REMOTE_PATH}/ - adb push ${QNN_SDK_PATH}/lib/hexagon-v75/unsigned/libQnnHtpV75Skel.so ${REMOTE_PATH}/ + adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnSystem.so ${REMOTE_PATH}/ + adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnCpu.so ${REMOTE_PATH}/ + adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnGpu.so ${REMOTE_PATH}/ + + adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnHtp.so ${REMOTE_PATH}/ + adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnHtpNetRunExtensions.so ${REMOTE_PATH}/ + adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnHtpPrepare.so ${REMOTE_PATH}/ + adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnHtpV75Stub.so ${REMOTE_PATH}/ + adb push ${QNN_SDK_PATH}/lib/hexagon-v75/unsigned/libQnnHtpV75Skel.so ${REMOTE_PATH}/ } @@ -129,32 +129,37 @@ function build_ggml_qnn() } -function run_llamacli() +function prepare_run_on_phone() { + if [ $# != 1 ]; then + print "invalid param" + return + fi + program=$1 + check_qnn_libs if [ -f ./out/android/bin/libggml-qnn.so ]; then adb push ./out/android/bin/*.so ${REMOTE_PATH}/ fi - adb push ./out/android/bin/llama-cli ${REMOTE_PATH}/ - adb shell chmod +x ${REMOTE_PATH}/llama-cli + adb push ./out/android/bin/${program} ${REMOTE_PATH}/ + adb shell chmod +x ${REMOTE_PATH}/${program} +} + +function run_llamacli() +{ + prepare_run_on_phone llama-cli adb shell "cd ${REMOTE_PATH} \ && export LD_LIBRARY_PATH=${REMOTE_PATH} \ - && ${REMOTE_PATH}/llama-cli -mg ${qnnbackend} -m ${GGUF_MODEL_NAME} -p \"introduce the movie Once Upon a Time in America briefly.\n\"" + && ${REMOTE_PATH}/llama-cli -mg ${qnnbackend} -no-cnv -m ${GGUF_MODEL_NAME} -p \"introduce the movie Once Upon a Time in America briefly.\n\"" } function run_llamabench() { - check_qnn_libs - - if [ -f ./out/android/bin/libggml-qnn.so ]; then - adb push ./out/android/bin/*.so ${REMOTE_PATH}/ - fi - adb push ./out/android/bin/llama-bench ${REMOTE_PATH}/ - adb shell chmod +x ${REMOTE_PATH}/llama-bench + prepare_run_on_phone llama-bench adb shell "cd ${REMOTE_PATH} \ && export LD_LIBRARY_PATH=${REMOTE_PATH} \ @@ -165,13 +170,7 @@ function run_llamabench() function run_test-backend-ops() { - check_qnn_libs - - if [ -f ./out/android/bin/libggml-qnn.so ]; then - adb push ./out/android/bin/*.so ${REMOTE_PATH}/ - fi - adb push ./out/android/bin/test-backend-ops ${REMOTE_PATH}/ - adb shell chmod +x ${REMOTE_PATH}/test-backend-ops + prepare_run_on_phone test-backend-ops adb shell "cd ${REMOTE_PATH} \ && export LD_LIBRARY_PATH=${REMOTE_PATH} \ @@ -179,6 +178,36 @@ function run_test-backend-ops() } +function run_ut_add() +{ + prepare_run_on_phone ggml-qnn-ut + + adb shell "cd ${REMOTE_PATH} \ + && export LD_LIBRARY_PATH=${REMOTE_PATH} \ + && ${REMOTE_PATH}/ggml-qnn-ut -t GGML_OP_ADD -b $qnnbackend" + +} + +function run_ut_mulmat() +{ + prepare_run_on_phone ggml-qnn-ut + + adb shell "cd ${REMOTE_PATH} \ + && export LD_LIBRARY_PATH=${REMOTE_PATH} \ + && ${REMOTE_PATH}/ggml-qnn-ut -t GGML_OP_MUL_MAT -b $qnnbackend" + +} + +function run_ut_mul() +{ + prepare_run_on_phone ggml-qnn-ut + + adb shell "cd ${REMOTE_PATH} \ + && export LD_LIBRARY_PATH=${REMOTE_PATH} \ + && ${REMOTE_PATH}/ggml-qnn-ut -t GGML_OP_MUL -b $qnnbackend" + +} + function show_usage() { @@ -186,6 +215,9 @@ function show_usage() echo " $0 build" echo " $0 updateqnnlib" echo " $0 run_testop" + echo " $0 run_ut_add 0 (QNN_CPU) / 1 (QNN_GPU) / 2 (QNN_NPU) / 3 (ggml)" + echo " $0 run_ut_mulmat 0 (QNN_CPU) / 1 (QNN_GPU) / 2 (QNN_NPU) / 3 (ggml)" + echo " $0 run_ut_mul 0 (QNN_CPU) / 1 (QNN_GPU) / 2 (QNN_NPU) / 3 (ggml)" echo " $0 run_llamacli 0 (QNN_CPU) / 1 (QNN_GPU) / 2 (QNN_NPU) / 3 (ggml)" echo " $0 run_llamabench 0 (QNN_CPU) / 1 (QNN_GPU) / 2 (QNN_NPU) / 3 (ggml)" echo -e "\n\n\n" @@ -213,6 +245,7 @@ elif [ $# == 1 ]; then elif [ "$1" == "run_testop" ]; then run_test-backend-ops exit 0 + elif [ "$1" == "updateqnnlib" ]; then update_qnn_libs exit 0 @@ -233,6 +266,15 @@ elif [ $# == 2 ]; then elif [ "$1" == "run_llamabench" ]; then run_llamabench exit 0 + elif [ "$1" == "run_ut_add" ]; then + run_ut_add + exit 0 + elif [ "$1" == "run_ut_mulmat" ]; then + run_ut_mulmat + exit 0 + elif [ "$1" == "run_ut_mul" ]; then + run_ut_mul + exit 0 fi else show_usage From 436707e9b7a43cb177830ba777850cb65c9418a9 Mon Sep 17 00:00:00 2001 From: zhouwg Date: Tue, 18 Feb 2025 09:53:57 +0800 Subject: [PATCH 13/76] ggml-qnn: merge QNN RPC feature from https://github.com/zhouwg/kantv/blob/ggml-qnn-quantize/core/ggml/llamacpp/ggml-qnn.cpp --- ggml/src/ggml-qnn/ggml-qnn.cpp | 2974 -------------------------------- 1 file changed, 2974 deletions(-) delete mode 100644 ggml/src/ggml-qnn/ggml-qnn.cpp diff --git a/ggml/src/ggml-qnn/ggml-qnn.cpp b/ggml/src/ggml-qnn/ggml-qnn.cpp deleted file mode 100644 index c830128f750c8..0000000000000 --- a/ggml/src/ggml-qnn/ggml-qnn.cpp +++ /dev/null @@ -1,2974 +0,0 @@ -/* - * Copyright (c) 2023-2024 The ggml authors - * - * Qualcomm QNN SDK and reference tech guides could be found at: - * https://www.qualcomm.com/developer/software/qualcomm-ai-engine-direct-sdk - * https://developer.qualcomm.com/software/hexagon-dsp-sdk/tools - * - * the implementation of ggml-qnn backend has six sections: - * section-1 does forward/external declaration, - * section-2 defines ggml-qnn internal log function - * section-3 does general helper macro / data structure / function - * section-4 does QNN helper macro / data structure / function - * section-5 does ggml-qnn backend helper macro / data structure / function / class - * section-6 does implementation of ggml-qnn backend according to ggml's backend subsystem - * - * currently provide following ggml ops' QNN backend implementation in ggml-qnn-ops.cpp: - * - GGML_OP_ADD: this is a simple skeleton, can expand other ggml ops according to expertise - * - GGML_OP_MUL: this is a simple skeleton, can expand other ggml ops according to expertise - * - GGML_OP_MUL_MAT:this is a complicated skeleton, can expand other complex ggml ops accordingly - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ -#include "ggml-qnn-impl.h" -#include "ggml-qnn-ops.h" -// ================================================================================================= -// section-1: forward/external declaration -// ================================================================================================= -static int free_qnn_tensor(Qnn_Tensor_t * tensor); -static enum ggml_status ggml_backend_qnn_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph); -typedef void (* ggmlqnn_op_func_t)(ggml_backend_qnn_context * ctx, ggml_tensor * op); - -// ================================================================================================= -// section-2: ggml-qnn internal troubleshooting function -// ================================================================================================= -void ggmlqnn_log_internal(ggml_log_level level, const char * file, const char * func, int line, const char * format, ...) { - static std::mutex ggmlqnn_log_internal_mutex; - static char s_ggmlqnn_log_internal_buf[GGML_QNN_LOGBUF_LEN]; - - GGML_UNUSED(file); - { - std::lock_guard lock(ggmlqnn_log_internal_mutex); - va_list args; - va_start(args, format); - int len_prefix = snprintf(s_ggmlqnn_log_internal_buf, GGML_QNN_LOGBUF_LEN, "[%s, %d]: ", func, line); - int len = vsnprintf(s_ggmlqnn_log_internal_buf + len_prefix, GGML_QNN_LOGBUF_LEN - len_prefix, format, args); - if (len < (GGML_QNN_LOGBUF_LEN - len_prefix)) { -#if (defined __ANDROID__) || (defined ANDROID) - //for Android application(standard APP or command line tool) - __android_log_print(ANDROID_LOG_INFO, "ggml-qnn", "%s\n", s_ggmlqnn_log_internal_buf); - if (GGML_LOG_LEVEL_INFO == level) { - printf("%s\n", s_ggmlqnn_log_internal_buf); - } -#else - //for Snapdragon based WoA(Windows on ARM) device or Linux - printf("%s\n", s_ggmlqnn_log_internal_buf); -#endif - } - va_end(args); - } -} - -// ================================================================================================= -// section-3: general helper macro / data structure / function -// ================================================================================================= -#if defined(_WIN32) -static const char * last_func = nullptr; -static long last_err; -void * dlopen(const char * dll, int flags) { - HINSTANCE h = LoadLibraryA(dll); - GGML_UNUSED(flags); - if (h == NULL) { - last_err = GetLastError(); - last_func = "dlopen"; - } - return h; -} - -int dlclose(void * h) { - if (!FreeLibrary((HINSTANCE)h)) { - last_err = GetLastError(); - last_func = "dlclose"; - return -1; - } - return 0; -} - -void * dlsym(void * h, const char * name) { - FARPROC p = GetProcAddress((HINSTANCE)h, name); - if (!p) { - last_err = GetLastError(); - last_func = "dlsym"; - } - return (void*)(intptr_t)p; -} - -const char * dlerror(void) { - static char str[512]; - if (!last_err) return nullptr; - - snprintf(str, 512, "%s error #%ld", last_func, last_err); - last_err = 0; - last_func = NULL; - - return str; -} -#endif - -static intptr_t ggmlqnn_align_to(size_t alignment, intptr_t offset) { - return offset % alignment == 0 ? offset - : offset + - (static_cast(alignment) - - offset % static_cast(alignment)); -} - -static size_t get_system_total_memory_in_bytes() { -#if defined(__ANDROID__) || defined(__linux__) - struct sysinfo info = {}; - if (0 == sysinfo(&info)) { - return (info.totalram + info.totalswap) * info.mem_unit; - } - auto pages = (size_t)sysconf(_SC_PHYS_PAGES); - auto page_size = (size_t)sysconf(_SC_PAGE_SIZE); - - return pages * page_size; -#elif defined(_WIN32) - //TODO: Snapdragon based WoA(Windows on ARM) - return 0; -#else -#error "ggml-qnn only support WoA, Android, Linux" -#endif -} - -static size_t get_system_free_memory_in_bytes() { -#if defined(__ANDROID__) || defined(__linux__) - struct sysinfo info = {}; - if (0 == sysinfo(&info)) { - return (info.freeram + info.freeswap) * info.mem_unit; - } - auto avail_pages = (size_t)sysconf(_SC_AVPHYS_PAGES); - auto page_size = (size_t)sysconf(_SC_PAGE_SIZE); - - return avail_pages * page_size; -#elif defined(_WIN32) - //TODO: Snapdragon based WoA(Windows on ARM) - return 0; -#else -#error "ggml-qnn only support WoA, Android, Linux" -#endif -} - -static size_t ggmlqnn_memscpy(void * dst, size_t dst_size, const void * src, size_t copy_size) { - if (!dst || !src || !dst_size || !copy_size) - return 0; - - size_t min_size = dst_size < copy_size ? dst_size : copy_size; - - memcpy(dst, src, min_size); - - return min_size; -} - -static char * ggmlqnn_strndup(const char * source, size_t maxlen) { - return strndup(source, maxlen); -} - -static void * ggmlqnn_host_malloc(size_t n) { -#if defined(__ANDROID__) || defined(__linux__) - void * data = nullptr; - int result = posix_memalign((void **)&data, sysconf(_SC_PAGESIZE), n); - if (result != 0) { - GGMLQNN_LOG_WARN("%s: error: posix_memalign failed\n", __func__); - return nullptr; - } -#elif defined(_WIN32) - //TODO: Snapdragon based WoA(Windows on ARM) - return nullptr; -#else -#error "ggml-qnn only support WoA, Android, Linux" -#endif - - return data; -} - -// ================================================================================================= -// section-4: QNN helper macro / data structure / function -// ================================================================================================= -#define QNN_TENSOR_GET_ID(tensor) get_qnn_tensorid(tensor) -#define QNN_TENSOR_GET_NAME(tensor) get_qnn_tensorname(tensor) -#define QNN_TENSOR_GET_TYPE(tensor) get_qnn_tensortype(tensor) -#define QNN_TENSOR_GET_DATA_FORMAT(tensor) get_qnn_tensor_dataformat(tensor) -#define QNN_TENSOR_GET_DATA_TYPE(tensor) get_qnn_tensor_datatype(tensor) -#define QNN_TENSOR_GET_QUANT_PARAMS(tensor) get_qnn_tensor_quantparams(tensor) -#define QNN_TENSOR_GET_RANK(tensor) get_qnn_tensor_rank(tensor) -#define QNN_TENSOR_GET_DIMENSIONS(tensor) get_qnn_tensor_dimensions(tensor) -#define QNN_TENSOR_GET_MEM_TYPE(tensor) get_qnn_tensor_memtype(tensor) -#define QNN_TENSOR_GET_CLIENT_BUF(tensor) get_qnn_tensor_clientbuf(tensor) -#define QNN_TENSOR_GET_MEM_HANDLE(tensor) get_qnn_tensor_memhandle(tensor) - -#define QNN_TENSOR_SET_ID(tensor, value) set_qnn_tensor_id(tensor, value) -#define QNN_TENSOR_SET_NAME(tensor, value) set_qnn_tensor_name(tensor, value) -#define QNN_TENSOR_SET_TYPE(tensor, value) set_qnn_tensor_type(tensor, value) -#define QNN_TENSOR_SET_DATA_FORMAT(tensor, value) set_qnn_tensor_dataformat(tensor, value) -#define QNN_TENSOR_SET_DATA_TYPE(tensor, value) set_qnn_tensor_datatype(tensor, value) -#define QNN_TENSOR_SET_QUANT_PARAMS(tensor, value) set_qnn_tensor_quantparams(tensor, value) -#define QNN_TENSOR_SET_RANK(tensor, value) set_qnn_tensor_rank(tensor, value) -#define QNN_TENSOR_SET_DIMENSIONS(tensor, value) set_qnn_tensor_dimensions(tensor, value) -#define QNN_TENSOR_SET_MEM_TYPE(tensor, value) set_qnn_tensor_memtype(tensor, value) -#define QNN_TENSOR_SET_CLIENT_BUF(tensor, value) set_qnn_tensor_clientbuf(tensor, value) -#define QNN_TENSOR_SET_MEM_HANDLE(tensor, value) set_qnn_tensor_memhandle(tensor, value) - -static inline uint32_t get_qnn_tensorid(const Qnn_Tensor_t & tensor) { - if (tensor.version == QNN_TENSOR_VERSION_1) { - return tensor.v1.id; - } - - return 0u; -} - -static inline const char * get_qnn_tensorname(const Qnn_Tensor_t & tensor) { - if (tensor.version == QNN_TENSOR_VERSION_1) { - return tensor.v1.name; - } - return nullptr; -} - -static inline Qnn_TensorType_t get_qnn_tensortype(const Qnn_Tensor_t & tensor) { - if (tensor.version == QNN_TENSOR_VERSION_1) { - return tensor.v1.type; - } - return QNN_TENSOR_TYPE_UNDEFINED; -} - -static inline Qnn_TensorDataFormat_t get_qnn_tensor_dataformat(const Qnn_Tensor_t & tensor) { - if (tensor.version == QNN_TENSOR_VERSION_1) { - return tensor.v1.dataFormat; - } - return QNN_TENSOR_DATA_FORMAT_FLAT_BUFFER; -} - -static inline Qnn_DataType_t get_qnn_tensor_datatype(const Qnn_Tensor_t & tensor) { - if (tensor.version == QNN_TENSOR_VERSION_1) { - return tensor.v1.dataType; - } - return QNN_DATATYPE_UNDEFINED; -} - -static inline Qnn_QuantizeParams_t get_qnn_tensor_quantparams(const Qnn_Tensor_t & tensor) { - if (tensor.version == QNN_TENSOR_VERSION_1) { - return tensor.v1.quantizeParams; - } - return QNN_QUANTIZE_PARAMS_INIT; -} - -static inline uint32_t get_qnn_tensor_rank(const Qnn_Tensor_t & tensor) { - if (tensor.version == QNN_TENSOR_VERSION_1) { - return tensor.v1.rank; - } - return 0u; -} - -static inline uint32_t * get_qnn_tensor_dimensions(const Qnn_Tensor_t & tensor) { - if (tensor.version == QNN_TENSOR_VERSION_1) { - return tensor.v1.dimensions; - } - return nullptr; -} - -static inline Qnn_TensorMemType_t get_qnn_tensor_memtype(const Qnn_Tensor_t & tensor) { - if (tensor.version == QNN_TENSOR_VERSION_1) { - return tensor.v1.memType; - } - return QNN_TENSORMEMTYPE_UNDEFINED; -} - -static inline void set_qnn_tensor_id(Qnn_Tensor_t & tensor, uint32_t id) { - if (tensor.version == QNN_TENSOR_VERSION_1) { - tensor.v1.id = id; - } -} - -static inline void set_qnn_tensor_name(Qnn_Tensor_t & tensor, const char * name) { - if (tensor.version == QNN_TENSOR_VERSION_1) { - tensor.v1.name = name; - } -} - -static inline void set_qnn_tensor_type(Qnn_Tensor_t & tensor, Qnn_TensorType_t type) { - if (tensor.version == QNN_TENSOR_VERSION_1) { - tensor.v1.type = type; - } -} - -static inline void set_qnn_tensor_dataformat(Qnn_Tensor_t & tensor, Qnn_TensorDataFormat_t format) { - if (tensor.version == QNN_TENSOR_VERSION_1) { - tensor.v1.dataFormat = format; - } -} - -static inline void set_qnn_tensor_datatype(Qnn_Tensor_t & tensor, Qnn_DataType_t dataType) { - if (tensor.version == QNN_TENSOR_VERSION_1) { - tensor.v1.dataType = dataType; - } -} - -static inline void set_qnn_tensor_quantparams(Qnn_Tensor_t & tensor, Qnn_QuantizeParams_t params) { - if (tensor.version == QNN_TENSOR_VERSION_1) { - tensor.v1.quantizeParams = params; - } -} - -static inline void set_qnn_tensor_rank(Qnn_Tensor_t & tensor, uint32_t rank) { - if (tensor.version == QNN_TENSOR_VERSION_1) { - tensor.v1.rank = rank; - } -} - -static inline void set_qnn_tensor_dimensions(Qnn_Tensor_t & tensor, uint32_t * dims) { - if (tensor.version == QNN_TENSOR_VERSION_1) { - tensor.v1.dimensions = dims; - } -} - -static inline void set_qnn_tensor_memtype(Qnn_Tensor_t & tensor, Qnn_TensorMemType_t memType) { - if (tensor.version == QNN_TENSOR_VERSION_1) { - tensor.v1.memType = memType; - } -} - -static inline void set_qnn_tensor_clientbuf(Qnn_Tensor_t & tensor, Qnn_ClientBuffer_t clientBuf) { - if (tensor.version == QNN_TENSOR_VERSION_1) { - tensor.v1.clientBuf = clientBuf; - } -} - -static inline void set_qnn_tensor_memhandle(Qnn_Tensor_t & tensor, Qnn_MemHandle_t handle) { - if (tensor.version == QNN_TENSOR_VERSION_1) { - tensor.v1.memHandle = handle; - } -} - -static int deep_copy_qnn_tensors(Qnn_Tensor_t & src, Qnn_Tensor_t & dst) { - int err = 0; - - dst.version = src.version; - QNN_TENSOR_SET_NAME(dst, ggmlqnn_strndup(QNN_TENSOR_GET_NAME(src), std::string(QNN_TENSOR_GET_NAME(src)).size())); - if (nullptr == QNN_TENSOR_GET_NAME(dst)) { - return 1; - } - QNN_TENSOR_SET_ID(dst, QNN_TENSOR_GET_ID(src)); - QNN_TENSOR_SET_TYPE(dst, QNN_TENSOR_GET_TYPE(src)); - QNN_TENSOR_SET_DATA_FORMAT(dst, QNN_TENSOR_GET_DATA_FORMAT(src)); - QNN_TENSOR_SET_DATA_TYPE(dst, QNN_TENSOR_GET_DATA_TYPE(src)); - QNN_TENSOR_SET_MEM_TYPE(dst, QNN_TENSOR_GET_MEM_TYPE(src)); - - if (QNN_TENSOR_GET_MEM_TYPE(src) == QNN_TENSORMEMTYPE_RAW) { - Qnn_ClientBuffer_t client_buf = {nullptr, 0}; - QNN_TENSOR_SET_CLIENT_BUF(dst, client_buf); - } else if (QNN_TENSOR_GET_MEM_TYPE(src) == QNN_TENSORMEMTYPE_MEMHANDLE) { - QNN_TENSOR_SET_MEM_HANDLE(dst, nullptr); - } else { - return 1; - } - - Qnn_QuantizeParams_t src_qparam = QNN_TENSOR_GET_QUANT_PARAMS(src); - Qnn_QuantizationEncoding_t encoding = src_qparam.quantizationEncoding; - if (encoding == QNN_QUANTIZATION_ENCODING_AXIS_SCALE_OFFSET) { - Qnn_QuantizeParams_t src_qparam_cpy = src_qparam; - Qnn_AxisScaleOffset_t & axis_scale_offset = src_qparam_cpy.axisScaleOffsetEncoding; - Qnn_ScaleOffset_t ** scale_offset = &axis_scale_offset.scaleOffset; - size_t scale_offset_size = axis_scale_offset.numScaleOffsets * sizeof(Qnn_ScaleOffset_t); - *scale_offset = (Qnn_ScaleOffset_t *)malloc(scale_offset_size); - ggmlqnn_memscpy(*scale_offset, - scale_offset_size, - src_qparam.axisScaleOffsetEncoding.scaleOffset, - scale_offset_size); - QNN_TENSOR_SET_QUANT_PARAMS(dst, src_qparam_cpy); - } else if (encoding == QNN_QUANTIZATION_ENCODING_BW_AXIS_SCALE_OFFSET) { - Qnn_QuantizeParams_t src_qparam_cpy = src_qparam; - Qnn_BwAxisScaleOffset_t & bwaxis_scale_offset = src_qparam_cpy.bwAxisScaleOffsetEncoding; - size_t scale_size = bwaxis_scale_offset.numElements * sizeof(float); - float ** scales = &bwaxis_scale_offset.scales; - int32_t ** offsets = &bwaxis_scale_offset.offsets; - *scales = (float *)malloc(scale_size); - ggmlqnn_memscpy(*scales, scale_size, src_qparam.bwAxisScaleOffsetEncoding.scales, scale_size); - - if (bwaxis_scale_offset.offsets != nullptr) { - size_t offset_size = bwaxis_scale_offset.numElements * sizeof(int32_t); - *offsets = (int32_t *)malloc(offset_size); - ggmlqnn_memscpy(*offsets, offset_size, src_qparam.bwAxisScaleOffsetEncoding.offsets, offset_size); - } - QNN_TENSOR_SET_QUANT_PARAMS(dst, src_qparam_cpy); - } else { - QNN_TENSOR_SET_QUANT_PARAMS(dst, src_qparam); - } - - uint32_t rank = QNN_TENSOR_GET_RANK(src); - QNN_TENSOR_SET_RANK(dst, rank); - size_t dim_size = GGML_MAX_DIMS * sizeof(uint32_t); - uint32_t * dimensions = (uint32_t *)malloc(dim_size); - if (nullptr == dimensions) { - GGMLQNN_LOG_WARN("deep_copy_qnn_tensors() allocation error while copying tensor %s\n", QNN_TENSOR_GET_NAME(src)); - return 1; - } - ggmlqnn_memscpy(dimensions, dim_size, QNN_TENSOR_GET_DIMENSIONS(src), dim_size); - QNN_TENSOR_SET_DIMENSIONS(dst, dimensions); - - return err; -} - -static int free_qnn_tensor(Qnn_Tensor_t * tensor) { - int err = 0; - free((void *) QNN_TENSOR_GET_NAME(*tensor)); - Qnn_QuantizeParams_t src_qparam = QNN_TENSOR_GET_QUANT_PARAMS(*tensor); - Qnn_QuantizationEncoding_t encoding = src_qparam.quantizationEncoding; - if (encoding == QNN_QUANTIZATION_ENCODING_AXIS_SCALE_OFFSET) { - free(src_qparam.axisScaleOffsetEncoding.scaleOffset); - } else if (encoding == QNN_QUANTIZATION_ENCODING_BW_AXIS_SCALE_OFFSET) { - free(src_qparam.bwAxisScaleOffsetEncoding.scales); - if (src_qparam.bwAxisScaleOffsetEncoding.offsets != nullptr) { - free(src_qparam.bwAxisScaleOffsetEncoding.offsets); - } - } - free(QNN_TENSOR_GET_DIMENSIONS(*tensor)); - free(tensor); - - return err; -} - -const char * ggmlqnn_get_error_string(Qnn_ErrorHandle_t qnn_error_code) { - // file:///opt/qcom/aistack/qairt/2.31.0.250130/docs/QNN/general/api_error_codes.html - switch (qnn_error_code) { - case QNN_SUCCESS: - return "QNN_SUCCESS"; - case QNN_COMMON_ERROR_GENERAL: - return "QNN_COMMON_ERROR_GENERAL"; - - // QnnGraph_Error_t - case QNN_GRAPH_ERROR_UNSUPPORTED_FEATURE: - return "QNN_GRAPH_ERROR_UNSUPPORTED_FEATURE"; - case QNN_GRAPH_ERROR_MEM_ALLOC: - return "QNN_GRAPH_ERROR_MEM_ALLOC"; - case QNN_GRAPH_ERROR_INVALID_ARGUMENT: - return "QNN_GRAPH_ERROR_INVALID_ARGUMENT"; - case QNN_GRAPH_ERROR_INVALID_HANDLE: - return "QNN_GRAPH_ERROR_INVALID_HANDLE"; - case QNN_GRAPH_ERROR_GRAPH_DOES_NOT_EXIST: - return "QNN_GRAPH_ERROR_GRAPH_DOES_NOT_EXIST"; - case QNN_GRAPH_ERROR_INVALID_NAME: - return "QNN_GRAPH_ERROR_INVALID_NAME"; - case QNN_GRAPH_ERROR_INVALID_TENSOR: - return "QNN_GRAPH_ERROR_INVALID_TENSOR"; - case QNN_GRAPH_ERROR_INVALID_OP_CONFIG: - return "QNN_GRAPH_ERROR_INVALID_OP_CONFIG"; - case QNN_GRAPH_ERROR_SET_PROFILE: - return "QNN_GRAPH_ERROR_SET_PROFILE"; - case QNN_GRAPH_ERROR_UNCONNECTED_NODE: - return "QNN_GRAPH_ERROR_UNCONNECTED_NODE"; - case QNN_GRAPH_ERROR_CREATE_FAILED: - return "QNN_GRAPH_ERROR_CREATE_FAILED"; - case QNN_GRAPH_ERROR_OPTIMIZATION_FAILED: - return "QNN_GRAPH_ERROR_OPTIMIZATION_FAILED"; - case QNN_GRAPH_ERROR_FINALIZE_FAILED: - return "QNN_GRAPH_ERROR_FINALIZE_FAILED"; - case QNN_GRAPH_ERROR_GRAPH_NOT_FINALIZED: - return "QNN_GRAPH_ERROR_GRAPH_NOT_FINALIZED"; - case QNN_GRAPH_ERROR_GRAPH_FINALIZED: - return "QNN_GRAPH_ERROR_GRAPH_FINALIZED"; - case QNN_GRAPH_ERROR_EXECUTION_ASYNC_FIFO_FULL: - return "QNN_GRAPH_ERROR_EXECUTION_ASYNC_FIFO_FULL"; - case QNN_GRAPH_ERROR_SIGNAL_IN_USE: - return "QNN_GRAPH_ERROR_SIGNAL_IN_USE"; - case QNN_GRAPH_ERROR_ABORTED: - return "QNN_GRAPH_ERROR_ABORTED"; - case QNN_GRAPH_ERROR_PROFILE_IN_USE: - return "QNN_GRAPH_ERROR_PROFILE_IN_USE"; - case QNN_GRAPH_ERROR_TIMED_OUT: - return "QNN_GRAPH_ERROR_TIMED_OUT"; - case QNN_GRAPH_ERROR_SUBGRAPH: - return "QNN_GRAPH_ERROR_SUBGRAPH"; - case QNN_GRAPH_ERROR_DISABLED: - return "QNN_GRAPH_ERROR_DISABLED"; - case QNN_GRAPH_ERROR_DYNAMIC_TENSOR_SHAPE: - return "QNN_GRAPH_ERROR_DYNAMIC_TENSOR_SHAPE"; - case QNN_GRAPH_ERROR_TENSOR_SPARSITY: - return "QNN_GRAPH_ERROR_TENSOR_SPARSITY"; - case QNN_GRAPH_ERROR_EARLY_TERMINATION: - return "QNN_GRAPH_ERROR_EARLY_TERMINATION"; - case QNN_GRAPH_ERROR_INVALID_CONTEXT: - return "QNN_GRAPH_ERROR_INVALID_CONTEXT"; - - //QQnnTensor_Error_t - //Invalid context/graph handle in creating tensor - case QNN_TENSOR_ERROR_INVALID_HANDLE: - return "QNN_TENSOR_ERROR_INVALID_HANDLE"; - //Tensor with specified credentials not registered with a context/graph - case QNN_TENSOR_ERROR_DOES_NOT_EXIST: - return "QNN_TENSOR_ERROR_DOES_NOT_EXIST"; - // (deprecated) Tensor has already been registered with backend - case QNN_TENSOR_ERROR_ALREADY_EXISTS: - return "QNN_TENSOR_ERROR_ALREADY_EXISTS"; - // Invalid tensor param. - case QNN_TENSOR_ERROR_INVALID_TENSOR_PARAM: - return "QNN_TENSOR_ERROR_INVALID_TENSOR_PARAM"; - // This tensor param is currently unsupported - case QNN_TENSOR_ERROR_UNSUPPORTED_TENSOR_PARAM: - return "QNN_TENSOR_ERROR_UNSUPPORTED_TENSOR_PARAM"; - // Tensor provided for update is invalid - case QNN_TENSOR_ERROR_INCOMPATIBLE_TENSOR_UPDATE: - return "QNN_TENSOR_ERROR_INCOMPATIBLE_TENSOR_UPDATE"; - - // QnnOpPackage_Error_t - case QNN_OP_PACKAGE_ERROR_LIBRARY_ALREADY_INITIALIZED: - return "QNN_OP_PACKAGE_ERROR_LIBRARY_ALREADY_INITIALIZED"; - case QNN_OP_PACKAGE_ERROR_LIBRARY_NOT_INITIALIZED: - return "QNN_OP_PACKAGE_ERROR_LIBRARY_NOT_INITIALIZED"; - case QNN_OP_PACKAGE_ERROR_INVALID_HANDLE: - return "QNN_OP_PACKAGE_ERROR_INVALID_HANDLE"; - case QNN_OP_PACKAGE_ERROR_INVALID_INFRASTRUCTURE: - return "QNN_OP_PACKAGE_ERROR_INVALID_INFRASTRUCTURE"; - case QNN_OP_PACKAGE_ERROR_INVALID_INFO: - return "QNN_OP_PACKAGE_ERROR_INVALID_INFO"; - case QNN_OP_PACKAGE_ERROR_VALIDATION_FAILURE: - return "QNN_OP_PACKAGE_ERROR_VALIDATION_FAILURE"; - case QNN_OP_PACKAGE_ERROR_INVALID_ARGUMENT: - return "QNN_OP_PACKAGE_ERROR_INVALID_ARGUMENT"; - - default: - return "unknown QNN error"; - } -} - -// helper function to create an operation config -Qnn_OpConfig_t ggmlqnn_create_op_config(const char * name, const char * package, const char * type, - Qnn_Param_t * params, uint32_t num_params, - Qnn_Tensor_t * inputs, uint32_t num_inputs, - Qnn_Tensor_t * outputs, uint32_t num_outputs) { - Qnn_OpConfigV1_t v1 = {name, package, type, - num_params, params, - num_inputs, inputs, - num_outputs, outputs - }; - Qnn_OpConfig_t opcfg = {QNN_OPCONFIG_VERSION_1, {v1}}; - - return opcfg; -} - -// ================================================================================================= -// section-5:ggml-qnn backend helper macro / data structure / function / class -// ================================================================================================= -//file:///opt/qcom/aistack/qairt/2.31.0.250130/docs/QNN/general/overview.html#tbl-supported-snapdragon-devices -static struct qcom_socinfo g_qnn_soc_info_table[] = { - /* Qualcomm SnapDragon 7 Gen 1 */ - [SM7450] = { - .soc_model = SM7450, - .htp_arch = V69, - .vtcm_size_in_mb = 8, - .soc_desc = "Qualcomm SnapDragon 7 Gen 1"}, - - /* Qualcomm SnapDragon 888 */ - [SM8350] = { - .soc_model = SM8350, - .htp_arch = V68, - .vtcm_size_in_mb = 8, - .soc_desc = "Qualcomm SnapDragon 888 "}, - - /* Qualcomm SnapDragon 8 Gen 1 */ - [SM8450] = { - .soc_model = SM8450, - .htp_arch = V69, - .vtcm_size_in_mb = 8, - .soc_desc = "Qualcomm SnapDragon 8 Gen 1"}, - - /* Qualcomm SnapDragon 8 Gen 1+ */ - [SM8475] = { - .soc_model = SM8475, - .htp_arch = V69, - .vtcm_size_in_mb = 8, - .soc_desc = "Qualcomm SnapDragon 8 Gen 1+"}, - - /* Qualcomm SnapDragon 8 Gen 2 */ - [SM8550] = { - .soc_model = SM8550, - .htp_arch = V73, - .vtcm_size_in_mb = 8, - .soc_desc = "Qualcomm SnapDragon 8 Gen 2"}, - - /* Qualcomm SnapDragon 8 Gen 3 */ - [SM8650] = { - .soc_model = SM8650, - .htp_arch = V75, - .vtcm_size_in_mb = 8, - .soc_desc = "Qualcomm SnapDragon 8 Gen 3 "}, - - /* Qualcomm SnapDragon 8 Gen 4 */ - [SM8750] = { - .soc_model = SM8750, - .htp_arch = V79, - .vtcm_size_in_mb = 8, - .soc_desc = "Qualcomm SnapDragon 8 Gen 4"}, - -#if defined(_WIN32) - /* Qualcomm SnapDragon 7c Gen 2 */ - [SC7280X] = { - .soc_model = SC7280X, - .htp_arch = V68, - .vtcm_size_in_mb = 8, - .soc_desc = "Qualcomm SnapDragon 7c Gen 2"}, - - /* Qualcomm SnapDragon 8cx Gen 3 */ - [SC8280X] = { - .soc_model = SC8280X, - .htp_arch = V68, - .vtcm_size_in_mb = 8, - .soc_desc = "Qualcomm SnapDragon 8cx Gen 3"}, - - /* Qualcomm SnapDragon 8cx Gen 4 */ - [SC8380XP] = { - .soc_model = SC8380XP, - .htp_arch = V73, - .vtcm_size_in_mb = 8, - .soc_desc = "Qualcomm SnapDragon 8cx Gen 4"}, -#endif - -}; - -//the following helper funcs are used to ensure every QNN tensor name is unique -static std::atomic g_ggmltensor_idx(0); -static void reset_idx() { - g_ggmltensor_idx = 0; -} - -static void inc_idx() { - g_ggmltensor_idx++; -} - -static int32_t get_idx() { - return g_ggmltensor_idx.load(); -} - -// file:///opt/qcom/aistack/qairt/2.31.0.250130/docs/QNN/general/quantization.html -// CPU - Choose a non-quantized model.Quantized models are currently incompatible with the CPU backend -// GPU - Choose a non-quantized model.Quantized models are currently incompatible with the GPU backend -// HTP - Choose a quantized model. Quantized models are required when running on the HTP backend -// DSP - Choose a quantized model. Quantized models are required when running on the DSP backend -// HTA - Choose a quantized model. Quantized models are required when running on the HTA backend -static struct ggml_backend_qnn_context g_qnn_mgr[GGML_QNN_MAX_DEVICES] = { - [QNN_BACKEND_CPU] = {.device = 0, - .threads = 1, - .name = "qnn-cpu", - .desc = "Qualcomm Kryo CPU", -#if defined(_WIN32) - .lib = "QnnCpu.dll", -#else - .lib = "libQnnCpu.so", -#endif - .instance = nullptr, - .backend = nullptr, - .raw_interface = {}, - .raw_system_interface = {}, - .socinfo = {}}, - - [QNN_BACKEND_GPU] = {.device = 1, - .threads = 1, - .name = "qnn-gpu", - .desc = "Qualcomm Adreno GPU", -#if defined(_WIN32) - .lib = "QnnGpu.dll", -#else - .lib = "libQnnGpu.so", -#endif - .instance = nullptr, - .backend = nullptr, - .raw_interface = {}, - .raw_system_interface = {}, - .socinfo = {}}, - - [QNN_BACKEND_NPU] = {.device = 2, - .threads = 1, - .name = "qnn-npu", - .desc = "Qualcomm NPU(Hexagon Tensor Processor)", -#if defined(_WIN32) - .lib = "QnnHtp.dll", -#else - .lib = "libQnnHtp.so", -#endif - .instance = nullptr, - .backend = nullptr, - .raw_interface = {}, - .raw_system_interface = {}, - .socinfo = {}}, -}; - -const qnn_op_caps_t ggmlqnn_k_op_caps[] = { - {}, // GGML_OP_NONE - {}, // GGML_OP_DUP - { - // GGML_OP_ADD - QNN_OP_ELEMENT_WISE_ADD, - 2, - }, - {}, // GGML_OP_ADD1 - {}, // GGML_OP_ACC - {}, // GGML_OP_SUB - { - // GGML_OP_MUL - QNN_OP_ELEMENT_WISE_MULTIPLY, - 2, - }, - {}, // GGML_OP_DIV - {}, // GGML_OP_SQR - {}, // GGML_OP_SQRT - {}, // GGML_OP_LOG - {}, // GGML_OP_SIN - {}, // GGML_OP_COS - {}, // GGML_OP_SUM - {}, // GGML_OP_SUM_ROWS - {}, // GGML_OP_MEAN - {}, // GGML_OP_ARGMAX - {}, // GGML_OP_COUNT_EQUAL - {}, // GGML_OP_REPEAT - {}, // GGML_OP_REPEAT_BACK - {}, // GGML_OP_CONCAT - {}, // GGML_OP_SILU_BACK - {}, // GGML_OP_NORM - {}, // GGML_OP_RMS_NORM - {}, // GGML_OP_RMS_NORM_BACK - {}, // GGML_OP_GROUP_NORM - { - // GGML_OP_MUL_MAT - QNN_OP_MAT_MUL, - 2, - }, - {}, // GGML_OP_MUL_MAT_ID - {}, // GGML_OP_OUT_PROD - {}, // GGML_OP_SCALE - {}, // GGML_OP_SET - {}, // GGML_OP_CPY - {}, // GGML_OP_CONT - {}, // GGML_OP_RESHAPE - {}, // GGML_OP_VIEW - {}, // GGML_OP_PERMUTE - {}, // GGML_OP_TRANSPOSE - {}, // GGML_OP_GET_ROWS - {}, // GGML_OP_GET_ROWS_BACK - {}, // GGML_OP_DIAG - {}, // GGML_OP_DIAG_MASK_INF - {}, // GGML_OP_DIAG_MASK_ZERO - {}, // GGML_OP_SOFT_MAX - {}, // GGML_OP_SOFT_MAX_BACK - {}, // GGML_OP_ROPE - {}, // GGML_OP_ROPE_BACK - {}, // GGML_OP_CLAMP - {}, // GGML_OP_CONV_TRANSPOSE_1D - {}, // GGML_OP_IM2COL - {}, // GGML_OP_IM2COL_BACK - {}, // GGML_OP_CONV_TRANSPOSE_2D - {}, // GGML_OP_POOL_1D - {}, // GGML_OP_POOL_2D - {}, // GGML_OP_POOL_2D_BACK - {}, // GGML_OP_UPSCALE - {}, // GGML_OP_PAD - {}, // GGML_OP_PAD_REFLECT_1D - {}, // GGML_OP_ARANGE - {}, // GGML_OP_TIMESTEP_EMBEDDING - {}, // GGML_OP_ARGSORT - {}, // GGML_OP_LEAKY_RELU - {}, // GGML_OP_FLASH_ATTN_EXT - {}, // GGML_OP_FLASH_ATTN_BACK - {}, // GGML_OP_SSM_CONV - {}, // GGML_OP_SSM_SCAN - {}, // GGML_OP_WIN_PART - {}, // GGML_OP_WIN_UNPART - {}, // GGML_OP_GET_REL_POS - {}, // GGML_OP_ADD_REL_POS - {}, // GGML_OP_RWKV_WKV6 - {}, // GGML_OP_GATED_LINEAR_ATTN - {}, // GGML_OP_UNARY - {}, // GGML_OP_MAP_UNARY - {}, // GGML_OP_MAP_BINARY - {}, // GGML_OP_MAP_CUSTOM1_F32 - {}, // GGML_OP_MAP_CUSTOM2_F32 - {}, // GGML_OP_MAP_CUSTOM3_F32 - {}, // GGML_OP_MAP_CUSTOM1 - {}, // GGML_OP_MAP_CUSTOM2 - {}, // GGML_OP_MAP_CUSTOM3 - {}, // GGML_OP_CROSS_ENTROPY_LOSS - {}, // GGML_OP_CROSS_ENTROPY_LOSS_BACK - {}, // GGML_OP_OPT_STEP_ADAMW - {}, // GGML_UNARY_OP_ABS - {}, // GGML_UNARY_OP_SGN - {}, // GGML_UNARY_OP_NEG - {}, // GGML_UNARY_OP_STEP - {}, // GGML_UNARY_OP_TANH - {}, // GGML_UNARY_OP_ELU - {}, // GGML_UNARY_OP_RELU - {}, // GGML_UNARY_OP_SIGMOID - {}, // GGML_UNARY_OP_GELU - {}, // GGML_UNARY_OP_GELU_QUICK - {}, // GGML_UNARY_OP_SILU - {}, // GGML_UNARY_OP_HARDSWISH - {}, // GGML_UNARY_OP_HARDSIGMOID - {}, // GGML_UNARY_OP_EXP -}; - -static const char * qnn_get_socmodel_desc(uint32_t soc_model) { - switch (soc_model) { - case SM7450: - return "SM7450"; - case SM8350: - return "SM8350"; - case SM8450: - return "SM8450"; - case SM8475: - return "SM8475"; - case SM8550: - return "SM8550"; - case SM8650: - return "SM8650"; - case SM8750: - return "SM8750"; - default: - return "unknown"; - } -} - -static const char * qnn_get_htparch_desc(size_t htp_arch) { - switch (htp_arch) { - case V68: - return "QCOM_HTP_V68"; - case V69: - return "QCOM_HTP_V69"; - case V73: - return "QCOM_HTP_V73"; - case V75: - return "QCOM_HTP_V75"; - case V79: - return "QCOM_HTP_V79"; - default: - return "unknown"; - } -} - -static struct qcom_socinfo * qnn_get_socinfo_from_socmodel(uint32_t soc_model) { - size_t items = sizeof(g_qnn_soc_info_table) / sizeof(g_qnn_soc_info_table[0]); - for (size_t idx = 0; idx < items; idx++) { - if (soc_model == g_qnn_soc_info_table[idx].soc_model) { - return &g_qnn_soc_info_table[idx]; - } - } - return nullptr; -} - - -static const char * ggml_get_type_name(ggml_type type) { - const struct ggml_type_traits * traits = ggml_get_type_traits(type); - return traits->type_name; -} - -static const char * get_ggml_type_name(ggml_type type) { - const auto * traits = ggml_get_type_traits(type); - return traits->type_name; -} - -// ref:explanation of k-quants, https://github.com/ggerganov/llama.cpp/pull/1684 -Qnn_DataType_t ggmlqnn_datatype_from_ggml_datatype(enum ggml_type ggmltype) { - switch (ggmltype) { - case GGML_TYPE_F16: - return QNN_DATATYPE_FLOAT_16; - case GGML_TYPE_F32: - return QNN_DATATYPE_FLOAT_32; - case GGML_TYPE_I8: - return QNN_DATATYPE_INT_8; - case GGML_TYPE_Q8_0: - return QNN_DATATYPE_SFIXED_POINT_8; - case GGML_TYPE_Q4_0: - return QNN_DATATYPE_SFIXED_POINT_4; - default: - break; - } - return QNN_DATATYPE_UNDEFINED; -} - -static ggml_type ggml_datatype_from_qnn_datatype(Qnn_DataType_t qnn_type) { - switch (qnn_type) { - case QNN_DATATYPE_FLOAT_32: - return GGML_TYPE_F32; - case QNN_DATATYPE_FLOAT_16: - return GGML_TYPE_F16; - case QNN_DATATYPE_UINT_32: - case QNN_DATATYPE_INT_32: - return GGML_TYPE_I32; - case QNN_DATATYPE_INT_16: - return GGML_TYPE_I16; - case QNN_DATATYPE_INT_8: - return GGML_TYPE_I8; - case QNN_DATATYPE_SFIXED_POINT_8: - return GGML_TYPE_Q8_0; - case QNN_DATATYPE_SFIXED_POINT_4: - return GGML_TYPE_Q4_0; - default: - break; - } - return GGML_TYPE_COUNT; -} - -static void get_qnn_dimensions_from_ggml_dimensions(uint32_t * qnn_dimensions, const uint32_t * ggml_dimensions, uint32_t rank) { - if (rank > GGML_MAX_DIMS) { - GGMLQNN_LOG_WARN("invalid params"); - return; - } - if (nullptr == qnn_dimensions || nullptr == ggml_dimensions) { - GGMLQNN_LOG_WARN("invalid params"); - return; - } - for (size_t idx = 0; idx < GGML_MAX_DIMS; idx++) - qnn_dimensions[idx] = ggml_dimensions[idx]; - - if (rank >= 2) { - qnn_dimensions[rank - 1] = ggml_dimensions[rank - 2]; - qnn_dimensions[rank - 2] = ggml_dimensions[rank - 1]; - } -} - -Qnn_Tensor_t * ggmlqnn_create_general_tensor(const ggml_tensor * tensor, const char * name, - Qnn_TensorType_t qnn_tensor_type, - Qnn_DataType_t qnn_data_type, - uint32_t rank, uint32_t * dims, - void * data, uint32_t data_size, - bool b_transpose) { - Qnn_ErrorHandle_t error = QNN_SUCCESS; - char tensor_name[GGML_MAX_NAME] = {}; - - //ensure the tensor name is unique - if (nullptr != name) { - snprintf(tensor_name, GGML_MAX_NAME, "tensor_%-8d", get_idx()); - } else { - snprintf(tensor_name, GGML_MAX_NAME, "tensor_%s%-8d", name, get_idx()); - } - GGMLQNN_LOG_DEBUG("init_tensor %d", get_idx()); - inc_idx(); - - uint32_t reverse_dims[GGML_MAX_DIMS] = {}; - uint32_t transpose_dims[GGML_MAX_DIMS] = {}; - uint32_t * tensor_dims = nullptr; - //case 1:use dims info from ggml tensor - if (nullptr != tensor) { - //there are different dimension order between ggml tensor and qnn tensor - for (size_t idx = 0; idx < rank; idx++) { - reverse_dims[idx] = (uint32_t)tensor->ne[rank - 1 - idx]; - } - tensor_dims = reverse_dims; - } - //case 2: use user's specified tensor_dims - if (nullptr != dims) { - tensor_dims = dims; - } - //case 3: transpose for dst tensor - if (b_transpose) { - GGML_ASSERT(tensor != nullptr); //ensure ggml_tensor is not nullptr for this special case - - get_qnn_dimensions_from_ggml_dimensions(transpose_dims, reverse_dims, ggml_n_dims(tensor)); - tensor_dims = transpose_dims; -#if 0 - for (size_t idx = 0; idx < 4; idx++) { - GGMLQNN_LOG_DEBUG("origin dim[%d]=%d\n", idx, reverse_dims[idx]); - } - for (size_t idx = 0; idx < 4; idx++) { - GGMLQNN_LOG_DEBUG("trans dim[%d]=%d\n", idx, transpose_dims[idx]); - } -#endif - } - - Qnn_Tensor_t qnn_tensor = { - .version= QNN_TENSOR_VERSION_1, - {.v1= { - .id = 0, - .name = tensor_name, - .type = qnn_tensor_type, - .dataFormat = QNN_TENSOR_DATA_FORMAT_FLAT_BUFFER, - .dataType = qnn_data_type, - .quantizeParams = {.encodingDefinition = QNN_DEFINITION_UNDEFINED, - .quantizationEncoding = QNN_QUANTIZATION_ENCODING_UNDEFINED, - {.scaleOffsetEncoding = {.scale = 0.0000000000000000f, .offset = 0}}}, - .rank = rank, - .dimensions = tensor_dims, - .memType = QNN_TENSORMEMTYPE_RAW, - .clientBuf = {.data = nullptr, .dataSize = 0} - } - } - }; - if (nullptr != name) { - QNN_VER_PTR(qnn_tensor)->name = name; - } - Qnn_Tensor_t * p_qnn_tensor = (Qnn_Tensor_t *)calloc(1, sizeof(Qnn_Tensor_t)); - if (nullptr == p_qnn_tensor) { - GGMLQNN_LOG_WARN("calloc failed"); - return nullptr; - } - error = deep_copy_qnn_tensors(qnn_tensor, * p_qnn_tensor); - if (error != QNN_SUCCESS) { - free(p_qnn_tensor); - GGMLQNN_LOG_WARN("init tensor failed"); - return nullptr; - } - QNN_VER_PTR(*p_qnn_tensor)->clientBuf = {data, data_size}; - - return p_qnn_tensor; -} - -Qnn_Tensor_t * ggmlqnn_create_compute_tensor(const ggml_tensor * tensor) { - uint32_t dimensions[] = {(uint32_t) tensor->ne[0], (uint32_t) tensor->ne[1], - (uint32_t) tensor->ne[2], (uint32_t) tensor->ne[3]}; - Qnn_DataType_t qnn_data_type = QNN_DATATYPE_FLOAT_32; - Qnn_TensorType_t qnn_tensor_type = QNN_TENSOR_TYPE_APP_WRITE; - - if (tensor->flags & GGML_TENSOR_FLAG_INPUT) { - qnn_tensor_type = QNN_TENSOR_TYPE_APP_WRITE; - } else if (tensor->flags & GGML_TENSOR_FLAG_OUTPUT) { - qnn_tensor_type = QNN_TENSOR_TYPE_APP_READ; - } - - qnn_data_type = ggmlqnn_datatype_from_ggml_datatype(tensor->type); - Qnn_Tensor_t * p_qnn_tensor = ggmlqnn_create_general_tensor(tensor, nullptr, - qnn_tensor_type, qnn_data_type, - ggml_n_dims(tensor), dimensions, - nullptr, 0); - - return p_qnn_tensor; -} - -void * ggmlqnn_type_trait(ggml_backend_qnn_context * ctx, ggml_tensor * op) { - const ggml_tensor * src0 = op->src[0]; - const ggml_tensor * src1 = op->src[1]; - ggml_tensor * dst = op; - const enum ggml_type src0_type = src0->type; - - GGML_TENSOR_BINARY_OP_LOCALS - GGML_ASSERT(ne0 == ne01); - GGML_ASSERT(ne1 == ne11); - GGML_ASSERT(ne2 == ne12); - GGML_ASSERT(ne3 == ne13); - GGML_ASSERT(nb00 == ggml_type_size(src0_type)); - GGML_ASSERT(nb10 == ggml_type_size(src1->type)); - - const int64_t ne_plane = ne01 * ne00; - const size_t desired_size = ((GGML_TYPE_F32 == src0_type) ? 0 : ne03 * ne02 * ne_plane * sizeof(float)); - ctx->desired_size = desired_size; - if (ctx->work_size < desired_size) { - ctx->work_data.reset(new char[desired_size]); - ctx->work_size = desired_size; - } - ctx->n_threads = std::thread::hardware_concurrency(); - void * wdata = ctx->work_data.get(); - // convert src0 to float - if (src0_type != GGML_TYPE_F32) { - const auto * type_traits = ggml_get_type_traits(src0_type); - ggml_to_float_t const to_float = type_traits->to_float; - - for (int64_t i03 = 0; i03 < ne03; i03++) { - for (int64_t i02 = 0; i02 < ne02; i02++) { - const void * x = (char *)src0->data + i02 * nb02 + i03 * nb03; - float * const wplane = (float *)wdata + i02 * ne_plane + i03 * ne02 * ne_plane; - - const int min_cols_per_thread = 4096; - const int min_rows_per_thread = std::max((int)(min_cols_per_thread / ne00), 1); - const int n_threads = std::max( - std::min(ctx->n_threads, (int)(ne01 / min_rows_per_thread)), 1); - for (int i = 1; i < n_threads; i++) { - const int64_t start = i * ne01 / n_threads; - const int64_t end = (i + 1) * ne01 / n_threads; - if (start < end) { - ctx->tasks.push_back(std::async(std::launch::async, [=]() { - for (int64_t i01 = start; i01 < end; i01++) { - to_float((const char *)x + i01 * nb01, wplane + i01 * ne00, ne00); - } - })); - } - } - { - // reuse the current thread for the first task - const int64_t start = 0; - const int64_t end = ne01 / n_threads; - for (int64_t i01 = start; i01 < end; i01++) { - to_float((const char *) x + i01 * nb01, wplane + i01 * ne00, ne00); - } - } - } - } - - // wait for all tasks to finish - for (auto &task: ctx->tasks) { - task.get(); - } - ctx->tasks.clear(); - } - return wdata; -} - -static void append_tensor_dimensions(const ggml_tensor * tensor, std::string & output) { - char buffer[256] = {}; - const char * type_name = get_ggml_type_name(tensor->type); - int len = 0; - switch (ggml_n_dims(tensor)) { - case 1: - len = snprintf(buffer, sizeof(buffer), "%ldx1%s", (long)tensor->ne[0], type_name); - break; - case 2: - len = snprintf(buffer, sizeof(buffer), "%ldx%ld%s", (long)tensor->ne[0], (long)tensor->ne[1], type_name); - break; - case 3: - len = snprintf(buffer, sizeof(buffer), "%ldx%ldx%ld%s", (long)tensor->ne[0], (long)tensor->ne[1], - (long)tensor->ne[2], type_name); - break; - case 4: - default: - len = snprintf(buffer, sizeof(buffer), "%ldx%ldx%ldx%ld%s", (long)tensor->ne[0], (long)tensor->ne[1], - (long)tensor->ne[2], (long)tensor->ne[3], type_name); - break; - } - GGML_ASSERT(len > 0 && len < (int)sizeof(buffer)); - output.append(buffer, len); -} - -size_t ggmlqnn_get_opcaps_size() { - return std::size(ggmlqnn_k_op_caps); -} - -size_t ggmlqnn_get_op_index(const ggml_tensor * tensor) { - if (tensor->op == GGML_OP_UNARY) { - return static_cast(GGML_OP_COUNT) + static_cast(ggml_get_unary_op(tensor)); - } - - return tensor->op; -} - -static size_t ggmlqnn_get_op_input_param_count(const ggml_tensor * op) { - auto op_index = ggmlqnn_get_op_index(op); - GGML_ASSERT(op_index < std::size(ggmlqnn_k_op_caps)); - return ggmlqnn_k_op_caps[op_index].input_param_count; -} - -void ggmlqnn_get_graphkey_from_op(const ggml_tensor * op, std::string & output) { - GGML_ASSERT(op->op != GGML_OP_NONE); - output += ggml_op_desc(op); - output += get_ggml_type_name(op->type); - size_t param_count = ggmlqnn_get_op_input_param_count(op); - for (size_t i = 0; i < param_count; ++i) { - auto * input = op->src[i]; - if (!input) { - break; - } - output += '_'; - append_tensor_dimensions(input, output); - } -} - -template -Fn load_qnn_functionpointers(void * handle, const char * function_name) { - return reinterpret_cast(dlsym(handle, function_name)); -} - -std::mutex qnn_instance::_init_mutex; -std::unordered_map qnn_instance::_loaded_lib_handle; -std::unordered_map qnn_instance::_lib_path_to_backend_id; -std::unordered_map qnn_instance::_loaded_backend; - -void * qnn_instance::alloc_rpcmem_internal(size_t bytes, size_t alignment) { - if (!_rpcmem_initialized) { - GGMLQNN_LOG_WARN("rpc memory not initialized\n"); - return nullptr; - } - - auto allocate_bytes = static_cast(bytes + alignment); - void * buf = _pfn_rpc_mem_alloc(RPCMEM_HEAP_ID_SYSTEM, RPCMEM_DEFAULT_FLAGS, allocate_bytes); - if (nullptr == buf) { - GGMLQNN_LOG_WARN("failed to allocate rpc memory\n"); - return nullptr; - } - - auto aligned_buf = reinterpret_cast(ggmlqnn_align_to(alignment, - reinterpret_cast(buf))); - bool status = _rpcmem_store_map.insert(std::pair(aligned_buf, buf)).second; - if (!status) { - GGMLQNN_LOG_WARN("failed to allocate rpc memory\n"); - _pfn_rpc_mem_free(buf); - } - return aligned_buf; -} - -void * qnn_instance::alloc_rpcmem(size_t bytes, size_t alignment) { - if (_rpcmem_usage > (_rpcmem_capacity - 8)) { // reserve 8Mbytes in rpc mempool - GGMLQNN_LOG_WARN("rpc mempool capcaity: %d MB, usage: %d MB", _rpcmem_capacity, _rpcmem_usage); - return nullptr; - } - - auto aligned_buf = alloc_rpcmem_internal(bytes, alignment); - if (nullptr == aligned_buf) - return nullptr; - _rpcmem_usage_map.insert(std::pair(aligned_buf, bytes)); - - size_t rpcmem_usage_in_bytes = _rpcmem_usage * (1 << 20); - rpcmem_usage_in_bytes += bytes; - _rpcmem_usage = rpcmem_usage_in_bytes / ( 1 << 20); - return aligned_buf; -} - -void qnn_instance::free_rpcmem(void * buf) { - size_t rpcbuffer_size = 0; - if (!_rpcmem_initialized) { - GGMLQNN_LOG_WARN("rpc memory not initialized\n"); - } else if (0 == _rpcmem_store_map.count(buf)) { - GGMLQNN_LOG_WARN("no allocated tensor\n"); - } else { - GGMLQNN_LOG_DEBUG("free rpc mem %p", _rpcmem_store_map[buf]); - for (std::unordered_map::iterator it = _rpcmem_usage_map.begin(); - it != _rpcmem_usage_map.end(); - it++) { - void * rpcbuffer = it->first; - if (buf == rpcbuffer) { - rpcbuffer_size = it->second; - size_t rpcmem_usage_in_bytes = _rpcmem_usage * (1 << 20); - rpcmem_usage_in_bytes -= rpcbuffer_size; - _rpcmem_usage = rpcmem_usage_in_bytes / ( 1 << 20); - } - } - if (rpcbuffer_size != 0) { - _rpcmem_usage_map.erase(buf); - } else { - GGMLQNN_LOG_WARN("it shouldn't happen, pls check why?"); - } - _pfn_rpc_mem_free(_rpcmem_store_map[buf]); - _rpcmem_store_map.erase(buf); - } -} - -void qnn_instance::free_rpcmem() { - if (_rpcmem_store_map.empty()) { - GGMLQNN_LOG_WARN("no rpcmem allocated\n"); - return; - } - - for (std::unordered_map::iterator it = _rpcmem_store_map.begin(); - it != _qnn_mem_set.end(); - it++) { - void * rpcbuffer = it->second; - GGMLQNN_LOG_DEBUG("free rpc buffer %p", rpcbuffer); - _pfn_rpc_mem_free(rpcbuffer); - } - _rpcmem_store_map.clear(); - _rpcmem_usage_map.clear(); - _rpcmem_usage = 0; -} - -int32_t qnn_instance::rpcmem_to_fd(void * buf) { - int32_t mem_fd = -1; - if (!is_rpcmem_initialized()) { - GGMLQNN_LOG_WARN("rpc memory not initialized\n"); - } else { - mem_fd = _pfn_rpc_mem_to_fd(buf); - } - - return mem_fd; -} - -int qnn_instance::register_rpcmem(void * p_data, Qnn_Tensor_t * p_tensor) { - if (nullptr == p_data || (nullptr == p_tensor)) { - GGMLQNN_LOG_WARN("invalid param\n"); - return 1; - } - - if (!is_rpcmem_initialized()) { - GGMLQNN_LOG_WARN("rpc memory not initialized\n"); - return 2; - } - - if (is_rpcmem_registered((QNN_VER_PTR(*p_tensor)->memHandle))) { - GGMLQNN_LOG_WARN("tensor %s has been registered shared memory\n", (QNN_VER_PTR(*p_tensor)->name)); - return 3; - } - - int32_t mem_fd = rpcmem_to_fd(p_data); - if (-1 == mem_fd) { - GGMLQNN_LOG_WARN("failed to get file descriptor\n"); - return 4; - } - GGMLQNN_LOG_DEBUG("mem_fd %d\n", mem_fd); - Qnn_MemDescriptor_t descriptor = { - {QNN_VER_PTR(*p_tensor)->rank, QNN_VER_PTR(*p_tensor)->dimensions, nullptr}, - QNN_VER_PTR(*p_tensor)->dataType, - QNN_MEM_TYPE_ION, - {{mem_fd}}}; - Qnn_MemHandle_t handle = nullptr; - int error = QNN_SUCCESS; - error = _qnn_interface.qnn_mem_register( - _qnn_context_handle, - &descriptor, - /*numDescriptors=*/1, - &handle); - if (error != QNN_SUCCESS) { - GGMLQNN_LOG_WARN("failed to register shared memory, error %d, %s\n", QNN_GET_ERROR_CODE(error), strerror(error)); - return 5; - } else { - GGMLQNN_LOG_INFO("tensor %s successfully register shared memory\n", (QNN_VER_PTR(*p_tensor)->name)); - } - QNN_VER_PTR(*p_tensor)->memHandle = handle; - _qnn_mem_set.insert((std::pair(p_data, handle))); - - return 0; -} - -Qnn_MemHandle_t qnn_instance::register_rpcmem(void * p_data, const uint32_t rank, uint32_t * dimensions, Qnn_DataType_t data_type) { - if (!p_data) { - GGMLQNN_LOG_WARN("invalid param"); - return nullptr; - } - - if (!is_rpcmem_initialized()) { - GGMLQNN_LOG_WARN("rpc memory not initialized"); - return nullptr; - } - - if (is_rpcmem_registered(p_data)) { - GGMLQNN_LOG_WARN("rpc memory already registered"); - return _qnn_rpc_buffer_to_handles[p_data]; - } - - auto mem_fd = rpcmem_to_fd(p_data); - if (mem_fd == -1) { - GGMLQNN_LOG_WARN("failed to get file descriptor"); - return nullptr; - } - - GGMLQNN_LOG_DEBUG("mem_fd %d", mem_fd); - Qnn_MemDescriptor_t descriptor = { - {rank, dimensions, nullptr}, - data_type, QNN_MEM_TYPE_ION, - {{mem_fd}} - }; - Qnn_MemHandle_t handle = nullptr; - auto error = _qnn_interface.qnn_mem_register(_qnn_context_handle, &descriptor, /*numDescriptors=*/1, &handle); - if (error != QNN_SUCCESS) { - GGMLQNN_LOG_WARN("failed to register shared memory, error %d, %s", QNN_GET_ERROR_CODE(error), strerror(error)); - return nullptr; - } - - _qnn_rpc_buffer_to_handles.insert({p_data, handle}); - GGMLQNN_LOG_DEBUG("successfully register shared memory handler: %p", handle); - return handle; -} - -void * qnn_instance::get_rpcmem_from_memhandle(Qnn_MemHandle_t mem_handle) { - for (std::unordered_map::iterator it = _qnn_mem_set.begin(); - it != _qnn_mem_set.end(); - it++) { - Qnn_MemHandle_t mem_handle = it->second; - if (it->second == mem_handle) { - return it->first; - } - } - GGMLQNN_LOG_WARN("can't find rpcmem from qnn mem handle %p", mem_handle); - return nullptr; -} - -void qnn_instance::unregister_rpcmem() { - Qnn_ErrorHandle_t error = QNN_SUCCESS; - - if (_qnn_mem_set.empty()) { - GGMLQNN_LOG_WARN("no rpcmem registered\n"); - } - - for (std::unordered_map::iterator it = _qnn_mem_set.begin(); - it != _qnn_mem_set.end(); - it++) { - Qnn_MemHandle_t mem_handle = it->second; - error = _qnn_interface.qnn_mem_de_register(&mem_handle, 1); - if (error != QNN_SUCCESS) { - GGMLQNN_LOG_WARN("failed to unregister shared memory, error %d\n", QNN_GET_ERROR_CODE(error)); - } else { - GGMLQNN_LOG_DEBUG("unregister shared memory ok"); - } - } - _qnn_mem_set.clear(); -} - -void qnn_instance::unregister_rpcmem(Qnn_MemHandle_t mem_handle) { - Qnn_ErrorHandle_t error = _qnn_interface.qnn_mem_de_register(&mem_handle, 1); - if (error != QNN_SUCCESS) { - GGMLQNN_LOG_WARN("failed to unregister shared memory, error %d", QNN_GET_ERROR_CODE(error)); - } - - auto it = std::find_if(_qnn_mem_set.begin(), _qnn_mem_set.end(), - [mem_handle](const auto &kv) { return kv.second == mem_handle; }); - if (it == _qnn_mem_set.end()) { - GGMLQNN_LOG_WARN("failed to find shared memory handler: %p", mem_handle); - return; - } - - _qnn_mem_set.erase(it); -} - -bool qnn_instance::is_rpcmem_allocated(void * buf) { - return _rpcmem_store_map.count(buf) != 0U; -} - -int qnn_instance::load_backend(std::string & lib_path, const QnnSaver_Config_t ** saver_config) { - Qnn_ErrorHandle_t error = QNN_SUCCESS; - GGMLQNN_LOG_DEBUG("lib_path:%s\n", lib_path.c_str()); - - void * lib_handle = dlopen(lib_path.c_str(), RTLD_NOW | RTLD_GLOBAL); - if (nullptr == lib_handle) { - GGMLQNN_LOG_WARN("can not open QNN library %s, with error: %s", lib_path.c_str(), dlerror()); - return 1; - } - - auto get_providers = load_qnn_functionpointers<_pfn_QnnInterface_getProviders *>( - lib_handle, - "QnnInterface_getProviders"); - if (nullptr == get_providers) { - GGMLQNN_LOG_WARN("can not load symbol QnnInterface_getProviders : %s", dlerror()); - return 2; - } - - // get QnnInterface Providers - std::uint32_t num_providers = 0; - const QnnInterface_t ** provider_list = nullptr; - error = get_providers(&provider_list, &num_providers); - if (error != QNN_SUCCESS) { - GGMLQNN_LOG_WARN("failed to get providers, error %d", QNN_GET_ERROR_CODE(error)); - return 3; - } - GGMLQNN_LOG_DEBUG("num_providers=%d\n", num_providers); - if (num_providers != _required_num_providers) { - GGMLQNN_LOG_WARN("providers is %d instead of required %d", num_providers, _required_num_providers); - return 4; - } - - if (nullptr == provider_list) { - GGMLQNN_LOG_WARN("failed to get qnn interface providers\n"); - return 5; - } - bool found_valid_interface = false; - QNN_INTERFACE_VER_TYPE qnn_interface; - for (size_t idx = 0; idx < num_providers; idx++) { - if (QNN_API_VERSION_MAJOR == provider_list[idx]->apiVersion.coreApiVersion.major && - QNN_API_VERSION_MINOR <= provider_list[idx]->apiVersion.coreApiVersion.minor) { - found_valid_interface = true; - qnn_interface = provider_list[idx]->QNN_INTERFACE_VER_NAME; - break; - } - } - - if (!found_valid_interface) { - GGMLQNN_LOG_WARN("unable to find a valid qnn interface\n"); - return 6; - } else { - GGMLQNN_LOG_INFO("find a valid qnn interface\n"); - } - set_qnn_raw_interface(qnn_interface); - - BackendIdType backend_id = provider_list[0]->backendId; - _lib_path_to_backend_id[lib_path] = backend_id; - if (_loaded_backend.count(backend_id) > 0) { - GGMLQNN_LOG_WARN("lib_path %s is loaded, but backend %d already exists\n", - lib_path.c_str(), backend_id); - } - _loaded_backend[backend_id] = provider_list[0]; - if (_loaded_lib_handle.count(backend_id) > 0) { - GGMLQNN_LOG_WARN("closing %p\n", _loaded_lib_handle[backend_id]); - int dlclose_error = dlclose(_loaded_lib_handle[backend_id]); - if (dlclose_error != 0) { - GGMLQNN_LOG_WARN("fail to close %p with error %s\n", _loaded_lib_handle[backend_id], dlerror()); - } - } - _loaded_lib_handle[backend_id] = lib_handle; - _backend_id = backend_id; - - auto saver_initialize = - load_qnn_functionpointers<_pfn_QnnSaver_initialize *>( - _loaded_lib_handle[backend_id], "QnnSaver_initialize"); - if (nullptr != saver_initialize) { - error = saver_initialize(saver_config); - if (error != QNN_SUCCESS) { - GGMLQNN_LOG_WARN("failed to saver_initialize，error %d", QNN_GET_ERROR_CODE(error)); - return 7; - } - } else { - GGMLQNN_LOG_WARN("saver_initialize is null\n"); - } - - return 0; -} - -int qnn_instance::unload_backend() { - int dlclose_error = 0; - for (auto & it : _loaded_lib_handle) { - dlclose_error = dlclose(it.second); - if (dlclose_error != 0) { - GGMLQNN_LOG_WARN("failed to close QNN backend %d, error %s\n", it.first, dlerror()); - } - } - - _loaded_lib_handle.clear(); - _lib_path_to_backend_id.clear(); - _loaded_backend.clear(); - - return 0; -} - -int qnn_instance::load_system() { - Qnn_ErrorHandle_t error = QNN_SUCCESS; - -#ifdef _WIN32 - std::string system_lib_path = _lib_path + "QnnSystem.dll"; -#else - std::string system_lib_path = _lib_path + "libQnnSystem.so"; -#endif - GGMLQNN_LOG_DEBUG("system_lib_path:%s\n", system_lib_path.c_str()); - - _system_lib_handle = dlopen(system_lib_path.c_str(), RTLD_NOW | RTLD_LOCAL); - if (nullptr == _system_lib_handle) { - GGMLQNN_LOG_WARN("can not open QNN library %s, error: %s\n", system_lib_path.c_str(), dlerror()); - //re-try with default path of QNN binary runtime lib - _lib_path = "/data/local/tmp/"; -#ifdef _WIN32 - system_lib_path = _lib_path + "QnnSystem.dll"; -#else - system_lib_path = _lib_path + "libQnnSystem.so"; -#endif - _system_lib_handle = dlopen(system_lib_path.c_str(), RTLD_NOW | RTLD_LOCAL); - if (nullptr == _system_lib_handle) { - GGMLQNN_LOG_WARN("can not open QNN library %s, error: %s\n", system_lib_path.c_str(), dlerror()); - return 1; - } - } - - auto * get_providers = reinterpret_cast<_pfn_QnnSystemInterface_getProviders *>(dlsym( - _system_lib_handle, "QnnSystemInterface_getProviders")); - if (nullptr == get_providers) { - GGMLQNN_LOG_WARN("can not load QNN symbol QnnSystemInterface_getProviders: %s\n", dlerror()); - return 2; - } - - uint32_t num_providers = 0; - const QnnSystemInterface_t ** provider_list = nullptr; - error = get_providers(&provider_list, &num_providers); - if (error != QNN_SUCCESS) { - GGMLQNN_LOG_WARN("failed to get providers, error %d\n", QNN_GET_ERROR_CODE(error)); - return 3; - } - - if (num_providers != _required_num_providers) { - GGMLQNN_LOG_WARN("providers is %d instead of required %d\n", num_providers, _required_num_providers); - return 4; - } - - if (nullptr == provider_list) { - GGMLQNN_LOG_WARN("can not get providers\n"); - return 5; - } - - QNN_SYSTEM_INTERFACE_VER_TYPE qnn_system_interface; - bool found_valid_system_interface = false; - for (size_t idx = 0; idx < num_providers; idx++) { - if (QNN_SYSTEM_API_VERSION_MAJOR == - provider_list[idx]->systemApiVersion.major && - QNN_SYSTEM_API_VERSION_MINOR <= - provider_list[idx]->systemApiVersion.minor) { - found_valid_system_interface = true; - qnn_system_interface = provider_list[idx]->QNN_SYSTEM_INTERFACE_VER_NAME; - break; - } - } - if (!found_valid_system_interface) { - GGMLQNN_LOG_WARN("unable to find a valid qnn system interface\n"); - return 6; - } else { - GGMLQNN_LOG_INFO("find a valid qnn system interface\n"); - } - set_qnn_raw_system_interface(qnn_system_interface); - - _qnn_interface.set_qnn_system_interface(provider_list[0]); - - _qnn_interface.qnn_system_context_create(&_qnn_system_handle); - if (nullptr == _qnn_system_handle) { - GGMLQNN_LOG_WARN("can not create QNN system contenxt\n"); - } else { - GGMLQNN_LOG_INFO("initialize qnn system successfully\n"); - } - - return 0; -} - -int qnn_instance::unload_system() { - int result = 0; - - if (nullptr == _system_lib_handle) { - GGMLQNN_LOG_DEBUG("system lib handle is null\n"); - return 1; - } - - if (nullptr != _qnn_system_handle) { - result = _qnn_interface.qnn_system_context_free(_qnn_system_handle); - if (result != QNN_SUCCESS) { - GGMLQNN_LOG_WARN("failed to free QNN system context\n"); - } - _qnn_system_handle = nullptr; - } - - int dlclose_error = dlclose(_system_lib_handle); - if (dlclose_error != 0) { - GGMLQNN_LOG_WARN("failed to close QnnSystem library, error %s\n", dlerror()); - return 2; - } - - _system_lib_handle = nullptr; - - return result; -} - -#if GGMLQNN_PRINT_QNN_INTERNAL_LOG -static void ggml_qnn_logcallback(const char * fmt, - QnnLog_Level_t level, - uint64_t timestamp, - va_list argp) { - - static std::mutex log_mutex; - static unsigned char s_ggml_qnn_logbuf[GGML_QNN_LOGBUF_LEN]; - - const char * log_level_desc = ""; - switch (level) { - case QNN_LOG_LEVEL_ERROR: - log_level_desc = " ERROR "; - break; - case QNN_LOG_LEVEL_WARN: - log_level_desc = "WARNING"; - break; - case QNN_LOG_LEVEL_INFO: - log_level_desc = " INFO "; - break; - case QNN_LOG_LEVEL_DEBUG: - log_level_desc = " DEBUG "; - break; - case QNN_LOG_LEVEL_VERBOSE: - log_level_desc = "VERBOSE"; - break; - case QNN_LOG_LEVEL_MAX: - log_level_desc = "UNKNOWN"; - break; - } - - double ms = (double) timestamp / 1000000.0; - { - std::lock_guard lock(log_mutex); - memset(s_ggml_qnn_logbuf, 0, GGML_QNN_LOGBUF_LEN); - vsnprintf(reinterpret_cast(s_ggml_qnn_logbuf), GGML_QNN_LOGBUF_LEN, fmt, argp); - GGMLQNN_LOG_DEBUG("%8.1fms [%-7s] %s\n", ms, log_level_desc, s_ggml_qnn_logbuf); - } -} -#else -static void ggml_qnn_logcallback(const char * fmt, - QnnLog_Level_t level, - uint64_t timestamp, - va_list argp) { - GGML_UNUSED(fmt); - GGML_UNUSED(level); - GGML_UNUSED(timestamp); - GGML_UNUSED(argp); -} -#endif - -int qnn_instance::qnn_init(const QnnSaver_Config_t ** saver_config) { - BackendIdType backend_id = QNN_BACKEND_ID_NULL; - GGMLQNN_LOG_DEBUG("enter qni_init\n"); - const std::lock_guard lock(_init_mutex); - if (0 != load_system()) { - GGMLQNN_LOG_WARN("can not load QNN system lib, pls check why?\n"); - return 1; - } else { - GGMLQNN_LOG_DEBUG("load QNN system lib successfully\n"); - } - - std::string backend_lib_path = _lib_path + _backend_name; - if (0 == _lib_path_to_backend_id.count(backend_lib_path)) { - int is_load_ok = load_backend(backend_lib_path, saver_config); - if (0 != is_load_ok) { - GGMLQNN_LOG_WARN("failed to load QNN backend\n"); - return 2; - } - } - - backend_id = _lib_path_to_backend_id[backend_lib_path]; - if (0 == _loaded_backend.count(backend_id) || - 0 == _loaded_lib_handle.count(backend_id)) { - GGMLQNN_LOG_WARN("library %s is loaded but loaded backend count=%zu, loaded lib_handle count=%zu\n", - backend_lib_path.c_str(), - _loaded_backend.count(backend_id), - _loaded_lib_handle.count(backend_id)); - return 3; - } - _qnn_interface.set_qnn_interface(_loaded_backend[backend_id]); -#if 1 - _qnn_interface.qnn_log_create(ggml_qnn_logcallback, _qnn_log_level, &_qnn_log_handle); -#else - _qnn_raw_interface.logCreate(ggml_qnn_logcallback, _qnn_log_level, &_qnn_log_handle); -#endif - if (nullptr == _qnn_log_handle) { - GGMLQNN_LOG_WARN("why failed to initialize qnn log\n"); //NPU backend not work on Qualcomm SoC based low-end phone - return 4; - } else { - GGMLQNN_LOG_DEBUG("initialize qnn log successfully\n"); - } - - std::vector temp_backend_config; - _qnn_interface.qnn_backend_create(_qnn_log_handle, - temp_backend_config.empty() ? nullptr : temp_backend_config.data(), - &_qnn_backend_handle); - if (nullptr == _qnn_backend_handle) { - GGMLQNN_LOG_WARN("why failed to initialize qnn backend\n"); - return 5; - } else { - GGMLQNN_LOG_DEBUG("initialize qnn backend successfully\n"); - } - - if (nullptr != _qnn_raw_interface.propertyHasCapability) { - auto qnnstatus = _qnn_raw_interface.propertyHasCapability(QNN_PROPERTY_GROUP_DEVICE); - if (QNN_PROPERTY_NOT_SUPPORTED == qnnstatus) { - GGMLQNN_LOG_WARN("device property is not supported\n"); - } - if (QNN_PROPERTY_ERROR_UNKNOWN_KEY == qnnstatus) { - GGMLQNN_LOG_WARN("device property is not known to backend\n"); - } - } - - auto qnnstatus = _qnn_raw_interface.deviceCreate( - _qnn_log_handle, nullptr, &_qnn_device_handle); - if (QNN_SUCCESS != qnnstatus && QNN_DEVICE_ERROR_UNSUPPORTED_FEATURE != qnnstatus) { - GGMLQNN_LOG_WARN("failed to create QNN device\n"); - } else { - GGMLQNN_LOG_INFO("create device successfully\n"); - } - - if (ggml_qnn_profile_level::profile_off != _profile_level) { - GGMLQNN_LOG_INFO("profiling turned on; level = %d", _profile_level); - if (ggml_qnn_profile_level::profile_basic == _profile_level) { - GGMLQNN_LOG_INFO("basic profiling requested. creating Qnn Profile object\n"); - if (QNN_PROFILE_NO_ERROR != _qnn_raw_interface.profileCreate( - _qnn_backend_handle, QNN_PROFILE_LEVEL_BASIC, &_qnn_profile_handle)) { - GGMLQNN_LOG_WARN("unable to create profile handle in the backend\n"); - return 6; - } else { - GGMLQNN_LOG_DEBUG("initialize qnn profile successfully\n"); - } - } else if (ggml_qnn_profile_level::profile_detail == _profile_level) { - GGMLQNN_LOG_INFO("detailed profiling requested. Creating Qnn Profile object\n"); - if (QNN_PROFILE_NO_ERROR != _qnn_raw_interface.profileCreate( - _qnn_backend_handle, QNN_PROFILE_LEVEL_DETAILED, &_qnn_profile_handle)) { - GGMLQNN_LOG_WARN("unable to create profile handle in the backend\n"); - return 7; - } else { - GGMLQNN_LOG_DEBUG("initialize qnn profile successfully\n"); - } - } - } - -#if defined(__ANDROID__) || defined(__linux__) - _rpc_lib_handle = dlopen("libcdsprpc.so", RTLD_NOW | RTLD_LOCAL); -#elif defined(_WIN32) - _rpc_lib_handle = dlopen("libcdsprpc.dll", RTLD_NOW | RTLD_LOCAL); -#else -#error "ggml-qnn only support WoA, Android, Linux" -#endif - if (nullptr == _rpc_lib_handle) { - GGMLQNN_LOG_WARN("failed to load qualcomm's rpc lib, error:%s\n", dlerror()); - return 8; - } else { - GGMLQNN_LOG_DEBUG("load rpcmem lib successfully\n"); - set_rpcmem_initialized(true); - } - _pfn_rpc_mem_init = reinterpret_cast(dlsym(_rpc_lib_handle, "rpcmem_init")); - _pfn_rpc_mem_deinit = reinterpret_cast(dlsym(_rpc_lib_handle, "rpcmem_deinit")); - _pfn_rpc_mem_alloc = reinterpret_cast(dlsym(_rpc_lib_handle,"rpcmem_alloc")); - _pfn_rpc_mem_free = reinterpret_cast(dlsym(_rpc_lib_handle, "rpcmem_free")); - _pfn_rpc_mem_to_fd = reinterpret_cast(dlsym(_rpc_lib_handle,"rpcmem_to_fd")); - if (nullptr == _pfn_rpc_mem_alloc || nullptr == _pfn_rpc_mem_free - || nullptr == _pfn_rpc_mem_to_fd) { - GGMLQNN_LOG_WARN("unable to access symbols in QNN RPC lib. dlerror(): %s", dlerror()); - dlclose(_rpc_lib_handle); - return 9; - } - - if (nullptr != _pfn_rpc_mem_init) // make Qualcomm's SoC based low-end phone happy - _pfn_rpc_mem_init(); - - std::vector temp_context_config; - _qnn_interface.qnn_context_create(_qnn_backend_handle, _qnn_device_handle, - temp_context_config.empty() ? nullptr : temp_context_config.data(), - &_qnn_context_handle); - if (nullptr == _qnn_context_handle) { - GGMLQNN_LOG_WARN("why failed to initialize qnn context, error:%s\n", strerror(errno)); - return 10; - } else { - GGMLQNN_LOG_DEBUG("initialize qnn context successfully\n"); - } - - if (_backend_name.find("Htp") != std::variant_npos) { - const QnnDevice_PlatformInfo_t * p_info = nullptr; - _qnn_raw_interface.deviceGetPlatformInfo(nullptr, &p_info); - GGMLQNN_LOG_INFO("device counts %d", p_info->v1.numHwDevices); - QnnDevice_HardwareDeviceInfo_t * infos = p_info->v1.hwDevices; - for (size_t i = 0; i < p_info->v1.numHwDevices; i++) { - GGMLQNN_LOG_INFO("deviceID:%d, deviceType:%d, numCores %d", infos[i].v1.deviceId, - infos[i].v1.deviceType, infos[i].v1.numCores); - QnnDevice_DeviceInfoExtension_t devinfo = infos[i].v1.deviceInfoExtension; - QnnHtpDevice_OnChipDeviceInfoExtension_t chipinfo = devinfo->onChipDevice; - QnnHtpDevice_Arch_t htp_arch = chipinfo.arch; - GGMLQNN_LOG_INFO("htp_type:%d(%s)", devinfo->devType, - (devinfo->devType == QNN_HTP_DEVICE_TYPE_ON_CHIP) ? "QNN_HTP_DEVICE_TYPE_ON_CHIP" : "QNN_HTP_DEVICE_TYPE_UNKNOWN"); - GGMLQNN_LOG_INFO("qualcomm soc_model:%d(%s), htp_arch:%d(%s), vtcm_size:%d MB", \ - chipinfo.socModel, qnn_get_socmodel_desc(chipinfo.socModel), \ - htp_arch, qnn_get_htparch_desc(htp_arch), chipinfo.vtcmSize); - struct qcom_socinfo * socinfo = qnn_get_socinfo_from_socmodel(chipinfo.socModel); - g_qnn_mgr[QNN_BACKEND_NPU].socinfo = { chipinfo.socModel, htp_arch, chipinfo.vtcmSize, {}}; - if (nullptr != socinfo) { - memcpy(g_qnn_mgr[QNN_BACKEND_NPU].socinfo.soc_desc, socinfo->soc_desc, sizeof(socinfo->soc_desc)); - GGMLQNN_LOG_INFO("soc info:%s", socinfo->soc_desc); - } else { - memcpy(g_qnn_mgr[QNN_BACKEND_NPU].socinfo.soc_desc, "unknown", 7); - GGMLQNN_LOG_INFO("soc info:unknown"); - } - } - _qnn_raw_interface.deviceFreePlatformInfo(nullptr, p_info); - - probe_device_meminfo(); - - if (0 != init_htp_perfinfra()) { - GGMLQNN_LOG_WARN("initialize HTP performance failure"); - } - if (0 != set_rpc_polling()) { - GGMLQNN_LOG_WARN("set RPC polling failure"); - } - if (0 != set_high_performance_mode()) { - GGMLQNN_LOG_WARN("set HTP high performance mode failure"); - } - } - - GGMLQNN_LOG_DEBUG("leave qni_init\n"); - - return 0; -} - -int qnn_instance::qnn_finalize() { - int ret_status = 0; - Qnn_ErrorHandle_t error = QNN_SUCCESS; - - GGMLQNN_LOG_DEBUG("enter %s\n", __func__); - reset_idx(); - - free_rpcmem(); - unregister_rpcmem(); - - if (nullptr != _pfn_rpc_mem_deinit) - _pfn_rpc_mem_deinit(); - - if (dlclose(_rpc_lib_handle) != 0) { - GGMLQNN_LOG_WARN("failed to unload qualcomm's rpc lib, error:%s\n", dlerror()); - } else { - GGMLQNN_LOG_DEBUG("succeed to close rpcmem lib\n"); - } - - if (nullptr != _qnn_context_handle) { - error = _qnn_interface.qnn_context_free(_qnn_context_handle, _qnn_profile_handle); - if (error != QNN_SUCCESS) { - GGMLQNN_LOG_WARN("failed to free QNN context_handle: ID %u, error %d\n", - _qnn_interface.get_backend_id(), QNN_GET_ERROR_CODE(error)); - - } - _qnn_context_handle = nullptr; - } - - if (nullptr != _qnn_profile_handle) { - error = _qnn_interface.qnn_profile_free(_qnn_profile_handle); - if (error != QNN_SUCCESS) { - GGMLQNN_LOG_WARN("failed to free QNN profile_handle: ID %u, error %d\n", - _qnn_interface.get_backend_id(), QNN_GET_ERROR_CODE(error)); - - } - _qnn_profile_handle = nullptr; - } - - if (nullptr != _qnn_device_handle) { - error = _qnn_interface.qnn_device_free(_qnn_device_handle); - if (error != QNN_SUCCESS) { - GGMLQNN_LOG_WARN("failed to free QNN device_handle: ID %u, error %d\n", - _qnn_interface.get_backend_id(), QNN_GET_ERROR_CODE(error)); - - } - _qnn_device_handle = nullptr; - } - - if (nullptr != _qnn_backend_handle) { - error = _qnn_interface.qnn_backend_free(_qnn_backend_handle); - if (error != QNN_SUCCESS) { - GGMLQNN_LOG_WARN("failed to free QNN backend_handle: ID %u, error %d\n", - _qnn_interface.get_backend_id(), QNN_GET_ERROR_CODE(error)); - } - _qnn_backend_handle = nullptr; - - } - - if (nullptr != _qnn_log_handle) { - error = _qnn_interface.qnn_log_free(_qnn_log_handle); - if (error != QNN_SUCCESS) { - GGMLQNN_LOG_WARN("failed to free QNN log_handle: ID %u, error %d\n", - _qnn_interface.get_backend_id(), QNN_GET_ERROR_CODE(error)); - } - _qnn_log_handle = nullptr; - } - - unload_backend(); - - unload_system(); - GGMLQNN_LOG_DEBUG("leave %s\n", __func__); - - return ret_status; -} - -int qnn_instance::init_qnn_graph(const std::string & graph_name, QNNBackend device, size_t vtcm_size_in_mb, size_t hvx_threads) { - _graph_name = graph_name; - _device_id = device; - - GGMLQNN_LOG_DEBUG("[%s][%s]created", ggml_backend_qnn_get_devname(device), graph_name.c_str()); - - Qnn_ErrorHandle_t error = QNN_SUCCESS; - Qnn_GraphHandle_t graph_handle = nullptr; - if (device == QNN_BACKEND_NPU) { - QnnHtpGraph_CustomConfig_t hvx_config; - hvx_config.option = QNN_HTP_GRAPH_CONFIG_OPTION_NUM_HVX_THREADS; - hvx_config.numHvxThreads = hvx_threads; - QnnGraph_Config_t graph_hvx_config; - graph_hvx_config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM; - graph_hvx_config.customConfig = &hvx_config; - - QnnHtpGraph_CustomConfig_t dlbc_config; - dlbc_config.option = QNN_HTP_GRAPH_CONFIG_OPTION_OPTIMIZATION; - dlbc_config.optimizationOption.type = QNN_HTP_GRAPH_OPTIMIZATION_TYPE_ENABLE_DLBC; - dlbc_config.optimizationOption.floatValue = 1.0; // set to 0.0 to turn off DLBC - QnnGraph_Config_t graph_dlbc_config; - graph_dlbc_config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM; - graph_dlbc_config.customConfig = &dlbc_config; - - QnnHtpGraph_CustomConfig_t opt_config; - opt_config.optimizationOption.type = QNN_HTP_GRAPH_OPTIMIZATION_TYPE_FINALIZE_OPTIMIZATION_FLAG; - opt_config.optimizationOption.floatValue = 1; // 1 / 3 - QnnGraph_Config_t graph_opt_config; - graph_opt_config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM; - graph_opt_config.customConfig = &opt_config; - - QnnHtpGraph_CustomConfig_t vtcm_config; - vtcm_config.option = QNN_HTP_GRAPH_CONFIG_OPTION_VTCM_SIZE; - vtcm_config.vtcmSizeInMB = vtcm_size_in_mb; - QnnGraph_Config_t graph_vtcm_config; - graph_vtcm_config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM; - graph_vtcm_config.customConfig = &vtcm_config; - - const QnnGraph_Config_t * graph_configs[] = {&graph_hvx_config, &graph_dlbc_config, &graph_vtcm_config, - &graph_opt_config, nullptr}; - error = _qnn_interface.qnn_graph_create(_qnn_context_handle, graph_name.c_str(), graph_configs, &graph_handle); - } else { - error = _qnn_interface.qnn_graph_create(_qnn_context_handle, graph_name.c_str(), nullptr, &graph_handle); - } - - if (error != QNN_SUCCESS) { - GGMLQNN_LOG_ERROR("[%s][%s]failed to create qnn graph, error: %s", - ggml_backend_qnn_get_devname(device), graph_name.c_str(), - ggmlqnn_get_error_string(error)); - return error; - } - - GGMLQNN_LOG_DEBUG("[%s]create graph %s succeed", ggml_backend_qnn_get_devname(device), graph_name.c_str()); - _qnn_graph_handle = graph_handle; - return QNN_SUCCESS; -} - -int qnn_instance::init_qnn_graph(const char * graph_name, bool debug, uint8_t do_node_validation, - const QnnGraph_Config_t ** graph_configs) { - int result = 0; - - if (nullptr == graph_name) { - GGMLQNN_LOG_WARN("graph name is null\n"); - return 1; - } - - if (!_graph_name.empty()) { - GGMLQNN_LOG_WARN("qnn model for graph %s already initialized\n", graph_name); - return 2; - } - - if (!do_node_validation) { - GGMLQNN_LOG_WARN("node validation disabled, backend will not perform op validation prior to adding node\n"); - } - - _graph_name = graph_name; - _debug_tensor = debug; - _do_node_validations = do_node_validation; - - result = _qnn_raw_interface.graphCreate(_qnn_context_handle, - graph_name, - graph_configs, - &_qnn_graph_handle); - if (result != QNN_GRAPH_NO_ERROR || nullptr == _qnn_graph_handle) { - GGMLQNN_LOG_WARN("failed to create graph in qnn context\n"); - return 3; - } else { - GGMLQNN_LOG_INFO("succeed to create graph %s, %p\n", graph_name, _qnn_graph_handle); - } - - return 0; -} - -int qnn_instance::finalize_qnn_graph() { - if (nullptr != _qnn_graph_handle) { - if (_qnn_raw_interface.graphFinalize(_qnn_graph_handle, - _qnn_profile_handle, nullptr) - != QNN_GRAPH_NO_ERROR) { - GGMLQNN_LOG_WARN("finalizing graph failure\n"); - return 1; - } - } else { - GGMLQNN_LOG_DEBUG("qnn graph handle is null\n"); - } - - return 0; -} - -int qnn_instance::init_htp_perfinfra() { - QnnDevice_Infrastructure_t device_infra = nullptr; - int error = _qnn_raw_interface.deviceGetInfrastructure(&device_infra); - if (error != QNN_SUCCESS) { - GGMLQNN_LOG_WARN("failed to get qnn device infra\n"); - return 1; - } - - QnnHtpDevice_Infrastructure_t * htp_infra = static_cast(device_infra); - QnnHtpDevice_PerfInfrastructure_t * htp_perfinfra = &htp_infra->perfInfra; - uint32_t power_configid = 1; - uint32_t device_id = 0; - uint32_t core_id = 0; - htp_perfinfra->createPowerConfigId(device_id, core_id, &power_configid); - _qnn_htp_perfinfra = htp_perfinfra; - _qnn_power_configid = power_configid; - - return 0; -} - -int qnn_instance::set_rpc_polling() { - if (_qnn_rpc_pollingtime > 0) { - QnnHtpPerfInfrastructure_PowerConfig_t rpc_pollingtime; - memset(&rpc_pollingtime, 0, sizeof(rpc_pollingtime)); - rpc_pollingtime.option = QNN_HTP_PERF_INFRASTRUCTURE_POWER_CONFIGOPTION_RPC_POLLING_TIME; - rpc_pollingtime.rpcPollingTimeConfig = _qnn_rpc_pollingtime; - const QnnHtpPerfInfrastructure_PowerConfig_t * power_configs[] = {&rpc_pollingtime, nullptr}; - if (_qnn_htp_perfinfra) { - _qnn_htp_perfinfra->setPowerConfig(_qnn_power_configid, power_configs); - } - } - return 0; -} - -int qnn_instance::set_high_performance_mode() { - if (nullptr == _qnn_htp_perfinfra) { - GGMLQNN_LOG_DEBUG("perf intra is null\n"); - return 1; - } - - QnnHtpPerfInfrastructure_PowerConfig_t power_config; - memset(&power_config, 0, sizeof(power_config)); - power_config.option = QNN_HTP_PERF_INFRASTRUCTURE_POWER_CONFIGOPTION_DCVS_V3; - power_config.dcvsV3Config.dcvsEnable = 0; - power_config.dcvsV3Config.setDcvsEnable = 1; - power_config.dcvsV3Config.contextId = _qnn_power_configid; - power_config.dcvsV3Config.powerMode = QNN_HTP_PERF_INFRASTRUCTURE_POWERMODE_PERFORMANCE_MODE; - power_config.dcvsV3Config.setSleepLatency = 1; // True to consider Latency parameter otherwise False - power_config.dcvsV3Config.setBusParams = 1; // True to consider Bus parameter otherwise False - power_config.dcvsV3Config.setCoreParams = 1; // True to consider Core parameter otherwise False - power_config.dcvsV3Config.sleepDisable = 0; // True to consider sleep/LPM modes, False to enable - power_config.dcvsV3Config.setSleepDisable = 0; // True to consider sleep disable/enable parameter otherwise False - // set Sleep latency parameter - uint32_t latencyValue = 40; - power_config.dcvsV3Config.sleepLatency = latencyValue; // range 40-2000 micro sec - // set Bus Clock Parameters (refer QnnHtpPerfInfrastructure_VoltageCorner_t enum) - power_config.dcvsV3Config.busVoltageCornerMin = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; - power_config.dcvsV3Config.busVoltageCornerTarget = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; - power_config.dcvsV3Config.busVoltageCornerMax = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; - // set Core Clock Parameters (refer QnnHtpPerfInfrastructure_VoltageCorner_t enum) - power_config.dcvsV3Config.coreVoltageCornerMin = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; - power_config.dcvsV3Config.coreVoltageCornerTarget = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; - power_config.dcvsV3Config.coreVoltageCornerMax = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; - // set power config with different performance parameters - const QnnHtpPerfInfrastructure_PowerConfig_t * power_configs[] = {&power_config, nullptr}; - - _qnn_htp_perfinfra->setPowerConfig(_qnn_power_configid, power_configs); - - return 0; -} - -void qnn_instance::probe_device_meminfo() { - size_t candidate_size = 0; - uint8_t * rpc_buffer = nullptr; - const int SIZE_IN_MB = (1 << 20); - size_t probe_slots[] = {1024, 1536, 2048 - 48, 2048}; - size_t probe_counts = sizeof(probe_slots) / sizeof(size_t); - for (size_t idx = 0; idx < probe_counts; idx++) { - rpc_buffer = static_cast(alloc_rpcmem_internal(probe_slots[idx] * SIZE_IN_MB, 4)); - if (nullptr == rpc_buffer) { - GGMLQNN_LOG_DEBUG("alloc rpcmem %d (MB) failure, %s\n", probe_slots[idx], strerror(errno)); - break; - } else { - candidate_size = probe_slots[idx]; - free_rpcmem(rpc_buffer); - rpc_buffer = nullptr; - } - } - if (candidate_size > _rpcmem_capacity) - _rpcmem_capacity = candidate_size; - - free_rpcmem(); - _rpcmem_usage = 0; - GGMLQNN_LOG_INFO("capacity of rpc ion memory %d MB\n", _rpcmem_capacity); -} - -uint8_t * ggmlqnn_create_rpc_buffer(qnn_instance * instance, const ggml_tensor * ggml_tensor, Qnn_Tensor_t * qnn_tensor, bool b_copydata) { - if (nullptr == instance || nullptr == ggml_tensor || nullptr == qnn_tensor) { - GGMLQNN_LOG_WARN("invalid params\n"); - return nullptr; - } - - uint8_t * qnn_rpcbuffer = static_cast(instance->alloc_rpcmem(ggml_nbytes(ggml_tensor), 4)); - if (nullptr == qnn_rpcbuffer) { - GGMLQNN_LOG_WARN("alloc rpcmem failure, %s\n", strerror(errno)); - return nullptr; - } else { - GGMLQNN_LOG_DEBUG("alloc rpcmem %p successfully\n", qnn_rpcbuffer); - } - if (b_copydata) - memcpy(qnn_rpcbuffer, ggml_tensor->data, ggml_nbytes(ggml_tensor)); - instance->register_rpcmem(qnn_rpcbuffer, qnn_tensor); - return qnn_rpcbuffer; -} - -void ggmlqnn_print_tensors_info(const char * func_name, ggml_backend_qnn_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - //skip sanity check of params - if (nullptr != func_name && nullptr != ctx) { - GGMLQNN_LOG_DEBUG("call %s in dev %s\n", func_name, ctx->name); - } - GGMLQNN_LOG_DEBUG("%-6s: type = %i (%s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi, %5zi)", - src0->name, - src0->type, ggml_type_name(src0->type), src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], - src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3]); - GGMLQNN_LOG_DEBUG("%-6s: type = %i (%s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi, %5zi)", - src1->name, - src1->type, ggml_type_name(src1->type), src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3], - src1->nb[0], src1->nb[1], src1->nb[2], src1->nb[3]); - GGMLQNN_LOG_DEBUG("%-6s: type = %i (%s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi, %5zi)", - dst->name, - dst->type, ggml_type_name(dst->type), dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], - dst->nb[0], dst->nb[1], dst->nb[2], dst->nb[3]); - GGMLQNN_LOG_DEBUG("\n"); -} - -static void dump_op_info(const struct ggml_tensor * tensor) { - //skip sanity check of params - const struct ggml_tensor * src0 = tensor->src[0]; - struct ggml_tensor * src1 = tensor->src[1]; - struct ggml_tensor * dst = const_cast(tensor); - GGMLQNN_LOG_DEBUG("op name:%s, tensor type:%s", ggml_op_name(tensor->op), ggml_type_name(tensor->type)); - ggmlqnn_print_tensors_info(nullptr, nullptr, src0, src1, dst); -} - -// ================================================================================================= -// section-6: implementation of ggml-qnn backend -// ================================================================================================= -//TODO: refine this function as it is a performance hotspot/bottleneck function -static bool ggml_qnn_can_handle_op(const ggml_backend_qnn_context * ctx, const struct ggml_tensor * tensor) { - if (tensor->op == GGML_OP_NONE) { - return true; - } - if (ggml_is_empty(tensor) || tensor->op == GGML_OP_RESHAPE - || tensor->op == GGML_OP_TRANSPOSE - || tensor->op == GGML_OP_VIEW - || tensor->op == GGML_OP_PERMUTE - ) { - return false; - } - - //TODO: add other op here - bool supported_op = ((tensor->op == GGML_OP_ADD) - || (tensor->op == GGML_OP_MUL_MAT) - || (tensor->op == GGML_OP_MUL) - ); - if (!supported_op) { - return false; - } - - struct ggml_tensor * src0 = tensor->src[0]; - struct ggml_tensor * src1 = tensor->src[1]; - - const int64_t ne00 = tensor->src[0]->ne[0]; - const int64_t ne01 = tensor->src[0]->ne[1]; - - const int64_t ne10 = tensor->src[1]->ne[0]; - const int64_t ne11 = tensor->src[1]->ne[1]; - - const int64_t ne0 = tensor->ne[0]; - const int64_t ne1 = tensor->ne[1]; - - const uint32_t src0_rank = ggml_n_dims(src0); - const uint32_t src1_rank = ggml_n_dims(src1); - GGML_UNUSED(ne01); - GGML_UNUSED(ne10); - GGML_UNUSED(ne11); - GGML_UNUSED(ne0); - GGML_UNUSED(ne1); - - if (tensor->op == GGML_OP_ADD) { - //dump_op_info(tensor); - if (!ggml_are_same_shape(src0, src1)) { - return false; - } - if (ne00 < 32) - return false; - return (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16) - && (src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16); - } - - if (tensor->op == GGML_OP_MUL_MAT) { - //dump_op_info(tensor); - if (src0_rank != src1_rank) // make QNN SDK happy - return false; - if (src0_rank < 2) // QNN's limitation, make QNN SDK happy - return false; - if (4 == src0_rank) //TODO: 4D matrix mulmat in CT - return false; - if ((src1->ne[2] != src0->ne[2]) || (src1->ne[3] != src0->ne[3])) // make QNN SDK happy - return false; - - if (ctx->device == QNN_BACKEND_NPU) - if (2 == src0_rank) - return (src0->type == GGML_TYPE_F32 - || src0->type == GGML_TYPE_Q4_0 || src0->type == GGML_TYPE_Q8_0 - || src0->type == GGML_TYPE_Q6_K || src0->type == GGML_TYPE_Q8_K - ) && (src1->type == GGML_TYPE_F32) && (tensor->type == GGML_TYPE_F32); - else - return (src0->type == GGML_TYPE_F32) && (src1->type == GGML_TYPE_F32) && (tensor->type == GGML_TYPE_F32); - else - return (src0->type == GGML_TYPE_F32 || ggml_is_quantized(src0->type)) - && (src1->type == GGML_TYPE_F32) && (tensor->type == GGML_TYPE_F32); - } - - if (tensor->op == GGML_OP_MUL) { - //dump_op_info(tensor); - if ((src0_rank != 2) || (src1_rank != 2)) //TODO: 3D and 4D matrix - return false; - return (src0->type == GGML_TYPE_F32) - && (src1->type == GGML_TYPE_F32) - && (tensor->type == src1->type); - } - - return false; -} - -static bool ggml_qnn_compute_forward(ggml_backend_t backend, struct ggml_tensor * dst) { - ggmlqnn_op_func_t func = nullptr; - ggml_backend_qnn_context * ctx = (ggml_backend_qnn_context *)backend->context; - - switch (dst->op) { - case GGML_OP_REPEAT: - ggml_qnn_repeat(ctx, dst); - break; - case GGML_OP_GET_ROWS: - ggml_qnn_get_rows(ctx, dst); - break; - case GGML_OP_DUP: - ggml_qnn_dup(ctx, dst); - break; - case GGML_OP_ADD: - func = ggml_qnn_general_node; - break; - case GGML_OP_ACC: - ggml_qnn_acc(ctx, dst); - break; - case GGML_OP_MUL: - func = ggml_qnn_general_node; - break; - case GGML_OP_DIV: - ggml_qnn_div(ctx, dst); - break; - case GGML_OP_UNARY: - switch (ggml_get_unary_op(dst)) { - case GGML_UNARY_OP_GELU: - break; - case GGML_UNARY_OP_SILU: - break; - case GGML_UNARY_OP_GELU_QUICK: - break; - case GGML_UNARY_OP_TANH: - break; - case GGML_UNARY_OP_RELU: - break; - case GGML_UNARY_OP_HARDSIGMOID: - break; - case GGML_UNARY_OP_HARDSWISH: - break; - default: - return false; - } - break; - case GGML_OP_NORM: - ggml_qnn_norm(ctx, dst); - break; - case GGML_OP_GROUP_NORM: - ggml_qnn_group_norm(ctx, dst); - break; - case GGML_OP_CONCAT: - ggml_qnn_concat(ctx, dst); - break; - case GGML_OP_UPSCALE: - ggml_qnn_upsample_nearest2d(ctx, dst); - break; - case GGML_OP_PAD: - ggml_qnn_pad(ctx, dst); - break; - case GGML_OP_ARANGE: - ggml_qnn_arange(ctx, dst); - break; - case GGML_OP_TIMESTEP_EMBEDDING: - ggml_qnn_timestep_embedding(ctx, dst); - break; - case GGML_OP_LEAKY_RELU: - ggml_qnn_leaky_relu(ctx, dst); - break; - case GGML_OP_RMS_NORM: - ggml_qnn_rms_norm(ctx, dst); - break; - case GGML_OP_MUL_MAT: - ggml_qnn_mul_mat(ctx, dst); - break; - case GGML_OP_MUL_MAT_ID: - return false; - case GGML_OP_SCALE: - ggml_qnn_scale(ctx, dst); - break; - case GGML_OP_SQR: - ggml_qnn_sqr(ctx, dst); - break; - case GGML_OP_CLAMP: - ggml_qnn_clamp(ctx, dst); - break; - case GGML_OP_CPY: - ggml_qnn_cpy(ctx, dst); - break; - case GGML_OP_CONT: - ggml_qnn_dup(ctx, dst); - break; - case GGML_OP_NONE: - case GGML_OP_RESHAPE: - case GGML_OP_VIEW: - case GGML_OP_PERMUTE: - case GGML_OP_TRANSPOSE: - break; - case GGML_OP_DIAG_MASK_INF: - ggml_qnn_diag_mask(ctx, dst, -INFINITY); - break; - case GGML_OP_SOFT_MAX: - ggml_qnn_softmax(ctx, dst); - break; - case GGML_OP_ROPE: - ggml_qnn_rope(ctx, dst); - break; - case GGML_OP_IM2COL: - ggml_qnn_im2col(ctx, dst); - break; - case GGML_OP_POOL_2D: - ggml_qnn_pool2d(ctx, dst); - break; - case GGML_OP_SUM_ROWS: - ggml_qnn_sum_rows(ctx, dst); - break; - case GGML_OP_ARGSORT: - ggml_qnn_argsort(ctx, dst); - break; - default: - return false; - } - - if (nullptr != func) - func(ctx, dst); - - return true; -} - -struct ggml_backend_qnn_buffer_context { - ~ggml_backend_qnn_buffer_context() { - if (buffer) { - free(buffer); - } - - for (auto * sub_buffer : sub_buffers) { - free(sub_buffer); - } - - for (auto * qnn_tensor : qnn_tensors) { - free_qnn_tensor(qnn_tensor); - } - - sub_buffers.clear(); - qnn_tensors.clear(); - } - void * buffer = nullptr; - - struct ggml_backend_qnn_context * backend_ctx = nullptr; - - size_t buffer_size = 0; - std::vector sub_buffers; - std::vector qnn_tensors; -}; - -static void ggml_backend_qnn_buffer_free_buffer(ggml_backend_buffer_t buffer) { - ggml_backend_qnn_buffer_context * ctx = (ggml_backend_qnn_buffer_context *)buffer->context; - delete ctx; -} - -static void * ggml_backend_qnn_buffer_get_base(ggml_backend_buffer_t buffer) { - ggml_backend_qnn_buffer_context * ctx = (ggml_backend_qnn_buffer_context *)buffer->context; - return ctx->buffer; -} - -static void ggml_backend_qnn_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) { - ggml_backend_qnn_buffer_context * ctx = (ggml_backend_qnn_buffer_context *)buffer->context; - GGML_UNUSED(tensor); - GGML_UNUSED(ctx); - return; -} - -static void ggml_backend_qnn_buffer_set_tensor(ggml_backend_buffer_t buffer, - ggml_tensor * tensor, const void * data, - size_t offset, size_t size) { - GGML_UNUSED(buffer); - - memcpy((char *)tensor->data + offset, data, size); -} - -static void ggml_backend_qnn_buffer_memset_tensor(ggml_backend_buffer_t buffer, - struct ggml_tensor * tensor, - uint8_t value, size_t offset, size_t size) { - GGML_UNUSED(buffer); - memset((char *)tensor->data + offset, value, size); -} - -static void ggml_backend_qnn_buffer_get_tensor(ggml_backend_buffer_t buffer, - const ggml_tensor * tensor, - void * data, size_t offset, size_t size) { - GGML_UNUSED(buffer); - memcpy(data, (const char *)tensor->data + offset, size); -} - -static bool ggml_backend_qnn_buffer_cpy_tensor(ggml_backend_buffer_t buffer, - const struct ggml_tensor * src, - struct ggml_tensor * dst) { - GGML_UNUSED(buffer); - if (ggml_backend_buffer_is_host(src->buffer)) { - memcpy(dst->data, src->data, ggml_nbytes(src)); - return true; - } - - return false; -} - -static void ggml_backend_qnn_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) { - ggml_backend_qnn_buffer_context * ctx = (ggml_backend_qnn_buffer_context *)buffer->context; - memset(ctx->buffer, value, ctx->buffer_size); -} - -static ggml_backend_buffer_i ggml_backend_qnn_buffer_interface = { - /* .free_buffer = */ ggml_backend_qnn_buffer_free_buffer, - /* .get_base = */ ggml_backend_qnn_buffer_get_base, - /* .init_tensor = */ ggml_backend_qnn_buffer_init_tensor, - /* .memset_tensor = */ ggml_backend_qnn_buffer_memset_tensor, - /* .set_tensor = */ ggml_backend_qnn_buffer_set_tensor, - /* .get_tensor = */ ggml_backend_qnn_buffer_get_tensor, - /* .cpy_tensor = */ ggml_backend_qnn_buffer_cpy_tensor, - /* .clear = */ ggml_backend_qnn_buffer_clear, - /* .reset = */ nullptr, -}; - -static const char * ggml_backend_qnn_buffer_type_name(ggml_backend_buffer_type_t buft) { - GGML_UNUSED(buft); - return "qnn-buffer"; -} - -static ggml_backend_buffer_t ggml_backend_qnn_buffer_type_alloc_buffer( - ggml_backend_buffer_type_t buft, size_t size) { - ggml_backend_qnn_buffer_context * ctx = new ggml_backend_qnn_buffer_context; - -#if defined(__ANDROID__) || defined(__linux__) - size_t size_page = sysconf(_SC_PAGESIZE); -#elif defined(_WIN32) - SYSTEM_INFO systeminfo; - GetSystemInfo(&systeminfo); - size_t size_page = systeminfo.dwPageSize; -#endif - size_t size_aligned = size; - if ((size_aligned % size_page) != 0) { - size_aligned += (size_page - (size_aligned % size_page)); - } - ctx->buffer = ggmlqnn_host_malloc(size_aligned); - ctx->buffer_size = size_aligned; - if (nullptr == ctx->buffer) { - GGMLQNN_LOG_WARN("%s: failed to allocate %.2f MiB\n", __func__, size / (1 << 20)); - return nullptr; - } - - return ggml_backend_buffer_init(buft, ggml_backend_qnn_buffer_interface, ctx, size); -} - -static size_t ggml_backend_qnn_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) { - GGML_UNUSED(buft); - return 32; -} - -//TODO:not used currently -static size_t ggml_backend_qnn_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) { - GGML_UNUSED(buft); - - return (2 * (1 << 20)); -} - -static bool ggml_backend_qnn_buffer_is_host(ggml_backend_buffer_type_t buft) { - GGML_UNUSED(buft); - return true; -} - -static const char * ggml_backend_qnn_name(ggml_backend_t backend) { - ggml_backend_qnn_context * ctx = (ggml_backend_qnn_context *) backend->context; - return g_qnn_mgr[ctx->device].name; -} - -static void ggml_backend_qnn_free(ggml_backend_t backend) { - GGMLQNN_LOG_DEBUG("enter %s", __func__ ); - ggml_backend_qnn_context * ctx = (ggml_backend_qnn_context *) backend->context; - GGMLQNN_LOG_DEBUG("idx %d, name:%s", ctx->device, g_qnn_mgr[ctx->device].name); - - qnn_instance * instance = (qnn_instance*)g_qnn_mgr[ctx->device].instance; - if (instance != nullptr) { - std::map>>::iterator graph_it; - - for (graph_it = instance->_qnn_graph_map.begin(); - graph_it != instance->_qnn_graph_map.end(); graph_it++) { - auto & graph_item = graph_it->second; - Qnn_GraphHandle_t & graph_handle = std::get<0>(graph_item); - qnn_tensors_t & tensors = std::get<1>(graph_item); - for (auto tensor_it = tensors.begin(); tensor_it != tensors.end(); ++tensor_it) { - free_qnn_tensor(*tensor_it); - } - GGML_UNUSED(graph_handle); - GGMLQNN_LOG_DEBUG("graph type:%s", graph_it->first.c_str()); - } - instance->_qnn_graph_map.clear(); - - instance->qnn_finalize(); - delete instance; - g_qnn_mgr[ctx->device].instance = nullptr; - } - - if (g_qnn_mgr[ctx->device].backend != nullptr) { - delete backend; - g_qnn_mgr[ctx->device].backend = nullptr; - } - GGMLQNN_LOG_DEBUG("leave %s", __func__ ); -} - -static enum ggml_status ggml_backend_qnn_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph) { - enum ggml_status result = GGML_STATUS_SUCCESS; - ggml_backend_qnn_context * ctx = (ggml_backend_qnn_context *) backend->context; - GGML_UNUSED(ctx); - - for (int i = 0; i < cgraph->n_nodes; i++) { - ggml_tensor * node = cgraph->nodes[i]; - if (ggml_is_empty(node) || node->op == GGML_OP_RESHAPE - || node->op == GGML_OP_TRANSPOSE || node->op == GGML_OP_VIEW - || node->op == GGML_OP_PERMUTE || node->op == GGML_OP_NONE) { - continue; - } - bool ok = ggml_qnn_compute_forward(backend, node); - if (!ok) { - GGMLQNN_LOG_DEBUG("%s: error: op not supported %s (%s)\n", - __func__, node->name, ggml_op_name(node->op)); - } - } - - return result; -} - -static const char * ggml_backend_qnn_device_get_name(ggml_backend_dev_t dev) { - struct ggml_backend_qnn_context *ctx = static_cast(dev->context); - if (nullptr == ctx) { - GGMLQNN_LOG_ERROR("pls check why ctx is null"); - return "unknown"; - } - return ctx->name; -} - -static const char * ggml_backend_qnn_device_get_description(ggml_backend_dev_t dev) { - struct ggml_backend_qnn_context * ctx = static_cast(dev->context); - static char qnn_device_desc[256]; - if (nullptr == ctx) { - GGMLQNN_LOG_ERROR("pls check why ctx is null"); - return "unknown"; - } - if (0 == strncmp(ctx->name, "qnn-npu", 7)) { - const char * soc_info = qnn_get_socmodel_desc(ctx->socinfo.soc_model); - const char * htp_arch = qnn_get_htparch_desc(ctx->socinfo.htp_arch); - std::string dev_desc = std::string(ctx->desc) - + std::string(soc_info) + "_" + std::string(htp_arch) - + "," + std::string(ctx->socinfo.soc_desc); - memset(qnn_device_desc, 0, 256); - memcpy(qnn_device_desc, dev_desc.c_str(), strlen(dev_desc.c_str())); - return qnn_device_desc; - } else { - return ctx->desc; - } -} - -static void ggml_backend_qnn_device_get_memory(ggml_backend_dev_t dev, size_t * free, size_t * total) { - struct ggml_backend_qnn_context * ctx = static_cast(dev->context); - if ((nullptr == ctx) || (ctx->device > QNN_BACKEND_GGML)) { - GGMLQNN_LOG_ERROR("pls check params"); - *free = 0; - *total = 0; - } - - if (QNN_BACKEND_CPU == ctx->device || QNN_BACKEND_GGML == ctx->device) { - *total = get_system_total_memory_in_bytes(); - *free = get_system_free_memory_in_bytes(); - } else if (QNN_BACKEND_GPU == ctx->device) { - //TODO: probe GPU info in Qualcomm Adreno GPU - *total = get_system_total_memory_in_bytes(); - *free = get_system_free_memory_in_bytes(); - } else if (QNN_BACKEND_NPU == ctx->device) { - size_t rpc_ion_memsize = ctx->instance->get_rpcmem_capacity(); - size_t rpc_ion_usage = ctx->instance->get_rpcmem_usage(); - GGMLQNN_LOG_DEBUG("rpc memsize %d", rpc_ion_memsize); - GGMLQNN_LOG_DEBUG("rpc usage %d", rpc_ion_usage); - *total = rpc_ion_memsize * (1 << 20); - *free = (rpc_ion_memsize - rpc_ion_usage) * (1 << 20); - } -} - -static enum ggml_backend_dev_type ggml_backend_qnn_device_get_type(ggml_backend_dev_t dev) { - GGML_UNUSED(dev); - return GGML_BACKEND_DEVICE_TYPE_ACCEL; -} - -static void ggml_backend_qnn_device_get_props(ggml_backend_dev_t dev, - struct ggml_backend_dev_props * props) { - props->name = ggml_backend_qnn_device_get_name(dev); - props->description = ggml_backend_qnn_device_get_description(dev); - props->type = ggml_backend_qnn_device_get_type(dev); - ggml_backend_qnn_device_get_memory(dev, &props->memory_free, &props->memory_total); - props->caps = { - /* .async = */ false, - /* .host_buffer = */ false, - /* .buffer_from_host_ptr = */ true, - /* .events = */ false, - }; -} - -static ggml_backend_t ggml_backend_qnn_device_init_backend(ggml_backend_dev_t dev, const char * params) { - GGML_UNUSED(dev); - if (nullptr == params) { - params = 0; - } - ggml_backend_t qnn_backend = ggml_backend_qnn_init((int) (intptr_t) params, - "/data/local/tmp/"); - - return qnn_backend; - -} - -static ggml_backend_buffer_type_t ggml_backend_qnn_buffer_type(size_t device_index) { - if (device_index >= GGML_QNN_MAX_DEVICES) { - GGMLQNN_LOG_DEBUG("ggml_backend_qnn_buffer_type error: device_index:%d is out of range [0, %d]\n", - device_index, GGML_QNN_MAX_DEVICES - 1); - return nullptr; - } - - static struct ggml_backend_buffer_type ggml_backend_buffer_type_qnn = { - /* .iface = */ { - /* .get_name = */ ggml_backend_qnn_buffer_type_name, - /* .alloc_buffer = */ ggml_backend_qnn_buffer_type_alloc_buffer, - /* .get_alignment = */ ggml_backend_qnn_buffer_type_get_alignment, - /* .get_max_size = */ ggml_backend_qnn_buffer_type_get_max_size, - /* .get_alloc_size = */ nullptr,// defaults to ggml_nbytes - /* .is_host = */ ggml_backend_qnn_buffer_is_host - }, - /* .device = */ nullptr, - /* .context = */ nullptr, - }; - - return &ggml_backend_buffer_type_qnn; -} - -static ggml_backend_buffer_type_t ggml_backend_qnn_device_get_buffer_type(ggml_backend_dev_t dev) { - ggml_backend_qnn_context * ctx = (ggml_backend_qnn_context *) dev->context; - return ggml_backend_qnn_buffer_type(ctx->device); -} - -static ggml_backend_buffer_t ggml_backend_qnn_device_buffer_from_host_ptr(ggml_backend_dev_t dev, - void * ptr, size_t size, size_t max_tensor_size) { - return ggml_backend_cpu_buffer_from_ptr(ptr, size); - - GGML_UNUSED(dev); - GGML_UNUSED(max_tensor_size); -} - -static bool ggml_backend_qnn_device_supports_op(ggml_backend_dev_t dev, const struct ggml_tensor * op) { - ggml_backend_qnn_context * ctx = (ggml_backend_qnn_context *) dev->context; - return (ggml_qnn_can_handle_op(ctx,op)); -} - -static bool ggml_backend_qnn_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) { - GGML_UNUSED(dev); - return ggml_backend_buft_is_host(buft); -} - -static struct ggml_backend_device_i ggml_backend_qnn_device_interface = { - /* .get_name = */ ggml_backend_qnn_device_get_name, - /* .get_description = */ ggml_backend_qnn_device_get_description, - /* .get_memory = */ ggml_backend_qnn_device_get_memory, - /* .get_type = */ ggml_backend_qnn_device_get_type, - /* .get_props = */ ggml_backend_qnn_device_get_props, - /* .init_backend = */ ggml_backend_qnn_device_init_backend, - /* .get_buffer_type = */ ggml_backend_qnn_device_get_buffer_type, - /* .get_host_buffer_type = */ nullptr, - /* .buffer_from_host_ptr = */ ggml_backend_qnn_device_buffer_from_host_ptr, - /* .supports_op = */ ggml_backend_qnn_device_supports_op, - /* .supports_buft = */ ggml_backend_qnn_device_supports_buft, - /* .offload_op = */ nullptr, - /* .event_new = */ nullptr, - /* .event_free = */ nullptr, - /* .event_synchronize = */ nullptr, -}; - -static ggml_backend_i ggml_backend_qnn_interface = { - /* .get_name = */ ggml_backend_qnn_name, - /* .free = */ ggml_backend_qnn_free, - /* .set_tensor_async = */ nullptr, - /* .get_tensor_async = */ nullptr, - /* .cpy_tensor_async = */ nullptr, - /* .synchronize = */ nullptr, - /* .graph_plan_create = */ nullptr, - /* .graph_plan_free = */ nullptr, - /* .graph_plan_update = */ nullptr, - /* .graph_plan_compute = */ nullptr, - /* .graph_compute = */ ggml_backend_qnn_graph_compute, - /* .event_record = */ nullptr, - /* .event_wait = */ nullptr, -}; - -//FIXME: this guid is not make sense -static ggml_guid_t ggml_backend_qnn_guid() { - static ggml_guid guid = { - 0x1a, 0x2b, 0x3c, 0x4d, 0x5e, 0x6f, 0x70, 0x81, - 0x92, 0xa3, 0xb4, 0xc5, 0xd6, 0xe7, 0xf8, 0x09 - }; - return &guid; -} - -bool ggml_backend_is_qnn(ggml_backend_t backend) { - return backend != nullptr && ggml_guid_matches(backend->guid, ggml_backend_qnn_guid()); -} - -void ggml_backend_qnn_set_n_threads(ggml_backend_t backend, int n_threads) { - GGML_ASSERT(ggml_backend_is_qnn(backend)); - - struct ggml_backend_qnn_context * ctx = (struct ggml_backend_qnn_context *)backend->context; - ctx->threads = n_threads; -} - -int ggml_backend_qnn_get_device_count() { - return GGML_QNN_MAX_DEVICES; -} - -struct ggml_backend_qnn_reg_context { - std::vector devices; -}; - -static const char * ggml_backend_qnn_reg_get_name(ggml_backend_reg_t reg) { - GGML_UNUSED(reg); - return "ggml-qnn"; -} - -static size_t ggml_backend_qnn_reg_get_device_count(ggml_backend_reg_t reg) { - GGML_UNUSED(reg); - return GGML_QNN_MAX_DEVICES; -} - -static ggml_backend_dev_t ggml_backend_qnn_reg_get_device(ggml_backend_reg_t reg, size_t index) { - GGML_UNUSED(reg); - GGML_UNUSED(index); - - GGMLQNN_LOG_DEBUG("index %d", index); - ggml_backend_qnn_reg_context * ctx = (ggml_backend_qnn_reg_context *)reg->context; - GGML_ASSERT(index < ctx->devices.size()); - return ctx->devices[index]; -} - -static void * ggml_backend_qnn_reg_get_proc_address(ggml_backend_reg_t reg, const char * name) { - GGML_UNUSED(reg); - - if (nullptr == name) - return nullptr; - - const char * slot_name = "ggml_backend_set_n_threads"; - //avoid buffer attack rather than strcmp - if (0 == std::memcmp(name, slot_name, strlen(slot_name))) { - return (void *)ggml_backend_qnn_set_n_threads; - } - return nullptr; -} - -static const ggml_backend_reg_i ggml_backend_qnn_reg_interface = { - /* .get_name = */ ggml_backend_qnn_reg_get_name, - /* .get_device_count = */ ggml_backend_qnn_reg_get_device_count, - /* .get_device = */ ggml_backend_qnn_reg_get_device, - /* .get_proc_address = */ ggml_backend_qnn_reg_get_proc_address, -}; - -ggml_backend_reg_t ggml_backend_qnn_reg() { - static ggml_backend_reg reg; - static bool initialized = false; - GGMLQNN_LOG_DEBUG("enter ggml_backend_qnn_reg"); - { - static std::mutex mutex; - std::lock_guard lock(mutex); - if (!initialized) { - ggml_backend_qnn_reg_context * ctx = new ggml_backend_qnn_reg_context; - - for (int i = 0; i < ggml_backend_qnn_get_device_count(); i++) { - ggml_backend_dev_t dev = new ggml_backend_device { - /* .iface = */ ggml_backend_qnn_device_interface, - /* .reg = */ ®, - /* .context = */ &g_qnn_mgr[i] - }; - ctx->devices.push_back(dev); - } - - reg = ggml_backend_reg { - /* .api_version = */ GGML_BACKEND_API_VERSION, - /* .iface = */ ggml_backend_qnn_reg_interface, - /* .context = */ ctx - }; - } - - initialized = true; - } - GGMLQNN_LOG_DEBUG("leave ggml_backend_qnn_reg"); - - return ® -} - -/** - * - * @param device 0: QNN_BACKEND_CPU 1: QNN_BACKEND_GPU 2: QNN_BACKEND_NPU - * @param qnn_lib_path QNN binrary runtime library path, such as "/data/local/tmp/" on Android or specified in JNI layer - * @return - */ -ggml_backend_t ggml_backend_qnn_init(size_t device, const char * qnn_lib_path) { - int result = 0; - - if (nullptr == qnn_lib_path) - return nullptr; - - GGMLQNN_LOG_DEBUG("device %d", device); - GGMLQNN_LOG_DEBUG("qnn_lib_path %s", qnn_lib_path); - if (device >= GGML_QNN_MAX_DEVICES) { - GGMLQNN_LOG_ERROR("invalid device %d", device); - return nullptr; - } - - if (nullptr != g_qnn_mgr[device].backend) { - GGMLQNN_LOG_WARN("qnn backend %d(%s) already loaded", device, ggml_backend_qnn_get_devname(device)); - return g_qnn_mgr[device].backend; - } - -#if defined(__ANDROID__) - std::string path = qnn_lib_path; - if (QNN_BACKEND_NPU == device) { - if (0 == setenv("LD_LIBRARY_PATH", - (path + - ":/vendor/dsp/cdsp:/vendor/lib64:/vendor/dsp/dsp:/vendor/dsp/images").c_str(), - 1)) { - GGMLQNN_LOG_INFO("QNN NPU backend setenv successfully"); - } else { - GGMLQNN_LOG_ERROR("QNN NPU backend setenv failure"); - } - if (0 == setenv("ADSP_LIBRARY_PATH", - (path + - ";/vendor/dsp/cdsp;/vendor/lib/rfsa/adsp;/system/lib/rfsa/adsp;/vendor/dsp/dsp;/vendor/dsp/images;/dsp").c_str(), - 1)) { - GGMLQNN_LOG_INFO("QNN NPU backend setenv successfully"); - } else { - GGMLQNN_LOG_ERROR("QNN NPU backend setenv failure"); - } - } else { - if (0 == setenv("LD_LIBRARY_PATH", - (path + - ":/vendor/dsp/cdsp:/vendor/lib64:/vendor/dsp/dsp:/vendor/dsp/images").c_str(), - 1)) { - GGMLQNN_LOG_INFO("%s backend setenv successfully\n", ggml_backend_qnn_get_devname(device)); - } else { - GGMLQNN_LOG_ERROR("%s backend setenv failure\n", ggml_backend_qnn_get_devname(device)); - } - } -#endif - - qnn_instance * instance = nullptr; - instance = new qnn_instance(qnn_lib_path, g_qnn_mgr[device].lib, ""); - result = instance->qnn_init(nullptr); - if (0 != result) { - GGMLQNN_LOG_WARN("init qnn subsystem failed with qnn backend %s, pls check why\n", ggml_backend_qnn_get_devname(device)); - delete instance; - return nullptr; - } - qnn_interface qnn_interface = instance->get_qnn_interface(); - if (!qnn_interface.is_loaded()) { - GGMLQNN_LOG_WARN("qnn subsystem failure\n"); - delete instance; - return nullptr; - } - - std::string device_name = ggml_backend_qnn_get_devname(device); - GGMLQNN_LOG_INFO("qnn device name %s", device_name.c_str()); - g_qnn_mgr[device].instance = instance; - g_qnn_mgr[device].raw_interface = instance->get_qnn_raw_interface(); - g_qnn_mgr[device].raw_system_interface = instance->get_qnn_raw_system_interface(); - - ggml_backend_t qnn_backend = new ggml_backend{ - /* .guid = */ ggml_backend_qnn_guid(), - /* .iface = */ ggml_backend_qnn_interface, - /* .device = */ ggml_backend_reg_dev_get(ggml_backend_qnn_reg(), device), - /* .context = */ &g_qnn_mgr[device] - }; - g_qnn_mgr[device].backend = qnn_backend; - - return qnn_backend; -} - -GGML_BACKEND_DL_IMPL(ggml_backend_qnn_reg) From 101f4f974b7beb4c734bbbff16c47236c2fe5ea5 Mon Sep 17 00:00:00 2001 From: zhouwg Date: Tue, 18 Feb 2025 17:35:04 +0800 Subject: [PATCH 14/76] ggml-qnn: sync from branch kantvai-ggmlqnn-npurpc --- ggml/src/ggml-qnn/ggml-qnn.cpp | 3994 ++++++++++++++++++++++++++++++++ 1 file changed, 3994 insertions(+) create mode 100644 ggml/src/ggml-qnn/ggml-qnn.cpp diff --git a/ggml/src/ggml-qnn/ggml-qnn.cpp b/ggml/src/ggml-qnn/ggml-qnn.cpp new file mode 100644 index 0000000000000..6f2949333908e --- /dev/null +++ b/ggml/src/ggml-qnn/ggml-qnn.cpp @@ -0,0 +1,3994 @@ +/* + * Copyright (c) 2023-2024 The ggml authors + * + * Qualcomm QNN SDK and reference tech guides could be found at: + * https://www.qualcomm.com/developer/software/qualcomm-ai-engine-direct-sdk + * https://developer.qualcomm.com/software/hexagon-dsp-sdk/tools + * + * the implementation of ggml-qnn backend has six sections: + * section-1 does forward/external declaration, + * section-2 defines ggml-qnn internal log function + * section-3 does general helper macro / data structure / function + * section-4 does QNN helper macro / data structure / function + * section-5 does ggml-qnn backend helper macro / data structure / function / class + * section-6 does implementation of ggml-qnn backend according to ggml's backend subsystem + * + * currently only provide GGML_OP_ADD's QNN backend implementation: + * - GGML_OP_ADD: this is skeleton, can expand other ggml ops according to expertise + * + * of course, can porting ggml-qnn to Windows on ARM as need. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if (defined __ANDROID__) || (defined ANDROID) +#include "android/log.h" +#endif + +#include "QnnTypes.h" +#include "QnnCommon.h" +#include "QnnContext.h" +#include "QnnBackend.h" +#include "QnnGraph.h" +#include "QnnProperty.h" +#include "QnnTensor.h" +#include "QnnInterface.h" +#include "Saver/QnnSaver.h" +#include "System/QnnSystemInterface.h" +#include "HTP/QnnHtpDevice.h" +#include "HTP/QnnHtpGraph.h" + +#include "ggml-qnn.h" +#include "ggml-impl.h" +#include "ggml-backend-impl.h" + +// ================================================================================================= +// section-1: forward/external declaration +// ================================================================================================= +class qnn_instance; +struct ggml_backend_qnn_context; +static int free_qnn_tensor(Qnn_Tensor_t * tensor); +static enum ggml_status ggml_backend_qnn_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph); +static void ggmlqnn_log_internal(ggml_log_level level, const char * file, const char * func, int line, const char * format, ...); + +// ================================================================================================= +// section-2: ggml-qnn internal troubleshooting function +// ================================================================================================= +#define GGMLQNN_DEBUG 1 // for troubleshooting QNN backend +#define GGML_QNN_LOGBUF_LEN 4096 +#define ENABLE_QNNBACKEND_PERF 1 // enable/disable op's perf info +#define GGMLQNN_PRINT_QNN_INTERNAL_LOG 0 // enable/disable QNN's internal log +#define GGMLQNN_PRINT_OP_ADD_LOG 1 // GGML_OP_ADD already verified with QNN-CPU / QNN-GPU / QNN-NPU +#define GGMLQNN_PRINT_OP_MUL_MAT_LOG 1 + +#define GGMLQNN_LOG_ERROR(...) ggmlqnn_log_internal(GGML_LOG_LEVEL_DEBUG, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) +#define GGMLQNN_LOG_WARN(...) ggmlqnn_log_internal(GGML_LOG_LEVEL_DEBUG , __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) +#define GGMLQNN_LOG_INFO(...) ggmlqnn_log_internal(GGML_LOG_LEVEL_DEBUG , __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) + +#if GGMLQNN_DEBUG +#define GGMLQNN_LOG_DEBUG(...) ggmlqnn_log_internal(GGML_LOG_LEVEL_DEBUG, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) +#else +#define GGMLQNN_LOG_DEBUG(...) +#endif +static void ggmlqnn_log_internal(ggml_log_level level, const char * file, const char * func, int line, const char * format, ...) { + static std::mutex ggmlqnn_log_internal_mutex; + static char s_ggmlqnn_log_internal_buf[GGML_QNN_LOGBUF_LEN]; + + { + std::lock_guard lock(ggmlqnn_log_internal_mutex); + va_list args; + va_start(args, format); + int len_prefix = snprintf(s_ggmlqnn_log_internal_buf, GGML_QNN_LOGBUF_LEN, "[%s, %d]: ", func, line); + int len = vsnprintf(s_ggmlqnn_log_internal_buf + len_prefix, GGML_QNN_LOGBUF_LEN - len_prefix, format, args); + if (len < (GGML_QNN_LOGBUF_LEN - len_prefix)) { +#if (defined __ANDROID__) || (defined ANDROID) + //for Android application(standard APP or command line tool) + __android_log_print(ANDROID_LOG_INFO, "ggml-qnn", "%s\n", s_ggmlqnn_log_internal_buf); +#endif +#if (defined __ANDROID__) || (defined ANDROID) + //do nothing when running on Snapdragon based Android device +#else + //for Snapdragon based WoA(Windows on ARM) device + printf("%s\n", s_ggmlqnn_log_internal_buf); +#endif + } + va_end(args); + } +} + +// ================================================================================================= +// section-3: general helper macro / data structure / function +// ================================================================================================= +#define DISABLE_COPY(class_name) \ + class_name(const class_name &) = delete; \ + void operator=(const class_name &) = delete + +#define DISABLE_MOVE(class_name) \ + class_name(class_name &&) = delete; \ + void operator=(class_name &&) = delete + +#define GGMLQNN_MEM_ADD(alignment) (sizeof (size_t) + alignment) +#define GGMLQNN_MEM_MASK(alignment) ((uintptr_t)alignment - 1) + +static intptr_t ggmlqnn_align_to(size_t alignment, intptr_t offset) { + return offset % alignment == 0 ? offset + : offset + + (static_cast(alignment) - + offset % static_cast(alignment)); +} + +static void * ggmlqnn_mallocz_aligned(size_t size, size_t alignment) { + uint8_t * buffer = NULL; + size_t * sp = NULL; + buffer = static_cast(calloc(1, size + GGMLQNN_MEM_ADD(alignment))); + if (!buffer) + return NULL; + sp = (size_t *)buffer; + *sp = size; + buffer = (uint8_t *)(((uintptr_t) buffer + GGMLQNN_MEM_ADD(alignment)) & ~GGMLQNN_MEM_MASK(alignment)); + buffer[-1] = buffer - (uint8_t *)sp; + return buffer; +} + +static void * ggmlqnn_malloc_aligned(size_t size, size_t alignment) { + uint8_t * buffer = NULL; + size_t * sp = NULL; + buffer = static_cast(malloc(size + GGMLQNN_MEM_ADD(alignment))); + if (!buffer) + return NULL; + sp = (size_t *)buffer; + *sp = size; + buffer = (uint8_t *)(((uintptr_t) buffer + GGMLQNN_MEM_ADD(alignment)) & ~GGMLQNN_MEM_MASK(alignment)); + buffer[-1] = buffer - (uint8_t *)sp; + return buffer; +} + +static void ggmqnn_free_aligned(void * ptr) { + uint8_t * old = (uint8_t *)ptr; + if (!old) + return; + old -= old[-1]; + free(old); +} + +static size_t get_system_total_memory_in_bytes() { + struct sysinfo info = {}; + if (sysinfo(&info) == 0) { + return (info.totalram + info.totalswap) * info.mem_unit; + } + + auto pages = (size_t)sysconf(_SC_PHYS_PAGES); + auto page_size = (size_t)sysconf(_SC_PAGE_SIZE); + + return pages * page_size; +} + +static size_t get_system_free_memory_in_bytes() { + struct sysinfo info = {}; + if (sysinfo(&info) == 0) { + return (info.freeram + info.freeswap) * info.mem_unit; + } + + auto avail_pages = (size_t)sysconf(_SC_AVPHYS_PAGES); + auto page_size = (size_t)sysconf(_SC_PAGE_SIZE); + + return avail_pages * page_size; +} + +static size_t ggmlqnn_memscpy(void * dst, size_t dst_size, const void * src, size_t copy_size) { + if (!dst || !src || !dst_size || !copy_size) + return 0; + + size_t min_size = dst_size < copy_size ? dst_size : copy_size; + + memcpy(dst, src, min_size); + + return min_size; +} + +static char * ggmlqnn_strndup(const char * source, size_t maxlen) { + return ::strndup(source, maxlen); +} + +static void * ggmlqnn_host_malloc(size_t n) { + void * data = NULL; + int result = posix_memalign((void **) &data, sysconf(_SC_PAGESIZE), n); + if (result != 0) { + GGMLQNN_LOG_WARN("%s: error: posix_memalign failed\n", __func__); + return NULL; + } + + return data; +} + +// ================================================================================================= +// section-4: QNN helper macro / data structure / function +// ================================================================================================= +#define VALIDATE(value, status) \ + do { \ + status = value; \ + if (status != QNN_SUCCESS) { \ + GGMLQNN_LOG_WARN("%s expected QNN_SUCCESS\n", #value); \ + return status; \ + } \ + } while (0) + +#define CHECK_QNN_API(error) \ + do { \ + if (QNN_SUCCESS != (error)) { \ + GGMLQNN_LOG_INFO("error = %d\n", (error)); \ + } \ + } while (0) + +#define VALIDATE_TENSOR_VERSION(tensor, err) VALIDATE(validate_tensor_version(tensor), err) + +#define VALIDATE_OP_CONFIG_VERSION(op, err) VALIDATE(validate_op_config_version(op), err) + +#define QNN_VER_PTR(x) (&((x).v1)) +#define QNN_OP_CFG_VALID(op_config) ((op_config).version == QNN_OPCONFIG_VERSION_1) + +#define QNN_OP_CFG_GET_NAME(op_config) get_qnn_oponfig_name(op_config) +#define QNN_OP_CFG_GET_PACKAGE_NAME(op_config) get_qnn_op_config_packagename(op_config) +#define QNN_OP_CFG_GET_TYPE_NAME(op_config) get_qnn_op_config_typename(op_config) +#define QNN_OP_CFG_GET_NUM_PARAMS(op_config) get_qnn_op_config_numparams(op_config) +#define QNN_OP_CFG_GET_PARAMS(op_config) get_qnn_op_config_params(op_config) +#define QNN_OP_CFG_GET_NUM_INPUTS(op_config) get_qnn_op_config_numinputs(op_config) +#define QNN_OP_CFG_GET_INPUTS(op_config) get_qnn_op_config_inputs(op_config) +#define QNN_OP_CFG_GET_NUM_OUTPUTS(op_config) get_qnn_op_config_numoutputs(op_config) +#define QNN_OP_CFG_GET_OUTPUTS(op_config) get_qnn_op_config_outputs(op_config) + +#define QNN_OP_CFG_SET_NAME(op_config, value) set_qnn_op_config_name(op_config, value) +#define QNN_OP_CFG_SET_PACKAGE_NAME(op_config, value) set_qnn_op_config_packagename(op_config, value) +#define QNN_OP_CFG_SET_TYPE_NAME(op_config, value) set_qnn_op_config_typename(op_config, value) + +#define QNN_OP_CFG_SET_PARAMS(op_config, num_of_params, params) \ + set_qnn_op_config_params(op_config, num_of_params, params) + +#define QNN_OP_CFG_SET_INPUTS(op_config, num_of_inputs, inputTensors) \ + set_qnn_op_config_inputs(op_config, num_of_inputs, inputTensors) + +#define QNN_OP_CFG_SET_OUTPUTS(op_config, num_of_outputs, output_tensors) \ + set_qnn_op_config_outputs(op_config, num_of_outputs, output_tensors) + +#define QNN_TENSOR_GET_ID(tensor) get_qnn_tensorid(tensor) +#define QNN_TENSOR_GET_NAME(tensor) get_qnn_tensorname(tensor) +#define QNN_TENSOR_GET_TYPE(tensor) get_qnn_tensortype(tensor) +#define QNN_TENSOR_GET_DATA_FORMAT(tensor) get_qnn_tensor_dataformat(tensor) +#define QNN_TENSOR_GET_DATA_TYPE(tensor) get_qnn_tensor_datatype(tensor) +#define QNN_TENSOR_GET_QUANT_PARAMS(tensor) get_qnn_tensor_quantparams(tensor) +#define QNN_TENSOR_GET_RANK(tensor) get_qnn_tensor_rank(tensor) +#define QNN_TENSOR_GET_DIMENSIONS(tensor) get_qnn_tensor_dimensions(tensor) +#define QNN_TENSOR_GET_MEM_TYPE(tensor) get_qnn_tensor_memtype(tensor) +#define QNN_TENSOR_GET_CLIENT_BUF(tensor) get_qnn_tensor_clientbuf(tensor) +#define QNN_TENSOR_GET_MEM_HANDLE(tensor) get_qnn_tensor_memhandle(tensor) + +#define QNN_TENSOR_SET_ID(tensor, value) set_qnn_tensor_id(tensor, value) +#define QNN_TENSOR_SET_NAME(tensor, value) set_qnn_tensor_name(tensor, value) +#define QNN_TENSOR_SET_TYPE(tensor, value) set_qnn_tensor_type(tensor, value) +#define QNN_TENSOR_SET_DATA_FORMAT(tensor, value) set_qnn_tensor_dataformat(tensor, value) +#define QNN_TENSOR_SET_DATA_TYPE(tensor, value) set_qnn_tensor_datatype(tensor, value) +#define QNN_TENSOR_SET_QUANT_PARAMS(tensor, value) set_qnn_tensor_quantparams(tensor, value) +#define QNN_TENSOR_SET_RANK(tensor, value) set_qnn_tensor_rank(tensor, value) +#define QNN_TENSOR_SET_DIMENSIONS(tensor, value) set_qnn_tensor_dimensions(tensor, value) +#define QNN_TENSOR_SET_MEM_TYPE(tensor, value) set_qnn_tensor_memtype(tensor, value) +#define QNN_TENSOR_SET_CLIENT_BUF(tensor, value) set_qnn_tensor_clientbuf(tensor, value) +#define QNN_TENSOR_SET_MEM_HANDLE(tensor, value) set_qnn_tensor_memhandle(tensor, value) + +static inline int validate_tensor_version(Qnn_Tensor_t tensor) { + if (tensor.version != QNN_TENSOR_VERSION_1) { + GGMLQNN_LOG_WARN("validate_tensor_version() tensor %s, got unsupported version %d\n", + tensor.v1.name, + tensor.version); + return 1; + } + return 0; +} + +[[maybe_unused]] static inline int validate_op_config_version(Qnn_OpConfig_t op_config) { + if (op_config.version != QNN_OPCONFIG_VERSION_1) { + GGMLQNN_LOG_WARN("validate_op_config_version() op %s, got unsupported version %d\n", + op_config.v1.name, + op_config.version); + return 1; + } + return 0; +} + +static inline const char * get_qnn_oponfig_name(const Qnn_OpConfig_t & op_config) { + if (op_config.version == QNN_OPCONFIG_VERSION_1) { + return op_config.v1.name; + } + return nullptr; +} + +[[maybe_unused]] static inline const char * get_qnn_oponfig_name(const Qnn_OpConfig_t * op_config) { + return get_qnn_oponfig_name(*op_config); +} + +static inline const char * get_qnn_op_config_packagename(const Qnn_OpConfig_t & op_config) { + if (op_config.version == QNN_OPCONFIG_VERSION_1) { + return op_config.v1.packageName; + } + return nullptr; +} + +[[maybe_unused]] static inline const char * get_qnn_op_config_packagename(const Qnn_OpConfig_t * op_config) { + return get_qnn_op_config_packagename(*op_config); +} + +static inline const char * get_qnn_op_config_typename(const Qnn_OpConfig_t & op_config) { + if (op_config.version == QNN_OPCONFIG_VERSION_1) { + return op_config.v1.typeName; + } + return nullptr; +} + +[[maybe_unused]] static inline const char * get_qnn_op_config_typename(const Qnn_OpConfig_t * op_config) { + return get_qnn_op_config_typename(*op_config); +} + +static inline uint32_t get_qnn_op_config_numparams(const Qnn_OpConfig_t & op_config) { + if (op_config.version == QNN_OPCONFIG_VERSION_1) { + return op_config.v1.numOfParams; + } + return 0u; +} + +[[maybe_unused]] static inline uint32_t get_qnn_op_config_numparams(const Qnn_OpConfig_t * op_config) { + return get_qnn_op_config_numparams(*op_config); +} + +static inline const Qnn_Param_t * get_qnn_op_config_params(const Qnn_OpConfig_t & op_config) { + if (op_config.version == QNN_OPCONFIG_VERSION_1) { + return op_config.v1.params; + } + return nullptr; +} + +[[maybe_unused]] static inline const Qnn_Param_t * get_qnn_op_config_params(const Qnn_OpConfig_t * op_config) { + return get_qnn_op_config_params(*op_config); +} + +static inline uint32_t get_qnn_op_config_numinputs(const Qnn_OpConfig_t & op_config) { + if (op_config.version == QNN_OPCONFIG_VERSION_1) { + return op_config.v1.numOfInputs; + } + return 0u; +} + +[[maybe_unused]] static inline uint32_t get_qnn_op_config_numinputs(const Qnn_OpConfig_t * op_config) { + return get_qnn_op_config_numinputs(*op_config); +} + +static inline const Qnn_Tensor_t * get_qnn_op_config_inputs(const Qnn_OpConfig_t & op_config) { + if (op_config.version == QNN_OPCONFIG_VERSION_1) { + return op_config.v1.inputTensors; + } + return nullptr; +} + +[[maybe_unused]] static inline const Qnn_Tensor_t * get_qnn_op_config_inputs(const Qnn_OpConfig_t * op_config) { + return get_qnn_op_config_inputs(*op_config); +} + +static inline uint32_t get_qnn_op_config_numoutputs(const Qnn_OpConfig_t & op_config) { + if (op_config.version == QNN_OPCONFIG_VERSION_1) { + return op_config.v1.numOfOutputs; + } + return 0u; +} + +[[maybe_unused]] static inline uint32_t get_qnn_op_config_numoutputs(const Qnn_OpConfig_t * op_config) { + return get_qnn_op_config_numoutputs(*op_config); +} + +static inline const Qnn_Tensor_t * get_qnn_op_config_outputs(const Qnn_OpConfig_t & op_config) { + if (op_config.version == QNN_OPCONFIG_VERSION_1) { + return op_config.v1.outputTensors; + } + return nullptr; +} + +[[maybe_unused]] static inline const Qnn_Tensor_t * get_qnn_op_config_outputs(const Qnn_OpConfig_t * op_config) { + return get_qnn_op_config_outputs(*op_config); +} + +static inline void set_qnn_op_config_name(Qnn_OpConfig_t & op_config, const char * name) { + if (op_config.version == QNN_OPCONFIG_VERSION_1) { + op_config.v1.name = name; + } +} + +[[maybe_unused]] static inline void set_qnn_op_config_name(Qnn_OpConfig_t * op_config, const char * name) { + set_qnn_op_config_name(*op_config, name); +} + +static inline void set_qnn_op_config_packagename(Qnn_OpConfig_t & op_config, const char * package_name) { + if (op_config.version == QNN_OPCONFIG_VERSION_1) { + op_config.v1.packageName = package_name; + } +} + +[[maybe_unused]] static inline void set_qnn_op_config_packagename(Qnn_OpConfig_t * op_config, const char * package_name) { + set_qnn_op_config_packagename(*op_config, package_name); +} + +static inline void set_qnn_op_config_typename(Qnn_OpConfig_t & op_config, const char * type_name) { + if (op_config.version == QNN_OPCONFIG_VERSION_1) { + op_config.v1.typeName = type_name; + } +} + +[[maybe_unused]] static inline void set_qnn_op_config_typename(Qnn_OpConfig_t * op_config, const char * type_name) { + set_qnn_op_config_typename(*op_config, type_name); +} + +static inline void set_qnn_op_config_params(Qnn_OpConfig_t & op_config, + uint32_t num_of_params, + Qnn_Param_t * params) { + if (op_config.version == QNN_OPCONFIG_VERSION_1) { + op_config.v1.numOfParams = num_of_params; + op_config.v1.params = params; + } +} + +[[maybe_unused]] static inline void set_qnn_op_config_params(Qnn_OpConfig_t * op_config, + uint32_t num_of_params, + Qnn_Param_t * params) { + set_qnn_op_config_params(*op_config, num_of_params, params); +} + +static inline void set_qnn_op_config_inputs(Qnn_OpConfig_t & op_config, + uint32_t num_of_inputs, + Qnn_Tensor_t * input_tensors) { + if (op_config.version == QNN_OPCONFIG_VERSION_1) { + op_config.v1.numOfInputs = num_of_inputs; + op_config.v1.inputTensors = input_tensors; + } +} + +[[maybe_unused]] static inline void set_qnn_op_config_inputs(Qnn_OpConfig_t * op_config, + uint32_t num_of_inputs, + Qnn_Tensor_t * input_tensors) { + set_qnn_op_config_inputs(*op_config, num_of_inputs, input_tensors); +} + +static inline void set_qnn_op_config_outputs(Qnn_OpConfig_t & op_config, + uint32_t num_of_outputs, + Qnn_Tensor_t * output_tensors) { + if (op_config.version == QNN_OPCONFIG_VERSION_1) { + op_config.v1.numOfOutputs = num_of_outputs; + op_config.v1.outputTensors = output_tensors; + } +} + +[[maybe_unused]] static inline void set_qnn_op_config_outputs(Qnn_OpConfig_t * op_config, + uint32_t num_of_outputs, + Qnn_Tensor_t * output_tensors) { + set_qnn_op_config_outputs(*op_config, num_of_outputs, output_tensors); +} + +static inline uint32_t get_qnn_tensorid(const Qnn_Tensor_t & tensor) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + return tensor.v1.id; + } + + return 0u; +} + +[[maybe_unused]] static inline uint32_t get_qnn_tensorid(const Qnn_Tensor_t * tensor) { + return get_qnn_tensorid(*tensor); +} + +static inline const char * get_qnn_tensorname(const Qnn_Tensor_t & tensor) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + return tensor.v1.name; + } + return nullptr; +} + +static inline const char * get_qnn_tensorname(const Qnn_Tensor_t * tensor) { + return get_qnn_tensorname(*tensor); +} + +static inline Qnn_TensorType_t get_qnn_tensortype(const Qnn_Tensor_t & tensor) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + return tensor.v1.type; + } + return QNN_TENSOR_TYPE_UNDEFINED; +} + +[[maybe_unused]] static inline Qnn_TensorType_t get_qnn_tensortype(const Qnn_Tensor_t * tensor) { + return get_qnn_tensortype(*tensor); +} + +static inline Qnn_TensorDataFormat_t get_qnn_tensor_dataformat(const Qnn_Tensor_t & tensor) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + return tensor.v1.dataFormat; + } + return QNN_TENSOR_DATA_FORMAT_FLAT_BUFFER; +} + +[[maybe_unused]] static inline Qnn_TensorDataFormat_t get_qnn_tensor_dataformat(const Qnn_Tensor_t * tensor) { + return get_qnn_tensor_dataformat(*tensor); +} + +static inline Qnn_DataType_t get_qnn_tensor_datatype(const Qnn_Tensor_t & tensor) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + return tensor.v1.dataType; + } + return QNN_DATATYPE_UNDEFINED; +} + +[[maybe_unused]] static inline Qnn_DataType_t get_qnn_tensor_datatype(const Qnn_Tensor_t * tensor) { + return get_qnn_tensor_datatype(*tensor); +} + +static inline Qnn_QuantizeParams_t get_qnn_tensor_quantparams(const Qnn_Tensor_t & tensor) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + return tensor.v1.quantizeParams; + } + return QNN_QUANTIZE_PARAMS_INIT; +} + +[[maybe_unused]] static inline Qnn_QuantizeParams_t get_qnn_tensor_quantparams(const Qnn_Tensor_t * tensor) { + return get_qnn_tensor_quantparams(*tensor); +} + +static inline uint32_t get_qnn_tensor_rank(const Qnn_Tensor_t & tensor) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + return tensor.v1.rank; + } + return 0u; +} + +[[maybe_unused]] static inline uint32_t get_qnn_tensor_rank(const Qnn_Tensor_t * tensor) { + return get_qnn_tensor_rank(*tensor); +} + +static inline uint32_t * get_qnn_tensor_dimensions(const Qnn_Tensor_t & tensor) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + return tensor.v1.dimensions; + } + return nullptr; +} + +[[maybe_unused]] static inline uint32_t * get_qnn_tensor_dimensions(const Qnn_Tensor_t * tensor) { + return get_qnn_tensor_dimensions(*tensor); +} + +static inline Qnn_TensorMemType_t get_qnn_tensor_memtype(const Qnn_Tensor_t & tensor) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + return tensor.v1.memType; + } + return QNN_TENSORMEMTYPE_UNDEFINED; +} + +[[maybe_unused]] static inline Qnn_TensorMemType_t get_qnn_tensor_memtype(const Qnn_Tensor_t * tensor) { + return get_qnn_tensor_memtype(*tensor); +} + +static inline Qnn_ClientBuffer_t get_qnn_tensor_clientbuf(const Qnn_Tensor_t & tensor) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + return tensor.v1.clientBuf; + } + return QNN_CLIENT_BUFFER_INIT; +} + +[[maybe_unused]] static inline Qnn_ClientBuffer_t get_qnn_tensor_clientbuf(const Qnn_Tensor_t * tensor) { + return get_qnn_tensor_clientbuf(*tensor); +} + +static inline Qnn_MemHandle_t get_qnn_tensor_memhandle(const Qnn_Tensor_t & tensor) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + return tensor.v1.memHandle; + } + return nullptr; +} + +[[maybe_unused]] static inline Qnn_MemHandle_t get_qnn_tensor_memhandle(const Qnn_Tensor_t * tensor) { + return get_qnn_tensor_memhandle(*tensor); +} + +static inline void set_qnn_tensor_id(Qnn_Tensor_t & tensor, uint32_t id) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + tensor.v1.id = id; + } +} + +[[maybe_unused]] static inline void set_qnn_tensor_id(Qnn_Tensor_t * tensor, uint32_t id) { + set_qnn_tensor_id(*tensor, id); +} + +static inline void set_qnn_tensor_name(Qnn_Tensor_t & tensor, const char * name) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + tensor.v1.name = name; + } +} + +[[maybe_unused]] static inline void set_qnn_tensor_name(Qnn_Tensor_t * tensor, const char * name) { + set_qnn_tensor_name(*tensor, name); +} + +static inline void set_qnn_tensor_type(Qnn_Tensor_t & tensor, Qnn_TensorType_t type) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + tensor.v1.type = type; + } +} + +[[maybe_unused]] static inline void set_qnn_tensor_type(Qnn_Tensor_t * tensor, Qnn_TensorType_t type) { + set_qnn_tensor_type(*tensor, type); +} + +static inline void set_qnn_tensor_dataformat(Qnn_Tensor_t & tensor, Qnn_TensorDataFormat_t format) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + tensor.v1.dataFormat = format; + } +} + +[[maybe_unused]] static inline void set_qnn_tensor_dataformat(Qnn_Tensor_t * tensor, Qnn_TensorDataFormat_t format) { + set_qnn_tensor_dataformat(*tensor, format); +} + +static inline void set_qnn_tensor_datatype(Qnn_Tensor_t & tensor, Qnn_DataType_t dataType) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + tensor.v1.dataType = dataType; + } +} + +[[maybe_unused]] static inline void set_qnn_tensor_datatype(Qnn_Tensor_t * tensor, Qnn_DataType_t dataType) { + set_qnn_tensor_datatype(*tensor, dataType); +} + +static inline void set_qnn_tensor_quantparams(Qnn_Tensor_t & tensor, Qnn_QuantizeParams_t params) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + tensor.v1.quantizeParams = params; + } +} + +[[maybe_unused]] static inline void set_qnn_tensor_quantparams(Qnn_Tensor_t * tensor, Qnn_QuantizeParams_t params) { + set_qnn_tensor_quantparams(*tensor, params); +} + +static inline void set_qnn_tensor_rank(Qnn_Tensor_t & tensor, uint32_t rank) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + tensor.v1.rank = rank; + } +} + +[[maybe_unused]] static inline void set_qnn_tensor_rank(Qnn_Tensor_t * tensor, uint32_t rank) { + set_qnn_tensor_rank(*tensor, rank); +} + +static inline void set_qnn_tensor_dimensions(Qnn_Tensor_t & tensor, uint32_t * dims) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + tensor.v1.dimensions = dims; + } +} + +[[maybe_unused]] static inline void set_qnn_tensor_dimensions(Qnn_Tensor_t * tensor, uint32_t * dims) { + set_qnn_tensor_dimensions(*tensor, dims); +} + +static inline void set_qnn_tensor_memtype(Qnn_Tensor_t & tensor, Qnn_TensorMemType_t memType) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + tensor.v1.memType = memType; + } +} + +[[maybe_unused]] static inline void set_qnn_tensor_memtype(Qnn_Tensor_t * tensor, Qnn_TensorMemType_t memType) { + set_qnn_tensor_memtype(*tensor, memType); +} + +static inline void set_qnn_tensor_clientbuf(Qnn_Tensor_t & tensor, Qnn_ClientBuffer_t clientBuf) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + tensor.v1.clientBuf = clientBuf; + } +} + +[[maybe_unused]] static inline void set_qnn_tensor_clientbuf(Qnn_Tensor_t * tensor, Qnn_ClientBuffer_t clientBuf) { + set_qnn_tensor_clientbuf(*tensor, clientBuf); +} + +static inline void set_qnn_tensor_memhandle(Qnn_Tensor_t & tensor, Qnn_MemHandle_t handle) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + tensor.v1.memHandle = handle; + } +} + +[[maybe_unused]] static inline void set_qnn_tensor_memhandle(Qnn_Tensor_t * tensor, Qnn_MemHandle_t handle) { + set_qnn_tensor_memhandle(*tensor, handle); +} + +inline static Qnn_Tensor_t qnn_tensor_init(Qnn_TensorVersion_t version) { + Qnn_Tensor_t tensor; + tensor.version = version; + if (version == QNN_TENSOR_VERSION_1) { + tensor.v1 = QNN_TENSOR_V1_INIT; + } else if (version == QNN_TENSOR_VERSION_2) { + tensor.v2 = QNN_TENSOR_V2_INIT; + } + return tensor; +} + +static int deep_copy_qnn_tensors(Qnn_Tensor_t & src, Qnn_Tensor_t & dst) { + int err = 0; + VALIDATE_TENSOR_VERSION(src, err); + + dst.version = src.version; + QNN_TENSOR_SET_NAME( + dst, ggmlqnn_strndup(QNN_TENSOR_GET_NAME(src), std::string(QNN_TENSOR_GET_NAME(src)).size())); + if (QNN_TENSOR_GET_NAME(dst) == nullptr) { + return 1; + } + QNN_TENSOR_SET_ID(dst, QNN_TENSOR_GET_ID(src)); + QNN_TENSOR_SET_TYPE(dst, QNN_TENSOR_GET_TYPE(src)); + QNN_TENSOR_SET_DATA_FORMAT(dst, QNN_TENSOR_GET_DATA_FORMAT(src)); + QNN_TENSOR_SET_DATA_TYPE(dst, QNN_TENSOR_GET_DATA_TYPE(src)); + QNN_TENSOR_SET_MEM_TYPE(dst, QNN_TENSOR_GET_MEM_TYPE(src)); + + if (QNN_TENSOR_GET_MEM_TYPE(src) == QNN_TENSORMEMTYPE_RAW) { + Qnn_ClientBuffer_t client_buf = {nullptr, 0}; + QNN_TENSOR_SET_CLIENT_BUF(dst, client_buf); + } else if (QNN_TENSOR_GET_MEM_TYPE(src) == QNN_TENSORMEMTYPE_MEMHANDLE) { + QNN_TENSOR_SET_MEM_HANDLE(dst, nullptr); + } else { + return 1; + } + + Qnn_QuantizeParams_t src_qparam = QNN_TENSOR_GET_QUANT_PARAMS(src); + Qnn_QuantizationEncoding_t encoding = src_qparam.quantizationEncoding; + if (encoding == QNN_QUANTIZATION_ENCODING_AXIS_SCALE_OFFSET) { + Qnn_QuantizeParams_t src_qparam_cpy = src_qparam; + Qnn_AxisScaleOffset_t & axis_scale_offset = src_qparam_cpy.axisScaleOffsetEncoding; + Qnn_ScaleOffset_t ** scale_offset = &axis_scale_offset.scaleOffset; + size_t scale_offset_size = axis_scale_offset.numScaleOffsets * sizeof(Qnn_ScaleOffset_t); + *scale_offset = (Qnn_ScaleOffset_t *)malloc(scale_offset_size); + ggmlqnn_memscpy(*scale_offset, + scale_offset_size, + src_qparam.axisScaleOffsetEncoding.scaleOffset, + scale_offset_size); + QNN_TENSOR_SET_QUANT_PARAMS(dst, src_qparam_cpy); + } else if (encoding == QNN_QUANTIZATION_ENCODING_BW_AXIS_SCALE_OFFSET) { + Qnn_QuantizeParams_t src_qparam_cpy = src_qparam; + Qnn_BwAxisScaleOffset_t & bwaxis_scale_offset = src_qparam_cpy.bwAxisScaleOffsetEncoding; + size_t scale_size = bwaxis_scale_offset.numElements * sizeof(float); + float ** scales = &bwaxis_scale_offset.scales; + int32_t ** offsets = &bwaxis_scale_offset.offsets; + *scales = (float *)malloc(scale_size); + ggmlqnn_memscpy(*scales, scale_size, src_qparam.bwAxisScaleOffsetEncoding.scales, scale_size); + + if (bwaxis_scale_offset.offsets != nullptr) { + size_t offset_size = bwaxis_scale_offset.numElements * sizeof(int32_t); + *offsets = (int32_t *)malloc(offset_size); + ggmlqnn_memscpy(*offsets, offset_size, src_qparam.bwAxisScaleOffsetEncoding.offsets, offset_size); + } + QNN_TENSOR_SET_QUANT_PARAMS(dst, src_qparam_cpy); + } else { + QNN_TENSOR_SET_QUANT_PARAMS(dst, src_qparam); + } + + uint32_t rank = QNN_TENSOR_GET_RANK(src); + QNN_TENSOR_SET_RANK(dst, rank); + size_t dim_size = rank * sizeof(uint32_t); + uint32_t * dimensions = (uint32_t *)malloc(dim_size); + GGMLQNN_LOG_DEBUG("tensor dims %p", dimensions); + if (dimensions == nullptr) { + GGMLQNN_LOG_WARN("deep_copy_qnn_tensors() allocation error while copying tensor %s\n", QNN_TENSOR_GET_NAME(src)); + return 1; + } + ggmlqnn_memscpy(dimensions, dim_size, QNN_TENSOR_GET_DIMENSIONS(src), dim_size); + QNN_TENSOR_SET_DIMENSIONS(dst, dimensions); + + return err; +} + +static int free_qnn_tensor(Qnn_Tensor_t * tensor) { + int err = 0; + VALIDATE_TENSOR_VERSION(*tensor, err); + free((void *) QNN_TENSOR_GET_NAME(*tensor)); + + Qnn_QuantizeParams_t src_qparam = QNN_TENSOR_GET_QUANT_PARAMS(*tensor); + Qnn_QuantizationEncoding_t encoding = src_qparam.quantizationEncoding; + if (encoding == QNN_QUANTIZATION_ENCODING_AXIS_SCALE_OFFSET) { + free(src_qparam.axisScaleOffsetEncoding.scaleOffset); + } else if (encoding == QNN_QUANTIZATION_ENCODING_BW_AXIS_SCALE_OFFSET) { + free(src_qparam.bwAxisScaleOffsetEncoding.scales); + if (src_qparam.bwAxisScaleOffsetEncoding.offsets != nullptr) { + free(src_qparam.bwAxisScaleOffsetEncoding.offsets); + } + } + free(QNN_TENSOR_GET_DIMENSIONS(*tensor)); + free(tensor); + + return err; +} + + +static size_t qnn_datatype_size(Qnn_DataType_t qnn_type) { + switch (qnn_type) { + case QNN_DATATYPE_FLOAT_32: + return sizeof(float); + case QNN_DATATYPE_FLOAT_16: + return sizeof(uint16_t); + case QNN_DATATYPE_UINT_32: + case QNN_DATATYPE_INT_32: + return sizeof(int32_t); + case QNN_DATATYPE_INT_16: + return sizeof(int16_t); + case QNN_DATATYPE_INT_8: + return sizeof(int8_t); + case QNN_DATATYPE_SFIXED_POINT_8: + return sizeof(int8_t); + case QNN_DATATYPE_SFIXED_POINT_4: + return sizeof(int8_t); + default: + break; + } + return 0; +} + +static const char * qnn_datatype_to_string(Qnn_DataType_t qnn_type) { + switch (qnn_type) { + case QNN_DATATYPE_FLOAT_32: + return "QNN_DATATYPE_FLOAT_32"; + case QNN_DATATYPE_FLOAT_16: + return "QNN_DATATYPE_FLOAT_16"; + case QNN_DATATYPE_UINT_32: + return "QNN_DATATYPE_UINT_32"; + case QNN_DATATYPE_INT_32: + return "QNN_DATATYPE_INT_32"; + case QNN_DATATYPE_INT_16: + return "QNN_DATATYPE_INT_16"; + case QNN_DATATYPE_INT_8: + return "QNN_DATATYPE_INT_8"; + case QNN_DATATYPE_SFIXED_POINT_8: + return "QNN_DATATYPE_SFIXED_POINT_8"; + case QNN_DATATYPE_SFIXED_POINT_4: + return "QNN_DATATYPE_SFIXED_POINT_4"; + default: + break; + } + return "QNN_DATATYPE_UNDEFINED"; +} + +static const char * qnn_get_error_string(Qnn_ErrorHandle_t qnn_error_code) { + // file:///opt/qcom/aistack/qairt/2.31.0.250130/docs/QNN/general/api_error_codes.html + switch (qnn_error_code) { + case QNN_SUCCESS: + return "QNN_SUCCESS"; + case QNN_COMMON_ERROR_GENERAL: + return "QNN_COMMON_ERROR_GENERAL"; + + // QnnGraph_Error_t + case QNN_GRAPH_ERROR_UNSUPPORTED_FEATURE: + return "QNN_GRAPH_ERROR_UNSUPPORTED_FEATURE"; + case QNN_GRAPH_ERROR_MEM_ALLOC: + return "QNN_GRAPH_ERROR_MEM_ALLOC"; + case QNN_GRAPH_ERROR_INVALID_ARGUMENT: + return "QNN_GRAPH_ERROR_INVALID_ARGUMENT"; + case QNN_GRAPH_ERROR_INVALID_HANDLE: + return "QNN_GRAPH_ERROR_INVALID_HANDLE"; + case QNN_GRAPH_ERROR_GRAPH_DOES_NOT_EXIST: + return "QNN_GRAPH_ERROR_GRAPH_DOES_NOT_EXIST"; + case QNN_GRAPH_ERROR_INVALID_NAME: + return "QNN_GRAPH_ERROR_INVALID_NAME"; + case QNN_GRAPH_ERROR_INVALID_TENSOR: + return "QNN_GRAPH_ERROR_INVALID_TENSOR"; + case QNN_GRAPH_ERROR_INVALID_OP_CONFIG: + return "QNN_GRAPH_ERROR_INVALID_OP_CONFIG"; + case QNN_GRAPH_ERROR_SET_PROFILE: + return "QNN_GRAPH_ERROR_SET_PROFILE"; + case QNN_GRAPH_ERROR_UNCONNECTED_NODE: + return "QNN_GRAPH_ERROR_UNCONNECTED_NODE"; + case QNN_GRAPH_ERROR_CREATE_FAILED: + return "QNN_GRAPH_ERROR_CREATE_FAILED"; + case QNN_GRAPH_ERROR_OPTIMIZATION_FAILED: + return "QNN_GRAPH_ERROR_OPTIMIZATION_FAILED"; + case QNN_GRAPH_ERROR_FINALIZE_FAILED: + return "QNN_GRAPH_ERROR_FINALIZE_FAILED"; + case QNN_GRAPH_ERROR_GRAPH_NOT_FINALIZED: + return "QNN_GRAPH_ERROR_GRAPH_NOT_FINALIZED"; + case QNN_GRAPH_ERROR_GRAPH_FINALIZED: + return "QNN_GRAPH_ERROR_GRAPH_FINALIZED"; + case QNN_GRAPH_ERROR_EXECUTION_ASYNC_FIFO_FULL: + return "QNN_GRAPH_ERROR_EXECUTION_ASYNC_FIFO_FULL"; + case QNN_GRAPH_ERROR_SIGNAL_IN_USE: + return "QNN_GRAPH_ERROR_SIGNAL_IN_USE"; + case QNN_GRAPH_ERROR_ABORTED: + return "QNN_GRAPH_ERROR_ABORTED"; + case QNN_GRAPH_ERROR_PROFILE_IN_USE: + return "QNN_GRAPH_ERROR_PROFILE_IN_USE"; + case QNN_GRAPH_ERROR_TIMED_OUT: + return "QNN_GRAPH_ERROR_TIMED_OUT"; + case QNN_GRAPH_ERROR_SUBGRAPH: + return "QNN_GRAPH_ERROR_SUBGRAPH"; + case QNN_GRAPH_ERROR_DISABLED: + return "QNN_GRAPH_ERROR_DISABLED"; + case QNN_GRAPH_ERROR_DYNAMIC_TENSOR_SHAPE: + return "QNN_GRAPH_ERROR_DYNAMIC_TENSOR_SHAPE"; + case QNN_GRAPH_ERROR_TENSOR_SPARSITY: + return "QNN_GRAPH_ERROR_TENSOR_SPARSITY"; + case QNN_GRAPH_ERROR_EARLY_TERMINATION: + return "QNN_GRAPH_ERROR_EARLY_TERMINATION"; + case QNN_GRAPH_ERROR_INVALID_CONTEXT: + return "QNN_GRAPH_ERROR_INVALID_CONTEXT"; + + //QQnnTensor_Error_t + //Invalid context/graph handle in creating tensor + case QNN_TENSOR_ERROR_INVALID_HANDLE: + return "QNN_TENSOR_ERROR_INVALID_HANDLE"; + //Tensor with specified credentials not registered with a context/graph + case QNN_TENSOR_ERROR_DOES_NOT_EXIST: + return "QNN_TENSOR_ERROR_DOES_NOT_EXIST"; + // (deprecated) Tensor has already been registered with backend + case QNN_TENSOR_ERROR_ALREADY_EXISTS: + return "QNN_TENSOR_ERROR_ALREADY_EXISTS"; + // Invalid tensor param. + case QNN_TENSOR_ERROR_INVALID_TENSOR_PARAM: + return "QNN_TENSOR_ERROR_INVALID_TENSOR_PARAM"; + // This tensor param is currently unsupported + case QNN_TENSOR_ERROR_UNSUPPORTED_TENSOR_PARAM: + return "QNN_TENSOR_ERROR_UNSUPPORTED_TENSOR_PARAM"; + // Tensor provided for update is invalid + case QNN_TENSOR_ERROR_INCOMPATIBLE_TENSOR_UPDATE: + return "QNN_TENSOR_ERROR_INCOMPATIBLE_TENSOR_UPDATE"; + + // QnnOpPackage_Error_t + case QNN_OP_PACKAGE_ERROR_LIBRARY_ALREADY_INITIALIZED: + return "QNN_OP_PACKAGE_ERROR_LIBRARY_ALREADY_INITIALIZED"; + case QNN_OP_PACKAGE_ERROR_LIBRARY_NOT_INITIALIZED: + return "QNN_OP_PACKAGE_ERROR_LIBRARY_NOT_INITIALIZED"; + case QNN_OP_PACKAGE_ERROR_INVALID_HANDLE: + return "QNN_OP_PACKAGE_ERROR_INVALID_HANDLE"; + case QNN_OP_PACKAGE_ERROR_INVALID_INFRASTRUCTURE: + return "QNN_OP_PACKAGE_ERROR_INVALID_INFRASTRUCTURE"; + case QNN_OP_PACKAGE_ERROR_INVALID_INFO: + return "QNN_OP_PACKAGE_ERROR_INVALID_INFO"; + case QNN_OP_PACKAGE_ERROR_VALIDATION_FAILURE: + return "QNN_OP_PACKAGE_ERROR_VALIDATION_FAILURE"; + case QNN_OP_PACKAGE_ERROR_INVALID_ARGUMENT: + return "QNN_OP_PACKAGE_ERROR_INVALID_ARGUMENT"; + + default: + return "unknown QNN error"; + } +} + +// ================================================================================================= +// section-5:ggml-qnn backend helper macro / data structure / function / class +// ================================================================================================= +#define RPCMEM_DEFAULT_FLAGS 1 +#define RPCMEM_HEAP_ID_SYSTEM 25 + +typedef void (* ggmlqnn_op_func_t)(ggml_backend_t backend, ggml_tensor * op); + +using pfn_rpc_mem_init = void (*)(void); +using pfn_rpc_mem_deinit = void (*)(void); +using pfn_rpc_mem_alloc = void *(*)(int, uint32_t, int); +using pfn_rpc_mem_free = void (*)(void *); +using pfn_rpc_mem_to_fd = int (*)(void *); +using _pfn_QnnSaver_initialize = decltype(QnnSaver_initialize); +using _pfn_QnnInterface_getProviders = decltype(QnnInterface_getProviders); +using _pfn_QnnSystemInterface_getProviders = decltype(QnnSystemInterface_getProviders); + +enum class ggml_qnn_profile_level { + profile_off = 0, + profile_basic = 1, + profile_detail = 2 +}; + +enum qcom_htp_arch { + NONE = 0, + V68 = 68, + V69 = 69, + V73 = 73, + V75 = 75, + V79 = 79, +}; + +enum qcom_chipset_soc_model { + UNKNOWN_SM = 0, + SM7450 = 41, // v69, 7 Gen1 + SM8350 = 30, // v68, 888 + SM8450 = 36, // v69, SD 8 Gen 1 + SM8475 = 42, // v69, SD 8+ Gen 1 + SM8550 = 43, // v73, SD 8 Gen 2 + SM8650 = 57, // v75, SD 8 Gen 3 + SM8750 = 69, // v79, SD 8 Gen 4 +}; + +struct qcom_socinfo { + uint32_t soc_model; + size_t htp_arch; + size_t vtcm_size_in_mb; + char soc_desc[GGML_MAX_NAME]; +}; + +//file:///opt/qcom/aistack/qairt/2.31.0.250130/docs/QNN/general/overview.html#tbl-supported-snapdragon-devices +static struct qcom_socinfo g_qnn_soc_info_table[] = { + /* Qualcomm SnapDragon 7 Gen 1 */ + [SM7450] = { + .soc_model = SM7450, + .htp_arch = V69, + .vtcm_size_in_mb = 8, + .soc_desc = "Qualcomm SnapDragon 7 Gen 1"}, + + /* Qualcomm SnapDragon 888 */ + [SM8350] = { + .soc_model = SM8350, + .htp_arch = V68, + .vtcm_size_in_mb = 8, + .soc_desc = "Qualcomm SnapDragon 888 "}, + + /* Qualcomm SnapDragon 8 Gen 1 */ + [SM8450] = { + .soc_model = SM8450, + .htp_arch = V69, + .vtcm_size_in_mb = 8, + .soc_desc = "Qualcomm SnapDragon 8 Gen 1"}, + + /* Qualcomm SnapDragon 8 Gen 1+ */ + [SM8475] = { + .soc_model = SM8475, + .htp_arch = V69, + .vtcm_size_in_mb = 8, + .soc_desc = "Qualcomm SnapDragon 8 Gen 1+"}, + + /* Qualcomm SnapDragon 8 Gen 2 */ + [SM8550] = { + .soc_model = SM8550, + .htp_arch = V73, + .vtcm_size_in_mb = 8, + .soc_desc = "Qualcomm SnapDragon 8 Gen 2"}, + + /* Qualcomm SnapDragon 8 Gen 3 */ + [SM8650] = { + .soc_model = SM8650, + .htp_arch = V75, + .vtcm_size_in_mb = 8, + .soc_desc = "Qualcomm SnapDragon 8 Gen 3 "}, + + /* Qualcomm SnapDragon 8 Gen 4 */ + [SM8750] = { + .soc_model = SM8750, + .htp_arch = V79, + .vtcm_size_in_mb = 8, + .soc_desc = "Qualcomm SnapDragon 8 Gen 4"}, + +}; + +struct ggml_backend_qnn_context { + int device; + int threads; + char name[GGML_MAX_NAME]; + char desc[GGML_MAX_NAME]; + char lib[GGML_MAX_NAME]; + qnn_instance * instance; + struct ggml_backend * backend; + QNN_INTERFACE_VER_TYPE raw_interface; + QNN_SYSTEM_INTERFACE_VER_TYPE raw_system_interface; + struct qcom_socinfo socinfo; + + //FIXME: should I move it from public member of class qnn_instance to here? + //std::map> _qnn_graph_map; +} ; + +//FIXME: the following global vars and three helper funcs should be removed in the future +static int32_t g_ggmltensor_idx = 0; +static void reset_idx() { + g_ggmltensor_idx = 0; +} + +static void inc_idx() { + g_ggmltensor_idx++; +} + +static int32_t get_idx() { + return g_ggmltensor_idx; +} + +// file:///opt/qcom/aistack/qairt/2.31.0.250130/docs/QNN/general/quantization.html +// CPU - Choose a non-quantized model.Quantized models are currently incompatible with the CPU backend +// GPU - Choose a non-quantized model.Quantized models are currently incompatible with the GPU backend +// HTP - Choose a quantized model. Quantized models are required when running on the HTP backend +// DSP - Choose a quantized model. Quantized models are required when running on the DSP backend +// HTA - Choose a quantized model. Quantized models are required when running on the HTA backend +static struct ggml_backend_qnn_context g_qnn_mgr[GGML_QNN_MAX_DEVICES] = { + [QNN_BACKEND_CPU] = {.device = 0, + .threads = 1, + .name = "qnn-cpu", + .desc = "Qualcomm Kryo CPU", + .lib = "libQnnCpu.so", + .instance = nullptr, + .backend = nullptr, + .raw_interface = {}, + .raw_system_interface = {}, + .socinfo = {}}, + + [QNN_BACKEND_GPU] = {.device = 1, + .threads = 1, + .name = "qnn-gpu", + .desc = "Qualcomm Adreno GPU", + .lib = "libQnnGpu.so", + .instance = nullptr, + .backend = nullptr, + .raw_interface = {}, + .raw_system_interface = {}, + .socinfo = {}}, + + [QNN_BACKEND_NPU] = {.device = 2, + .threads = 1, + .name = "qnn-npu", + .desc = "Qualcomm NPU(Hexagon Tensor Processor)", + .lib = "libQnnHtp.so", + .instance = nullptr, + .backend = nullptr, + .raw_interface = {}, + .raw_system_interface = {}, + .socinfo = {}}, +}; + + +struct qnn_op_caps_t { + const char * qnn_op_name = nullptr; + const size_t input_param_count = 0; + const char * qnn_param_name = nullptr; +}; + +static const qnn_op_caps_t kOpCaps[] = { + {}, // GGML_OP_NONE + {}, // GGML_OP_DUP + { + // GGML_OP_ADD + QNN_OP_ELEMENT_WISE_ADD, // qnn_op_name + 2, // input_param_count + }, + {}, // GGML_OP_ADD1 + {}, // GGML_OP_ACC + {}, // GGML_OP_SUB + {}, // GGML_OP_MUL + {}, // GGML_OP_DIV + {}, // GGML_OP_SQR + {}, // GGML_OP_SQRT + {}, // GGML_OP_LOG + {}, // GGML_OP_SIN + {}, // GGML_OP_COS + {}, // GGML_OP_SUM + {}, // GGML_OP_SUM_ROWS + {}, // GGML_OP_MEAN + {}, // GGML_OP_ARGMAX + {}, // GGML_OP_COUNT_EQUAL + {}, // GGML_OP_REPEAT + {}, // GGML_OP_REPEAT_BACK + {}, // GGML_OP_CONCAT + {}, // GGML_OP_SILU_BACK + {}, // GGML_OP_NORM + {}, // GGML_OP_RMS_NORM + {}, // GGML_OP_RMS_NORM_BACK + {}, // GGML_OP_GROUP_NORM + { + // GGML_OP_MUL_MAT + QNN_OP_MAT_MUL, // qnn_op_name + 2, // input_param_count + }, + {}, // GGML_OP_MUL_MAT_ID + {}, // GGML_OP_OUT_PROD + {}, // GGML_OP_SCALE + {}, // GGML_OP_SET + {}, // GGML_OP_CPY + {}, // GGML_OP_CONT + {}, // GGML_OP_RESHAPE + {}, // GGML_OP_VIEW + {}, // GGML_OP_PERMUTE + {}, // GGML_OP_TRANSPOSE + {}, // GGML_OP_GET_ROWS + {}, // GGML_OP_GET_ROWS_BACK + {}, // GGML_OP_DIAG + {}, // GGML_OP_DIAG_MASK_INF + {}, // GGML_OP_DIAG_MASK_ZERO + {}, // GGML_OP_SOFT_MAX + {}, // GGML_OP_SOFT_MAX_BACK + {}, // GGML_OP_ROPE + {}, // GGML_OP_ROPE_BACK + {}, // GGML_OP_CLAMP + {}, // GGML_OP_CONV_TRANSPOSE_1D + {}, // GGML_OP_IM2COL + {}, // GGML_OP_IM2COL_BACK + {}, // GGML_OP_CONV_TRANSPOSE_2D + {}, // GGML_OP_POOL_1D + {}, // GGML_OP_POOL_2D + {}, // GGML_OP_POOL_2D_BACK + {}, // GGML_OP_UPSCALE + {}, // GGML_OP_PAD + {}, // GGML_OP_PAD_REFLECT_1D + {}, // GGML_OP_ARANGE + {}, // GGML_OP_TIMESTEP_EMBEDDING + {}, // GGML_OP_ARGSORT + {}, // GGML_OP_LEAKY_RELU + {}, // GGML_OP_FLASH_ATTN_EXT + {}, // GGML_OP_FLASH_ATTN_BACK + {}, // GGML_OP_SSM_CONV + {}, // GGML_OP_SSM_SCAN + {}, // GGML_OP_WIN_PART + {}, // GGML_OP_WIN_UNPART + {}, // GGML_OP_GET_REL_POS + {}, // GGML_OP_ADD_REL_POS + {}, // GGML_OP_RWKV_WKV6 + {}, // GGML_OP_GATED_LINEAR_ATTN + {}, // GGML_OP_UNARY + {}, // GGML_OP_MAP_UNARY + {}, // GGML_OP_MAP_BINARY + {}, // GGML_OP_MAP_CUSTOM1_F32 + {}, // GGML_OP_MAP_CUSTOM2_F32 + {}, // GGML_OP_MAP_CUSTOM3_F32 + {}, // GGML_OP_MAP_CUSTOM1 + {}, // GGML_OP_MAP_CUSTOM2 + {}, // GGML_OP_MAP_CUSTOM3 + {}, // GGML_OP_CROSS_ENTROPY_LOSS + {}, // GGML_OP_CROSS_ENTROPY_LOSS_BACK + {}, // GGML_OP_OPT_STEP_ADAMW + {}, // GGML_UNARY_OP_ABS + {}, // GGML_UNARY_OP_SGN + {}, // GGML_UNARY_OP_NEG + {}, // GGML_UNARY_OP_STEP + {}, // GGML_UNARY_OP_TANH + {}, // GGML_UNARY_OP_ELU + {}, // GGML_UNARY_OP_RELU + {}, // GGML_UNARY_OP_SIGMOID + {}, // GGML_UNARY_OP_GELU + {}, // GGML_UNARY_OP_GELU_QUICK + {}, // GGML_UNARY_OP_SILU + {}, // GGML_UNARY_OP_HARDSWISH + {}, // GGML_UNARY_OP_HARDSIGMOID + {}, // GGML_UNARY_OP_EXP +}; + +static const char * qnn_get_socmodel_desc(uint32_t soc_model) { + switch (soc_model) { + case SM7450: + return "SM7450"; + case SM8350: + return "SM8350"; + case SM8450: + return "SM8450"; + case SM8475: + return "SM8475"; + case SM8550: + return "SM8550"; + case SM8650: + return "SM8650"; + case SM8750: + return "SM8750"; + default: + return "unknown"; + } +} + +static const char * qnn_get_htparch_desc(size_t htp_arch) { + switch (htp_arch) { + case V68: + return "QCOM_HTP_V68"; + case V69: + return "QCOM_HTP_V69"; + case V73: + return "QCOM_HTP_V73"; + case V75: + return "QCOM_HTP_V75"; + case V79: + return "QCOM_HTP_V79"; + default: + return "unknown"; + } +} + +static struct qcom_socinfo * qnn_get_socinfo_from_socmodel(uint32_t soc_model) { + size_t items = sizeof(g_qnn_soc_info_table) / sizeof(g_qnn_soc_info_table[0]); + for (size_t idx = 0; idx < items; idx++) { + if (soc_model == g_qnn_soc_info_table[idx].soc_model) { + return &g_qnn_soc_info_table[idx]; + } + } + return nullptr; +} + +static bool ggmlqnn_is_valid_params(ggml_backend_qnn_context * ctx, const ggml_tensor * src0, + const ggml_tensor * src1, ggml_tensor * dst) { + if ((nullptr == ctx) || (nullptr == src0) || (nullptr == src1) || (nullptr == dst)) { + GGMLQNN_LOG_WARN("invalid params\n"); + return false; + } + + qnn_instance * instance = ctx->instance; + if (nullptr == instance) { + GGMLQNN_LOG_WARN("invalid params\n"); + return false; + } + + return true; +} + +#define GGMLQNN_CHECK_PARAMS(ctx, src0, src1, dst) \ + do { \ + if (!ggmlqnn_is_valid_params((ctx), (src0), (src1), (dst))) { \ + return; \ + } \ + } while (0) + +static uint32_t ggml_get_tensor_rank(const ggml_tensor * tensor) { + /* + uint32_t rank = 0; + for (int i = 0; i < GGML_MAX_DIMS; i++) { + if ((0 != tensor->ne[i]) && (1 != tensor->ne[i])) { + rank++; + } + } + return rank; + */ + return ggml_n_dims(tensor); +} + +static uint32_t ggml_get_tensor_data_size(const ggml_tensor * tensor) { + /* + size_t data_size = ggml_row_size(tensor->type, tensor->ne[0]); + size_t n_dims = ggml_get_tensor_rank(tensor); + for (int i = 1; i < n_dims; i++) { + data_size *= tensor->ne[i]; + } + + return data_size; + */ + return ggml_nbytes(tensor); +} + +static const char * ggml_get_type_name(ggml_type type) { + const struct ggml_type_traits * traits = ggml_get_type_traits(type); + return traits->type_name; +} + +Qnn_Tensor_t * ggml_qnn_create_tensor(const ggml_tensor * tensor) { + Qnn_ErrorHandle_t error = QNN_SUCCESS; + char tensor_name[GGML_MAX_NAME] = {0}; + + //FIXME:remove get_idx() and inc_idx() in the future but ensure the tensor name is unique + snprintf(tensor_name, GGML_MAX_NAME, "tensor_%-8d", get_idx()); + GGMLQNN_LOG_DEBUG("init_tensor %d", get_idx()); + inc_idx(); + + uint32_t dimensions[] = {(uint32_t) tensor->ne[0], (uint32_t) tensor->ne[1], + (uint32_t) tensor->ne[2], (uint32_t) tensor->ne[3]}; + Qnn_DataType_t qnn_data_type = QNN_DATATYPE_FLOAT_32; + Qnn_TensorType_t qnn_tensor_type = QNN_TENSOR_TYPE_APP_WRITE; + + if (tensor->flags & GGML_TENSOR_FLAG_INPUT) { + qnn_tensor_type = QNN_TENSOR_TYPE_APP_WRITE; + } else if (tensor->flags & GGML_TENSOR_FLAG_OUTPUT) { + qnn_tensor_type = QNN_TENSOR_TYPE_APP_READ; + } + Qnn_Tensor_t qnn_tensor = { + .version= QNN_TENSOR_VERSION_1, + {.v1= { + .id = 0, + .name = tensor_name, + .type = qnn_tensor_type, + .dataFormat = QNN_TENSOR_DATA_FORMAT_FLAT_BUFFER, + .dataType = qnn_data_type, + .quantizeParams = {QNN_DEFINITION_UNDEFINED, + QNN_QUANTIZATION_ENCODING_UNDEFINED, + {.scaleOffsetEncoding = {.scale = 0.0000000000000000f, .offset = 0}}}, + .rank = ggml_get_tensor_rank(tensor), + .dimensions = dimensions, + .memType = QNN_TENSORMEMTYPE_RAW, + {.clientBuf = {.data = nullptr, + .dataSize = 0}}}} + }; + Qnn_Tensor_t * p_qnn_tensor = (Qnn_Tensor_t *)calloc(1, sizeof(Qnn_Tensor_t)); + if (nullptr == p_qnn_tensor) { + GGMLQNN_LOG_WARN("calloc failed"); + return nullptr; + } + error = deep_copy_qnn_tensors(qnn_tensor, * p_qnn_tensor); + if (error != QNN_SUCCESS) { + free(p_qnn_tensor); + GGMLQNN_LOG_WARN("init tensor failed"); + return nullptr; + } + + return p_qnn_tensor; +} + +//TODO: +// ref:explanation of k-quants, https://github.com/ggerganov/llama.cpp/pull/1684 +static Qnn_DataType_t qnn_datatype_from_ggml_datatype(enum ggml_type ggmltype) { + switch (ggmltype) { + case GGML_TYPE_F16: + return QNN_DATATYPE_FLOAT_16; + case GGML_TYPE_F32: + return QNN_DATATYPE_FLOAT_32; + case GGML_TYPE_I8: + return QNN_DATATYPE_INT_8; + case GGML_TYPE_Q8_0: + return QNN_DATATYPE_SFIXED_POINT_8; + case GGML_TYPE_Q4_0: + return QNN_DATATYPE_SFIXED_POINT_4; + default: + break; + } + return QNN_DATATYPE_UNDEFINED; +} + +//TODO: +static ggml_type ggml_datatype_from_qnn_datatype(Qnn_DataType_t qnn_type) { + switch (qnn_type) { + case QNN_DATATYPE_FLOAT_32: + return GGML_TYPE_F32; + case QNN_DATATYPE_FLOAT_16: + return GGML_TYPE_F16; + case QNN_DATATYPE_UINT_32: + case QNN_DATATYPE_INT_32: + return GGML_TYPE_I32; + case QNN_DATATYPE_INT_16: + return GGML_TYPE_I16; + case QNN_DATATYPE_INT_8: + return GGML_TYPE_I8; + case QNN_DATATYPE_SFIXED_POINT_8: + return GGML_TYPE_Q8_0; + case QNN_DATATYPE_SFIXED_POINT_4: + return GGML_TYPE_Q4_0; + default: + break; + } + return GGML_TYPE_COUNT; +} + +//TODO: add more ops +static const char * qnn_opname_from_ggmlop(enum ggml_op ggmlop) { + switch (ggmlop) { + case GGML_OP_ADD: + return QNN_OP_ELEMENT_WISE_ADD; + case GGML_OP_MUL_MAT: + return QNN_OP_MAT_MUL; + default: + break; + } + return nullptr; +} + +static const char * get_ggml_type_name(ggml_type type) { + const auto * traits = ggml_get_type_traits(type); + return traits->type_name; +} + +static void append_tensor_dimensions(const ggml_tensor * tensor, std::string & output) { + char buffer[256] = {}; + const char * type_name = get_ggml_type_name(tensor->type); + int len = 0; + switch (ggml_n_dims(tensor)) { + case 1: + len = snprintf(buffer, sizeof(buffer), "%ldx1%s", (long)tensor->ne[0], type_name); + break; + case 2: + len = snprintf(buffer, sizeof(buffer), "%ldx%ld%s", (long)tensor->ne[0], (long)tensor->ne[1], type_name); + break; + case 3: + len = snprintf(buffer, sizeof(buffer), "%ldx%ldx%ld%s", (long)tensor->ne[0], (long)tensor->ne[1], + (long)tensor->ne[2], type_name); + break; + case 4: + default: + len = snprintf(buffer, sizeof(buffer), "%ldx%ldx%ldx%ld%s", (long)tensor->ne[0], (long)tensor->ne[1], + (long)tensor->ne[2], (long)tensor->ne[3], type_name); + break; + } + GGML_ASSERT(len > 0 && len < (int)sizeof(buffer)); + output.append(buffer, len); +} + +constexpr const size_t kGgmlUnaryOpStart = GGML_OP_COUNT; + +static size_t get_qnn_op_index(const ggml_tensor * tensor) { + if (tensor->op == GGML_OP_UNARY) { + return kGgmlUnaryOpStart + ggml_get_unary_op(tensor); + } + + return tensor->op; +} + +static size_t get_qnn_op_input_param_count(const ggml_tensor * op) { + auto op_index = get_qnn_op_index(op); + GGML_ASSERT(op_index < std::size(kOpCaps)); + return kOpCaps[op_index].input_param_count; +} + +static void get_graph_key_from_op(const ggml_tensor * op, std::string & output) { + GGML_ASSERT(op->op != GGML_OP_NONE); + output += ggml_op_desc(op); + output += get_ggml_type_name(op->type); + size_t param_count = get_qnn_op_input_param_count(op); + for (size_t i = 0; i < param_count; ++i) { + auto * input = op->src[i]; + if (!input) { + break; + } + output += '_'; + append_tensor_dimensions(input, output); + } +} + +#if ENABLE_QNNBACKEND_PERF +class qnn_perf { +public: + qnn_perf(const std::string & perf_name) : _perf_name(std::move(perf_name)) {}; + qnn_perf() = delete; + qnn_perf(const qnn_perf & ) = delete; + qnn_perf & operator= (const qnn_perf & ) = delete; + + void start() { + _begin_time = ggml_time_us(); + } + + void info() { + _end_time = ggml_time_us(); + _duration = (_end_time - _begin_time); + GGMLQNN_LOG_DEBUG("duration of %s : %lld microseconds\n", _perf_name.c_str(), _duration); + } + +private: + int64_t _begin_time = 0LL; + int64_t _end_time = 0LL; + int64_t _duration = 0LL; + std::string _perf_name; +}; +#else +class qnn_perf { +public: + qnn_perf(const std::string & perf_name) {} + qnn_perf() = delete; + qnn_perf(const qnn_perf & ) = delete; + qnn_perf & operator= (const qnn_perf & ) = delete; + + void start() {} + void info() {} +}; +#endif + +template +Fn load_qnn_functionpointers(void * handle, const char * function_name) { + return reinterpret_cast(dlsym(handle, function_name)); +} + +class qnn_interface { + +#define DEFINE_SHIM_FUNCTION_INTERFACE(F, pointer_name) \ + template \ + inline auto qnn_##F(Args... args) const { \ + return (_qnn_interface->QNN_INTERFACE_VER_NAME.pointer_name)( \ + std::forward(args)...); \ + } + + +#define DEFINE_SHIM_FUNCTION_SYS_INTERFACE(F, pointer_name) \ + template \ + inline auto qnn_##F(Args... args) const { \ + return (_qnn_sys_interface->QNN_SYSTEM_INTERFACE_VER_NAME.pointer_name)( \ + std::forward(args)...); \ + } + + friend class qnn_instance; + +public: + qnn_interface() = default; + + // QnnBackend + DEFINE_SHIM_FUNCTION_INTERFACE(backend_create, backendCreate); + + DEFINE_SHIM_FUNCTION_INTERFACE(backend_free, backendFree); + + DEFINE_SHIM_FUNCTION_INTERFACE(backend_register_op_package, backendRegisterOpPackage); + + DEFINE_SHIM_FUNCTION_INTERFACE(backend_validate_op_config, backendValidateOpConfig); + + DEFINE_SHIM_FUNCTION_INTERFACE(backend_get_api_version, backendGetApiVersion); + + // QnnDevice + DEFINE_SHIM_FUNCTION_INTERFACE(device_create, deviceCreate); + + DEFINE_SHIM_FUNCTION_INTERFACE(device_free, deviceFree); + + DEFINE_SHIM_FUNCTION_INTERFACE(device_get_infrastructure, deviceGetInfrastructure); + + DEFINE_SHIM_FUNCTION_INTERFACE(device_get_platform_info, deviceGetPlatformInfo); + + DEFINE_SHIM_FUNCTION_INTERFACE(device_get_info, deviceGetInfo); + + // QnnContext + DEFINE_SHIM_FUNCTION_INTERFACE(context_create, contextCreate); + + DEFINE_SHIM_FUNCTION_INTERFACE(context_get_binary_size, contextGetBinarySize); + + DEFINE_SHIM_FUNCTION_INTERFACE(context_get_binary, contextGetBinary); + + DEFINE_SHIM_FUNCTION_INTERFACE(context_create_from_binary, contextCreateFromBinary); + + DEFINE_SHIM_FUNCTION_INTERFACE(context_free, contextFree); + + // QnnGraph + DEFINE_SHIM_FUNCTION_INTERFACE(graph_create, graphCreate); + + DEFINE_SHIM_FUNCTION_INTERFACE(graph_add_node, graphAddNode); + + DEFINE_SHIM_FUNCTION_INTERFACE(graph_finalize, graphFinalize); + + DEFINE_SHIM_FUNCTION_INTERFACE(graph_execute, graphExecute); + + DEFINE_SHIM_FUNCTION_INTERFACE(graph_retrieve, graphRetrieve); + + // QnnLog + DEFINE_SHIM_FUNCTION_INTERFACE(log_create, logCreate); + + DEFINE_SHIM_FUNCTION_INTERFACE(log_free, logFree); + + DEFINE_SHIM_FUNCTION_INTERFACE(log_set_log_level, logSetLogLevel); + + // QnnProfile + DEFINE_SHIM_FUNCTION_INTERFACE(profile_create, profileCreate); + + DEFINE_SHIM_FUNCTION_INTERFACE(profile_get_events, profileGetEvents); + + DEFINE_SHIM_FUNCTION_INTERFACE(profile_get_sub_events, profileGetSubEvents); + + DEFINE_SHIM_FUNCTION_INTERFACE(profile_get_event_data, profileGetEventData); + + DEFINE_SHIM_FUNCTION_INTERFACE(profile_free, profileFree); + + // QnnMem + DEFINE_SHIM_FUNCTION_INTERFACE(mem_register, memRegister); + + DEFINE_SHIM_FUNCTION_INTERFACE(mem_de_register, memDeRegister); + + // QnnProperty + DEFINE_SHIM_FUNCTION_INTERFACE(property_has_capability, propertyHasCapability); + + // QnnTensor + DEFINE_SHIM_FUNCTION_INTERFACE(tensor_create_context_tensor, tensorCreateContextTensor); + + DEFINE_SHIM_FUNCTION_INTERFACE(tensor_create_graph_tensor, tensorCreateGraphTensor); + + // QnnSystem + DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_create, systemContextCreate); + + DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_get_binary_info, systemContextGetBinaryInfo); + + DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_free, systemContextFree); + + void set_qnn_interface(const QnnInterface_t * qnn_interface) { + _qnn_interface = qnn_interface; + } + + void set_qnn_system_interface(const QnnSystemInterface_t * qnn_sys_interface) { + _qnn_sys_interface = qnn_sys_interface; + } + + uint32_t get_backend_id() const { + return _qnn_interface->backendId; + } + + bool is_loaded() const { + return ((_qnn_sys_interface != nullptr) && (_qnn_interface != nullptr)); + } + +private: + const QnnInterface_t *_qnn_interface = nullptr; + + const QnnSystemInterface_t *_qnn_sys_interface = nullptr; +}; + +class qnn_instance { +public: + using BackendIdType = decltype(QnnInterface_t{}.backendId); + + explicit qnn_instance(const std::string & lib_path, const std::string & backend_name, + const std::string & model_name) : + _lib_path(std::move(lib_path)), + _backend_name(std::move(backend_name)), + _model_name(std::move(model_name)) {}; + + ~qnn_instance() { + } + + int qnn_init(const QnnSaver_Config_t ** saver_config); + + int qnn_finalize(); + + const qnn_interface &get_qnn_interface() { + if (!_qnn_interface.is_loaded()) { + GGMLQNN_LOG_WARN("pls check why _qnn_interface is not loaded\n"); + } + return _qnn_interface; + } + + const QNN_INTERFACE_VER_TYPE &get_qnn_raw_interface() { + if (!_qnn_interface.is_loaded()) { + GGMLQNN_LOG_WARN("pls check why _qnn_interface is not loaded\n"); + } + return _qnn_raw_interface; + } + + const QNN_SYSTEM_INTERFACE_VER_TYPE &get_qnn_raw_system_interface() { + if (!_qnn_interface.is_loaded()) { + GGMLQNN_LOG_WARN("pls check why _qnn_interface is not loaded\n"); + } + return _qnn_raw_system_interface; + } + + const Qnn_LogHandle_t get_qnn_log_handle() { return _qnn_log_handle; } + + const Qnn_ProfileHandle_t get_qnn_profile_handle() { return _qnn_profile_handle; } + + const Qnn_DeviceHandle_t get_qnn_device_handle() { return _qnn_device_handle; } + + const Qnn_BackendHandle_t get_qnn_backend_handle() { return _qnn_backend_handle; } + + const Qnn_ContextHandle_t get_qnn_context_handle() { return _qnn_context_handle; } + + const QnnSystemContext_Handle_t get_qnn_system_handle() { return _qnn_system_handle; } + + const Qnn_GraphHandle_t get_qnn_graph_handle() { return _qnn_graph_handle; } + + int init_qnn_graph(const char * graph_name, + bool debug, + uint8_t do_node_validation = 1, + const QnnGraph_Config_t ** graph_configs = nullptr + ); + int init_qnn_graph(const std::string &graph_name, QNNBackend device, size_t vtcm_size_in_mb); + + int finalize_qnn_graph(); + + bool is_valid_graph() const { return _qnn_graph_handle != nullptr; } + + int init_htp_perfinfra() { + QnnDevice_Infrastructure_t device_infra = nullptr; + int error = _qnn_raw_interface.deviceGetInfrastructure(&device_infra); + if (error != QNN_SUCCESS) { + GGMLQNN_LOG_WARN("failed to get qnn device infra\n"); + return 1; + } + + QnnHtpDevice_Infrastructure_t *htp_infra = static_cast(device_infra); + QnnHtpDevice_PerfInfrastructure_t *htp_perfinfra = &htp_infra->perfInfra; + uint32_t power_configid = 1; + uint32_t device_id = 0; + uint32_t core_id = 0; + htp_perfinfra->createPowerConfigId(device_id, core_id, &power_configid); + _qnn_htp_perfinfra = htp_perfinfra; + _qnn_power_configid = power_configid; + + return 0; + } + + int set_rpc_polling() { + if (_qnn_rpc_pollingtime > 0) { + QnnHtpPerfInfrastructure_PowerConfig_t rpc_pollingtime; + memset(&rpc_pollingtime, 0, sizeof(rpc_pollingtime)); + rpc_pollingtime.option = + QNN_HTP_PERF_INFRASTRUCTURE_POWER_CONFIGOPTION_RPC_POLLING_TIME; + rpc_pollingtime.rpcPollingTimeConfig = _qnn_rpc_pollingtime; + const QnnHtpPerfInfrastructure_PowerConfig_t * power_configs[] = {&rpc_pollingtime, nullptr}; + if (_qnn_htp_perfinfra) { + _qnn_htp_perfinfra->setPowerConfig(_qnn_power_configid, power_configs); + } + } + return 0; + } + + int set_high_performance_mode() { + if (nullptr == _qnn_htp_perfinfra) { + GGMLQNN_LOG_DEBUG("perf intra is null\n"); + return 1; + } + + QnnHtpPerfInfrastructure_PowerConfig_t power_config; + memset(&power_config, 0, sizeof(power_config)); + power_config.option = QNN_HTP_PERF_INFRASTRUCTURE_POWER_CONFIGOPTION_DCVS_V3; + power_config.dcvsV3Config.dcvsEnable = 0; + power_config.dcvsV3Config.setDcvsEnable = 1; + power_config.dcvsV3Config.contextId = _qnn_power_configid; + power_config.dcvsV3Config.powerMode = QNN_HTP_PERF_INFRASTRUCTURE_POWERMODE_PERFORMANCE_MODE; + power_config.dcvsV3Config.setSleepLatency = 1; // True to consider Latency parameter otherwise False + power_config.dcvsV3Config.setBusParams = 1; // True to consider Bus parameter otherwise False + power_config.dcvsV3Config.setCoreParams = 1; // True to consider Core parameter otherwise False + power_config.dcvsV3Config.sleepDisable = 0; // True to consider sleep/LPM modes, False to enable + power_config.dcvsV3Config.setSleepDisable = 0; // True to consider sleep disable/enable parameter otherwise False + // set Sleep latency parameter + uint32_t latencyValue = 40; + power_config.dcvsV3Config.sleepLatency = latencyValue; // range 40-2000 micro sec + // set Bus Clock Parameters (refer QnnHtpPerfInfrastructure_VoltageCorner_t enum) + power_config.dcvsV3Config.busVoltageCornerMin = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; + power_config.dcvsV3Config.busVoltageCornerTarget = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; + power_config.dcvsV3Config.busVoltageCornerMax = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; + // set Core Clock Parameters (refer QnnHtpPerfInfrastructure_VoltageCorner_t enum) + power_config.dcvsV3Config.coreVoltageCornerMin = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; + power_config.dcvsV3Config.coreVoltageCornerTarget = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; + power_config.dcvsV3Config.coreVoltageCornerMax = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; + // set power config with different performance parameters + const QnnHtpPerfInfrastructure_PowerConfig_t * power_configs[] = {&power_config, nullptr}; + + _qnn_htp_perfinfra->setPowerConfig(_qnn_power_configid, power_configs); + + return 0; + } + + std::string & get_qnn_graph_name() { return _graph_name; } + + bool is_rpcmem_initialized() { + return _rpcmem_initialized; + } + + void set_rpcmem_initialized(bool initialized) { + _rpcmem_initialized = initialized; + } + + size_t get_rpcmem_capacity() { return _rpcmem_capacity; } + + int32_t rpcmem_to_fd(void * buf); + + int register_rpcmem(void * p_data, Qnn_Tensor_t * p_tensor); + Qnn_MemHandle_t register_rpcmem(void * p_data, const uint32_t rank, uint32_t * dimensions, Qnn_DataType_t data_type); + + void unregister_rpcmem(); + void unregister_rpcmem(Qnn_MemHandle_t mem_handle); + + void * alloc_rpcmem(size_t bytes, size_t alignment); + void * get_rpcmem_from_memhandle(Qnn_MemHandle_t mem_handle); + + void free_rpcmem(void * buf); + void free_rpcmem(); + + bool is_rpcmem_allocated(void * buf); + + bool is_rpcmem_registered(Qnn_MemHandle_t handle) { + return _qnn_mem_set.count(handle) != 0U; + } + + bool enable_qnn_rpc() { + return _enable_qnn_rpc; + } + +public: + std::map> _qnn_graph_map; + +private: + int load_system(); + + int unload_system(); + + int load_backend(std::string & lib_path, const QnnSaver_Config_t ** saver_config); + + int unload_backend(); + + void set_qnn_raw_interface(QNN_INTERFACE_VER_TYPE & raw_interface) { + _qnn_raw_interface = raw_interface; + } + + void set_qnn_raw_system_interface(QNN_SYSTEM_INTERFACE_VER_TYPE & raw_interface) { + _qnn_raw_system_interface = raw_interface; + } + +private: + static constexpr const int _required_num_providers = 1; + +private: + std::string _lib_path; + std::string _backend_name; + std::string _model_name; // name of prebuilt QNN model, might be used in the future + BackendIdType _backend_id; + + bool _debug_tensor = false; // flag to indicate if requested graph is to be run in debug mode + bool _do_node_validations = true; // flag to indicate whether all add_node calls need to be validated + QnnLog_Level_t _qnn_log_level = QNN_LOG_LEVEL_DEBUG; + + ggml_qnn_profile_level _profile_level = ggml_qnn_profile_level::profile_detail; + + qnn_interface _qnn_interface; + + void *_system_lib_handle = nullptr; + + Qnn_GraphHandle_t _qnn_graph_handle = nullptr; + + Qnn_LogHandle_t _qnn_log_handle = nullptr; + + Qnn_ProfileHandle_t _qnn_profile_handle = nullptr; + + Qnn_DeviceHandle_t _qnn_device_handle = nullptr; + + Qnn_BackendHandle_t _qnn_backend_handle = nullptr; + + Qnn_ContextHandle_t _qnn_context_handle = nullptr; + + QnnSystemContext_Handle_t _qnn_system_handle = nullptr; + + QnnHtpDevice_PerfInfrastructure_t *_qnn_htp_perfinfra = nullptr; + uint32_t _qnn_power_configid = 1; + uint32_t _qnn_rpc_pollingtime = 9999; // 0-10000 us for high performing + + QNN_INTERFACE_VER_TYPE _qnn_raw_interface; + QNN_SYSTEM_INTERFACE_VER_TYPE _qnn_raw_system_interface; + + std::unordered_map _qnn_mem_set; + std::unordered_map _qnn_rpc_buffer_to_handles; + + + static std::mutex _init_mutex; + static std::unordered_map _loaded_lib_handle; + static std::unordered_map _lib_path_to_backend_id; + static std::unordered_map _loaded_backend; + + void * _rpc_lib_handle = nullptr; + std::atomic_bool _rpcmem_initialized{false}; + pfn_rpc_mem_alloc _pfn_rpc_mem_alloc; + pfn_rpc_mem_free _pfn_rpc_mem_free; + pfn_rpc_mem_to_fd _pfn_rpc_mem_to_fd; + pfn_rpc_mem_init _pfn_rpc_mem_init; + pfn_rpc_mem_deinit _pfn_rpc_mem_deinit; + std::unordered_map _rpcmem_store_map; + size_t _rpcmem_capacity = 512; + + std::string _graph_name; + QNNBackend _device_id; + bool _enable_qnn_rpc = false; //FIXME:unknown issue with QNN RPC feature + + DISABLE_COPY(qnn_instance); + DISABLE_MOVE(qnn_instance); +}; + +std::mutex qnn_instance::_init_mutex; +std::unordered_map qnn_instance::_loaded_lib_handle; +std::unordered_map qnn_instance::_lib_path_to_backend_id; +std::unordered_map qnn_instance::_loaded_backend; + +void * qnn_instance::alloc_rpcmem(size_t bytes, size_t alignment) { + if (!_rpcmem_initialized) { + GGMLQNN_LOG_WARN("rpc memory not initialized\n"); + return nullptr; + } + + auto allocate_bytes = static_cast(bytes + alignment); + void * buf = _pfn_rpc_mem_alloc(RPCMEM_HEAP_ID_SYSTEM, RPCMEM_DEFAULT_FLAGS, allocate_bytes); + if (buf == nullptr) { + GGMLQNN_LOG_WARN("failed to allocate rpc memory\n"); + return nullptr; + } + + auto aligned_buf = reinterpret_cast(ggmlqnn_align_to(alignment, + reinterpret_cast(buf))); + bool status = _rpcmem_store_map.insert(std::pair(aligned_buf, buf)).second; + if (!status) { + GGMLQNN_LOG_WARN("failed to allocate rpc memory\n"); + _pfn_rpc_mem_free(buf); + } + + return aligned_buf; +} + +void qnn_instance::free_rpcmem(void * buf) { + if (!_rpcmem_initialized) { + GGMLQNN_LOG_WARN("rpc memory not initialized\n"); + } else if (0 == _rpcmem_store_map.count(buf)) { + GGMLQNN_LOG_WARN("no allocated tensor\n"); + } else { + GGMLQNN_LOG_DEBUG("free rpc mem %p", _rpcmem_store_map[buf]); + _pfn_rpc_mem_free(_rpcmem_store_map[buf]); + _rpcmem_store_map.erase(buf); + } +} + +void qnn_instance::free_rpcmem() { + Qnn_ErrorHandle_t error = QNN_SUCCESS; + + if (_rpcmem_store_map.empty()) { + GGMLQNN_LOG_WARN("no rpcmem allocated\n"); + return; + } + + for (std::unordered_map::iterator it = _rpcmem_store_map.begin(); + it != _qnn_mem_set.end(); + it++) { + void * rpcbuffer = it->second; + GGMLQNN_LOG_DEBUG("free rpc buffer %p", rpcbuffer); + _pfn_rpc_mem_free(rpcbuffer); + } + _rpcmem_store_map.clear(); +} + +int32_t qnn_instance::rpcmem_to_fd(void * buf) { + int32_t mem_fd = -1; + if (!is_rpcmem_initialized()) { + GGMLQNN_LOG_WARN("rpc memory not initialized\n"); + } else { + mem_fd = _pfn_rpc_mem_to_fd(buf); + } + + return mem_fd; +} + +int qnn_instance::register_rpcmem(void * p_data, Qnn_Tensor_t * p_tensor) { + if (nullptr == p_data || (nullptr == p_tensor)) { + GGMLQNN_LOG_WARN("invalid param\n"); + return 1; + } + + if (!is_rpcmem_initialized()) { + GGMLQNN_LOG_WARN("rpc memory not initialized\n"); + return 2; + } + + if (is_rpcmem_registered((QNN_VER_PTR(*p_tensor)->memHandle))) { + GGMLQNN_LOG_WARN("tensor %s has been registered shared memory\n", (QNN_VER_PTR(*p_tensor)->name)); + return 4; + } + + int32_t mem_fd = rpcmem_to_fd(p_data); + if (-1 == mem_fd) { + GGMLQNN_LOG_WARN("failed to get file descriptor\n"); + return 5; + } + GGMLQNN_LOG_DEBUG("mem_fd %d\n", mem_fd); + Qnn_MemDescriptor_t descriptor = { + {QNN_VER_PTR(*p_tensor)->rank, QNN_VER_PTR(*p_tensor)->dimensions, nullptr}, + QNN_VER_PTR(*p_tensor)->dataType, + QNN_MEM_TYPE_ION, + {{mem_fd}}}; + Qnn_MemHandle_t handle = nullptr; + int error = QNN_SUCCESS; + error = _qnn_interface.qnn_mem_register( + _qnn_context_handle, + &descriptor, + /*numDescriptors=*/1, + &handle); + if (error != QNN_SUCCESS) { + GGMLQNN_LOG_WARN("failed to register shared memory, error %d, %s\n", QNN_GET_ERROR_CODE(error), + strerror(error)); + return 6; + } else { + GGMLQNN_LOG_INFO("tensor %s successfully register shared memory\n", (QNN_VER_PTR(*p_tensor)->name)); + } + QNN_VER_PTR(*p_tensor)->memHandle = handle; + _qnn_mem_set.insert((std::pair(p_data, handle))); + + return 0; +} + +Qnn_MemHandle_t qnn_instance::register_rpcmem(void * p_data, const uint32_t rank, uint32_t * dimensions, Qnn_DataType_t data_type) { + if (!p_data) { + GGMLQNN_LOG_WARN("invalid param"); + return nullptr; + } + + if (!is_rpcmem_initialized()) { + GGMLQNN_LOG_WARN("rpc memory not initialized"); + return nullptr; + } + + if (is_rpcmem_registered(p_data)) { + GGMLQNN_LOG_WARN("rpc memory already registered"); + return _qnn_rpc_buffer_to_handles[p_data]; + } + + auto mem_fd = rpcmem_to_fd(p_data); + if (mem_fd == -1) { + GGMLQNN_LOG_WARN("failed to get file descriptor"); + return nullptr; + } + + GGMLQNN_LOG_DEBUG("mem_fd %d", mem_fd); + Qnn_MemDescriptor_t descriptor = {{rank, dimensions, nullptr}, data_type, QNN_MEM_TYPE_ION, {{mem_fd}}}; + Qnn_MemHandle_t handle = nullptr; + auto error = _qnn_interface.qnn_mem_register(_qnn_context_handle, &descriptor, + /*numDescriptors=*/1, &handle); + if (error != QNN_SUCCESS) { + GGMLQNN_LOG_WARN("failed to register shared memory, error %d, %s", QNN_GET_ERROR_CODE(error), strerror(error)); + return nullptr; + } + + _qnn_rpc_buffer_to_handles.insert({p_data, handle}); + GGMLQNN_LOG_DEBUG("successfully register shared memory handler: %p", handle); + return handle; +} + +void * qnn_instance::get_rpcmem_from_memhandle(Qnn_MemHandle_t mem_handle) { + for (std::unordered_map::iterator it = _qnn_mem_set.begin(); + it != _qnn_mem_set.end(); + it++) { + Qnn_MemHandle_t mem_handle = it->second; + if (it->second == mem_handle) { + return it->first; + } + } + GGMLQNN_LOG_WARN("can't find rpcmem from qnn mem handle %p", mem_handle); + return nullptr; +} + +void qnn_instance::unregister_rpcmem() { + Qnn_ErrorHandle_t error = QNN_SUCCESS; + + if (_qnn_mem_set.empty()) { + GGMLQNN_LOG_WARN("no rpcmem registered\n"); + } + + for (std::unordered_map::iterator it = _qnn_mem_set.begin(); + it != _qnn_mem_set.end(); + it++) { + Qnn_MemHandle_t mem_handle = it->second; + error = _qnn_interface.qnn_mem_de_register(&mem_handle, 1); + if (error != QNN_SUCCESS) { + GGMLQNN_LOG_WARN("failed to unregister shared memory, error %d\n", + QNN_GET_ERROR_CODE(error)); + } else { + GGMLQNN_LOG_DEBUG("unregister shared memory ok"); + } + } + _qnn_mem_set.clear(); +} + +void qnn_instance::unregister_rpcmem(Qnn_MemHandle_t mem_handle) { + Qnn_ErrorHandle_t error = _qnn_interface.qnn_mem_de_register(&mem_handle, 1); + if (error != QNN_SUCCESS) { + GGMLQNN_LOG_WARN("failed to unregister shared memory, error %d", QNN_GET_ERROR_CODE(error)); + } + + auto it = std::find_if(_qnn_mem_set.begin(), _qnn_mem_set.end(), + [mem_handle](const auto &kv) { return kv.second == mem_handle; }); + if (it == _qnn_mem_set.end()) { + GGMLQNN_LOG_WARN("failed to find shared memory handler: %p", mem_handle); + return; + } + + _qnn_mem_set.erase(it); +} + +bool qnn_instance::is_rpcmem_allocated(void * buf) { + return _rpcmem_store_map.count(buf) != 0U; +} + +int qnn_instance::load_backend(std::string & lib_path, const QnnSaver_Config_t ** saver_config) { + Qnn_ErrorHandle_t error = QNN_SUCCESS; + GGMLQNN_LOG_DEBUG("lib_path:%s\n", lib_path.c_str()); + + void *lib_handle = dlopen(lib_path.c_str(), RTLD_NOW | RTLD_GLOBAL); + if (nullptr == lib_handle) { + GGMLQNN_LOG_WARN("can not open QNN library %s, with error: %s", lib_path.c_str(), dlerror()); + return 1; + } + + auto get_providers = + load_qnn_functionpointers<_pfn_QnnInterface_getProviders *>(lib_handle, + "QnnInterface_getProviders"); + if (nullptr == get_providers) { + GGMLQNN_LOG_WARN("can not load symbol QnnInterface_getProviders : %s", dlerror()); + return 2; + } + + // get QnnInterface Providers + std::uint32_t num_providers = 0; + const QnnInterface_t **provider_list = nullptr; + error = get_providers(&provider_list, &num_providers); + if (error != QNN_SUCCESS) { + GGMLQNN_LOG_WARN("failed to get providers, error %d", QNN_GET_ERROR_CODE(error)); + return 3; + } + GGMLQNN_LOG_DEBUG("num_providers=%d\n", num_providers); + if (num_providers != _required_num_providers) { + GGMLQNN_LOG_WARN("providers is %d instead of required %d", num_providers, _required_num_providers); + return 4; + } + + if (nullptr == provider_list) { + GGMLQNN_LOG_WARN("failed to get qnn interface providers\n"); + return 5; + } + bool found_valid_interface = false; + QNN_INTERFACE_VER_TYPE qnn_interface; + for (size_t idx = 0; idx < num_providers; idx++) { + if (QNN_API_VERSION_MAJOR == provider_list[idx]->apiVersion.coreApiVersion.major && + QNN_API_VERSION_MINOR <= provider_list[idx]->apiVersion.coreApiVersion.minor) { + found_valid_interface = true; + qnn_interface = provider_list[idx]->QNN_INTERFACE_VER_NAME; + break; + } + } + + if (!found_valid_interface) { + GGMLQNN_LOG_WARN("unable to find a valid qnn interface\n"); + return 6; + } else { + GGMLQNN_LOG_INFO("find a valid qnn interface\n"); + } + set_qnn_raw_interface(qnn_interface); + + BackendIdType backend_id = provider_list[0]->backendId; + _lib_path_to_backend_id[lib_path] = backend_id; + if (_loaded_backend.count(backend_id) > 0) { + GGMLQNN_LOG_WARN("lib_path %s is loaded, but backend %d already exists\n", + lib_path.c_str(), backend_id); + } + _loaded_backend[backend_id] = provider_list[0]; + if (_loaded_lib_handle.count(backend_id) > 0) { + GGMLQNN_LOG_WARN("closing %p\n", _loaded_lib_handle[backend_id]); + int dlclose_error = dlclose(_loaded_lib_handle[backend_id]); + if (dlclose_error != 0) { + GGMLQNN_LOG_WARN("fail to close %p with error %s\n", _loaded_lib_handle[backend_id], dlerror()); + } + } + _loaded_lib_handle[backend_id] = lib_handle; + _backend_id = backend_id; + +#if 0 // keep them here for further use + QnnSaver_Config_t outputdir_cfg; + outputdir_cfg.option = QNN_SAVER_CONFIG_OPTION_OUTPUT_DIRECTORY; + outputdir_cfg.outputDirectory = "/data/local/tmp/"; + QnnSaver_Config_t backendid_cfg; + backendid_cfg.option = QNN_SAVER_CONFIG_OPTION_BACKEND_ID; + backendid_cfg.backendId = _backend_id; + const QnnSaver_Config_t *saverCfg[] = {&outputdir_cfg, &backendid_cfg, nullptr}; + if (0 == QnnSaver_initialize(saverCfg)) { + GGMLQNN_LOG_INFO("QnnSaver_initialize successfully"); + } else { + GGMLQNN_LOG_WARN("QnnSaver_initialize failure"); + } +#endif + auto saver_initialize = + load_qnn_functionpointers<_pfn_QnnSaver_initialize *>( + _loaded_lib_handle[backend_id], "QnnSaver_initialize"); + if (nullptr != saver_initialize) { + error = saver_initialize(saver_config); + if (error != QNN_SUCCESS) { + GGMLQNN_LOG_WARN("failed to saver_initialize，error %d", QNN_GET_ERROR_CODE(error)); + return 7; + } + } else { + GGMLQNN_LOG_WARN("saver_initialize is null\n"); + } + + return 0; +} + +int qnn_instance::unload_backend() { + int dlclose_error = 0; + for (auto &it : _loaded_lib_handle) { + dlclose_error = dlclose(it.second); + if (dlclose_error != 0) { + GGMLQNN_LOG_WARN("failed to close QNN backend %d, error %s\n", it.first, dlerror()); + } + } + + _loaded_lib_handle.clear(); + _lib_path_to_backend_id.clear(); + _loaded_backend.clear(); + + return 0; +} + +int qnn_instance::load_system() { + Qnn_ErrorHandle_t error = QNN_SUCCESS; + + std::string system_lib_path = _lib_path + "libQnnSystem.so"; + GGMLQNN_LOG_DEBUG("system_lib_path:%s\n", system_lib_path.c_str()); + + _system_lib_handle = dlopen(system_lib_path.c_str(), RTLD_NOW | RTLD_LOCAL); + if (nullptr == _system_lib_handle) { + GGMLQNN_LOG_WARN("can not open QNN library %s, error: %s\n", system_lib_path.c_str(), dlerror()); + //re-try with default path of QNN binary runtime lib + _lib_path = "/data/local/tmp/"; + system_lib_path = _lib_path + "libQnnSystem.so"; + _system_lib_handle = dlopen(system_lib_path.c_str(), RTLD_NOW | RTLD_LOCAL); + if (nullptr == _system_lib_handle) { + GGMLQNN_LOG_WARN("can not open QNN library %s, error: %s\n", system_lib_path.c_str(), dlerror()); + return 1; + } + } + + auto * get_providers = reinterpret_cast<_pfn_QnnSystemInterface_getProviders *>(dlsym( + _system_lib_handle, "QnnSystemInterface_getProviders")); + if (nullptr == get_providers) { + GGMLQNN_LOG_WARN("can not load QNN symbol QnnSystemInterface_getProviders: %s\n", dlerror()); + return 2; + } + + uint32_t num_providers = 0; + const QnnSystemInterface_t ** provider_list = nullptr; + error = get_providers(&provider_list, &num_providers); + if (error != QNN_SUCCESS) { + GGMLQNN_LOG_WARN("failed to get providers, error %d\n", QNN_GET_ERROR_CODE(error)); + return 3; + } + + if (num_providers != _required_num_providers) { + GGMLQNN_LOG_WARN("providers is %d instead of required %d\n", num_providers, _required_num_providers); + return 4; + } + + if (nullptr == provider_list) { + GGMLQNN_LOG_WARN("can not get providers\n"); + return 5; + } + + QNN_SYSTEM_INTERFACE_VER_TYPE qnn_system_interface; + bool found_valid_system_interface = false; + for (size_t idx = 0; idx < num_providers; idx++) { + if (QNN_SYSTEM_API_VERSION_MAJOR == + provider_list[idx]->systemApiVersion.major && + QNN_SYSTEM_API_VERSION_MINOR <= + provider_list[idx]->systemApiVersion.minor) { + found_valid_system_interface = true; + qnn_system_interface = provider_list[idx]->QNN_SYSTEM_INTERFACE_VER_NAME; + break; + } + } + if (!found_valid_system_interface) { + GGMLQNN_LOG_WARN("unable to find a valid qnn system interface\n"); + return 6; + } else { + GGMLQNN_LOG_INFO("find a valid qnn system interface\n"); + } + set_qnn_raw_system_interface(qnn_system_interface); + + _qnn_interface.set_qnn_system_interface(provider_list[0]); + + _qnn_interface.qnn_system_context_create(&_qnn_system_handle); + if (nullptr == _qnn_system_handle) { + GGMLQNN_LOG_WARN("can not create QNN system contenxt\n"); + } else { + GGMLQNN_LOG_INFO("initialize qnn system successfully\n"); + } + + return 0; +} + +int qnn_instance::unload_system() { + int result = 0; + + if (nullptr == _system_lib_handle) { + GGMLQNN_LOG_DEBUG("system lib handle is null\n"); + return 1; + } + + if (nullptr != _qnn_system_handle) { + result = _qnn_interface.qnn_system_context_free(_qnn_system_handle); + if (result != QNN_SUCCESS) { + GGMLQNN_LOG_WARN("failed to free QNN system context\n"); + } + _qnn_system_handle = nullptr; + } + + int dlclose_error = dlclose(_system_lib_handle); + if (dlclose_error != 0) { + GGMLQNN_LOG_WARN("failed to close QnnSystem library, error %s\n", dlerror()); + return 2; + } + + _system_lib_handle = nullptr; + + return result; +} + +static void ggml_qnn_logcallback(const char * fmt, + QnnLog_Level_t level, + uint64_t timestamp, + va_list argp) { + + static std::mutex log_mutex; + static unsigned char s_ggml_qnn_logbuf[GGML_QNN_LOGBUF_LEN]; + + const char * log_level_desc = ""; + switch (level) { + case QNN_LOG_LEVEL_ERROR: + log_level_desc = " ERROR "; + break; + case QNN_LOG_LEVEL_WARN: + log_level_desc = "WARNING"; + break; + case QNN_LOG_LEVEL_INFO: + log_level_desc = " INFO "; + break; + case QNN_LOG_LEVEL_DEBUG: + log_level_desc = " DEBUG "; + break; + case QNN_LOG_LEVEL_VERBOSE: + log_level_desc = "VERBOSE"; + break; + case QNN_LOG_LEVEL_MAX: + log_level_desc = "UNKNOWN"; + break; + } + + double ms = (double) timestamp / 1000000.0; + + { + std::lock_guard lock(log_mutex); + + memset(s_ggml_qnn_logbuf, 0, GGML_QNN_LOGBUF_LEN); + vsnprintf(reinterpret_cast(s_ggml_qnn_logbuf), GGML_QNN_LOGBUF_LEN, fmt, argp); +#if GGMLQNN_PRINT_QNN_INTERNAL_LOG + GGMLQNN_LOG_INFO("%8.1fms [%-7s] %s\n", ms, log_level_desc, s_ggml_qnn_logbuf); +#endif + } +} + +int qnn_instance::qnn_init(const QnnSaver_Config_t ** saver_config) { + BackendIdType backend_id = QNN_BACKEND_ID_NULL; + GGMLQNN_LOG_DEBUG("enter qni_init\n"); + + const std::lock_guard lock(_init_mutex); + + if (0 != load_system()) { + GGMLQNN_LOG_WARN("can not load QNN system lib, pls check why?\n"); + return 1; + } else { + GGMLQNN_LOG_DEBUG("load QNN system lib successfully\n"); + } + + std::string bakend_lib_path = _lib_path + _backend_name; + if (0 == _lib_path_to_backend_id.count(bakend_lib_path)) { + int is_load_ok = load_backend(bakend_lib_path, saver_config); + if (0 != is_load_ok) { + GGMLQNN_LOG_WARN("failed to load QNN backend\n"); + return 2; + } + } + + backend_id = _lib_path_to_backend_id[bakend_lib_path]; + if (0 == _loaded_backend.count(backend_id) || + 0 == _loaded_lib_handle.count(backend_id)) { + GGMLQNN_LOG_WARN("library %s is loaded but loaded backend count=%zu, loaded lib_handle count=%zu\n", + bakend_lib_path.c_str(), + _loaded_backend.count(backend_id), + _loaded_lib_handle.count(backend_id)); + return 3; + } + + _qnn_interface.set_qnn_interface(_loaded_backend[backend_id]); + +#if 1 + _qnn_interface.qnn_log_create(ggml_qnn_logcallback, _qnn_log_level, &_qnn_log_handle); +#else + _qnn_raw_interface.logCreate(ggml_qnn_logcallback, _qnn_log_level, &_qnn_log_handle); +#endif + if (nullptr == _qnn_log_handle) { + GGMLQNN_LOG_WARN("why failed to initialize qnn log\n"); //NPU backend not work on Qualcomm SoC based low-end phone + return 4; + } else { + GGMLQNN_LOG_DEBUG("initialize qnn log successfully\n"); + } + + std::vector temp_backend_config; + _qnn_interface.qnn_backend_create(_qnn_log_handle, + temp_backend_config.empty() ? nullptr : temp_backend_config.data(), + &_qnn_backend_handle); + if (nullptr == _qnn_backend_handle) { + GGMLQNN_LOG_WARN("why failed to initialize qnn backend\n"); + return 5; + } else { + GGMLQNN_LOG_DEBUG("initialize qnn backend successfully\n"); + } + + if (nullptr != _qnn_raw_interface.propertyHasCapability) { + auto qnnstatus = _qnn_raw_interface.propertyHasCapability(QNN_PROPERTY_GROUP_DEVICE); + if (QNN_PROPERTY_NOT_SUPPORTED == qnnstatus) { + GGMLQNN_LOG_WARN("device property is not supported\n"); + } + if (QNN_PROPERTY_ERROR_UNKNOWN_KEY == qnnstatus) { + GGMLQNN_LOG_WARN("device property is not known to backend\n"); + } + } + + auto qnnstatus = _qnn_raw_interface.deviceCreate( + _qnn_log_handle, nullptr, &_qnn_device_handle); + if (QNN_SUCCESS != qnnstatus && QNN_DEVICE_ERROR_UNSUPPORTED_FEATURE != qnnstatus) { + GGMLQNN_LOG_WARN("failed to create QNN device\n"); + } else { + GGMLQNN_LOG_INFO("create device successfully\n"); + } + + if (ggml_qnn_profile_level::profile_off != _profile_level) { + GGMLQNN_LOG_INFO("profiling turned on; level = %d", _profile_level); + if (ggml_qnn_profile_level::profile_basic == _profile_level) { + GGMLQNN_LOG_INFO("basic profiling requested. creating Qnn Profile object\n"); + if (QNN_PROFILE_NO_ERROR != _qnn_raw_interface.profileCreate( + _qnn_backend_handle, QNN_PROFILE_LEVEL_BASIC, &_qnn_profile_handle)) { + GGMLQNN_LOG_WARN("unable to create profile handle in the backend\n"); + return 7; + } else { + GGMLQNN_LOG_DEBUG("initialize qnn profile successfully\n"); + } + } else if (ggml_qnn_profile_level::profile_detail == _profile_level) { + GGMLQNN_LOG_INFO("detailed profiling requested. Creating Qnn Profile object\n"); + if (QNN_PROFILE_NO_ERROR != _qnn_raw_interface.profileCreate( + _qnn_backend_handle, QNN_PROFILE_LEVEL_DETAILED, &_qnn_profile_handle)) { + GGMLQNN_LOG_WARN("unable to create profile handle in the backend\n"); + return 7; + } else { + GGMLQNN_LOG_DEBUG("initialize qnn profile successfully\n"); + } + } + } + + _rpc_lib_handle = dlopen("libcdsprpc.so", RTLD_NOW | RTLD_LOCAL); + if (nullptr == _rpc_lib_handle) { + GGMLQNN_LOG_WARN("failed to load qualcomm's rpc lib, error:%s\n", dlerror()); + return 9; + } else { + GGMLQNN_LOG_DEBUG("load rpcmem lib successfully\n"); + set_rpcmem_initialized(true); + } + _pfn_rpc_mem_init = reinterpret_cast(dlsym(_rpc_lib_handle, "rpcmem_init")); + _pfn_rpc_mem_deinit = reinterpret_cast(dlsym(_rpc_lib_handle, "rpcmem_deinit")); + _pfn_rpc_mem_alloc = reinterpret_cast(dlsym(_rpc_lib_handle,"rpcmem_alloc")); + _pfn_rpc_mem_free = reinterpret_cast(dlsym(_rpc_lib_handle, "rpcmem_free")); + _pfn_rpc_mem_to_fd = reinterpret_cast(dlsym(_rpc_lib_handle,"rpcmem_to_fd")); + if (nullptr == _pfn_rpc_mem_alloc || nullptr == _pfn_rpc_mem_free + || nullptr == _pfn_rpc_mem_to_fd) { + GGMLQNN_LOG_WARN("unable to access symbols in QNN RPC lib. dlerror(): %s", dlerror()); + dlclose(_rpc_lib_handle); + return 10; + } + + if (nullptr != _pfn_rpc_mem_init) // make Qualcomm's SoC based low-end phone happy + _pfn_rpc_mem_init(); + + std::vector temp_context_config; + _qnn_interface.qnn_context_create(_qnn_backend_handle, _qnn_device_handle, + temp_context_config.empty() ? nullptr : temp_context_config.data(), + &_qnn_context_handle); + if (nullptr == _qnn_context_handle) { + GGMLQNN_LOG_WARN("why failed to initialize qnn context, error:%s\n", strerror(errno)); + return 8; + } else { + GGMLQNN_LOG_DEBUG("initialize qnn context successfully\n"); + } + + if (_backend_name.find("Htp") != std::variant_npos) { + const QnnDevice_PlatformInfo_t * p_info = nullptr; + _qnn_raw_interface.deviceGetPlatformInfo(nullptr, &p_info); + GGMLQNN_LOG_INFO("device counts %d", p_info->v1.numHwDevices); + QnnDevice_HardwareDeviceInfo_t * infos = p_info->v1.hwDevices; + for (int i = 0; i < p_info->v1.numHwDevices; i++) { + GGMLQNN_LOG_INFO("deviceID:%d, deviceType:%d, numCores %d", infos[i].v1.deviceId, + infos[i].v1.deviceType, infos[i].v1.numCores); + QnnDevice_DeviceInfoExtension_t devinfo = infos[i].v1.deviceInfoExtension; + QnnHtpDevice_OnChipDeviceInfoExtension_t chipinfo = devinfo->onChipDevice; + QnnHtpDevice_Arch_t htp_arch = chipinfo.arch; + GGMLQNN_LOG_INFO("htp_type:%d(%s)", devinfo->devType, + (devinfo->devType == QNN_HTP_DEVICE_TYPE_ON_CHIP) ? "QNN_HTP_DEVICE_TYPE_ON_CHIP" : "QNN_HTP_DEVICE_TYPE_UNKNOWN"); + GGMLQNN_LOG_INFO("qualcomm soc_model:%d(%s), htp_arch:%d(%s), vtcm_size:%d MB", \ + chipinfo.socModel, qnn_get_socmodel_desc(chipinfo.socModel), \ + htp_arch, qnn_get_htparch_desc(htp_arch), chipinfo.vtcmSize); + struct qcom_socinfo * socinfo = qnn_get_socinfo_from_socmodel(chipinfo.socModel); + g_qnn_mgr[QNN_BACKEND_NPU].socinfo = { chipinfo.socModel, htp_arch, chipinfo.vtcmSize }; + if (nullptr != socinfo) { + memcpy(g_qnn_mgr[QNN_BACKEND_NPU].socinfo.soc_desc, socinfo->soc_desc, sizeof(socinfo->soc_desc)); + GGMLQNN_LOG_INFO("soc info:%s", socinfo->soc_desc); + } else { + memcpy(g_qnn_mgr[QNN_BACKEND_NPU].socinfo.soc_desc, "unknown", 7); + GGMLQNN_LOG_INFO("soc info:unknown"); + } + } + _qnn_raw_interface.deviceFreePlatformInfo(nullptr, p_info); + + size_t candidate_size = 0; + uint8_t * rpc_buffer = nullptr; + const int SIZE_IN_MB = (1 << 20); + size_t probe_slots[] = {1024, 1536, 2048 - 48, 2048}; + size_t probe_counts = sizeof(probe_slots) / sizeof(size_t); + for (size_t idx = 0; idx < probe_counts; idx++) { + rpc_buffer = static_cast(alloc_rpcmem(probe_slots[idx] * SIZE_IN_MB, 4)); + if (nullptr == rpc_buffer) { + GGMLQNN_LOG_DEBUG("alloc rpcmem %d (MB) failure, %s\n", probe_slots[idx], strerror(errno)); + break; + } else { + candidate_size = probe_slots[idx]; + free_rpcmem(rpc_buffer); + rpc_buffer = nullptr; + } + } + if (candidate_size > _rpcmem_capacity) + _rpcmem_capacity = candidate_size; + GGMLQNN_LOG_INFO("capacity of rpc ion memory %d MB\n", _rpcmem_capacity); + + if (0 != init_htp_perfinfra()) { + GGMLQNN_LOG_WARN("initialize HTP performance failure"); + } + if (0 != set_rpc_polling()) { + GGMLQNN_LOG_WARN("set RPC polling failure"); + } + if (0 != set_high_performance_mode()) { + GGMLQNN_LOG_WARN("set HTP high performance mode failure"); + } + } + + GGMLQNN_LOG_DEBUG("leave qni_init\n"); + + return 0; +} + +int qnn_instance::qnn_finalize() { + int ret_status = 0; + Qnn_ErrorHandle_t error = QNN_SUCCESS; + + GGMLQNN_LOG_DEBUG("enter %s\n", __func__); + //FIXME:should be removed in the future + reset_idx(); + + free_rpcmem(); + unregister_rpcmem(); + + if (nullptr != _pfn_rpc_mem_deinit) + _pfn_rpc_mem_deinit(); + + if (dlclose(_rpc_lib_handle) != 0) { + GGMLQNN_LOG_WARN("failed to unload qualcomm's rpc lib, error:%s\n", dlerror()); + } else { + GGMLQNN_LOG_DEBUG("succeed to close rpcmem lib\n"); + } + + if (nullptr != _qnn_context_handle) { + error = _qnn_interface.qnn_context_free(_qnn_context_handle, _qnn_profile_handle); + if (error != QNN_SUCCESS) { + GGMLQNN_LOG_WARN("failed to free QNN context_handle: ID %u, error %d\n", + _qnn_interface.get_backend_id(), QNN_GET_ERROR_CODE(error)); + + } + _qnn_context_handle = nullptr; + } + + if (nullptr != _qnn_profile_handle) { + error = _qnn_interface.qnn_profile_free(_qnn_profile_handle); + if (error != QNN_SUCCESS) { + GGMLQNN_LOG_WARN("failed to free QNN profile_handle: ID %u, error %d\n", + _qnn_interface.get_backend_id(), QNN_GET_ERROR_CODE(error)); + + } + _qnn_profile_handle = nullptr; + } + + if (nullptr != _qnn_device_handle) { + error = _qnn_interface.qnn_device_free(_qnn_device_handle); + if (error != QNN_SUCCESS) { + GGMLQNN_LOG_WARN("failed to free QNN device_handle: ID %u, error %d\n", + _qnn_interface.get_backend_id(), QNN_GET_ERROR_CODE(error)); + + } + _qnn_device_handle = nullptr; + } + + if (nullptr != _qnn_backend_handle) { + error = _qnn_interface.qnn_backend_free(_qnn_backend_handle); + if (error != QNN_SUCCESS) { + GGMLQNN_LOG_WARN("failed to free QNN backend_handle: ID %u, error %d\n", + _qnn_interface.get_backend_id(), QNN_GET_ERROR_CODE(error)); + } + _qnn_backend_handle = nullptr; + + } + + if (nullptr != _qnn_log_handle) { + error = _qnn_interface.qnn_log_free(_qnn_log_handle); + if (error != QNN_SUCCESS) { + GGMLQNN_LOG_WARN("failed to free QNN log_handle: ID %u, error %d\n", + _qnn_interface.get_backend_id(), QNN_GET_ERROR_CODE(error)); + } + _qnn_log_handle = nullptr; + } + + unload_backend(); + + unload_system(); + GGMLQNN_LOG_DEBUG("leave %s\n", __func__); + + return ret_status; +} + +int qnn_instance::init_qnn_graph(const std::string & graph_name, QNNBackend device, size_t vtcm_size_in_mb) { + _graph_name = graph_name; + _device_id = device; + + GGMLQNN_LOG_DEBUG("[%s][%s]created", ggml_backend_qnn_get_devname(device), graph_name.c_str()); + + Qnn_ErrorHandle_t error = QNN_SUCCESS; + Qnn_GraphHandle_t graph_handle = nullptr; + if (device == QNN_BACKEND_NPU) { + QnnHtpGraph_CustomConfig_t hvx_config; + hvx_config.option = QNN_HTP_GRAPH_CONFIG_OPTION_NUM_HVX_THREADS; + hvx_config.numHvxThreads = 8; + QnnGraph_Config_t graph_hvx_config; + graph_hvx_config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM; + graph_hvx_config.customConfig = &hvx_config; + + QnnHtpGraph_CustomConfig_t dlbc_config; + dlbc_config.option = QNN_HTP_GRAPH_CONFIG_OPTION_OPTIMIZATION; + dlbc_config.optimizationOption.type = QNN_HTP_GRAPH_OPTIMIZATION_TYPE_ENABLE_DLBC; + dlbc_config.optimizationOption.floatValue = 1.0; // set to 0.0 to turn off DLBC + QnnGraph_Config_t graph_dlbc_config; + graph_dlbc_config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM; + graph_dlbc_config.customConfig = &dlbc_config; + + QnnHtpGraph_CustomConfig_t opt_config; + opt_config.optimizationOption.type = QNN_HTP_GRAPH_OPTIMIZATION_TYPE_FINALIZE_OPTIMIZATION_FLAG; + opt_config.optimizationOption.floatValue = 1; // 1 / 3 + QnnGraph_Config_t graph_opt_config; + graph_opt_config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM; + graph_opt_config.customConfig = &opt_config; + + QnnHtpGraph_CustomConfig_t vtcm_config; + vtcm_config.option = QNN_HTP_GRAPH_CONFIG_OPTION_VTCM_SIZE; + vtcm_config.vtcmSizeInMB = vtcm_size_in_mb; + QnnGraph_Config_t graph_vtcm_config; + graph_vtcm_config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM; + graph_vtcm_config.customConfig = &vtcm_config; + + const QnnGraph_Config_t * graph_configs[] = {&graph_hvx_config, &graph_dlbc_config, &graph_vtcm_config, + &graph_opt_config, nullptr}; + error = _qnn_interface.qnn_graph_create(_qnn_context_handle, graph_name.c_str(), graph_configs, &graph_handle); + } else { + error = _qnn_interface.qnn_graph_create(_qnn_context_handle, graph_name.c_str(), nullptr, &graph_handle); + } + + if (error != QNN_SUCCESS) { + GGMLQNN_LOG_ERROR("[%s][%s]failed to create qnn graph, error: %s", + ggml_backend_qnn_get_devname(device), graph_name.c_str(), + qnn_get_error_string(error)); + return error; + } + + GGMLQNN_LOG_INFO("[%s]create graph %s succeed", ggml_backend_qnn_get_devname(device), graph_name.c_str()); + _qnn_graph_handle = graph_handle; + return QNN_SUCCESS; +} + +int qnn_instance::init_qnn_graph(const char * graph_name, bool debug, uint8_t do_node_validation, + const QnnGraph_Config_t ** graph_configs) { + int result = 0; + + if (nullptr == graph_name) { + GGMLQNN_LOG_WARN("graph name is null\n"); + return 1; + } + + if (!_graph_name.empty()) { + GGMLQNN_LOG_WARN("qnn model for graph %s already initialized\n", graph_name); + return 2; + } + + if (!do_node_validation) { + GGMLQNN_LOG_WARN("node validation disabled, backend will not perform op validation prior to adding node\n"); + } + + _graph_name = graph_name; + _debug_tensor = debug; + _do_node_validations = do_node_validation; + + result = _qnn_raw_interface.graphCreate(_qnn_context_handle, + graph_name, + graph_configs, + &_qnn_graph_handle); + if (result != QNN_GRAPH_NO_ERROR || nullptr == _qnn_graph_handle) { + GGMLQNN_LOG_WARN("failed to create graph in qnn context\n"); + return 3; + } else { + GGMLQNN_LOG_INFO("succeed to create graph %s, %p\n", graph_name, _qnn_graph_handle); + } + + return 0; +} + +int qnn_instance::finalize_qnn_graph() { + if (nullptr != _qnn_graph_handle) { + if (_qnn_raw_interface.graphFinalize(_qnn_graph_handle, + _qnn_profile_handle, nullptr) + != QNN_GRAPH_NO_ERROR) { + GGMLQNN_LOG_WARN("finalizing graph failure\n"); + return 1; + } + } else { + GGMLQNN_LOG_DEBUG("qnn graph handle is null\n"); + } + + return 0; +} + +static uint8_t * create_rpc_buffer(qnn_instance * instance, const ggml_tensor * ggml_tensor, Qnn_Tensor_t * qnn_tensor, bool b_copydata) { + if (nullptr == instance || nullptr == ggml_tensor || nullptr == qnn_tensor) { + GGMLQNN_LOG_WARN("invalid params\n"); + return nullptr; + } + + uint8_t * qnn_rpcbuffer = static_cast(instance->alloc_rpcmem(ggml_nbytes(ggml_tensor), 4)); + if (nullptr == qnn_rpcbuffer) { + GGMLQNN_LOG_WARN("alloc rpcmem failure, %s\n", strerror(errno)); + return nullptr; + } else { + GGMLQNN_LOG_DEBUG("alloc rpcmem %p successfully\n", qnn_rpcbuffer); + } + if (b_copydata) + memcpy(qnn_rpcbuffer, ggml_tensor->data, ggml_nbytes(ggml_tensor)); + instance->register_rpcmem(qnn_rpcbuffer, qnn_tensor); + return qnn_rpcbuffer; +} + +static Qnn_ErrorHandle_t create_htp_graph(ggml_backend_qnn_context * ctx, const std::string & graph_name, Qnn_GraphHandle_t * graph_handle) { + Qnn_ErrorHandle_t error = QNN_SUCCESS; + if (nullptr == ctx) + return QNN_MIN_ERROR_COMMON; + + qnn_instance * instance = ctx->instance; + QNN_INTERFACE_VER_TYPE qnn_raw_interface = ctx->raw_interface; + + QnnHtpGraph_CustomConfig_t hvx_config; + hvx_config.option = QNN_HTP_GRAPH_CONFIG_OPTION_NUM_HVX_THREADS; + hvx_config.numHvxThreads = 4; + QnnGraph_Config_t graph_hvx_config; + graph_hvx_config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM; + graph_hvx_config.customConfig = &hvx_config; + + QnnHtpGraph_CustomConfig_t dlbc_config; + dlbc_config.option = QNN_HTP_GRAPH_CONFIG_OPTION_OPTIMIZATION; + dlbc_config.optimizationOption.type = QNN_HTP_GRAPH_OPTIMIZATION_TYPE_ENABLE_DLBC; + dlbc_config.optimizationOption.floatValue = 1.0; // set to 0.0 to turn off DLBC + QnnGraph_Config_t graph_dlbc_config; + graph_dlbc_config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM; + graph_dlbc_config.customConfig = &dlbc_config; + + QnnHtpGraph_CustomConfig_t opt_config; + opt_config.optimizationOption.type = QNN_HTP_GRAPH_OPTIMIZATION_TYPE_FINALIZE_OPTIMIZATION_FLAG; + opt_config.optimizationOption.floatValue = 3; // 1 or 3 + QnnGraph_Config_t graph_opt_config; + graph_opt_config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM; + graph_opt_config.customConfig = &opt_config; + + QnnHtpGraph_CustomConfig_t vtcm_config; + vtcm_config.option = QNN_HTP_GRAPH_CONFIG_OPTION_VTCM_SIZE; + vtcm_config.vtcmSizeInMB = ctx->socinfo.vtcm_size_in_mb; + QnnGraph_Config_t graph_vtcm_config; + graph_vtcm_config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM; + graph_vtcm_config.customConfig = &vtcm_config; + + QnnHtpGraph_CustomConfig_t precision_config; + precision_config.option = QNN_HTP_GRAPH_CONFIG_OPTION_PRECISION; + precision_config.precision = QNN_PRECISION_FLOAT16; + QnnGraph_Config_t graph_precision_config; + graph_precision_config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM; + graph_precision_config.customConfig = &precision_config; + + const QnnGraph_Config_t * p_graphconfig[] = {&graph_hvx_config, + &graph_dlbc_config, + &graph_vtcm_config, + &graph_opt_config, + &graph_precision_config, + NULL}; + error = qnn_raw_interface.graphCreate(instance->get_qnn_context_handle(), + graph_name.c_str(), + p_graphconfig, graph_handle); + return error; +} + +static void print_tensors_info(const char * func_name, ggml_backend_qnn_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + //skip sanity check of params + GGMLQNN_LOG_DEBUG("call %s in dev %s\n", func_name, ctx->name); + GGMLQNN_LOG_DEBUG("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", + src0->name, + src0->type, ggml_type_name(src0->type), src0->ne[0], src0->ne[1], src0->ne[2], + src0->nb[0], src0->nb[1], src0->nb[2]); + GGMLQNN_LOG_DEBUG("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", + src1->name, + src1->type, ggml_type_name(src1->type), src1->ne[0], src1->ne[1], src1->ne[2], + src1->nb[0], src1->nb[1], src1->nb[2]); + GGMLQNN_LOG_DEBUG("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", + dst->name, + dst->type, ggml_type_name(dst->type), dst->ne[0], dst->ne[1], dst->ne[2], dst->nb[0], + dst->nb[1], dst->nb[2]); + GGMLQNN_LOG_DEBUG("%d, %d, %d, %d", src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3]); + GGMLQNN_LOG_DEBUG("tensor0 name %s", src0->name); + GGMLQNN_LOG_DEBUG("tensor1 name %s", src1->name); + GGMLQNN_LOG_DEBUG("tensor2 name %s", dst->name); +} + +static void dump_tensors_info(const struct ggml_tensor * tensor) { + //skip sanity check of params + struct ggml_tensor * src0 = tensor->src[0]; + struct ggml_tensor * src1 = tensor->src[1]; + GGMLQNN_LOG_DEBUG("op name:%s, tensor type:%s", ggml_op_name(tensor->op), + ggml_type_name(tensor->type)); + GGMLQNN_LOG_DEBUG("src0 type:%s", ggml_type_name(tensor->src[0]->type)); + GGMLQNN_LOG_DEBUG("src1 type:%s", ggml_type_name(tensor->src[1]->type)); + GGMLQNN_LOG_DEBUG( + "src0 %15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", + src0->name, + src0->type, ggml_type_name(src0->type), src0->ne[0], src0->ne[1], src0->ne[2], + src0->nb[0], src0->nb[1], src0->nb[2]); + GGMLQNN_LOG_DEBUG( + "src1 %15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", + src1->name, + src1->type, ggml_type_name(src1->type), src1->ne[0], src1->ne[1], src1->ne[2], + src1->nb[0], src1->nb[1], src1->nb[2]); + GGMLQNN_LOG_DEBUG( + " %15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", + tensor->name, + tensor->type, ggml_type_name(tensor->type), tensor->ne[0], tensor->ne[1], + tensor->ne[2], + tensor->nb[0], + tensor->nb[1], tensor->nb[2]); +} + +// ================================================================================================= +// section-6: implementation of ggml-qnn backend +// ================================================================================================= +static bool ggml_qnn_can_handle_op(const struct ggml_tensor * tensor) { + if (tensor->op == GGML_OP_NONE) { + return true; + } + if (ggml_is_empty(tensor) || tensor->op == GGML_OP_RESHAPE + || tensor->op == GGML_OP_TRANSPOSE || tensor->op == GGML_OP_VIEW + || tensor->op == GGML_OP_PERMUTE) { + return false; + } + + bool supported_op = ((tensor->op == GGML_OP_ADD) || (tensor->op == GGML_OP_MUL_MAT)); + if (!supported_op) { + return false; + } + + struct ggml_tensor * src0 = tensor->src[0]; + struct ggml_tensor * src1 = tensor->src[1]; + + int64_t ne00 = tensor->src[0]->ne[0]; + int64_t ne01 = tensor->src[0]->ne[1]; + + int64_t ne10 = tensor->src[1]->ne[0]; + int64_t ne11 = tensor->src[1]->ne[1]; + + int64_t ne0 = tensor->ne[0]; + int64_t ne1 = tensor->ne[1]; + + if (tensor->op == GGML_OP_ADD) { + if (!ggml_are_same_shape(src0, src1)) { + return false; + } + + if (ne00 < 32) + return false; + +#if GGMLQNN_PRINT_OP_ADD_LOG + dump_tensors_info(tensor); +#endif + return (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16) + && (src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16); + + } + + if (tensor->op == GGML_OP_MUL_MAT) { +#if GGMLQNN_PRINT_OP_MUL_MAT_LOG + dump_tensors_info(tensor); +#endif + //FIXME: 2048 is an experimental value between ASR inference and LLM inference because + // it's better only offload big matrix to QNN backend + if (ne01 <= 2048) { + return false; + } +#if 0 + //TODO: offload mul_mat to QNN backend + //need to process type trait in func ggml_qnn_mul_mat(...): + //src0: q4_0, q6_k, ... + //src1: f32 + //dst : f32 + return (src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16) + && (tensor->type == GGML_TYPE_F32 || tensor->type == GGML_TYPE_F16); +#else + //fall back to ggml cpu backend + return (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16) + && (src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16) + && (src0->type == src1->type) && (src0->type == tensor->type); +#endif + } + + //TODO:for other op + return (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16) + && (src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16) + && (src0->type == src1->type) && (src0->type == tensor->type); +} + +static void ggml_qnn_add(ggml_backend_t backend, ggml_tensor * op) { + Qnn_ErrorHandle_t error = QNN_SUCCESS; + enum ggml_status result = GGML_STATUS_SUCCESS; + bool graph_initialized = false; + qnn_instance * instance = nullptr; + ggml_backend_qnn_context * ctx = (ggml_backend_qnn_context *)backend->context; + std::string graph_name = "ggml_op_qnn_add"; + qnn_perf op_perf = qnn_perf("ggml_qnn_add"); + Qnn_GraphHandle_t graph_handle = nullptr; + Qnn_Tensor_t * tensor_0 = nullptr; + Qnn_Tensor_t * tensor_1 = nullptr; + Qnn_Tensor_t * tensor_2 = nullptr; + Qnn_Param_t qnn_params[] = {}; + enum ggml_op ggmlop = GGML_OP_ADD; + Qnn_DataType_t src0_qnn_type = QNN_DATATYPE_FLOAT_32; + Qnn_DataType_t src1_qnn_type = QNN_DATATYPE_FLOAT_32; + Qnn_DataType_t dst_qnn_type = QNN_DATATYPE_FLOAT_32; + const ggml_tensor * src0 = op->src[0]; + const ggml_tensor * src1 = op->src[1]; + ggml_tensor * dst = op; + + uint8_t * qnn_rpcbuffer_0 = nullptr; + uint8_t * qnn_rpcbuffer_1 = nullptr; + uint8_t * qnn_rpcbuffer_2 = nullptr; + + GGMLQNN_CHECK_PARAMS(ctx, src0, src1, dst); + + instance = ctx->instance; + QNN_INTERFACE_VER_TYPE qnn_raw_interface = ctx->raw_interface; + + op_perf.start(); + + std::string map_entry; + get_graph_key_from_op(op, map_entry); + if (instance->_qnn_graph_map.find(map_entry) != instance->_qnn_graph_map.end()) { + graph_initialized = true; + auto & graph_item = instance->_qnn_graph_map[map_entry]; + graph_handle = std::get<0>(graph_item); + tensor_0 = std::get<1>(graph_item); + tensor_1 = std::get<2>(graph_item); + tensor_2 = std::get<3>(graph_item); + } else { + tensor_0 = ggml_qnn_create_tensor(src0); + tensor_1 = ggml_qnn_create_tensor(src1); + tensor_2 = ggml_qnn_create_tensor(dst); + } + + print_tensors_info(__func__, ctx, src0, src1, dst); + + QNN_VER_PTR(*tensor_0)->type = QNN_TENSOR_TYPE_APP_WRITE; + QNN_VER_PTR(*tensor_1)->type = QNN_TENSOR_TYPE_APP_WRITE; + QNN_VER_PTR(*tensor_2)->type = QNN_TENSOR_TYPE_APP_READ; + + src0_qnn_type = qnn_datatype_from_ggml_datatype(src0->type); + src1_qnn_type = qnn_datatype_from_ggml_datatype(src1->type); + dst_qnn_type = qnn_datatype_from_ggml_datatype(dst->type); + + uint32_t * tensor_0_dimensions = QNN_VER_PTR(*tensor_0)->dimensions; + uint32_t * tensor_1_dimensions = QNN_VER_PTR(*tensor_1)->dimensions; + uint32_t * tensor_2_dimensions = QNN_VER_PTR(*tensor_2)->dimensions; + + bool enable_npu_rpc = instance->enable_qnn_rpc() && ctx->device == QNN_BACKEND_NPU; + + if (!graph_initialized) { + graph_name = map_entry; + GGMLQNN_LOG_DEBUG("graph name %s", graph_name.c_str()); + if (ctx->device == QNN_BACKEND_NPU) { + error = create_htp_graph(ctx, graph_name, &graph_handle); + } else { + error = qnn_raw_interface.graphCreate(instance->get_qnn_context_handle(), + graph_name.c_str(), + nullptr, &graph_handle); + } + if (QNN_SUCCESS != error) { + GGMLQNN_LOG_INFO("can't create qnn graph handle with graph name %s, error = %d\n", graph_name.c_str(), error); + return; + } + + if (enable_npu_rpc) { + QNN_VER_PTR(*tensor_0)->memType = QNN_TENSORMEMTYPE_MEMHANDLE; + QNN_VER_PTR(*tensor_0)->clientBuf = {.data=nullptr, .dataSize=0}; + + QNN_VER_PTR(*tensor_1)->memType = QNN_TENSORMEMTYPE_MEMHANDLE; + QNN_VER_PTR(*tensor_1)->clientBuf = {.data=nullptr, .dataSize=0}; + + QNN_VER_PTR(*tensor_2)->memType = QNN_TENSORMEMTYPE_MEMHANDLE; + QNN_VER_PTR(*tensor_2)->clientBuf = {.data=nullptr, .dataSize=0}; + } + + CHECK_QNN_API(error = qnn_raw_interface.tensorCreateGraphTensor(graph_handle, tensor_0)); + CHECK_QNN_API(error = qnn_raw_interface.tensorCreateGraphTensor(graph_handle, tensor_1)); + CHECK_QNN_API(error = qnn_raw_interface.tensorCreateGraphTensor(graph_handle, tensor_2)); + + if (enable_npu_rpc) { + qnn_rpcbuffer_0 = create_rpc_buffer(instance, src0, tensor_0, true); + qnn_rpcbuffer_1 = create_rpc_buffer(instance, src1, tensor_1, true); + qnn_rpcbuffer_2 = create_rpc_buffer(instance, dst, tensor_2, false); + if (nullptr == qnn_rpcbuffer_0 || nullptr == qnn_rpcbuffer_1 || nullptr == qnn_rpcbuffer_2) { + GGMLQNN_LOG_INFO("create rpc buffer failure\n"); + //FIXME: potential memory leak althought it shouldn't happen + return; + } + } else { + QNN_VER_PTR(*tensor_0)->clientBuf = {src0->data, ggml_get_tensor_data_size(src0)}; + QNN_VER_PTR(*tensor_1)->clientBuf = {src1->data, ggml_get_tensor_data_size(src1)}; + QNN_VER_PTR(*tensor_2)->clientBuf = {dst->data, ggml_get_tensor_data_size(dst)}; + } + + Qnn_Tensor_t tensor_inputs[] = { + *tensor_0, + *tensor_1 + }; + Qnn_Tensor_t tensor_outputs[] = { + *tensor_2 + }; + Qnn_OpConfig_t op_config = { + (Qnn_OpConfigVersion_t) 1, .v1 = { + "ggml_op_add", + QNN_OP_PACKAGE_NAME_QTI_AISW, + QNN_OP_ELEMENT_WISE_ADD, + 0, + qnn_params, + 2, + tensor_inputs, + 1, + tensor_outputs + } + }; + CHECK_QNN_API(error = qnn_raw_interface.graphAddNode(graph_handle, op_config)); + CHECK_QNN_API(error = qnn_raw_interface.graphFinalize(graph_handle, nullptr, nullptr)); + error = qnn_raw_interface.graphExecute(graph_handle, + tensor_inputs, 2, + tensor_outputs, 1, + nullptr, nullptr); + CHECK_QNN_API(error); + + if (enable_npu_rpc) { + uint8_t * qnn_rpcbuffer = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*tensor_2)->memHandle)); + GGMLQNN_LOG_INFO("qnn_rpcbuffer = %p\n", qnn_rpcbuffer); + if (nullptr != qnn_rpcbuffer) { + memcpy(dst->data, qnn_rpcbuffer, ggml_nbytes(dst)); + } + } + + auto graph_item = std::make_tuple(graph_handle, tensor_0, tensor_1, tensor_2); + instance->_qnn_graph_map[map_entry] = graph_item; + + } else { + + uint32_t dimensions_input_0[] = {(uint32_t) src0->ne[0], (uint32_t) src0->ne[1], + (uint32_t) src0->ne[2], (uint32_t) src0->ne[3]}; + uint32_t dimensions_input_1[] = {(uint32_t) src1->ne[0], (uint32_t) src1->ne[1], + (uint32_t) src1->ne[2], (uint32_t) src1->ne[3]}; + uint32_t dimensions_output[] = {(uint32_t) dst->ne[0], (uint32_t) dst->ne[1], + (uint32_t) dst->ne[2], (uint32_t) dst->ne[3]}; + + QNN_VER_PTR(*tensor_0)->dimensions = dimensions_input_0; + QNN_VER_PTR(*tensor_0)->rank = ggml_get_tensor_rank(src0); + QNN_VER_PTR(*tensor_0)->dataType = src0_qnn_type; + + QNN_VER_PTR(*tensor_1)->dimensions = dimensions_input_1; + QNN_VER_PTR(*tensor_1)->rank = ggml_get_tensor_rank(src1); + QNN_VER_PTR(*tensor_1)->dataType = src1_qnn_type; + + QNN_VER_PTR(*tensor_2)->dimensions = dimensions_output; + QNN_VER_PTR(*tensor_2)->rank = ggml_get_tensor_rank(dst); + QNN_VER_PTR(*tensor_2)->dataType = dst_qnn_type; + + if (enable_npu_rpc) { + //FIXME:why failure with test-backend-ops + uint8_t * qnn_buffer_0 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*tensor_0)->memHandle)); + GGMLQNN_LOG_INFO("qnn_rpcbuffer_0 = %p\n", qnn_rpcbuffer_0); + if (nullptr != qnn_buffer_0) { + memcpy(qnn_buffer_0, src0->data, ggml_nbytes(src0)); + } + + uint8_t * qnn_buffer_1 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*tensor_1)->memHandle)); + GGMLQNN_LOG_INFO("qnn_rpcbuffer_1 = %p\n", qnn_rpcbuffer_1); + if (nullptr != qnn_buffer_1) { + memcpy(qnn_buffer_1, src1->data, ggml_nbytes(src1)); + } + } else { + QNN_VER_PTR(*tensor_0)->clientBuf = {src0->data, ggml_get_tensor_data_size(src0)}; + QNN_VER_PTR(*tensor_1)->clientBuf = {src1->data, ggml_get_tensor_data_size(src1)}; + QNN_VER_PTR(*tensor_2)->clientBuf = {dst->data, ggml_get_tensor_data_size(dst)}; + } + + Qnn_Tensor_t tensor_inputs[] = { + *tensor_0, + *tensor_1 + }; + Qnn_Tensor_t tensor_outputs[] = { + *tensor_2 + }; + error = qnn_raw_interface.graphExecute(graph_handle, + tensor_inputs, 2, + tensor_outputs, 1, + nullptr, nullptr); + CHECK_QNN_API(error); + + if (enable_npu_rpc) { + //FIXME:why failure with test-backend-ops + uint8_t * qnn_buffer_2 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*tensor_2)->memHandle)); + if (nullptr != qnn_buffer_2) { + memcpy(dst->data, qnn_buffer_2, ggml_nbytes(dst)); + } + } + } + + //avoid memory leak in func free_qnn_tensor + QNN_VER_PTR(*tensor_0)->dimensions = tensor_0_dimensions; + QNN_VER_PTR(*tensor_1)->dimensions = tensor_1_dimensions; + QNN_VER_PTR(*tensor_2)->dimensions = tensor_2_dimensions; +#if GGMLQNN_PRINT_OP_ADD_LOG + op_perf.info(); +#endif +} + +//TODO: +/* + * the logic of ggml_qnn_mul_mat is similar to ggml_qnn_add,but type trait and matrix transpose are required + * for offload mulmat to QNN backend, so it's a standalone function. + * + * MUL_MAT take most of the compute time (about 95%).so to speed up llama inference, we should focus on MUL_MAT. + * + * we have three kinds of MUL_MAT to compute: + * mul_mat_f32: both src0 and src1 are F32, this will be naturally handled in QNN backend + * mul_mat_f16_f32: src0 is F16 and src1 is F32, f16 in src0 -> f32 in src0', then src0' * src1 + * mul_mat_q_f32: src0 is quantized (Q4_0, Q4_1, ...) and src1 is F32, src0 -> f32 in src0', then src0' * src1 +*/ +static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { + Qnn_ErrorHandle_t error = QNN_SUCCESS; + bool graph_initialized = false; + qnn_perf op_perf = qnn_perf("ggml_qnn_mul_mat"); + qnn_instance * instance = nullptr; + ggml_backend_qnn_context * ctx = (ggml_backend_qnn_context *) backend->context; + + std::string graph_name = "ggml_op_qnn_mul_mat"; + Qnn_GraphHandle_t graph_handle = nullptr; + Qnn_Tensor_t * tensor_0 = nullptr; + Qnn_Tensor_t * tensor_1 = nullptr; + Qnn_Tensor_t * tensor_2 = nullptr; + + Qnn_Param_t qnn_params[] = {}; + + enum ggml_op ggmlop = GGML_OP_ADD; + Qnn_DataType_t src0_qnn_type = QNN_DATATYPE_FLOAT_32; + Qnn_DataType_t src1_qnn_type = QNN_DATATYPE_FLOAT_32; + Qnn_DataType_t dst_qnn_type = QNN_DATATYPE_FLOAT_32; + const ggml_tensor * src0 = op->src[0]; + const ggml_tensor * src1 = op->src[1]; + ggml_tensor * dst = op; + + GGMLQNN_CHECK_PARAMS(ctx, src0, src1, dst); + + instance = ctx->instance; + QNN_INTERFACE_VER_TYPE qnn_raw_interface = ctx->raw_interface; + + op_perf.start(); + + std::string map_entry; + get_graph_key_from_op(op, map_entry); + if (instance->_qnn_graph_map.find(map_entry) != instance->_qnn_graph_map.end()) { + graph_initialized = true; + auto & graph_item = instance->_qnn_graph_map[map_entry]; + graph_handle = std::get<0>(graph_item); + tensor_0 = std::get<1>(graph_item); + tensor_1 = std::get<2>(graph_item); + tensor_2 = std::get<3>(graph_item); + } else { + tensor_0 = ggml_qnn_create_tensor(src0); + tensor_1 = ggml_qnn_create_tensor(src1); + tensor_2 = ggml_qnn_create_tensor(dst); + } + + print_tensors_info(__func__, ctx, src0, src1, dst); + + QNN_VER_PTR(*tensor_0)->type = QNN_TENSOR_TYPE_APP_WRITE; + QNN_VER_PTR(*tensor_1)->type = QNN_TENSOR_TYPE_APP_WRITE; + QNN_VER_PTR(*tensor_2)->type = QNN_TENSOR_TYPE_APP_READ; + + src0_qnn_type = qnn_datatype_from_ggml_datatype(src0->type); + src1_qnn_type = qnn_datatype_from_ggml_datatype(src1->type); + dst_qnn_type = qnn_datatype_from_ggml_datatype(dst->type); + + uint32_t * tensor_0_dimensions = QNN_VER_PTR(*tensor_0)->dimensions; + uint32_t * tensor_1_dimensions = QNN_VER_PTR(*tensor_1)->dimensions; + uint32_t * tensor_2_dimensions = QNN_VER_PTR(*tensor_2)->dimensions; + + if (!graph_initialized) { + graph_name = map_entry; + GGMLQNN_LOG_DEBUG("graph name %s", graph_name.c_str()); + error = qnn_raw_interface.graphCreate(instance->get_qnn_context_handle(), + graph_name.c_str(), nullptr, &graph_handle); + if (QNN_SUCCESS != error) { + GGMLQNN_LOG_INFO("can't create qnn graph handle with graph name %s, error = %d\n", graph_name.c_str(), error); + return; + } + CHECK_QNN_API(error = qnn_raw_interface.tensorCreateGraphTensor(graph_handle, tensor_0)); + CHECK_QNN_API(error = qnn_raw_interface.tensorCreateGraphTensor(graph_handle, tensor_1)); + CHECK_QNN_API(error = qnn_raw_interface.tensorCreateGraphTensor(graph_handle, tensor_2)); + + QNN_VER_PTR(*tensor_0)->clientBuf = {src0->data, ggml_get_tensor_data_size(src0)}; + QNN_VER_PTR(*tensor_1)->clientBuf = {src1->data, ggml_get_tensor_data_size(src1)}; + QNN_VER_PTR(*tensor_2)->clientBuf = {dst->data, ggml_get_tensor_data_size(dst)}; + + Qnn_Tensor_t tensor_inputs[] = { + *tensor_0, + *tensor_1 + }; + Qnn_Tensor_t tensor_outputs[] = { + *tensor_2 + }; + Qnn_OpConfig_t op_config = { + (Qnn_OpConfigVersion_t) 1, .v1 = { + "ggml_op_mul_mat", + QNN_OP_PACKAGE_NAME_QTI_AISW, + QNN_OP_MAT_MUL, + 0, + qnn_params, + 2, + tensor_inputs, + 1, + tensor_outputs + } + }; + CHECK_QNN_API(error = qnn_raw_interface.graphAddNode(graph_handle, op_config)); + CHECK_QNN_API(error = qnn_raw_interface.graphFinalize(graph_handle, nullptr, nullptr)); + error = qnn_raw_interface.graphExecute(graph_handle, + tensor_inputs, 2, + tensor_outputs, 1, + nullptr, nullptr); + CHECK_QNN_API(error); + auto graph_item = std::make_tuple(graph_handle, tensor_0, tensor_1, tensor_2); + instance->_qnn_graph_map[map_entry] = graph_item; + + } else { + + uint32_t dimensions_input_0[] = {(uint32_t) src0->ne[0], (uint32_t) src0->ne[1], + (uint32_t) src0->ne[2], (uint32_t) src0->ne[3]}; + uint32_t dimensions_input_1[] = {(uint32_t) src1->ne[0], (uint32_t) src1->ne[1], + (uint32_t) src1->ne[2], (uint32_t) src1->ne[3]}; + uint32_t dimensions_output[] = {(uint32_t) dst->ne[0], (uint32_t) dst->ne[1], + (uint32_t) dst->ne[2], (uint32_t) dst->ne[3]}; + QNN_VER_PTR(*tensor_0)->dimensions = dimensions_input_0; + QNN_VER_PTR(*tensor_0)->rank = ggml_get_tensor_rank(src0); + QNN_VER_PTR(*tensor_0)->dataType = src0_qnn_type; + + QNN_VER_PTR(*tensor_1)->dimensions = dimensions_input_1; + QNN_VER_PTR(*tensor_1)->rank = ggml_get_tensor_rank(src1); + QNN_VER_PTR(*tensor_1)->dataType = src1_qnn_type; + + QNN_VER_PTR(*tensor_2)->dimensions = dimensions_output; + QNN_VER_PTR(*tensor_2)->rank = ggml_get_tensor_rank(dst); + QNN_VER_PTR(*tensor_2)->dataType = dst_qnn_type; + + QNN_VER_PTR(*tensor_0)->clientBuf = {src0->data, ggml_get_tensor_data_size(src0)}; + QNN_VER_PTR(*tensor_1)->clientBuf = {src1->data, ggml_get_tensor_data_size(src1)}; + QNN_VER_PTR(*tensor_2)->clientBuf = {dst->data, ggml_get_tensor_data_size(dst)}; + + Qnn_Tensor_t tensor_inputs[] = { + *tensor_0, + *tensor_1 + }; + Qnn_Tensor_t tensor_outputs[] = { + *tensor_2 + }; + error = qnn_raw_interface.graphExecute(graph_handle, + tensor_inputs, 2, + tensor_outputs, 1, + nullptr, nullptr); + CHECK_QNN_API(error); + } + + //avoid memory leak in func free_qnn_tensor + QNN_VER_PTR(*tensor_0)->dimensions = tensor_0_dimensions; + QNN_VER_PTR(*tensor_1)->dimensions = tensor_1_dimensions; + QNN_VER_PTR(*tensor_2)->dimensions = tensor_2_dimensions; + + op_perf.info(); +} + +static bool ggml_qnn_compute_forward(ggml_backend_t backend, struct ggml_tensor * tensor) { + ggmlqnn_op_func_t func = nullptr; + + switch (tensor->op) { + case GGML_OP_ADD: + func = ggml_qnn_add; + break; + + case GGML_OP_MUL_MAT: + func = ggml_qnn_mul_mat; + break; + + default: + return false; + } + + if (nullptr != func) + func(backend, tensor); + + return true; +} + +struct ggml_backend_qnn_buffer_context { + ~ggml_backend_qnn_buffer_context() { + if (buffer) { + free(buffer); + } + + for (auto * sub_buffer : sub_buffers) { + free(sub_buffer); + } + + for (auto * qnn_tensor : qnn_tensors) { + free_qnn_tensor(qnn_tensor); + } + + sub_buffers.clear(); + qnn_tensors.clear(); + } + void * buffer = nullptr; + + struct ggml_backend_qnn_context * backend_ctx = nullptr; + + size_t buffer_size = 0; + std::vector sub_buffers; + std::vector qnn_tensors; +}; + +static void ggml_backend_qnn_buffer_free_buffer(ggml_backend_buffer_t buffer) { + ggml_backend_qnn_buffer_context * ctx = (ggml_backend_qnn_buffer_context *)buffer->context; + delete ctx; +} + +static void * ggml_backend_qnn_buffer_get_base(ggml_backend_buffer_t buffer) { + ggml_backend_qnn_buffer_context * ctx = (ggml_backend_qnn_buffer_context *)buffer->context; + + return ctx->buffer; +} + +static void ggml_backend_qnn_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) { + Qnn_ErrorHandle_t error = QNN_SUCCESS; + ggml_backend_qnn_buffer_context * ctx = (ggml_backend_qnn_buffer_context *)buffer->context; + GGML_UNUSED(error); + GGML_UNUSED(ctx); + return; +} + +static void ggml_backend_qnn_buffer_set_tensor(ggml_backend_buffer_t buffer, + ggml_tensor * tensor, const void * data, + size_t offset, size_t size) { + GGML_UNUSED(buffer); + + memcpy((char *)tensor->data + offset, data, size); +} + +static void ggml_backend_qnn_buffer_memset_tensor(ggml_backend_buffer_t buffer, + struct ggml_tensor * tensor, + uint8_t value, size_t offset, size_t size) { + GGML_UNUSED(buffer); + memset((char *)tensor->data + offset, value, size); +} + +static void ggml_backend_qnn_buffer_get_tensor(ggml_backend_buffer_t buffer, + const ggml_tensor * tensor, + void * data, size_t offset, size_t size) { + GGML_UNUSED(buffer); + memcpy(data, (const char *)tensor->data + offset, size); +} + +static bool ggml_backend_qnn_buffer_cpy_tensor(ggml_backend_buffer_t buffer, + const struct ggml_tensor * src, + struct ggml_tensor * dst) { + GGML_UNUSED(buffer); + if (ggml_backend_buffer_is_host(src->buffer)) { + memcpy(dst->data, src->data, ggml_nbytes(src)); + return true; + } + + return false; +} + +static void ggml_backend_qnn_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) { + ggml_backend_qnn_buffer_context * ctx = (ggml_backend_qnn_buffer_context *)buffer->context; + memset(ctx->buffer, value, ctx->buffer_size); +} + +[[maybe_unused]]static void ggml_backend_qnn_buffer_reset(ggml_backend_buffer_t buffer) { + ggml_backend_qnn_buffer_context * ctx = (ggml_backend_qnn_buffer_context *)buffer->context; + for (auto * sub_buffer : ctx->sub_buffers) { + free(sub_buffer); + } + ctx->sub_buffers.clear(); +} + +static ggml_backend_buffer_i ggml_backend_qnn_buffer_interface = { + /* .free_buffer = */ ggml_backend_qnn_buffer_free_buffer, + /* .get_base = */ ggml_backend_qnn_buffer_get_base, + /* .init_tensor = */ ggml_backend_qnn_buffer_init_tensor, + /* .memset_tensor = */ ggml_backend_qnn_buffer_memset_tensor, + /* .set_tensor = */ ggml_backend_qnn_buffer_set_tensor, + /* .get_tensor = */ ggml_backend_qnn_buffer_get_tensor, + /* .cpy_tensor = */ ggml_backend_qnn_buffer_cpy_tensor, + /* .clear = */ ggml_backend_qnn_buffer_clear, + /* .reset = */ NULL, +}; + +static const char * ggml_backend_qnn_buffer_type_name(ggml_backend_buffer_type_t buft) { + return "qnn-buffer"; +} + +static ggml_backend_buffer_t ggml_backend_qnn_buffer_type_alloc_buffer( + ggml_backend_buffer_type_t buft, size_t size) { + ggml_backend_qnn_buffer_context * ctx = new ggml_backend_qnn_buffer_context; + + size_t size_page = sysconf(_SC_PAGESIZE); + size_t size_aligned = size; + if ((size_aligned % size_page) != 0) { + size_aligned += (size_page - (size_aligned % size_page)); + } + ctx->buffer = ggmlqnn_host_malloc(size_aligned); + ctx->buffer_size = size_aligned; + if (nullptr == ctx->buffer) { + GGMLQNN_LOG_WARN("%s: failed to allocate %.2f MiB\n", __func__, size / (1 << 20)); + return nullptr; + } + + return ggml_backend_buffer_init(buft, ggml_backend_qnn_buffer_interface, ctx, size); +} + +static size_t ggml_backend_qnn_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) { + GGML_UNUSED(buft); + return 32; +} + +//FIXME: this value is an experimental value on Xiaomi14 +static size_t ggml_backend_qnn_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) { + GGML_UNUSED(buft); + + return (2 * (1 << 30)); +} + +static bool ggml_backend_qnn_buffer_is_host(ggml_backend_buffer_type_t buft) { + GGML_UNUSED(buft); + return true; +} + +static const char * ggml_backend_qnn_name(ggml_backend_t backend) { + ggml_backend_qnn_context * ctx = (ggml_backend_qnn_context *) backend->context; + return g_qnn_mgr[ctx->device].name; +} + +static void ggml_backend_qnn_free(ggml_backend_t backend) { + GGMLQNN_LOG_DEBUG("enter %s", __func__ ); + ggml_backend_qnn_context * ctx = (ggml_backend_qnn_context *) backend->context; + GGMLQNN_LOG_DEBUG("idx %d, name:%s", ctx->device, g_qnn_mgr[ctx->device].name); + + qnn_instance * instance = (qnn_instance*)g_qnn_mgr[ctx->device].instance; + if (instance != nullptr) { + std::map>::iterator graph_it; + + for (graph_it = instance->_qnn_graph_map.begin(); + graph_it != instance->_qnn_graph_map.end(); graph_it++) { + auto & graph_item = graph_it->second; + Qnn_GraphHandle_t & graph_handle = std::get<0>(graph_item); + Qnn_Tensor_t * tensor_0 = std::get<1>(graph_item); + Qnn_Tensor_t * tensor_1 = std::get<2>(graph_item); + Qnn_Tensor_t * tensor_2 = std::get<3>(graph_item); + GGML_UNUSED(graph_handle); + GGMLQNN_LOG_DEBUG("graph type:%s", graph_it->first.c_str()); + free_qnn_tensor(tensor_0); + free_qnn_tensor(tensor_1); + free_qnn_tensor(tensor_2); + } + instance->_qnn_graph_map.clear(); + + instance->qnn_finalize(); + delete instance; + g_qnn_mgr[ctx->device].instance = nullptr; + } + + if (g_qnn_mgr[ctx->device].backend != nullptr) { + delete backend; + g_qnn_mgr[ctx->device].backend = nullptr; + } + GGMLQNN_LOG_DEBUG("leave %s", __func__ ); +} + +static enum ggml_status ggml_backend_qnn_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph) { + enum ggml_status result = GGML_STATUS_SUCCESS; + ggml_backend_qnn_context * ctx = (ggml_backend_qnn_context *) backend->context; + GGML_UNUSED(ctx); + + GGMLQNN_LOG_DEBUG("cgraph->n_nodes %d", cgraph->n_nodes); + for (int i = 0; i < cgraph->n_nodes; i++) { + ggml_tensor * node = cgraph->nodes[i]; + if (ggml_is_empty(node) || node->op == GGML_OP_RESHAPE + || node->op == GGML_OP_TRANSPOSE || node->op == GGML_OP_VIEW + || node->op == GGML_OP_PERMUTE || node->op == GGML_OP_NONE) { + continue; + } + bool ok = ggml_qnn_compute_forward(backend, node); + if (!ok) { + GGMLQNN_LOG_DEBUG("%s: error: op not supported %s (%s)\n", + __func__, node->name, ggml_op_name(node->op)); + } + } + + return result; +} + +static const char * ggml_backend_qnn_device_get_name(ggml_backend_dev_t dev) { + struct ggml_backend_qnn_context *ctx = static_cast(dev->context); + if (nullptr == ctx) { + GGMLQNN_LOG_ERROR("pls check why ctx is null"); + return "unknown"; + } + return ctx->name; + + GGML_UNUSED(dev); +} + +static const char * ggml_backend_qnn_device_get_description(ggml_backend_dev_t dev) { + struct ggml_backend_qnn_context * ctx = static_cast(dev->context); + if (nullptr == ctx) { + GGMLQNN_LOG_ERROR("pls check why ctx is null"); + return "unknown"; + } + if (0 == strncmp(ctx->name, "qnn-npu", 7)) { + const char * soc_info = qnn_get_socmodel_desc(ctx->socinfo.soc_model); + const char * htp_arch = qnn_get_htparch_desc(ctx->socinfo.htp_arch); + std::string dev_desc = std::string(ctx->desc) + + std::string(soc_info) + "_" + std::string(htp_arch) + + "," + std::string(ctx->socinfo.soc_desc); + return dev_desc.c_str(); + } else { + return ctx->desc; + } +} + +static void ggml_backend_qnn_device_get_memory(ggml_backend_dev_t dev, size_t * free, size_t * total) { + //FIXME:this is NOT QNN device memory info + *free = get_system_free_memory_in_bytes(); + *total = get_system_total_memory_in_bytes(); + GGML_UNUSED(dev); +} + +static enum ggml_backend_dev_type ggml_backend_qnn_device_get_type(ggml_backend_dev_t dev) { + GGML_UNUSED(dev); + return GGML_BACKEND_DEVICE_TYPE_ACCEL; +} + +static void ggml_backend_qnn_device_get_props(ggml_backend_dev_t dev, + struct ggml_backend_dev_props * props) { + props->name = ggml_backend_qnn_device_get_name(dev); + props->description = ggml_backend_qnn_device_get_description(dev); + props->type = ggml_backend_qnn_device_get_type(dev); + ggml_backend_qnn_device_get_memory(dev, &props->memory_free, &props->memory_total); + props->caps = { + /* .async = */ false, + /* .host_buffer = */ false, + /* .buffer_from_host_ptr = */ true, + /* .events = */ false, + }; +} + +static ggml_backend_t ggml_backend_qnn_device_init_backend(ggml_backend_dev_t dev, const char * params) { + GGML_UNUSED(dev); + if (nullptr == params) { + params = 0; + } + ggml_backend_t qnn_backend = ggml_backend_qnn_init((int) (intptr_t) params, + "/data/local/tmp/"); + + return qnn_backend; + +} + +ggml_backend_buffer_type_t ggml_backend_qnn_buffer_type(size_t device_index) { + if (device_index >= GGML_QNN_MAX_DEVICES) { + GGMLQNN_LOG_DEBUG("ggml_backend_qnn_buffer_type error: device_index:%d is out of range [0, %d]\n", + device_index, GGML_QNN_MAX_DEVICES - 1); + return nullptr; + } + + static struct ggml_backend_buffer_type ggml_backend_buffer_type_qnn = { + /* .iface = */ { + /* .get_name = */ ggml_backend_qnn_buffer_type_name, + /* .alloc_buffer = */ ggml_backend_qnn_buffer_type_alloc_buffer, + /* .get_alignment = */ ggml_backend_qnn_buffer_type_get_alignment, + /* .get_max_size = */ ggml_backend_qnn_buffer_type_get_max_size, + /* .get_alloc_size = */ NULL,// defaults to ggml_nbytes + /* .is_host = */ ggml_backend_qnn_buffer_is_host + }, + /* .context = */ NULL, + }; + + return &ggml_backend_buffer_type_qnn; +} + +static ggml_backend_buffer_type_t ggml_backend_qnn_device_get_buffer_type(ggml_backend_dev_t dev) { + ggml_backend_qnn_context * ctx = (ggml_backend_qnn_context *) dev->context; + return ggml_backend_qnn_buffer_type(ctx->device); +} + +static ggml_backend_buffer_t ggml_backend_qnn_device_buffer_from_host_ptr(ggml_backend_dev_t dev, + void * ptr, size_t size, size_t max_tensor_size) { + return ggml_backend_cpu_buffer_from_ptr(ptr, size); + + GGML_UNUSED(dev); + GGML_UNUSED(max_tensor_size); +} + + +static bool ggml_backend_qnn_device_supports_op(ggml_backend_dev_t dev, const struct ggml_tensor * op) { + ggml_backend_qnn_context * ctx = (ggml_backend_qnn_context *) dev->context; + return (ggml_qnn_can_handle_op(op)); +} + +static bool ggml_backend_qnn_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) { + GGML_UNUSED(dev); + return ggml_backend_buft_is_host(buft); +} + +static struct ggml_backend_device_i ggml_backend_qnn_device_interface = { + /* .get_name = */ ggml_backend_qnn_device_get_name, + /* .get_description = */ ggml_backend_qnn_device_get_description, + /* .get_memory = */ ggml_backend_qnn_device_get_memory, + /* .get_type = */ ggml_backend_qnn_device_get_type, + /* .get_props = */ ggml_backend_qnn_device_get_props, + /* .init_backend = */ ggml_backend_qnn_device_init_backend, + /* .get_buffer_type = */ ggml_backend_qnn_device_get_buffer_type, + /* .get_host_buffer_type = */ NULL, + /* .buffer_from_host_ptr = */ ggml_backend_qnn_device_buffer_from_host_ptr, + /* .supports_op = */ ggml_backend_qnn_device_supports_op, + /* .supports_buft = */ ggml_backend_qnn_device_supports_buft, + /* .offload_op = */ NULL, + /* .event_new = */ NULL, + /* .event_free = */ NULL, + /* .event_synchronize = */ NULL, +}; + +static ggml_backend_i ggml_backend_qnn_interface = { + /* .get_name = */ ggml_backend_qnn_name, + /* .free = */ ggml_backend_qnn_free, + /* .set_tensor_async = */ nullptr, + /* .get_tensor_async = */ nullptr, + /* .cpy_tensor_async = */ nullptr, + /* .synchronize = */ nullptr, + /* .graph_plan_create = */ nullptr, + /* .graph_plan_free = */ nullptr, + /* .graph_plan_update = */ nullptr, + /* .graph_plan_compute = */ nullptr, + /* .graph_compute = */ ggml_backend_qnn_graph_compute, + /* .event_record = */ nullptr, + /* .event_wait = */ nullptr, +}; + +//FIXME: this guid is not make sense +static ggml_guid_t ggml_backend_qnn_guid() { + static ggml_guid guid = { + 0x1a, 0x2b, 0x3c, 0x4d, 0x5e, 0x6f, 0x70, 0x81, + 0x92, 0xa3, 0xb4, 0xc5, 0xd6, 0xe7, 0xf8, 0x09 + }; + return &guid; +} + +bool ggml_backend_is_qnn(ggml_backend_t backend) { + return backend != nullptr && ggml_guid_matches(backend->guid, ggml_backend_qnn_guid()); +} + +void ggml_backend_qnn_set_n_threads(ggml_backend_t backend, int n_threads) { + GGML_ASSERT(ggml_backend_is_qnn(backend)); + + struct ggml_backend_qnn_context * ctx = (struct ggml_backend_qnn_context *)backend->context; + ctx->threads = n_threads; +} + +int ggml_backend_qnn_get_device_count() { + return GGML_QNN_MAX_DEVICES; +} + +struct ggml_backend_qnn_reg_context { + std::vector devices; +}; + +static const char * ggml_backend_qnn_reg_get_name(ggml_backend_reg_t reg) { + return "ggml-qnn"; + + GGML_UNUSED(reg); +} + +static size_t ggml_backend_qnn_reg_get_device_count(ggml_backend_reg_t reg) { + GGML_UNUSED(reg); + return GGML_QNN_MAX_DEVICES; +} + +static ggml_backend_dev_t ggml_backend_qnn_reg_get_device(ggml_backend_reg_t reg, size_t index) { + GGML_UNUSED(reg); + GGML_UNUSED(index); + + GGMLQNN_LOG_DEBUG("index %d", index); + ggml_backend_qnn_reg_context * ctx = (ggml_backend_qnn_reg_context *)reg->context; + GGML_ASSERT(index < ctx->devices.size()); + return ctx->devices[index]; +} + +static void * ggml_backend_qnn_reg_get_proc_address(ggml_backend_reg_t reg, const char * name) { + GGML_UNUSED(reg); + + if (std::strcmp(name, "ggml_backend_set_n_threads") == 0) { + return (void *)ggml_backend_qnn_set_n_threads; + } + return NULL; +} + +static const ggml_backend_reg_i ggml_backend_qnn_reg_interface = { + /* .get_name = */ ggml_backend_qnn_reg_get_name, + /* .get_device_count = */ ggml_backend_qnn_reg_get_device_count, + /* .get_device = */ ggml_backend_qnn_reg_get_device, + /* .get_proc_address = */ ggml_backend_qnn_reg_get_proc_address, +}; + +ggml_backend_reg_t ggml_backend_qnn_reg() { + static ggml_backend_reg reg; + static bool initialized = false; + GGMLQNN_LOG_DEBUG("enter ggml_backend_qnn_reg"); + { + static std::mutex mutex; + std::lock_guard lock(mutex); + if (!initialized) { + ggml_backend_qnn_reg_context * ctx = new ggml_backend_qnn_reg_context; + + for (int i = 0; i < ggml_backend_qnn_get_device_count(); i++) { + ggml_backend_dev_t dev = new ggml_backend_device { + /* .iface = */ ggml_backend_qnn_device_interface, + /* .reg = */ ®, + /* .context = */ &g_qnn_mgr[i] + }; + ctx->devices.push_back(dev); + } + + reg = ggml_backend_reg { + /* .api_version = */ GGML_BACKEND_API_VERSION, + /* .iface = */ ggml_backend_qnn_reg_interface, + /* .context = */ ctx + }; + } + + initialized = true; + } + GGMLQNN_LOG_DEBUG("leave ggml_backend_qnn_reg"); + + return ® +} + +/** + * + * @param device 0: QNN_BACKEND_CPU 1: QNN_BACKEND_GPU 2: QNN_BACKEND_NPU + * @param qnn_lib_path QNN binrary runtime library path, such as "/data/local/tmp/" on Android or specified in JNI layer + * @return + */ +ggml_backend_t ggml_backend_qnn_init(size_t device, const char * qnn_lib_path) { + int result = 0; + + if (nullptr == qnn_lib_path) + return nullptr; + + GGMLQNN_LOG_DEBUG("device %d", device); + GGMLQNN_LOG_DEBUG("qnn_lib_path %s", qnn_lib_path); + if (device >= GGML_QNN_MAX_DEVICES) { + GGMLQNN_LOG_ERROR("invalid device %d", device); + return nullptr; + } + + if (nullptr != g_qnn_mgr[device].backend) { + GGMLQNN_LOG_WARN("qnn backend %d(%s) already loaded", device, ggml_backend_qnn_get_devname(device)); + return g_qnn_mgr[device].backend; + } + + std::string path = qnn_lib_path; + if (QNN_BACKEND_NPU == device) { + if (0 == setenv("LD_LIBRARY_PATH", + (path + + ":/vendor/dsp/cdsp:/vendor/lib64:/vendor/dsp/dsp:/vendor/dsp/images").c_str(), + 1)) { + GGMLQNN_LOG_INFO("QNN NPU backend setenv successfully"); + } else { + GGMLQNN_LOG_ERROR("QNN NPU backend setenv failure"); + } + if (0 == setenv("ADSP_LIBRARY_PATH", + (path + + ";/vendor/dsp/cdsp;/vendor/lib/rfsa/adsp;/system/lib/rfsa/adsp;/vendor/dsp/dsp;/vendor/dsp/images;/dsp").c_str(), + 1)) { + GGMLQNN_LOG_INFO("QNN NPU backend setenv successfully"); + } else { + GGMLQNN_LOG_ERROR("QNN NPU backend setenv failure"); + } + } else { + if (0 == setenv("LD_LIBRARY_PATH", + (path + + ":/vendor/dsp/cdsp:/vendor/lib64:/vendor/dsp/dsp:/vendor/dsp/images").c_str(), + 1)) { + GGMLQNN_LOG_INFO("%s backend setenv successfully\n", ggml_backend_qnn_get_devname(device)); + } else { + GGMLQNN_LOG_ERROR("%s backend setenv failure\n", ggml_backend_qnn_get_devname(device)); + } + } + + qnn_instance * instance = nullptr; + instance = new qnn_instance(qnn_lib_path, g_qnn_mgr[device].lib, ""); + result = instance->qnn_init(nullptr); + if (0 != result) { + GGMLQNN_LOG_WARN("init qnn subsystem failed with qnn backend %s, pls check why\n", ggml_backend_qnn_get_devname(device)); + delete instance; + return nullptr; + } + qnn_interface qnn_interface = instance->get_qnn_interface(); + if (!qnn_interface.is_loaded()) { + GGMLQNN_LOG_WARN("qnn subsystem failure\n"); + delete instance; + return nullptr; + } + + std::string device_name = ggml_backend_qnn_get_devname(device); + GGMLQNN_LOG_INFO("qnn device name %s", device_name.c_str()); + g_qnn_mgr[device].instance = instance; + g_qnn_mgr[device].raw_interface = instance->get_qnn_raw_interface(); + g_qnn_mgr[device].raw_system_interface = instance->get_qnn_raw_system_interface(); + + ggml_backend_t qnn_backend = new ggml_backend{ + /* .guid = */ ggml_backend_qnn_guid(), + /* .iface = */ ggml_backend_qnn_interface, + /* .device = */ ggml_backend_reg_dev_get(ggml_backend_qnn_reg(), device), + /* .context = */ &g_qnn_mgr[device] + }; + g_qnn_mgr[device].backend = qnn_backend; + + return qnn_backend; +} + +GGML_BACKEND_DL_IMPL(ggml_backend_qnn_reg) From 4001f7978b62cb1a0290c6beab3484635473b897 Mon Sep 17 00:00:00 2001 From: zhouwg Date: Wed, 19 Feb 2025 21:58:55 +0800 Subject: [PATCH 15/76] ggml-qnn: a concise approach to offload mulmat to QNN backend(sync from branch kantvai-ggmlqnn-npurpc, https://github.com/kantv-ai/llama.cpp/wiki/offloading-mulmat-to-QNN-backend) --- ggml/src/ggml-qnn/ggml-qnn.cpp | 626 ++++++++++++++++++++------------- 1 file changed, 377 insertions(+), 249 deletions(-) diff --git a/ggml/src/ggml-qnn/ggml-qnn.cpp b/ggml/src/ggml-qnn/ggml-qnn.cpp index 6f2949333908e..a1aca7940bf4f 100644 --- a/ggml/src/ggml-qnn/ggml-qnn.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn.cpp @@ -13,8 +13,9 @@ * section-5 does ggml-qnn backend helper macro / data structure / function / class * section-6 does implementation of ggml-qnn backend according to ggml's backend subsystem * - * currently only provide GGML_OP_ADD's QNN backend implementation: - * - GGML_OP_ADD: this is skeleton, can expand other ggml ops according to expertise + * currently only provide OPs' QNN backend implementation of GGML_OP_ADD & GGML_OP_MUL_MAT: + * - GGML_OP_ADD: this is a simple skeleton, can expand other ggml ops according to expertise + * - GGML_OP_MUL_MAT:this is a complicated skeleton, can expand other complex op accordingly * * of course, can porting ggml-qnn to Windows on ARM as need. * @@ -257,20 +258,25 @@ static void * ggmlqnn_host_malloc(size_t n) { // ================================================================================================= // section-4: QNN helper macro / data structure / function // ================================================================================================= -#define VALIDATE(value, status) \ - do { \ - status = value; \ - if (status != QNN_SUCCESS) { \ - GGMLQNN_LOG_WARN("%s expected QNN_SUCCESS\n", #value); \ - return status; \ - } \ +#define VALIDATE(value, status) \ + do { \ + status = value; \ + if (status != QNN_SUCCESS) { \ + GGMLQNN_LOG_WARN("%s expected QNN_SUCCESS\n", #value); \ + return status; \ + } \ } while (0) -#define CHECK_QNN_API(error) \ - do { \ - if (QNN_SUCCESS != (error)) { \ - GGMLQNN_LOG_INFO("error = %d\n", (error)); \ - } \ +#define CHECK_QNN_API(error, result) \ + do { \ + error = (result); \ + if (QNN_SUCCESS != error) { \ + if (error == QNN_COMMON_ERROR_NOT_SUPPORTED) { \ + GGMLQNN_LOG_WARN("WARNING: QNN feature/API not supported\n"); \ + } else { \ + GGMLQNN_LOG_INFO("QNN API error = %d(%s)\n", error, qnn_get_error_string(error)); \ + } \ + } \ } while (0) #define VALIDATE_TENSOR_VERSION(tensor, err) VALIDATE(validate_tensor_version(tensor), err) @@ -823,9 +829,8 @@ static int deep_copy_qnn_tensors(Qnn_Tensor_t & src, Qnn_Tensor_t & dst) { uint32_t rank = QNN_TENSOR_GET_RANK(src); QNN_TENSOR_SET_RANK(dst, rank); - size_t dim_size = rank * sizeof(uint32_t); + size_t dim_size = GGML_MAX_DIMS * sizeof(uint32_t); uint32_t * dimensions = (uint32_t *)malloc(dim_size); - GGMLQNN_LOG_DEBUG("tensor dims %p", dimensions); if (dimensions == nullptr) { GGMLQNN_LOG_WARN("deep_copy_qnn_tensors() allocation error while copying tensor %s\n", QNN_TENSOR_GET_NAME(src)); return 1; @@ -1025,6 +1030,9 @@ using _pfn_QnnSaver_initialize = decltype(QnnSaver_init using _pfn_QnnInterface_getProviders = decltype(QnnInterface_getProviders); using _pfn_QnnSystemInterface_getProviders = decltype(QnnSystemInterface_getProviders); +using qnn_res_t = std::tuple>; +using qnn_tensors_t = std::vector< Qnn_Tensor_t *>; + enum class ggml_qnn_profile_level { profile_off = 0, profile_basic = 1, @@ -1122,12 +1130,9 @@ struct ggml_backend_qnn_context { QNN_INTERFACE_VER_TYPE raw_interface; QNN_SYSTEM_INTERFACE_VER_TYPE raw_system_interface; struct qcom_socinfo socinfo; - - //FIXME: should I move it from public member of class qnn_instance to here? - //std::map> _qnn_graph_map; } ; -//FIXME: the following global vars and three helper funcs should be removed in the future +//TODO: the following global vars and three helper funcs should be removed in the future static int32_t g_ggmltensor_idx = 0; static void reset_idx() { g_ggmltensor_idx = 0; @@ -1399,11 +1404,11 @@ static const char * ggml_get_type_name(ggml_type type) { return traits->type_name; } -Qnn_Tensor_t * ggml_qnn_create_tensor(const ggml_tensor * tensor) { +static Qnn_Tensor_t * ggml_qnn_create_compute_tensor(const ggml_tensor * tensor) { Qnn_ErrorHandle_t error = QNN_SUCCESS; char tensor_name[GGML_MAX_NAME] = {0}; - //FIXME:remove get_idx() and inc_idx() in the future but ensure the tensor name is unique + //TODO:remove get_idx() and inc_idx() in the future but ensure the tensor name is unique snprintf(tensor_name, GGML_MAX_NAME, "tensor_%-8d", get_idx()); GGMLQNN_LOG_DEBUG("init_tensor %d", get_idx()); inc_idx(); @@ -1450,6 +1455,73 @@ Qnn_Tensor_t * ggml_qnn_create_tensor(const ggml_tensor * tensor) { return p_qnn_tensor; } +static Qnn_Tensor_t * ggml_qnn_create_mulmat_tensor(const ggml_tensor * tensor, const char * name, Qnn_TensorType_t qnn_tensor_type, + Qnn_DataType_t qnn_data_type, uint32_t rank, uint32_t * dims, void * data, uint32_t data_size) { + Qnn_ErrorHandle_t error = QNN_SUCCESS; + char tensor_name[GGML_MAX_NAME] = {0}; + + //TODO:remove get_idx() and inc_idx() in the future but ensure the tensor name is unique + if (nullptr != name) { + snprintf(tensor_name, GGML_MAX_NAME, "tensor_%-8d", get_idx()); + } else { + snprintf(tensor_name, GGML_MAX_NAME, "tensor_%s%-8d", name, get_idx()); + } + GGMLQNN_LOG_DEBUG("init_tensor %d", get_idx()); + inc_idx(); + + //there are different dimension order between ggml tensor and qnn tensor + uint32_t dimensions_transpose[GGML_MAX_DIMS] = {}; + uint32_t * tensor_dims = nullptr; + + if (nullptr != tensor) { + dimensions_transpose[0] = (uint32_t) tensor->ne[1]; + dimensions_transpose[1] = (uint32_t) tensor->ne[0]; + dimensions_transpose[2] = (uint32_t) tensor->ne[2]; + dimensions_transpose[3] = (uint32_t) tensor->ne[3]; + tensor_dims = dimensions_transpose; + } + if (nullptr != dims) { + tensor_dims = dims; + } + + Qnn_Tensor_t qnn_tensor = { + .version= QNN_TENSOR_VERSION_1, + {.v1= { + .id = 0, + .name = tensor_name, + .type = qnn_tensor_type, + .dataFormat = QNN_TENSOR_DATA_FORMAT_FLAT_BUFFER, + .dataType = qnn_data_type, + .quantizeParams = {QNN_DEFINITION_UNDEFINED, + QNN_QUANTIZATION_ENCODING_UNDEFINED, + {.scaleOffsetEncoding = {.scale = 0.0000000000000000f, .offset = 0}}}, + .rank = rank, + .dimensions = tensor_dims, + .memType = QNN_TENSORMEMTYPE_RAW, + {.clientBuf = {nullptr, 0} + } + } + } + }; + if (nullptr != name) { + QNN_VER_PTR(qnn_tensor)->name = name; + } + Qnn_Tensor_t * p_qnn_tensor = (Qnn_Tensor_t *)calloc(1, sizeof(Qnn_Tensor_t)); + if (nullptr == p_qnn_tensor) { + GGMLQNN_LOG_WARN("calloc failed"); + return nullptr; + } + error = deep_copy_qnn_tensors(qnn_tensor, * p_qnn_tensor); + if (error != QNN_SUCCESS) { + free(p_qnn_tensor); + GGMLQNN_LOG_WARN("init tensor failed"); + return nullptr; + } + QNN_VER_PTR(*p_qnn_tensor)->clientBuf = {data, data_size}; + + return p_qnn_tensor; +} + //TODO: // ref:explanation of k-quants, https://github.com/ggerganov/llama.cpp/pull/1684 static Qnn_DataType_t qnn_datatype_from_ggml_datatype(enum ggml_type ggmltype) { @@ -1908,7 +1980,7 @@ class qnn_instance { } public: - std::map> _qnn_graph_map; + std::map>> _qnn_graph_map; private: int load_system(); @@ -1988,7 +2060,7 @@ class qnn_instance { std::string _graph_name; QNNBackend _device_id; - bool _enable_qnn_rpc = false; //FIXME:unknown issue with QNN RPC feature + bool _enable_qnn_rpc = false; //TODO:unknown issue with QNN RPC feature DISABLE_COPY(qnn_instance); DISABLE_MOVE(qnn_instance); @@ -2207,7 +2279,7 @@ int qnn_instance::load_backend(std::string & lib_path, const QnnSaver_Config_t * Qnn_ErrorHandle_t error = QNN_SUCCESS; GGMLQNN_LOG_DEBUG("lib_path:%s\n", lib_path.c_str()); - void *lib_handle = dlopen(lib_path.c_str(), RTLD_NOW | RTLD_GLOBAL); + void * lib_handle = dlopen(lib_path.c_str(), RTLD_NOW | RTLD_GLOBAL); if (nullptr == lib_handle) { GGMLQNN_LOG_WARN("can not open QNN library %s, with error: %s", lib_path.c_str(), dlerror()); return 1; @@ -2223,7 +2295,7 @@ int qnn_instance::load_backend(std::string & lib_path, const QnnSaver_Config_t * // get QnnInterface Providers std::uint32_t num_providers = 0; - const QnnInterface_t **provider_list = nullptr; + const QnnInterface_t ** provider_list = nullptr; error = get_providers(&provider_list, &num_providers); if (error != QNN_SUCCESS) { GGMLQNN_LOG_WARN("failed to get providers, error %d", QNN_GET_ERROR_CODE(error)); @@ -2282,8 +2354,9 @@ int qnn_instance::load_backend(std::string & lib_path, const QnnSaver_Config_t * QnnSaver_Config_t backendid_cfg; backendid_cfg.option = QNN_SAVER_CONFIG_OPTION_BACKEND_ID; backendid_cfg.backendId = _backend_id; - const QnnSaver_Config_t *saverCfg[] = {&outputdir_cfg, &backendid_cfg, nullptr}; - if (0 == QnnSaver_initialize(saverCfg)) { + + const QnnSaver_Config_t * saver_cfg[] = {&outputdir_cfg, &backendid_cfg, nullptr}; + if (0 == QnnSaver_initialize(saver_cfg)) { GGMLQNN_LOG_INFO("QnnSaver_initialize successfully"); } else { GGMLQNN_LOG_WARN("QnnSaver_initialize failure"); @@ -2668,7 +2741,7 @@ int qnn_instance::qnn_finalize() { Qnn_ErrorHandle_t error = QNN_SUCCESS; GGMLQNN_LOG_DEBUG("enter %s\n", __func__); - //FIXME:should be removed in the future + //TODO:should be removed in the future reset_idx(); free_rpcmem(); @@ -2971,6 +3044,20 @@ static void dump_tensors_info(const struct ggml_tensor * tensor) { tensor->nb[1], tensor->nb[2]); } +//TODO: currently only support offloading 2D matrix to QNN backend +static void get_qnn_dimensions_from_ggml_dimensions(uint32_t * qnn_dimensions, uint32_t * ggml_dimensions, uint32_t rank) { + if (rank > GGML_MAX_DIMS) { + GGMLQNN_LOG_WARN("invalid params"); + return; + } + if (nullptr == qnn_dimensions || nullptr == ggml_dimensions) { + GGMLQNN_LOG_WARN("invalid params"); + return; + } + qnn_dimensions[0] = ggml_dimensions[1]; + qnn_dimensions[1] = ggml_dimensions[0]; +} + // ================================================================================================= // section-6: implementation of ggml-qnn backend // ================================================================================================= @@ -3010,7 +3097,7 @@ static bool ggml_qnn_can_handle_op(const struct ggml_tensor * tensor) { return false; #if GGMLQNN_PRINT_OP_ADD_LOG - dump_tensors_info(tensor); + //dump_tensors_info(tensor); #endif return (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16) && (src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16); @@ -3019,27 +3106,21 @@ static bool ggml_qnn_can_handle_op(const struct ggml_tensor * tensor) { if (tensor->op == GGML_OP_MUL_MAT) { #if GGMLQNN_PRINT_OP_MUL_MAT_LOG - dump_tensors_info(tensor); + //dump_tensors_info(tensor); #endif - //FIXME: 2048 is an experimental value between ASR inference and LLM inference because - // it's better only offload big matrix to QNN backend - if (ne01 <= 2048) { + uint32_t src0_rank = ggml_get_tensor_rank(src0); + uint32_t src1_rank = ggml_get_tensor_rank(src1); + + if ((src0_rank != 2) || (src1_rank != 2)) //TODO: only support offload 2D matrix mulmat to QNN backend return false; - } -#if 0 - //TODO: offload mul_mat to QNN backend - //need to process type trait in func ggml_qnn_mul_mat(...): + + //TODO: support more data type in func ggml_qnn_mul_mat(...): //src0: q4_0, q6_k, ... //src1: f32 //dst : f32 - return (src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16) - && (tensor->type == GGML_TYPE_F32 || tensor->type == GGML_TYPE_F16); -#else - //fall back to ggml cpu backend return (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16) && (src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16) && (src0->type == src1->type) && (src0->type == tensor->type); -#endif } //TODO:for other op @@ -3054,65 +3135,51 @@ static void ggml_qnn_add(ggml_backend_t backend, ggml_tensor * op) { bool graph_initialized = false; qnn_instance * instance = nullptr; ggml_backend_qnn_context * ctx = (ggml_backend_qnn_context *)backend->context; - std::string graph_name = "ggml_op_qnn_add"; qnn_perf op_perf = qnn_perf("ggml_qnn_add"); Qnn_GraphHandle_t graph_handle = nullptr; - Qnn_Tensor_t * tensor_0 = nullptr; - Qnn_Tensor_t * tensor_1 = nullptr; - Qnn_Tensor_t * tensor_2 = nullptr; + Qnn_Tensor_t * p_tensor0 = nullptr; + Qnn_Tensor_t * p_tensor1 = nullptr; + Qnn_Tensor_t * p_tensor2 = nullptr; Qnn_Param_t qnn_params[] = {}; - enum ggml_op ggmlop = GGML_OP_ADD; - Qnn_DataType_t src0_qnn_type = QNN_DATATYPE_FLOAT_32; - Qnn_DataType_t src1_qnn_type = QNN_DATATYPE_FLOAT_32; - Qnn_DataType_t dst_qnn_type = QNN_DATATYPE_FLOAT_32; const ggml_tensor * src0 = op->src[0]; const ggml_tensor * src1 = op->src[1]; ggml_tensor * dst = op; - uint8_t * qnn_rpcbuffer_0 = nullptr; - uint8_t * qnn_rpcbuffer_1 = nullptr; - uint8_t * qnn_rpcbuffer_2 = nullptr; - GGMLQNN_CHECK_PARAMS(ctx, src0, src1, dst); - instance = ctx->instance; QNN_INTERFACE_VER_TYPE qnn_raw_interface = ctx->raw_interface; - op_perf.start(); - std::string map_entry; - get_graph_key_from_op(op, map_entry); - if (instance->_qnn_graph_map.find(map_entry) != instance->_qnn_graph_map.end()) { + std::string graph_name; + get_graph_key_from_op(op, graph_name); + if (instance->_qnn_graph_map.find(graph_name) != instance->_qnn_graph_map.end()) { graph_initialized = true; - auto & graph_item = instance->_qnn_graph_map[map_entry]; + qnn_res_t & graph_item = instance->_qnn_graph_map[graph_name]; graph_handle = std::get<0>(graph_item); - tensor_0 = std::get<1>(graph_item); - tensor_1 = std::get<2>(graph_item); - tensor_2 = std::get<3>(graph_item); + qnn_tensors_t & tensor = std::get<1>(graph_item); + p_tensor0 = tensor[0]; + p_tensor1 = tensor[1]; + p_tensor2 = tensor[2]; } else { - tensor_0 = ggml_qnn_create_tensor(src0); - tensor_1 = ggml_qnn_create_tensor(src1); - tensor_2 = ggml_qnn_create_tensor(dst); + p_tensor0 = ggml_qnn_create_compute_tensor(src0); + p_tensor1 = ggml_qnn_create_compute_tensor(src1); + p_tensor2 = ggml_qnn_create_compute_tensor(dst); } - print_tensors_info(__func__, ctx, src0, src1, dst); - QNN_VER_PTR(*tensor_0)->type = QNN_TENSOR_TYPE_APP_WRITE; - QNN_VER_PTR(*tensor_1)->type = QNN_TENSOR_TYPE_APP_WRITE; - QNN_VER_PTR(*tensor_2)->type = QNN_TENSOR_TYPE_APP_READ; - - src0_qnn_type = qnn_datatype_from_ggml_datatype(src0->type); - src1_qnn_type = qnn_datatype_from_ggml_datatype(src1->type); - dst_qnn_type = qnn_datatype_from_ggml_datatype(dst->type); + //ensure QNN tensor has correct tensor type + QNN_VER_PTR(*p_tensor0)->type = QNN_TENSOR_TYPE_APP_WRITE; + QNN_VER_PTR(*p_tensor1)->type = QNN_TENSOR_TYPE_APP_WRITE; + QNN_VER_PTR(*p_tensor2)->type = QNN_TENSOR_TYPE_APP_READ; - uint32_t * tensor_0_dimensions = QNN_VER_PTR(*tensor_0)->dimensions; - uint32_t * tensor_1_dimensions = QNN_VER_PTR(*tensor_1)->dimensions; - uint32_t * tensor_2_dimensions = QNN_VER_PTR(*tensor_2)->dimensions; + //save the original dimensions of qnn tensors + uint32_t * tensor_0_dimensions = QNN_VER_PTR(*p_tensor0)->dimensions; + uint32_t * tensor_1_dimensions = QNN_VER_PTR(*p_tensor1)->dimensions; + uint32_t * tensor_2_dimensions = QNN_VER_PTR(*p_tensor2)->dimensions; bool enable_npu_rpc = instance->enable_qnn_rpc() && ctx->device == QNN_BACKEND_NPU; if (!graph_initialized) { - graph_name = map_entry; GGMLQNN_LOG_DEBUG("graph name %s", graph_name.c_str()); if (ctx->device == QNN_BACKEND_NPU) { error = create_htp_graph(ctx, graph_name, &graph_handle); @@ -3127,44 +3194,44 @@ static void ggml_qnn_add(ggml_backend_t backend, ggml_tensor * op) { } if (enable_npu_rpc) { - QNN_VER_PTR(*tensor_0)->memType = QNN_TENSORMEMTYPE_MEMHANDLE; - QNN_VER_PTR(*tensor_0)->clientBuf = {.data=nullptr, .dataSize=0}; + QNN_VER_PTR(*p_tensor0)->memType = QNN_TENSORMEMTYPE_MEMHANDLE; + QNN_VER_PTR(*p_tensor0)->clientBuf = {.data=nullptr, .dataSize=0}; - QNN_VER_PTR(*tensor_1)->memType = QNN_TENSORMEMTYPE_MEMHANDLE; - QNN_VER_PTR(*tensor_1)->clientBuf = {.data=nullptr, .dataSize=0}; + QNN_VER_PTR(*p_tensor1)->memType = QNN_TENSORMEMTYPE_MEMHANDLE; + QNN_VER_PTR(*p_tensor1)->clientBuf = {.data=nullptr, .dataSize=0}; - QNN_VER_PTR(*tensor_2)->memType = QNN_TENSORMEMTYPE_MEMHANDLE; - QNN_VER_PTR(*tensor_2)->clientBuf = {.data=nullptr, .dataSize=0}; + QNN_VER_PTR(*p_tensor2)->memType = QNN_TENSORMEMTYPE_MEMHANDLE; + QNN_VER_PTR(*p_tensor2)->clientBuf = {.data=nullptr, .dataSize=0}; } - CHECK_QNN_API(error = qnn_raw_interface.tensorCreateGraphTensor(graph_handle, tensor_0)); - CHECK_QNN_API(error = qnn_raw_interface.tensorCreateGraphTensor(graph_handle, tensor_1)); - CHECK_QNN_API(error = qnn_raw_interface.tensorCreateGraphTensor(graph_handle, tensor_2)); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor0)); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor1)); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor2)); if (enable_npu_rpc) { - qnn_rpcbuffer_0 = create_rpc_buffer(instance, src0, tensor_0, true); - qnn_rpcbuffer_1 = create_rpc_buffer(instance, src1, tensor_1, true); - qnn_rpcbuffer_2 = create_rpc_buffer(instance, dst, tensor_2, false); + uint8_t * qnn_rpcbuffer_0 = create_rpc_buffer(instance, src0, p_tensor0, true); + uint8_t * qnn_rpcbuffer_1 = create_rpc_buffer(instance, src1, p_tensor1, true); + uint8_t * qnn_rpcbuffer_2 = create_rpc_buffer(instance, dst, p_tensor2, false); if (nullptr == qnn_rpcbuffer_0 || nullptr == qnn_rpcbuffer_1 || nullptr == qnn_rpcbuffer_2) { GGMLQNN_LOG_INFO("create rpc buffer failure\n"); - //FIXME: potential memory leak althought it shouldn't happen + //TODO: potential memory leak although it shouldn't happen return; } } else { - QNN_VER_PTR(*tensor_0)->clientBuf = {src0->data, ggml_get_tensor_data_size(src0)}; - QNN_VER_PTR(*tensor_1)->clientBuf = {src1->data, ggml_get_tensor_data_size(src1)}; - QNN_VER_PTR(*tensor_2)->clientBuf = {dst->data, ggml_get_tensor_data_size(dst)}; + QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggml_get_tensor_data_size(src0)}; + QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggml_get_tensor_data_size(src1)}; + QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggml_get_tensor_data_size(dst)}; } Qnn_Tensor_t tensor_inputs[] = { - *tensor_0, - *tensor_1 + *p_tensor0, + *p_tensor1 }; Qnn_Tensor_t tensor_outputs[] = { - *tensor_2 + *p_tensor2 }; Qnn_OpConfig_t op_config = { - (Qnn_OpConfigVersion_t) 1, .v1 = { + QNN_OPCONFIG_VERSION_1, .v1 = { "ggml_op_add", QNN_OP_PACKAGE_NAME_QTI_AISW, QNN_OP_ELEMENT_WISE_ADD, @@ -3176,26 +3243,38 @@ static void ggml_qnn_add(ggml_backend_t backend, ggml_tensor * op) { tensor_outputs } }; - CHECK_QNN_API(error = qnn_raw_interface.graphAddNode(graph_handle, op_config)); - CHECK_QNN_API(error = qnn_raw_interface.graphFinalize(graph_handle, nullptr, nullptr)); - error = qnn_raw_interface.graphExecute(graph_handle, + CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, op_config)); + CHECK_QNN_API(error, qnn_raw_interface.graphFinalize(graph_handle, nullptr, nullptr)); + CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, tensor_inputs, 2, tensor_outputs, 1, - nullptr, nullptr); - CHECK_QNN_API(error); + nullptr, nullptr)); if (enable_npu_rpc) { - uint8_t * qnn_rpcbuffer = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*tensor_2)->memHandle)); + uint8_t * qnn_rpcbuffer = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*p_tensor2)->memHandle)); GGMLQNN_LOG_INFO("qnn_rpcbuffer = %p\n", qnn_rpcbuffer); if (nullptr != qnn_rpcbuffer) { memcpy(dst->data, qnn_rpcbuffer, ggml_nbytes(dst)); } } - auto graph_item = std::make_tuple(graph_handle, tensor_0, tensor_1, tensor_2); - instance->_qnn_graph_map[map_entry] = graph_item; + qnn_tensors_t ggml_op_add_tensors; + ggml_op_add_tensors.reserve(3); + ggml_op_add_tensors.push_back(p_tensor0); + ggml_op_add_tensors.push_back(p_tensor1); + ggml_op_add_tensors.push_back(p_tensor2); + + auto graph_item = std::make_tuple(graph_handle, ggml_op_add_tensors); + instance->_qnn_graph_map[graph_name] = graph_item; } else { + Qnn_DataType_t src0_qnn_type = QNN_DATATYPE_FLOAT_32; + Qnn_DataType_t src1_qnn_type = QNN_DATATYPE_FLOAT_32; + Qnn_DataType_t dst_qnn_type = QNN_DATATYPE_FLOAT_32; + + src0_qnn_type = qnn_datatype_from_ggml_datatype(src0->type); + src1_qnn_type = qnn_datatype_from_ggml_datatype(src1->type); + dst_qnn_type = qnn_datatype_from_ggml_datatype(dst->type); uint32_t dimensions_input_0[] = {(uint32_t) src0->ne[0], (uint32_t) src0->ne[1], (uint32_t) src0->ne[2], (uint32_t) src0->ne[3]}; @@ -3204,76 +3283,76 @@ static void ggml_qnn_add(ggml_backend_t backend, ggml_tensor * op) { uint32_t dimensions_output[] = {(uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->ne[2], (uint32_t) dst->ne[3]}; - QNN_VER_PTR(*tensor_0)->dimensions = dimensions_input_0; - QNN_VER_PTR(*tensor_0)->rank = ggml_get_tensor_rank(src0); - QNN_VER_PTR(*tensor_0)->dataType = src0_qnn_type; + QNN_VER_PTR(*p_tensor0)->dimensions = dimensions_input_0; + QNN_VER_PTR(*p_tensor0)->rank = ggml_get_tensor_rank(src0); + QNN_VER_PTR(*p_tensor0)->dataType = src0_qnn_type; - QNN_VER_PTR(*tensor_1)->dimensions = dimensions_input_1; - QNN_VER_PTR(*tensor_1)->rank = ggml_get_tensor_rank(src1); - QNN_VER_PTR(*tensor_1)->dataType = src1_qnn_type; + QNN_VER_PTR(*p_tensor1)->dimensions = dimensions_input_1; + QNN_VER_PTR(*p_tensor1)->rank = ggml_get_tensor_rank(src1); + QNN_VER_PTR(*p_tensor1)->dataType = src1_qnn_type; - QNN_VER_PTR(*tensor_2)->dimensions = dimensions_output; - QNN_VER_PTR(*tensor_2)->rank = ggml_get_tensor_rank(dst); - QNN_VER_PTR(*tensor_2)->dataType = dst_qnn_type; + QNN_VER_PTR(*p_tensor2)->dimensions = dimensions_output; + QNN_VER_PTR(*p_tensor2)->rank = ggml_get_tensor_rank(dst); + QNN_VER_PTR(*p_tensor2)->dataType = dst_qnn_type; if (enable_npu_rpc) { - //FIXME:why failure with test-backend-ops - uint8_t * qnn_buffer_0 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*tensor_0)->memHandle)); - GGMLQNN_LOG_INFO("qnn_rpcbuffer_0 = %p\n", qnn_rpcbuffer_0); + //TODO: NPU RPC feature will failed with test-backend-ops + uint8_t * qnn_buffer_0 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*p_tensor0)->memHandle)); + GGMLQNN_LOG_INFO("qnn_rpcbuffer_0 = %p\n", qnn_buffer_0); if (nullptr != qnn_buffer_0) { memcpy(qnn_buffer_0, src0->data, ggml_nbytes(src0)); } - uint8_t * qnn_buffer_1 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*tensor_1)->memHandle)); - GGMLQNN_LOG_INFO("qnn_rpcbuffer_1 = %p\n", qnn_rpcbuffer_1); + uint8_t * qnn_buffer_1 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*p_tensor1)->memHandle)); + GGMLQNN_LOG_INFO("qnn_rpcbuffer_1 = %p\n", qnn_buffer_1); if (nullptr != qnn_buffer_1) { memcpy(qnn_buffer_1, src1->data, ggml_nbytes(src1)); } } else { - QNN_VER_PTR(*tensor_0)->clientBuf = {src0->data, ggml_get_tensor_data_size(src0)}; - QNN_VER_PTR(*tensor_1)->clientBuf = {src1->data, ggml_get_tensor_data_size(src1)}; - QNN_VER_PTR(*tensor_2)->clientBuf = {dst->data, ggml_get_tensor_data_size(dst)}; + QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggml_get_tensor_data_size(src0)}; + QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggml_get_tensor_data_size(src1)}; + QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggml_get_tensor_data_size(dst)}; } Qnn_Tensor_t tensor_inputs[] = { - *tensor_0, - *tensor_1 + *p_tensor0, + *p_tensor1 }; Qnn_Tensor_t tensor_outputs[] = { - *tensor_2 + *p_tensor2 }; - error = qnn_raw_interface.graphExecute(graph_handle, + CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, tensor_inputs, 2, tensor_outputs, 1, - nullptr, nullptr); - CHECK_QNN_API(error); + nullptr, nullptr)); if (enable_npu_rpc) { - //FIXME:why failure with test-backend-ops - uint8_t * qnn_buffer_2 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*tensor_2)->memHandle)); + //TODO:NPU RPC feature will failed with test-backend-ops + uint8_t * qnn_buffer_2 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*p_tensor2)->memHandle)); if (nullptr != qnn_buffer_2) { memcpy(dst->data, qnn_buffer_2, ggml_nbytes(dst)); } } } - //avoid memory leak in func free_qnn_tensor - QNN_VER_PTR(*tensor_0)->dimensions = tensor_0_dimensions; - QNN_VER_PTR(*tensor_1)->dimensions = tensor_1_dimensions; - QNN_VER_PTR(*tensor_2)->dimensions = tensor_2_dimensions; + // restore the original dimensions of qnn tensors to avoid memory leak in func free_qnn_tensor + QNN_VER_PTR(*p_tensor0)->dimensions = tensor_0_dimensions; + QNN_VER_PTR(*p_tensor1)->dimensions = tensor_1_dimensions; + QNN_VER_PTR(*p_tensor2)->dimensions = tensor_2_dimensions; #if GGMLQNN_PRINT_OP_ADD_LOG op_perf.info(); #endif } -//TODO: /* - * the logic of ggml_qnn_mul_mat is similar to ggml_qnn_add,but type trait and matrix transpose are required - * for offload mulmat to QNN backend, so it's a standalone function. + * the logic of ggml_qnn_mul_mat is similar to ggml_qnn_add but much more complicated than ggml_qnn_add, + * matrix transpose and type trait are required for offload mulmat to QNN backend, + * so it's a standalone function. accordingly, this is another typical skeleton for offload other + * ggml ops to QNN backend * * MUL_MAT take most of the compute time (about 95%).so to speed up llama inference, we should focus on MUL_MAT. * - * we have three kinds of MUL_MAT to compute: + * have three kinds of MUL_MAT to compute: * mul_mat_f32: both src0 and src1 are F32, this will be naturally handled in QNN backend * mul_mat_f16_f32: src0 is F16 and src1 is F32, f16 in src0 -> f32 in src0', then src0' * src1 * mul_mat_q_f32: src0 is quantized (Q4_0, Q4_1, ...) and src1 is F32, src0 -> f32 in src0', then src0' * src1 @@ -3284,148 +3363,200 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { qnn_perf op_perf = qnn_perf("ggml_qnn_mul_mat"); qnn_instance * instance = nullptr; ggml_backend_qnn_context * ctx = (ggml_backend_qnn_context *) backend->context; - - std::string graph_name = "ggml_op_qnn_mul_mat"; Qnn_GraphHandle_t graph_handle = nullptr; - Qnn_Tensor_t * tensor_0 = nullptr; - Qnn_Tensor_t * tensor_1 = nullptr; - Qnn_Tensor_t * tensor_2 = nullptr; - - Qnn_Param_t qnn_params[] = {}; - - enum ggml_op ggmlop = GGML_OP_ADD; - Qnn_DataType_t src0_qnn_type = QNN_DATATYPE_FLOAT_32; - Qnn_DataType_t src1_qnn_type = QNN_DATATYPE_FLOAT_32; - Qnn_DataType_t dst_qnn_type = QNN_DATATYPE_FLOAT_32; + Qnn_Tensor_t * p_tensor0 = nullptr; + Qnn_Tensor_t * p_tensor1 = nullptr; + Qnn_Tensor_t * p_tensor2 = nullptr; + Qnn_Tensor_t * p_param_tensor = nullptr; + Qnn_Tensor_t * p_tensor2_transpose = nullptr; const ggml_tensor * src0 = op->src[0]; const ggml_tensor * src1 = op->src[1]; - ggml_tensor * dst = op; + ggml_tensor * dst = op; GGMLQNN_CHECK_PARAMS(ctx, src0, src1, dst); - instance = ctx->instance; QNN_INTERFACE_VER_TYPE qnn_raw_interface = ctx->raw_interface; - op_perf.start(); - std::string map_entry; - get_graph_key_from_op(op, map_entry); - if (instance->_qnn_graph_map.find(map_entry) != instance->_qnn_graph_map.end()) { - graph_initialized = true; - auto & graph_item = instance->_qnn_graph_map[map_entry]; - graph_handle = std::get<0>(graph_item); - tensor_0 = std::get<1>(graph_item); - tensor_1 = std::get<2>(graph_item); - tensor_2 = std::get<3>(graph_item); + std::string graph_name; + get_graph_key_from_op(op, graph_name); + if (instance->_qnn_graph_map.find(graph_name) != instance->_qnn_graph_map.end()) { + graph_initialized = true; + qnn_res_t & graph_item = instance->_qnn_graph_map[graph_name]; + graph_handle = std::get<0>(graph_item); + qnn_tensors_t & tensors = std::get<1>(graph_item); + p_tensor0 = tensors[0]; + p_tensor1 = tensors[1]; + p_tensor2 = tensors[2]; + p_param_tensor = tensors[3]; + p_tensor2_transpose = tensors[4]; } else { - tensor_0 = ggml_qnn_create_tensor(src0); - tensor_1 = ggml_qnn_create_tensor(src1); - tensor_2 = ggml_qnn_create_tensor(dst); + p_tensor0 = ggml_qnn_create_mulmat_tensor(src0, nullptr, QNN_TENSOR_TYPE_APP_WRITE,QNN_DATATYPE_FLOAT_32, 2, nullptr, nullptr, 0); + p_tensor1 = ggml_qnn_create_mulmat_tensor(src1, nullptr, QNN_TENSOR_TYPE_APP_WRITE,QNN_DATATYPE_FLOAT_32, 2, nullptr, nullptr, 0); + p_tensor2 = ggml_qnn_create_mulmat_tensor(dst, nullptr, QNN_TENSOR_TYPE_APP_READ,QNN_DATATYPE_FLOAT_32, 2, nullptr, nullptr, 0); } print_tensors_info(__func__, ctx, src0, src1, dst); - QNN_VER_PTR(*tensor_0)->type = QNN_TENSOR_TYPE_APP_WRITE; - QNN_VER_PTR(*tensor_1)->type = QNN_TENSOR_TYPE_APP_WRITE; - QNN_VER_PTR(*tensor_2)->type = QNN_TENSOR_TYPE_APP_READ; - - src0_qnn_type = qnn_datatype_from_ggml_datatype(src0->type); - src1_qnn_type = qnn_datatype_from_ggml_datatype(src1->type); - dst_qnn_type = qnn_datatype_from_ggml_datatype(dst->type); + //ensure QNN tensor has correct tensor type + QNN_VER_PTR(*p_tensor0)->type = QNN_TENSOR_TYPE_APP_WRITE; + QNN_VER_PTR(*p_tensor1)->type = QNN_TENSOR_TYPE_APP_WRITE; + QNN_VER_PTR(*p_tensor2)->type = QNN_TENSOR_TYPE_APP_READ; - uint32_t * tensor_0_dimensions = QNN_VER_PTR(*tensor_0)->dimensions; - uint32_t * tensor_1_dimensions = QNN_VER_PTR(*tensor_1)->dimensions; - uint32_t * tensor_2_dimensions = QNN_VER_PTR(*tensor_2)->dimensions; + //save the original dimensions of qnn tensors + uint32_t * tensor_0_dimensions = QNN_VER_PTR(*p_tensor0)->dimensions; + uint32_t * tensor_1_dimensions = QNN_VER_PTR(*p_tensor1)->dimensions; + uint32_t * tensor_2_dimensions = QNN_VER_PTR(*p_tensor2)->dimensions; if (!graph_initialized) { - graph_name = map_entry; GGMLQNN_LOG_DEBUG("graph name %s", graph_name.c_str()); + /* + there are two key-points in properly handling how to offload mulmat to the QNN backend in ggml-qnn + 1. transpose + a 3x2 f32 matrix which means 3 rows and 2 columns. in ggml, it could be created from: + struct ggml_tensor* matrix = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 2, 3); + which like this: + +---+---+ + | 0 | 1 | + +---+---+ + | 2 | 3 | + +---+---+ + | 4 | 5 | + +---+---+ + with + ne[0] = 2 + ne[1] = 3 + there are different dimension order between ggml tensor and qnn tensor + + 2. QNN's MatMul can only support input tensors with rank >= 2 + + there is gap between ggml mulmat and QNN mulmat,we need to perform a transpose operation when offloading mulmat to QNN backend. + */ + + //step-1: create qnn graph error = qnn_raw_interface.graphCreate(instance->get_qnn_context_handle(), graph_name.c_str(), nullptr, &graph_handle); if (QNN_SUCCESS != error) { GGMLQNN_LOG_INFO("can't create qnn graph handle with graph name %s, error = %d\n", graph_name.c_str(), error); return; } - CHECK_QNN_API(error = qnn_raw_interface.tensorCreateGraphTensor(graph_handle, tensor_0)); - CHECK_QNN_API(error = qnn_raw_interface.tensorCreateGraphTensor(graph_handle, tensor_1)); - CHECK_QNN_API(error = qnn_raw_interface.tensorCreateGraphTensor(graph_handle, tensor_2)); - - QNN_VER_PTR(*tensor_0)->clientBuf = {src0->data, ggml_get_tensor_data_size(src0)}; - QNN_VER_PTR(*tensor_1)->clientBuf = {src1->data, ggml_get_tensor_data_size(src1)}; - QNN_VER_PTR(*tensor_2)->clientBuf = {dst->data, ggml_get_tensor_data_size(dst)}; + //step-2: create param tensor for mulmat of 2d matrix + uint32_t param_tensor_dims[] = {2}; + uint32_t param_tensor_data[2] = {1, 0}; + p_param_tensor = ggml_qnn_create_mulmat_tensor(nullptr, "param", QNN_TENSOR_TYPE_STATIC,QNN_DATATYPE_UINT_32, 1, param_tensor_dims, param_tensor_data, 8); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_param_tensor)); + + //step-3: create compute tensor from ggml tensor + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor0)); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor1)); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor2)); + + QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggml_get_tensor_data_size(src0)}; + QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggml_get_tensor_data_size(src1)}; + QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggml_get_tensor_data_size(dst)}; + + //step-4: create a transpose tensor + uint32_t tensor2_transpose_dims[GGML_MAX_DIMS] = {}; + p_tensor2_transpose = ggml_qnn_create_mulmat_tensor(dst,"transpose",QNN_TENSOR_TYPE_NATIVE,QNN_DATATYPE_FLOAT_32, 2, nullptr, nullptr, 0); + get_qnn_dimensions_from_ggml_dimensions(tensor2_transpose_dims, tensor_2_dimensions,ggml_get_tensor_rank(dst)); + //save pointer because the dimensions of tensor p_tensor2_transpose will be changed later + uint32_t * tensor2_dimensions_transpose = QNN_VER_PTR(*p_tensor2_transpose)->dimensions; + //update dimensions of tensor p_tensor2_transpose to make QNN SDK happy + QNN_VER_PTR(*p_tensor2_transpose)->dimensions = tensor2_transpose_dims; + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor2_transpose)); + + //step-5: compose qnn graph: add mat_mul node + Qnn_Param_t out_0_params[] = { + {QNN_PARAMTYPE_SCALAR, + QNN_OP_MAT_MUL_PARAM_TRANSPOSE_IN1, + .scalarParam = {QNN_DATATYPE_BOOL_8, .bool8Value = 1} + } + }; - Qnn_Tensor_t tensor_inputs[] = { - *tensor_0, - *tensor_1 + Qnn_Tensor_t out_0_inputs[] = {*p_tensor0,*p_tensor1}; + Qnn_Tensor_t out_0_outputs[] = {*p_tensor2_transpose}; + Qnn_OpConfig_t out_0 = { + QNN_OPCONFIG_VERSION_1, .v1 = + {"ggmlqnn_mulmat_opconfig", QNN_OP_PACKAGE_NAME_QTI_AISW, QNN_OP_MAT_MUL, + 1, + out_0_params, + 2, + out_0_inputs, + 1, + out_0_outputs} }; - Qnn_Tensor_t tensor_outputs[] = { - *tensor_2 + CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle,out_0)); + + //step-5: compose qnn graph: add transpose node + Qnn_Param_t out_trans1_0_params[] = { + {(Qnn_ParamType_t) 1, + "perm", .tensorParam = *p_param_tensor + } }; - Qnn_OpConfig_t op_config = { - (Qnn_OpConfigVersion_t) 1, .v1 = { - "ggml_op_mul_mat", - QNN_OP_PACKAGE_NAME_QTI_AISW, - QNN_OP_MAT_MUL, - 0, - qnn_params, - 2, - tensor_inputs, + Qnn_Tensor_t out_trans1_0_inputs[] = {*p_tensor2_transpose}; + Qnn_Tensor_t out_trans1_0_outputs[] = {*p_tensor2}; + Qnn_OpConfig_t out_trans1_0 = { + QNN_OPCONFIG_VERSION_1, + .v1 = {"ggmlqnn_mulmat_transpose_opconfig", + "qti.aisw", + QNN_OP_TRANSPOSE, 1, + out_trans1_0_params, 1, - tensor_outputs - } + out_trans1_0_inputs, + 1, + out_trans1_0_outputs} }; - CHECK_QNN_API(error = qnn_raw_interface.graphAddNode(graph_handle, op_config)); - CHECK_QNN_API(error = qnn_raw_interface.graphFinalize(graph_handle, nullptr, nullptr)); - error = qnn_raw_interface.graphExecute(graph_handle, - tensor_inputs, 2, - tensor_outputs, 1, - nullptr, nullptr); - CHECK_QNN_API(error); - auto graph_item = std::make_tuple(graph_handle, tensor_0, tensor_1, tensor_2); - instance->_qnn_graph_map[map_entry] = graph_item; + CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle,out_trans1_0)); + + //step-6: finalize qnn graph and execute qnn graph + CHECK_QNN_API(error, qnn_raw_interface.graphFinalize(graph_handle, NULL, NULL)); + Qnn_Tensor_t input_tensors_0[] = {*p_tensor0,*p_tensor1}; + Qnn_Tensor_t output_tensors_0[] = {*p_tensor2}; + CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, + input_tensors_0, 2, + output_tensors_0, 1, + NULL, NULL)); + + qnn_tensors_t ggml_op_mulmat_tensors; + ggml_op_mulmat_tensors.reserve(5); + ggml_op_mulmat_tensors.push_back(p_tensor0); + ggml_op_mulmat_tensors.push_back(p_tensor1); + ggml_op_mulmat_tensors.push_back(p_tensor2); + ggml_op_mulmat_tensors.push_back(p_param_tensor); + ggml_op_mulmat_tensors.push_back(p_tensor2_transpose); + + auto graph_item = std::make_tuple(graph_handle, ggml_op_mulmat_tensors); + instance->_qnn_graph_map[graph_name] = graph_item; + + //avoid cleanup these resource to make test_backend_ops happy + //free_qnn_tensor(p_param_tensor); + //restore pointer to avoid memory leak + QNN_VER_PTR(*p_tensor2_transpose)->dimensions = tensor2_dimensions_transpose; + //free_qnn_tensor(p_tensor2_transpose); } else { - uint32_t dimensions_input_0[] = {(uint32_t) src0->ne[0], (uint32_t) src0->ne[1], - (uint32_t) src0->ne[2], (uint32_t) src0->ne[3]}; - uint32_t dimensions_input_1[] = {(uint32_t) src1->ne[0], (uint32_t) src1->ne[1], - (uint32_t) src1->ne[2], (uint32_t) src1->ne[3]}; - uint32_t dimensions_output[] = {(uint32_t) dst->ne[0], (uint32_t) dst->ne[1], - (uint32_t) dst->ne[2], (uint32_t) dst->ne[3]}; - QNN_VER_PTR(*tensor_0)->dimensions = dimensions_input_0; - QNN_VER_PTR(*tensor_0)->rank = ggml_get_tensor_rank(src0); - QNN_VER_PTR(*tensor_0)->dataType = src0_qnn_type; - - QNN_VER_PTR(*tensor_1)->dimensions = dimensions_input_1; - QNN_VER_PTR(*tensor_1)->rank = ggml_get_tensor_rank(src1); - QNN_VER_PTR(*tensor_1)->dataType = src1_qnn_type; - - QNN_VER_PTR(*tensor_2)->dimensions = dimensions_output; - QNN_VER_PTR(*tensor_2)->rank = ggml_get_tensor_rank(dst); - QNN_VER_PTR(*tensor_2)->dataType = dst_qnn_type; - - QNN_VER_PTR(*tensor_0)->clientBuf = {src0->data, ggml_get_tensor_data_size(src0)}; - QNN_VER_PTR(*tensor_1)->clientBuf = {src1->data, ggml_get_tensor_data_size(src1)}; - QNN_VER_PTR(*tensor_2)->clientBuf = {dst->data, ggml_get_tensor_data_size(dst)}; + QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggml_get_tensor_data_size(src0)}; + QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggml_get_tensor_data_size(src1)}; + QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggml_get_tensor_data_size(dst)}; Qnn_Tensor_t tensor_inputs[] = { - *tensor_0, - *tensor_1 + *p_tensor0, + *p_tensor1 }; Qnn_Tensor_t tensor_outputs[] = { - *tensor_2 + *p_tensor2 }; - error = qnn_raw_interface.graphExecute(graph_handle, + CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, tensor_inputs, 2, tensor_outputs, 1, - nullptr, nullptr); - CHECK_QNN_API(error); + nullptr, nullptr)); } - //avoid memory leak in func free_qnn_tensor - QNN_VER_PTR(*tensor_0)->dimensions = tensor_0_dimensions; - QNN_VER_PTR(*tensor_1)->dimensions = tensor_1_dimensions; - QNN_VER_PTR(*tensor_2)->dimensions = tensor_2_dimensions; + // restore the original dimensions of qnn tensors to avoid memory leak in func free_qnn_tensor + QNN_VER_PTR(*p_tensor0)->dimensions = tensor_0_dimensions; + QNN_VER_PTR(*p_tensor1)->dimensions = tensor_1_dimensions; + QNN_VER_PTR(*p_tensor2)->dimensions = tensor_2_dimensions; op_perf.info(); } @@ -3608,21 +3739,18 @@ static void ggml_backend_qnn_free(ggml_backend_t backend) { qnn_instance * instance = (qnn_instance*)g_qnn_mgr[ctx->device].instance; if (instance != nullptr) { - std::map>::iterator graph_it; + std::map>>::iterator graph_it; for (graph_it = instance->_qnn_graph_map.begin(); graph_it != instance->_qnn_graph_map.end(); graph_it++) { auto & graph_item = graph_it->second; Qnn_GraphHandle_t & graph_handle = std::get<0>(graph_item); - Qnn_Tensor_t * tensor_0 = std::get<1>(graph_item); - Qnn_Tensor_t * tensor_1 = std::get<2>(graph_item); - Qnn_Tensor_t * tensor_2 = std::get<3>(graph_item); + qnn_tensors_t & tensors = std::get<1>(graph_item); + for (auto tensor_it = tensors.begin(); tensor_it != tensors.end(); ++tensor_it) { + free_qnn_tensor(*tensor_it); + } GGML_UNUSED(graph_handle); GGMLQNN_LOG_DEBUG("graph type:%s", graph_it->first.c_str()); - free_qnn_tensor(tensor_0); - free_qnn_tensor(tensor_1); - free_qnn_tensor(tensor_2); } instance->_qnn_graph_map.clear(); From 4b7ef3ac08d8fb423f012e7478026e1596f6e3d2 Mon Sep 17 00:00:00 2001 From: zhouwg Date: Thu, 20 Feb 2025 08:39:15 +0800 Subject: [PATCH 16/76] ggml-qnn: remove redundant codes --- ggml/src/ggml-qnn/ggml-qnn.cpp | 298 +++++++++++---------------------- 1 file changed, 97 insertions(+), 201 deletions(-) diff --git a/ggml/src/ggml-qnn/ggml-qnn.cpp b/ggml/src/ggml-qnn/ggml-qnn.cpp index a1aca7940bf4f..37c947f412f1f 100644 --- a/ggml/src/ggml-qnn/ggml-qnn.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn.cpp @@ -1404,58 +1404,69 @@ static const char * ggml_get_type_name(ggml_type type) { return traits->type_name; } -static Qnn_Tensor_t * ggml_qnn_create_compute_tensor(const ggml_tensor * tensor) { - Qnn_ErrorHandle_t error = QNN_SUCCESS; - char tensor_name[GGML_MAX_NAME] = {0}; - - //TODO:remove get_idx() and inc_idx() in the future but ensure the tensor name is unique - snprintf(tensor_name, GGML_MAX_NAME, "tensor_%-8d", get_idx()); - GGMLQNN_LOG_DEBUG("init_tensor %d", get_idx()); - inc_idx(); - - uint32_t dimensions[] = {(uint32_t) tensor->ne[0], (uint32_t) tensor->ne[1], - (uint32_t) tensor->ne[2], (uint32_t) tensor->ne[3]}; - Qnn_DataType_t qnn_data_type = QNN_DATATYPE_FLOAT_32; - Qnn_TensorType_t qnn_tensor_type = QNN_TENSOR_TYPE_APP_WRITE; +static const char * get_ggml_type_name(ggml_type type) { + const auto * traits = ggml_get_type_traits(type); + return traits->type_name; +} - if (tensor->flags & GGML_TENSOR_FLAG_INPUT) { - qnn_tensor_type = QNN_TENSOR_TYPE_APP_WRITE; - } else if (tensor->flags & GGML_TENSOR_FLAG_OUTPUT) { - qnn_tensor_type = QNN_TENSOR_TYPE_APP_READ; - } - Qnn_Tensor_t qnn_tensor = { - .version= QNN_TENSOR_VERSION_1, - {.v1= { - .id = 0, - .name = tensor_name, - .type = qnn_tensor_type, - .dataFormat = QNN_TENSOR_DATA_FORMAT_FLAT_BUFFER, - .dataType = qnn_data_type, - .quantizeParams = {QNN_DEFINITION_UNDEFINED, - QNN_QUANTIZATION_ENCODING_UNDEFINED, - {.scaleOffsetEncoding = {.scale = 0.0000000000000000f, .offset = 0}}}, - .rank = ggml_get_tensor_rank(tensor), - .dimensions = dimensions, - .memType = QNN_TENSORMEMTYPE_RAW, - {.clientBuf = {.data = nullptr, - .dataSize = 0}}}} - }; - Qnn_Tensor_t * p_qnn_tensor = (Qnn_Tensor_t *)calloc(1, sizeof(Qnn_Tensor_t)); - if (nullptr == p_qnn_tensor) { - GGMLQNN_LOG_WARN("calloc failed"); - return nullptr; +//TODO: +// ref:explanation of k-quants, https://github.com/ggerganov/llama.cpp/pull/1684 +static Qnn_DataType_t qnn_datatype_from_ggml_datatype(enum ggml_type ggmltype) { + switch (ggmltype) { + case GGML_TYPE_F16: + return QNN_DATATYPE_FLOAT_16; + case GGML_TYPE_F32: + return QNN_DATATYPE_FLOAT_32; + case GGML_TYPE_I8: + return QNN_DATATYPE_INT_8; + case GGML_TYPE_Q8_0: + return QNN_DATATYPE_SFIXED_POINT_8; + case GGML_TYPE_Q4_0: + return QNN_DATATYPE_SFIXED_POINT_4; + default: + break; } - error = deep_copy_qnn_tensors(qnn_tensor, * p_qnn_tensor); - if (error != QNN_SUCCESS) { - free(p_qnn_tensor); - GGMLQNN_LOG_WARN("init tensor failed"); - return nullptr; + return QNN_DATATYPE_UNDEFINED; +} + +//TODO: +static ggml_type ggml_datatype_from_qnn_datatype(Qnn_DataType_t qnn_type) { + switch (qnn_type) { + case QNN_DATATYPE_FLOAT_32: + return GGML_TYPE_F32; + case QNN_DATATYPE_FLOAT_16: + return GGML_TYPE_F16; + case QNN_DATATYPE_UINT_32: + case QNN_DATATYPE_INT_32: + return GGML_TYPE_I32; + case QNN_DATATYPE_INT_16: + return GGML_TYPE_I16; + case QNN_DATATYPE_INT_8: + return GGML_TYPE_I8; + case QNN_DATATYPE_SFIXED_POINT_8: + return GGML_TYPE_Q8_0; + case QNN_DATATYPE_SFIXED_POINT_4: + return GGML_TYPE_Q4_0; + default: + break; } + return GGML_TYPE_COUNT; +} - return p_qnn_tensor; +//TODO: add more ops +static const char * qnn_opname_from_ggmlop(enum ggml_op ggmlop) { + switch (ggmlop) { + case GGML_OP_ADD: + return QNN_OP_ELEMENT_WISE_ADD; + case GGML_OP_MUL_MAT: + return QNN_OP_MAT_MUL; + default: + break; + } + return nullptr; } -static Qnn_Tensor_t * ggml_qnn_create_mulmat_tensor(const ggml_tensor * tensor, const char * name, Qnn_TensorType_t qnn_tensor_type, +static Qnn_Tensor_t * ggml_qnn_create_general_tensor(const ggml_tensor * tensor, const char * name, Qnn_TensorType_t qnn_tensor_type, Qnn_DataType_t qnn_data_type, uint32_t rank, uint32_t * dims, void * data, uint32_t data_size) { Qnn_ErrorHandle_t error = QNN_SUCCESS; char tensor_name[GGML_MAX_NAME] = {0}; @@ -1480,6 +1491,7 @@ static Qnn_Tensor_t * ggml_qnn_create_mulmat_tensor(const ggml_tensor * tensor, dimensions_transpose[3] = (uint32_t) tensor->ne[3]; tensor_dims = dimensions_transpose; } + //re-assign tensor_dims if (nullptr != dims) { tensor_dims = dims; } @@ -1522,66 +1534,25 @@ static Qnn_Tensor_t * ggml_qnn_create_mulmat_tensor(const ggml_tensor * tensor, return p_qnn_tensor; } -//TODO: -// ref:explanation of k-quants, https://github.com/ggerganov/llama.cpp/pull/1684 -static Qnn_DataType_t qnn_datatype_from_ggml_datatype(enum ggml_type ggmltype) { - switch (ggmltype) { - case GGML_TYPE_F16: - return QNN_DATATYPE_FLOAT_16; - case GGML_TYPE_F32: - return QNN_DATATYPE_FLOAT_32; - case GGML_TYPE_I8: - return QNN_DATATYPE_INT_8; - case GGML_TYPE_Q8_0: - return QNN_DATATYPE_SFIXED_POINT_8; - case GGML_TYPE_Q4_0: - return QNN_DATATYPE_SFIXED_POINT_4; - default: - break; - } - return QNN_DATATYPE_UNDEFINED; -} +static Qnn_Tensor_t * ggml_qnn_create_compute_tensor(const ggml_tensor * tensor) { + uint32_t dimensions[] = {(uint32_t) tensor->ne[0], (uint32_t) tensor->ne[1], + (uint32_t) tensor->ne[2], (uint32_t) tensor->ne[3]}; + Qnn_DataType_t qnn_data_type = QNN_DATATYPE_FLOAT_32; + Qnn_TensorType_t qnn_tensor_type = QNN_TENSOR_TYPE_APP_WRITE; -//TODO: -static ggml_type ggml_datatype_from_qnn_datatype(Qnn_DataType_t qnn_type) { - switch (qnn_type) { - case QNN_DATATYPE_FLOAT_32: - return GGML_TYPE_F32; - case QNN_DATATYPE_FLOAT_16: - return GGML_TYPE_F16; - case QNN_DATATYPE_UINT_32: - case QNN_DATATYPE_INT_32: - return GGML_TYPE_I32; - case QNN_DATATYPE_INT_16: - return GGML_TYPE_I16; - case QNN_DATATYPE_INT_8: - return GGML_TYPE_I8; - case QNN_DATATYPE_SFIXED_POINT_8: - return GGML_TYPE_Q8_0; - case QNN_DATATYPE_SFIXED_POINT_4: - return GGML_TYPE_Q4_0; - default: - break; + if (tensor->flags & GGML_TENSOR_FLAG_INPUT) { + qnn_tensor_type = QNN_TENSOR_TYPE_APP_WRITE; + } else if (tensor->flags & GGML_TENSOR_FLAG_OUTPUT) { + qnn_tensor_type = QNN_TENSOR_TYPE_APP_READ; } - return GGML_TYPE_COUNT; -} -//TODO: add more ops -static const char * qnn_opname_from_ggmlop(enum ggml_op ggmlop) { - switch (ggmlop) { - case GGML_OP_ADD: - return QNN_OP_ELEMENT_WISE_ADD; - case GGML_OP_MUL_MAT: - return QNN_OP_MAT_MUL; - default: - break; - } - return nullptr; -} + qnn_data_type = qnn_datatype_from_ggml_datatype(tensor->type); + Qnn_Tensor_t * p_qnn_tensor = ggml_qnn_create_general_tensor(tensor, nullptr, + qnn_tensor_type, qnn_data_type, + ggml_n_dims(tensor), dimensions, + nullptr, 0); -static const char * get_ggml_type_name(ggml_type type) { - const auto * traits = ggml_get_type_traits(type); - return traits->type_name; + return p_qnn_tensor; } static void append_tensor_dimensions(const ggml_tensor * tensor, std::string & output) { @@ -1865,7 +1836,7 @@ class qnn_instance { uint8_t do_node_validation = 1, const QnnGraph_Config_t ** graph_configs = nullptr ); - int init_qnn_graph(const std::string &graph_name, QNNBackend device, size_t vtcm_size_in_mb); + int init_qnn_graph(const std::string &graph_name, QNNBackend device, size_t vtcm_size_in_mb = 8, size_t hvx_threads = 8); int finalize_qnn_graph(); @@ -2813,7 +2784,7 @@ int qnn_instance::qnn_finalize() { return ret_status; } -int qnn_instance::init_qnn_graph(const std::string & graph_name, QNNBackend device, size_t vtcm_size_in_mb) { +int qnn_instance::init_qnn_graph(const std::string & graph_name, QNNBackend device, size_t vtcm_size_in_mb, size_t hvx_threads) { _graph_name = graph_name; _device_id = device; @@ -2824,7 +2795,7 @@ int qnn_instance::init_qnn_graph(const std::string & graph_name, QNNBackend devi if (device == QNN_BACKEND_NPU) { QnnHtpGraph_CustomConfig_t hvx_config; hvx_config.option = QNN_HTP_GRAPH_CONFIG_OPTION_NUM_HVX_THREADS; - hvx_config.numHvxThreads = 8; + hvx_config.numHvxThreads = hvx_threads; QnnGraph_Config_t graph_hvx_config; graph_hvx_config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM; graph_hvx_config.customConfig = &hvx_config; @@ -2940,65 +2911,11 @@ static uint8_t * create_rpc_buffer(qnn_instance * instance, const ggml_tensor * return qnn_rpcbuffer; } -static Qnn_ErrorHandle_t create_htp_graph(ggml_backend_qnn_context * ctx, const std::string & graph_name, Qnn_GraphHandle_t * graph_handle) { - Qnn_ErrorHandle_t error = QNN_SUCCESS; - if (nullptr == ctx) - return QNN_MIN_ERROR_COMMON; - - qnn_instance * instance = ctx->instance; - QNN_INTERFACE_VER_TYPE qnn_raw_interface = ctx->raw_interface; - - QnnHtpGraph_CustomConfig_t hvx_config; - hvx_config.option = QNN_HTP_GRAPH_CONFIG_OPTION_NUM_HVX_THREADS; - hvx_config.numHvxThreads = 4; - QnnGraph_Config_t graph_hvx_config; - graph_hvx_config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM; - graph_hvx_config.customConfig = &hvx_config; - - QnnHtpGraph_CustomConfig_t dlbc_config; - dlbc_config.option = QNN_HTP_GRAPH_CONFIG_OPTION_OPTIMIZATION; - dlbc_config.optimizationOption.type = QNN_HTP_GRAPH_OPTIMIZATION_TYPE_ENABLE_DLBC; - dlbc_config.optimizationOption.floatValue = 1.0; // set to 0.0 to turn off DLBC - QnnGraph_Config_t graph_dlbc_config; - graph_dlbc_config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM; - graph_dlbc_config.customConfig = &dlbc_config; - - QnnHtpGraph_CustomConfig_t opt_config; - opt_config.optimizationOption.type = QNN_HTP_GRAPH_OPTIMIZATION_TYPE_FINALIZE_OPTIMIZATION_FLAG; - opt_config.optimizationOption.floatValue = 3; // 1 or 3 - QnnGraph_Config_t graph_opt_config; - graph_opt_config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM; - graph_opt_config.customConfig = &opt_config; - - QnnHtpGraph_CustomConfig_t vtcm_config; - vtcm_config.option = QNN_HTP_GRAPH_CONFIG_OPTION_VTCM_SIZE; - vtcm_config.vtcmSizeInMB = ctx->socinfo.vtcm_size_in_mb; - QnnGraph_Config_t graph_vtcm_config; - graph_vtcm_config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM; - graph_vtcm_config.customConfig = &vtcm_config; - - QnnHtpGraph_CustomConfig_t precision_config; - precision_config.option = QNN_HTP_GRAPH_CONFIG_OPTION_PRECISION; - precision_config.precision = QNN_PRECISION_FLOAT16; - QnnGraph_Config_t graph_precision_config; - graph_precision_config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM; - graph_precision_config.customConfig = &precision_config; - - const QnnGraph_Config_t * p_graphconfig[] = {&graph_hvx_config, - &graph_dlbc_config, - &graph_vtcm_config, - &graph_opt_config, - &graph_precision_config, - NULL}; - error = qnn_raw_interface.graphCreate(instance->get_qnn_context_handle(), - graph_name.c_str(), - p_graphconfig, graph_handle); - return error; -} - static void print_tensors_info(const char * func_name, ggml_backend_qnn_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { //skip sanity check of params - GGMLQNN_LOG_DEBUG("call %s in dev %s\n", func_name, ctx->name); + if (nullptr != func_name && nullptr != ctx) { + GGMLQNN_LOG_DEBUG("call %s in dev %s\n", func_name, ctx->name); + } GGMLQNN_LOG_DEBUG("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", src0->name, src0->type, ggml_type_name(src0->type), src0->ne[0], src0->ne[1], src0->ne[2], @@ -3019,29 +2936,14 @@ static void print_tensors_info(const char * func_name, ggml_backend_qnn_context static void dump_tensors_info(const struct ggml_tensor * tensor) { //skip sanity check of params - struct ggml_tensor * src0 = tensor->src[0]; + const struct ggml_tensor * src0 = tensor->src[0]; struct ggml_tensor * src1 = tensor->src[1]; + struct ggml_tensor * dst = const_cast(tensor); GGMLQNN_LOG_DEBUG("op name:%s, tensor type:%s", ggml_op_name(tensor->op), ggml_type_name(tensor->type)); GGMLQNN_LOG_DEBUG("src0 type:%s", ggml_type_name(tensor->src[0]->type)); GGMLQNN_LOG_DEBUG("src1 type:%s", ggml_type_name(tensor->src[1]->type)); - GGMLQNN_LOG_DEBUG( - "src0 %15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", - src0->name, - src0->type, ggml_type_name(src0->type), src0->ne[0], src0->ne[1], src0->ne[2], - src0->nb[0], src0->nb[1], src0->nb[2]); - GGMLQNN_LOG_DEBUG( - "src1 %15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", - src1->name, - src1->type, ggml_type_name(src1->type), src1->ne[0], src1->ne[1], src1->ne[2], - src1->nb[0], src1->nb[1], src1->nb[2]); - GGMLQNN_LOG_DEBUG( - " %15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", - tensor->name, - tensor->type, ggml_type_name(tensor->type), tensor->ne[0], tensor->ne[1], - tensor->ne[2], - tensor->nb[0], - tensor->nb[1], tensor->nb[2]); + print_tensors_info(nullptr, nullptr, src0, src1, dst); } //TODO: currently only support offloading 2D matrix to QNN backend @@ -3089,25 +2991,20 @@ static bool ggml_qnn_can_handle_op(const struct ggml_tensor * tensor) { int64_t ne1 = tensor->ne[1]; if (tensor->op == GGML_OP_ADD) { + //dump_tensors_info(tensor); if (!ggml_are_same_shape(src0, src1)) { return false; } if (ne00 < 32) return false; - -#if GGMLQNN_PRINT_OP_ADD_LOG - //dump_tensors_info(tensor); -#endif + return (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16) && (src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16); - } if (tensor->op == GGML_OP_MUL_MAT) { -#if GGMLQNN_PRINT_OP_MUL_MAT_LOG - //dump_tensors_info(tensor); -#endif + dump_tensors_info(tensor); uint32_t src0_rank = ggml_get_tensor_rank(src0); uint32_t src1_rank = ggml_get_tensor_rank(src1); @@ -3181,17 +3078,12 @@ static void ggml_qnn_add(ggml_backend_t backend, ggml_tensor * op) { if (!graph_initialized) { GGMLQNN_LOG_DEBUG("graph name %s", graph_name.c_str()); - if (ctx->device == QNN_BACKEND_NPU) { - error = create_htp_graph(ctx, graph_name, &graph_handle); - } else { - error = qnn_raw_interface.graphCreate(instance->get_qnn_context_handle(), - graph_name.c_str(), - nullptr, &graph_handle); - } + error = instance->init_qnn_graph(graph_name, static_cast(ctx->device), 8); if (QNN_SUCCESS != error) { GGMLQNN_LOG_INFO("can't create qnn graph handle with graph name %s, error = %d\n", graph_name.c_str(), error); return; } + graph_handle = instance->get_qnn_graph_handle(); if (enable_npu_rpc) { QNN_VER_PTR(*p_tensor0)->memType = QNN_TENSORMEMTYPE_MEMHANDLE; @@ -3391,9 +3283,9 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { p_param_tensor = tensors[3]; p_tensor2_transpose = tensors[4]; } else { - p_tensor0 = ggml_qnn_create_mulmat_tensor(src0, nullptr, QNN_TENSOR_TYPE_APP_WRITE,QNN_DATATYPE_FLOAT_32, 2, nullptr, nullptr, 0); - p_tensor1 = ggml_qnn_create_mulmat_tensor(src1, nullptr, QNN_TENSOR_TYPE_APP_WRITE,QNN_DATATYPE_FLOAT_32, 2, nullptr, nullptr, 0); - p_tensor2 = ggml_qnn_create_mulmat_tensor(dst, nullptr, QNN_TENSOR_TYPE_APP_READ,QNN_DATATYPE_FLOAT_32, 2, nullptr, nullptr, 0); + p_tensor0 = ggml_qnn_create_general_tensor(src0, nullptr, QNN_TENSOR_TYPE_APP_WRITE,QNN_DATATYPE_FLOAT_32, 2, nullptr, nullptr, 0); + p_tensor1 = ggml_qnn_create_general_tensor(src1, nullptr, QNN_TENSOR_TYPE_APP_WRITE,QNN_DATATYPE_FLOAT_32, 2, nullptr, nullptr, 0); + p_tensor2 = ggml_qnn_create_general_tensor(dst, nullptr, QNN_TENSOR_TYPE_APP_READ,QNN_DATATYPE_FLOAT_32, 2, nullptr, nullptr, 0); } print_tensors_info(__func__, ctx, src0, src1, dst); @@ -3443,7 +3335,7 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { //step-2: create param tensor for mulmat of 2d matrix uint32_t param_tensor_dims[] = {2}; uint32_t param_tensor_data[2] = {1, 0}; - p_param_tensor = ggml_qnn_create_mulmat_tensor(nullptr, "param", QNN_TENSOR_TYPE_STATIC,QNN_DATATYPE_UINT_32, 1, param_tensor_dims, param_tensor_data, 8); + p_param_tensor = ggml_qnn_create_general_tensor(nullptr, "param", QNN_TENSOR_TYPE_STATIC,QNN_DATATYPE_UINT_32, 1, param_tensor_dims, param_tensor_data, 8); CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_param_tensor)); //step-3: create compute tensor from ggml tensor @@ -3457,8 +3349,8 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { //step-4: create a transpose tensor uint32_t tensor2_transpose_dims[GGML_MAX_DIMS] = {}; - p_tensor2_transpose = ggml_qnn_create_mulmat_tensor(dst,"transpose",QNN_TENSOR_TYPE_NATIVE,QNN_DATATYPE_FLOAT_32, 2, nullptr, nullptr, 0); - get_qnn_dimensions_from_ggml_dimensions(tensor2_transpose_dims, tensor_2_dimensions,ggml_get_tensor_rank(dst)); + p_tensor2_transpose = ggml_qnn_create_general_tensor(dst, "transpose", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, 2, nullptr, nullptr, 0); + get_qnn_dimensions_from_ggml_dimensions(tensor2_transpose_dims, tensor_2_dimensions, ggml_get_tensor_rank(dst)); //save pointer because the dimensions of tensor p_tensor2_transpose will be changed later uint32_t * tensor2_dimensions_transpose = QNN_VER_PTR(*p_tensor2_transpose)->dimensions; //update dimensions of tensor p_tensor2_transpose to make QNN SDK happy @@ -3547,6 +3439,10 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { Qnn_Tensor_t tensor_outputs[] = { *p_tensor2 }; + //attention: + // this is the second technical approach of "how to utilize the Hexagon NPU maximally" through + // QNN SDK, details could be found at + // https://github.com/kantv-ai/llama.cpp/wiki/mapping-ggml-compute-graph-to-QNN-compute-graph CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, tensor_inputs, 2, tensor_outputs, 1, From 779a8d4850393a4fe5105e8ea2b4b9fb05f9e4a5 Mon Sep 17 00:00:00 2001 From: zhouwg Date: Thu, 20 Feb 2025 12:33:38 +0800 Subject: [PATCH 17/76] ggml-qnn: sync from branch kantvai-ggmlqnn-npurpc --- ggml/src/ggml-qnn/ggml-qnn.cpp | 282 ++++++++++++++++++++++----------- 1 file changed, 186 insertions(+), 96 deletions(-) diff --git a/ggml/src/ggml-qnn/ggml-qnn.cpp b/ggml/src/ggml-qnn/ggml-qnn.cpp index 37c947f412f1f..ee273503b9e8a 100644 --- a/ggml/src/ggml-qnn/ggml-qnn.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn.cpp @@ -13,9 +13,10 @@ * section-5 does ggml-qnn backend helper macro / data structure / function / class * section-6 does implementation of ggml-qnn backend according to ggml's backend subsystem * - * currently only provide OPs' QNN backend implementation of GGML_OP_ADD & GGML_OP_MUL_MAT: + * currently provide following ggml ops' QNN backend implementation: * - GGML_OP_ADD: this is a simple skeleton, can expand other ggml ops according to expertise - * - GGML_OP_MUL_MAT:this is a complicated skeleton, can expand other complex op accordingly + * - GGML_OP_MUL: this is a simple skeleton, can expand other ggml ops according to expertise + * - GGML_OP_MUL_MAT:this is a complicated skeleton, can expand other complex ggml ops accordingly * * of course, can porting ggml-qnn to Windows on ARM as need. * @@ -95,7 +96,6 @@ #include "ggml-qnn.h" #include "ggml-impl.h" #include "ggml-backend-impl.h" - // ================================================================================================= // section-1: forward/external declaration // ================================================================================================= @@ -110,9 +110,9 @@ static void ggmlqnn_log_internal(ggml_log_level level, const char * file, const // ================================================================================================= #define GGMLQNN_DEBUG 1 // for troubleshooting QNN backend #define GGML_QNN_LOGBUF_LEN 4096 -#define ENABLE_QNNBACKEND_PERF 1 // enable/disable op's perf info +#define ENABLE_QNNBACKEND_PERF 0 // enable/disable op's perf info #define GGMLQNN_PRINT_QNN_INTERNAL_LOG 0 // enable/disable QNN's internal log -#define GGMLQNN_PRINT_OP_ADD_LOG 1 // GGML_OP_ADD already verified with QNN-CPU / QNN-GPU / QNN-NPU +#define GGMLQNN_PRINT_OP_ADD_LOG 0 // GGML_OP_ADD already verified with QNN-CPU / QNN-GPU / QNN-NPU #define GGMLQNN_PRINT_OP_MUL_MAT_LOG 1 #define GGMLQNN_LOG_ERROR(...) ggmlqnn_log_internal(GGML_LOG_LEVEL_DEBUG, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) @@ -1187,25 +1187,28 @@ static struct ggml_backend_qnn_context g_qnn_mgr[GGML_QNN_MAX_DEVICES] = { .socinfo = {}}, }; - struct qnn_op_caps_t { const char * qnn_op_name = nullptr; const size_t input_param_count = 0; const char * qnn_param_name = nullptr; }; -static const qnn_op_caps_t kOpCaps[] = { +static const qnn_op_caps_t k_op_caps[] = { {}, // GGML_OP_NONE {}, // GGML_OP_DUP { // GGML_OP_ADD - QNN_OP_ELEMENT_WISE_ADD, // qnn_op_name - 2, // input_param_count + QNN_OP_ELEMENT_WISE_ADD, + 2, }, {}, // GGML_OP_ADD1 {}, // GGML_OP_ACC {}, // GGML_OP_SUB - {}, // GGML_OP_MUL + { + // GGML_OP_MUL + QNN_OP_ELEMENT_WISE_MULTIPLY, + 2, + }, {}, // GGML_OP_DIV {}, // GGML_OP_SQR {}, // GGML_OP_SQRT @@ -1227,8 +1230,8 @@ static const qnn_op_caps_t kOpCaps[] = { {}, // GGML_OP_GROUP_NORM { // GGML_OP_MUL_MAT - QNN_OP_MAT_MUL, // qnn_op_name - 2, // input_param_count + QNN_OP_MAT_MUL, + 2, }, {}, // GGML_OP_MUL_MAT_ID {}, // GGML_OP_OUT_PROD @@ -1580,11 +1583,9 @@ static void append_tensor_dimensions(const ggml_tensor * tensor, std::string & o output.append(buffer, len); } -constexpr const size_t kGgmlUnaryOpStart = GGML_OP_COUNT; - static size_t get_qnn_op_index(const ggml_tensor * tensor) { if (tensor->op == GGML_OP_UNARY) { - return kGgmlUnaryOpStart + ggml_get_unary_op(tensor); + return GGML_OP_COUNT + ggml_get_unary_op(tensor); } return tensor->op; @@ -1592,8 +1593,8 @@ static size_t get_qnn_op_index(const ggml_tensor * tensor) { static size_t get_qnn_op_input_param_count(const ggml_tensor * op) { auto op_index = get_qnn_op_index(op); - GGML_ASSERT(op_index < std::size(kOpCaps)); - return kOpCaps[op_index].input_param_count; + GGML_ASSERT(op_index < std::size(k_op_caps)); + return k_op_caps[op_index].input_param_count; } static void get_graph_key_from_op(const ggml_tensor * op, std::string & output) { @@ -1796,21 +1797,21 @@ class qnn_instance { int qnn_finalize(); - const qnn_interface &get_qnn_interface() { + const qnn_interface & get_qnn_interface() { if (!_qnn_interface.is_loaded()) { GGMLQNN_LOG_WARN("pls check why _qnn_interface is not loaded\n"); } return _qnn_interface; } - const QNN_INTERFACE_VER_TYPE &get_qnn_raw_interface() { + const QNN_INTERFACE_VER_TYPE & get_qnn_raw_interface() { if (!_qnn_interface.is_loaded()) { GGMLQNN_LOG_WARN("pls check why _qnn_interface is not loaded\n"); } return _qnn_raw_interface; } - const QNN_SYSTEM_INTERFACE_VER_TYPE &get_qnn_raw_system_interface() { + const QNN_SYSTEM_INTERFACE_VER_TYPE & get_qnn_raw_system_interface() { if (!_qnn_interface.is_loaded()) { GGMLQNN_LOG_WARN("pls check why _qnn_interface is not loaded\n"); } @@ -1836,7 +1837,7 @@ class qnn_instance { uint8_t do_node_validation = 1, const QnnGraph_Config_t ** graph_configs = nullptr ); - int init_qnn_graph(const std::string &graph_name, QNNBackend device, size_t vtcm_size_in_mb = 8, size_t hvx_threads = 8); + int init_qnn_graph(const std::string & graph_name, QNNBackend device, size_t vtcm_size_in_mb = 8, size_t hvx_threads = 8); int finalize_qnn_graph(); @@ -1850,8 +1851,8 @@ class qnn_instance { return 1; } - QnnHtpDevice_Infrastructure_t *htp_infra = static_cast(device_infra); - QnnHtpDevice_PerfInfrastructure_t *htp_perfinfra = &htp_infra->perfInfra; + QnnHtpDevice_Infrastructure_t * htp_infra = static_cast(device_infra); + QnnHtpDevice_PerfInfrastructure_t * htp_perfinfra = &htp_infra->perfInfra; uint32_t power_configid = 1; uint32_t device_id = 0; uint32_t core_id = 0; @@ -1925,6 +1926,7 @@ class qnn_instance { } size_t get_rpcmem_capacity() { return _rpcmem_capacity; } + size_t get_rpcmem_usage() { return _rpcmem_usage; } int32_t rpcmem_to_fd(void * buf); @@ -1950,6 +1952,32 @@ class qnn_instance { return _enable_qnn_rpc; } + void probe_device_meminfo() { + size_t candidate_size = 0; + uint8_t *rpc_buffer = nullptr; + const int SIZE_IN_MB = (1 << 20); + size_t probe_slots[] = {1024, 1536, 2048 - 48, 2048}; + size_t probe_counts = sizeof(probe_slots) / sizeof(size_t); + for (size_t idx = 0; idx < probe_counts; idx++) { + rpc_buffer = static_cast(alloc_rpcmem_internal(probe_slots[idx] * SIZE_IN_MB, 4)); + if (nullptr == rpc_buffer) { + GGMLQNN_LOG_DEBUG("alloc rpcmem %d (MB) failure, %s\n", probe_slots[idx], + strerror(errno)); + break; + } else { + candidate_size = probe_slots[idx]; + free_rpcmem(rpc_buffer); + rpc_buffer = nullptr; + } + } + if (candidate_size > _rpcmem_capacity) + _rpcmem_capacity = candidate_size; + + free_rpcmem(); + _rpcmem_usage = 0; + GGMLQNN_LOG_INFO("capacity of rpc ion memory %d MB\n", _rpcmem_capacity); + } + public: std::map>> _qnn_graph_map; @@ -1969,6 +1997,8 @@ class qnn_instance { void set_qnn_raw_system_interface(QNN_SYSTEM_INTERFACE_VER_TYPE & raw_interface) { _qnn_raw_system_interface = raw_interface; } + + void * alloc_rpcmem_internal(size_t bytes, size_t alignment); private: static constexpr const int _required_num_providers = 1; @@ -1987,7 +2017,7 @@ class qnn_instance { qnn_interface _qnn_interface; - void *_system_lib_handle = nullptr; + void * _system_lib_handle = nullptr; Qnn_GraphHandle_t _qnn_graph_handle = nullptr; @@ -2013,7 +2043,6 @@ class qnn_instance { std::unordered_map _qnn_mem_set; std::unordered_map _qnn_rpc_buffer_to_handles; - static std::mutex _init_mutex; static std::unordered_map _loaded_lib_handle; static std::unordered_map _lib_path_to_backend_id; @@ -2027,7 +2056,9 @@ class qnn_instance { pfn_rpc_mem_init _pfn_rpc_mem_init; pfn_rpc_mem_deinit _pfn_rpc_mem_deinit; std::unordered_map _rpcmem_store_map; - size_t _rpcmem_capacity = 512; + std::unordered_map _rpcmem_usage_map; + size_t _rpcmem_capacity = 512; // mempool size in Mbytes + size_t _rpcmem_usage = 0; // mempool usage in MBytes std::string _graph_name; QNNBackend _device_id; @@ -2042,7 +2073,7 @@ std::unordered_map qnn_instance::_loaded_li std::unordered_map qnn_instance::_lib_path_to_backend_id; std::unordered_map qnn_instance::_loaded_backend; -void * qnn_instance::alloc_rpcmem(size_t bytes, size_t alignment) { +void * qnn_instance::alloc_rpcmem_internal(size_t bytes, size_t alignment) { if (!_rpcmem_initialized) { GGMLQNN_LOG_WARN("rpc memory not initialized\n"); return nullptr; @@ -2062,17 +2093,50 @@ void * qnn_instance::alloc_rpcmem(size_t bytes, size_t alignment) { GGMLQNN_LOG_WARN("failed to allocate rpc memory\n"); _pfn_rpc_mem_free(buf); } + return aligned_buf; +} + +void * qnn_instance::alloc_rpcmem(size_t bytes, size_t alignment) { + if (_rpcmem_usage > (_rpcmem_capacity - 8)) { // reserve 8Mbytes in rpc mempool + GGMLQNN_LOG_WARN("rpc mempool capcaity: %d MB, usage: %d MB", _rpcmem_capacity, _rpcmem_usage); + return nullptr; + } + + auto aligned_buf = alloc_rpcmem_internal(bytes, alignment); + if (nullptr == aligned_buf) + return nullptr; + _rpcmem_usage_map.insert(std::pair(aligned_buf, bytes)); + size_t rpcmem_usage_in_bytes = _rpcmem_usage * (1 << 20); + rpcmem_usage_in_bytes += bytes; + _rpcmem_usage = rpcmem_usage_in_bytes / ( 1 << 20); return aligned_buf; } void qnn_instance::free_rpcmem(void * buf) { + size_t rpcbuffer_size = 0; if (!_rpcmem_initialized) { GGMLQNN_LOG_WARN("rpc memory not initialized\n"); } else if (0 == _rpcmem_store_map.count(buf)) { GGMLQNN_LOG_WARN("no allocated tensor\n"); } else { GGMLQNN_LOG_DEBUG("free rpc mem %p", _rpcmem_store_map[buf]); + for (std::unordered_map::iterator it = _rpcmem_usage_map.begin(); + it != _rpcmem_usage_map.end(); + it++) { + void * rpcbuffer = it->first; + if (buf == rpcbuffer) { + rpcbuffer_size = it->second; + size_t rpcmem_usage_in_bytes = _rpcmem_usage * (1 << 20); + rpcmem_usage_in_bytes -= rpcbuffer_size; + _rpcmem_usage = rpcmem_usage_in_bytes / ( 1 << 20); + } + } + if (rpcbuffer_size != 0) { + _rpcmem_usage_map.erase(buf); + } else { + GGMLQNN_LOG_WARN("it shouldn't happen, pls check why?"); + } _pfn_rpc_mem_free(_rpcmem_store_map[buf]); _rpcmem_store_map.erase(buf); } @@ -2094,6 +2158,8 @@ void qnn_instance::free_rpcmem() { _pfn_rpc_mem_free(rpcbuffer); } _rpcmem_store_map.clear(); + _rpcmem_usage_map.clear(); + _rpcmem_usage = 0; } int32_t qnn_instance::rpcmem_to_fd(void * buf) { @@ -2177,7 +2243,11 @@ Qnn_MemHandle_t qnn_instance::register_rpcmem(void * p_data, const uint32_t ran } GGMLQNN_LOG_DEBUG("mem_fd %d", mem_fd); - Qnn_MemDescriptor_t descriptor = {{rank, dimensions, nullptr}, data_type, QNN_MEM_TYPE_ION, {{mem_fd}}}; + Qnn_MemDescriptor_t descriptor = { + {rank, dimensions, nullptr}, + data_type, QNN_MEM_TYPE_ION, + {{mem_fd}} + }; Qnn_MemHandle_t handle = nullptr; auto error = _qnn_interface.qnn_mem_register(_qnn_context_handle, &descriptor, /*numDescriptors=*/1, &handle); @@ -2318,7 +2388,7 @@ int qnn_instance::load_backend(std::string & lib_path, const QnnSaver_Config_t * _loaded_lib_handle[backend_id] = lib_handle; _backend_id = backend_id; -#if 0 // keep them here for further use +#if 0 // leave them here for further use QnnSaver_Config_t outputdir_cfg; outputdir_cfg.option = QNN_SAVER_CONFIG_OPTION_OUTPUT_DIRECTORY; outputdir_cfg.outputDirectory = "/data/local/tmp/"; @@ -2468,6 +2538,7 @@ int qnn_instance::unload_system() { return result; } +#if GGMLQNN_PRINT_QNN_INTERNAL_LOG static void ggml_qnn_logcallback(const char * fmt, QnnLog_Level_t level, uint64_t timestamp, @@ -2499,24 +2570,25 @@ static void ggml_qnn_logcallback(const char * fmt, } double ms = (double) timestamp / 1000000.0; - { std::lock_guard lock(log_mutex); - memset(s_ggml_qnn_logbuf, 0, GGML_QNN_LOGBUF_LEN); vsnprintf(reinterpret_cast(s_ggml_qnn_logbuf), GGML_QNN_LOGBUF_LEN, fmt, argp); -#if GGMLQNN_PRINT_QNN_INTERNAL_LOG GGMLQNN_LOG_INFO("%8.1fms [%-7s] %s\n", ms, log_level_desc, s_ggml_qnn_logbuf); -#endif } } +#else +static void ggml_qnn_logcallback(const char * fmt, + QnnLog_Level_t level, + uint64_t timestamp, + va_list argp) { +} +#endif int qnn_instance::qnn_init(const QnnSaver_Config_t ** saver_config) { BackendIdType backend_id = QNN_BACKEND_ID_NULL; GGMLQNN_LOG_DEBUG("enter qni_init\n"); - const std::lock_guard lock(_init_mutex); - if (0 != load_system()) { GGMLQNN_LOG_WARN("can not load QNN system lib, pls check why?\n"); return 1; @@ -2542,9 +2614,7 @@ int qnn_instance::qnn_init(const QnnSaver_Config_t ** saver_config) { _loaded_lib_handle.count(backend_id)); return 3; } - _qnn_interface.set_qnn_interface(_loaded_backend[backend_id]); - #if 1 _qnn_interface.qnn_log_create(ggml_qnn_logcallback, _qnn_log_level, &_qnn_log_handle); #else @@ -2671,25 +2741,7 @@ int qnn_instance::qnn_init(const QnnSaver_Config_t ** saver_config) { } _qnn_raw_interface.deviceFreePlatformInfo(nullptr, p_info); - size_t candidate_size = 0; - uint8_t * rpc_buffer = nullptr; - const int SIZE_IN_MB = (1 << 20); - size_t probe_slots[] = {1024, 1536, 2048 - 48, 2048}; - size_t probe_counts = sizeof(probe_slots) / sizeof(size_t); - for (size_t idx = 0; idx < probe_counts; idx++) { - rpc_buffer = static_cast(alloc_rpcmem(probe_slots[idx] * SIZE_IN_MB, 4)); - if (nullptr == rpc_buffer) { - GGMLQNN_LOG_DEBUG("alloc rpcmem %d (MB) failure, %s\n", probe_slots[idx], strerror(errno)); - break; - } else { - candidate_size = probe_slots[idx]; - free_rpcmem(rpc_buffer); - rpc_buffer = nullptr; - } - } - if (candidate_size > _rpcmem_capacity) - _rpcmem_capacity = candidate_size; - GGMLQNN_LOG_INFO("capacity of rpc ion memory %d MB\n", _rpcmem_capacity); + probe_device_meminfo(); if (0 != init_htp_perfinfra()) { GGMLQNN_LOG_WARN("initialize HTP performance failure"); @@ -2963,6 +3015,7 @@ static void get_qnn_dimensions_from_ggml_dimensions(uint32_t * qnn_dimensions, u // ================================================================================================= // section-6: implementation of ggml-qnn backend // ================================================================================================= +//TODO: refine this function as it is a performance hotspot/bottleneck function static bool ggml_qnn_can_handle_op(const struct ggml_tensor * tensor) { if (tensor->op == GGML_OP_NONE) { return true; @@ -2973,7 +3026,9 @@ static bool ggml_qnn_can_handle_op(const struct ggml_tensor * tensor) { return false; } - bool supported_op = ((tensor->op == GGML_OP_ADD) || (tensor->op == GGML_OP_MUL_MAT)); + //TODO: support other op + bool supported_op = ((tensor->op == GGML_OP_ADD) || (tensor->op == GGML_OP_MUL_MAT) + || (tensor->op == GGML_OP_MUL)); if (!supported_op) { return false; } @@ -2981,37 +3036,34 @@ static bool ggml_qnn_can_handle_op(const struct ggml_tensor * tensor) { struct ggml_tensor * src0 = tensor->src[0]; struct ggml_tensor * src1 = tensor->src[1]; - int64_t ne00 = tensor->src[0]->ne[0]; - int64_t ne01 = tensor->src[0]->ne[1]; + const int64_t ne00 = tensor->src[0]->ne[0]; + const int64_t ne01 = tensor->src[0]->ne[1]; - int64_t ne10 = tensor->src[1]->ne[0]; - int64_t ne11 = tensor->src[1]->ne[1]; + const int64_t ne10 = tensor->src[1]->ne[0]; + const int64_t ne11 = tensor->src[1]->ne[1]; - int64_t ne0 = tensor->ne[0]; - int64_t ne1 = tensor->ne[1]; + const int64_t ne0 = tensor->ne[0]; + const int64_t ne1 = tensor->ne[1]; + + const uint32_t src0_rank = ggml_get_tensor_rank(src0); + const uint32_t src1_rank = ggml_get_tensor_rank(src1); if (tensor->op == GGML_OP_ADD) { //dump_tensors_info(tensor); if (!ggml_are_same_shape(src0, src1)) { return false; } - if (ne00 < 32) return false; - return (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16) && (src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16); } if (tensor->op == GGML_OP_MUL_MAT) { - dump_tensors_info(tensor); - uint32_t src0_rank = ggml_get_tensor_rank(src0); - uint32_t src1_rank = ggml_get_tensor_rank(src1); - + //dump_tensors_info(tensor); if ((src0_rank != 2) || (src1_rank != 2)) //TODO: only support offload 2D matrix mulmat to QNN backend return false; - - //TODO: support more data type in func ggml_qnn_mul_mat(...): + //TODO: support more data type in func ggml_qnn_mul_mat(...) //src0: q4_0, q6_k, ... //src1: f32 //dst : f32 @@ -3020,19 +3072,30 @@ static bool ggml_qnn_can_handle_op(const struct ggml_tensor * tensor) { && (src0->type == src1->type) && (src0->type == tensor->type); } - //TODO:for other op + if (tensor->op == GGML_OP_MUL) { + dump_tensors_info(tensor); + if ((src0_rank != 2) || (src1_rank != 2)) //TODO: only support offload 2D matrix mul to QNN backend + return false; + return (src0->type == GGML_TYPE_F32) + && (src1->type == GGML_TYPE_F32) + && (tensor->type == src1->type); + } + return (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16) && (src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16) && (src0->type == src1->type) && (src0->type == tensor->type); } -static void ggml_qnn_add(ggml_backend_t backend, ggml_tensor * op) { +/* + * provide a general skeleton to offload ggml op to QNN backend: a single node contains 2 input + * tensor and 1 output tensor +*/ +static void ggml_qnn_general_node(ggml_backend_t backend, ggml_tensor * op) { Qnn_ErrorHandle_t error = QNN_SUCCESS; enum ggml_status result = GGML_STATUS_SUCCESS; bool graph_initialized = false; qnn_instance * instance = nullptr; ggml_backend_qnn_context * ctx = (ggml_backend_qnn_context *)backend->context; - qnn_perf op_perf = qnn_perf("ggml_qnn_add"); Qnn_GraphHandle_t graph_handle = nullptr; Qnn_Tensor_t * p_tensor0 = nullptr; Qnn_Tensor_t * p_tensor1 = nullptr; @@ -3045,6 +3108,14 @@ static void ggml_qnn_add(ggml_backend_t backend, ggml_tensor * op) { GGMLQNN_CHECK_PARAMS(ctx, src0, src1, dst); instance = ctx->instance; QNN_INTERFACE_VER_TYPE qnn_raw_interface = ctx->raw_interface; + + size_t qnn_op_index = get_qnn_op_index(op); + GGML_ASSERT(qnn_op_index < std::size(k_op_caps)); + const char * qnn_op_name = k_op_caps[qnn_op_index].qnn_op_name; + std::string ggml_op_name_string = std::string("ggml_") + ggml_op_name(op->op); + const char * ggml_op_name = ggml_op_name_string.c_str(); + + qnn_perf op_perf = qnn_perf(ggml_op_name); op_perf.start(); std::string graph_name; @@ -3124,9 +3195,9 @@ static void ggml_qnn_add(ggml_backend_t backend, ggml_tensor * op) { }; Qnn_OpConfig_t op_config = { QNN_OPCONFIG_VERSION_1, .v1 = { - "ggml_op_add", + ggml_op_name, QNN_OP_PACKAGE_NAME_QTI_AISW, - QNN_OP_ELEMENT_WISE_ADD, + qnn_op_name, 0, qnn_params, 2, @@ -3138,9 +3209,9 @@ static void ggml_qnn_add(ggml_backend_t backend, ggml_tensor * op) { CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, op_config)); CHECK_QNN_API(error, qnn_raw_interface.graphFinalize(graph_handle, nullptr, nullptr)); CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, - tensor_inputs, 2, - tensor_outputs, 1, - nullptr, nullptr)); + tensor_inputs, 2, + tensor_outputs, 1, + nullptr, nullptr)); if (enable_npu_rpc) { uint8_t * qnn_rpcbuffer = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*p_tensor2)->memHandle)); @@ -3214,9 +3285,9 @@ static void ggml_qnn_add(ggml_backend_t backend, ggml_tensor * op) { *p_tensor2 }; CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, - tensor_inputs, 2, - tensor_outputs, 1, - nullptr, nullptr)); + tensor_inputs, 2, + tensor_outputs, 1, + nullptr, nullptr)); if (enable_npu_rpc) { //TODO:NPU RPC feature will failed with test-backend-ops @@ -3231,18 +3302,17 @@ static void ggml_qnn_add(ggml_backend_t backend, ggml_tensor * op) { QNN_VER_PTR(*p_tensor0)->dimensions = tensor_0_dimensions; QNN_VER_PTR(*p_tensor1)->dimensions = tensor_1_dimensions; QNN_VER_PTR(*p_tensor2)->dimensions = tensor_2_dimensions; -#if GGMLQNN_PRINT_OP_ADD_LOG op_perf.info(); -#endif } /* - * the logic of ggml_qnn_mul_mat is similar to ggml_qnn_add but much more complicated than ggml_qnn_add, + * the logic of ggml_qnn_mul_mat is similar to ggml_qnn_general_node but much more complicated + * than ggml_qnn_general_node. * matrix transpose and type trait are required for offload mulmat to QNN backend, * so it's a standalone function. accordingly, this is another typical skeleton for offload other * ggml ops to QNN backend * - * MUL_MAT take most of the compute time (about 95%).so to speed up llama inference, we should focus on MUL_MAT. + * MUL_MAT take most of the compute time (about 95%).so to speed up llama inference, should focus on MUL_MAT. * * have three kinds of MUL_MAT to compute: * mul_mat_f32: both src0 and src1 are F32, this will be naturally handled in QNN backend @@ -3288,7 +3358,7 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { p_tensor2 = ggml_qnn_create_general_tensor(dst, nullptr, QNN_TENSOR_TYPE_APP_READ,QNN_DATATYPE_FLOAT_32, 2, nullptr, nullptr, 0); } - print_tensors_info(__func__, ctx, src0, src1, dst); + //print_tensors_info(__func__, ctx, src0, src1, dst); //ensure QNN tensor has correct tensor type QNN_VER_PTR(*p_tensor0)->type = QNN_TENSOR_TYPE_APP_WRITE; @@ -3444,8 +3514,8 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { // QNN SDK, details could be found at // https://github.com/kantv-ai/llama.cpp/wiki/mapping-ggml-compute-graph-to-QNN-compute-graph CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, - tensor_inputs, 2, - tensor_outputs, 1, + tensor_inputs, 2, + tensor_outputs, 1, nullptr, nullptr)); } @@ -3453,7 +3523,6 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { QNN_VER_PTR(*p_tensor0)->dimensions = tensor_0_dimensions; QNN_VER_PTR(*p_tensor1)->dimensions = tensor_1_dimensions; QNN_VER_PTR(*p_tensor2)->dimensions = tensor_2_dimensions; - op_perf.info(); } @@ -3462,13 +3531,17 @@ static bool ggml_qnn_compute_forward(ggml_backend_t backend, struct ggml_tensor switch (tensor->op) { case GGML_OP_ADD: - func = ggml_qnn_add; + func = ggml_qnn_general_node; break; case GGML_OP_MUL_MAT: func = ggml_qnn_mul_mat; break; + case GGML_OP_MUL: + func = ggml_qnn_general_node; + break; + default: return false; } @@ -3667,7 +3740,6 @@ static enum ggml_status ggml_backend_qnn_graph_compute(ggml_backend_t backend, s ggml_backend_qnn_context * ctx = (ggml_backend_qnn_context *) backend->context; GGML_UNUSED(ctx); - GGMLQNN_LOG_DEBUG("cgraph->n_nodes %d", cgraph->n_nodes); for (int i = 0; i < cgraph->n_nodes; i++) { ggml_tensor * node = cgraph->nodes[i]; if (ggml_is_empty(node) || node->op == GGML_OP_RESHAPE @@ -3715,10 +3787,28 @@ static const char * ggml_backend_qnn_device_get_description(ggml_backend_dev_t d } static void ggml_backend_qnn_device_get_memory(ggml_backend_dev_t dev, size_t * free, size_t * total) { - //FIXME:this is NOT QNN device memory info - *free = get_system_free_memory_in_bytes(); - *total = get_system_total_memory_in_bytes(); - GGML_UNUSED(dev); + struct ggml_backend_qnn_context * ctx = static_cast(dev->context); + if ((nullptr == ctx) || (ctx->device > QNN_BACKEND_GGML)) { + GGMLQNN_LOG_ERROR("pls check params"); + *free = 0; + *total = 0; + } + + if (QNN_BACKEND_CPU == ctx->device || QNN_BACKEND_GGML == ctx->device) { + *total = get_system_total_memory_in_bytes(); + *free = get_system_free_memory_in_bytes(); + } else if (QNN_BACKEND_GPU == ctx->device) { + //TODO: probe GPU info in Qualcomm Adreno GPU + *total = get_system_total_memory_in_bytes(); + *free = get_system_free_memory_in_bytes(); + } else if (QNN_BACKEND_NPU == ctx->device) { + size_t rpc_ion_memsize = ctx->instance->get_rpcmem_capacity(); + size_t rpc_ion_usage = ctx->instance->get_rpcmem_usage(); + GGMLQNN_LOG_DEBUG("rpc memsize %d", rpc_ion_memsize); + GGMLQNN_LOG_DEBUG("rpc usage %d", rpc_ion_usage); + *total = rpc_ion_memsize * (1 << 20); + *free = (rpc_ion_memsize - rpc_ion_usage) * (1 << 20); + } } static enum ggml_backend_dev_type ggml_backend_qnn_device_get_type(ggml_backend_dev_t dev) { From a19204dc9689a798109b8e994fe017e242b6a131 Mon Sep 17 00:00:00 2001 From: zhouwg Date: Thu, 20 Feb 2025 22:20:15 +0800 Subject: [PATCH 18/76] ggml-qnn: sync from branch kantvai-ggmlqnn-npurpc --- ggml/src/ggml-qnn/ggml-qnn.cpp | 106 +++++++++++++++++++-------------- 1 file changed, 60 insertions(+), 46 deletions(-) diff --git a/ggml/src/ggml-qnn/ggml-qnn.cpp b/ggml/src/ggml-qnn/ggml-qnn.cpp index ee273503b9e8a..9ef502421c051 100644 --- a/ggml/src/ggml-qnn/ggml-qnn.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn.cpp @@ -1483,15 +1483,13 @@ static Qnn_Tensor_t * ggml_qnn_create_general_tensor(const ggml_tensor * tensor, GGMLQNN_LOG_DEBUG("init_tensor %d", get_idx()); inc_idx(); - //there are different dimension order between ggml tensor and qnn tensor uint32_t dimensions_transpose[GGML_MAX_DIMS] = {}; uint32_t * tensor_dims = nullptr; - if (nullptr != tensor) { - dimensions_transpose[0] = (uint32_t) tensor->ne[1]; - dimensions_transpose[1] = (uint32_t) tensor->ne[0]; - dimensions_transpose[2] = (uint32_t) tensor->ne[2]; - dimensions_transpose[3] = (uint32_t) tensor->ne[3]; + //there are different dimension order between ggml tensor and qnn tensor + for (size_t idx = 0; idx < rank; idx++) { + dimensions_transpose[idx] = (uint32_t)tensor->ne[rank - 1 - idx]; + } tensor_dims = dimensions_transpose; } //re-assign tensor_dims @@ -2058,7 +2056,7 @@ class qnn_instance { std::unordered_map _rpcmem_store_map; std::unordered_map _rpcmem_usage_map; size_t _rpcmem_capacity = 512; // mempool size in Mbytes - size_t _rpcmem_usage = 0; // mempool usage in MBytes + size_t _rpcmem_usage = 0; // mempool usage in Mbytes std::string _graph_name; QNNBackend _device_id; @@ -2968,33 +2966,27 @@ static void print_tensors_info(const char * func_name, ggml_backend_qnn_context if (nullptr != func_name && nullptr != ctx) { GGMLQNN_LOG_DEBUG("call %s in dev %s\n", func_name, ctx->name); } - GGMLQNN_LOG_DEBUG("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", + GGMLQNN_LOG_DEBUG("%-6s: type = %i (%s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi, %5zi)", src0->name, - src0->type, ggml_type_name(src0->type), src0->ne[0], src0->ne[1], src0->ne[2], - src0->nb[0], src0->nb[1], src0->nb[2]); - GGMLQNN_LOG_DEBUG("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", + src0->type, ggml_type_name(src0->type), src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], + src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3]); + GGMLQNN_LOG_DEBUG("%-6s: type = %i (%s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi, %5zi)", src1->name, - src1->type, ggml_type_name(src1->type), src1->ne[0], src1->ne[1], src1->ne[2], - src1->nb[0], src1->nb[1], src1->nb[2]); - GGMLQNN_LOG_DEBUG("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", + src1->type, ggml_type_name(src1->type), src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3], + src1->nb[0], src1->nb[1], src1->nb[2], src1->nb[3]); + GGMLQNN_LOG_DEBUG("%-6s: type = %i (%s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi, %5zi)", dst->name, - dst->type, ggml_type_name(dst->type), dst->ne[0], dst->ne[1], dst->ne[2], dst->nb[0], - dst->nb[1], dst->nb[2]); - GGMLQNN_LOG_DEBUG("%d, %d, %d, %d", src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3]); - GGMLQNN_LOG_DEBUG("tensor0 name %s", src0->name); - GGMLQNN_LOG_DEBUG("tensor1 name %s", src1->name); - GGMLQNN_LOG_DEBUG("tensor2 name %s", dst->name); + dst->type, ggml_type_name(dst->type), dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], + dst->nb[0], dst->nb[1], dst->nb[2], dst->nb[3]); + GGMLQNN_LOG_DEBUG("\n"); } -static void dump_tensors_info(const struct ggml_tensor * tensor) { +static void dump_op_info(const struct ggml_tensor * tensor) { //skip sanity check of params const struct ggml_tensor * src0 = tensor->src[0]; - struct ggml_tensor * src1 = tensor->src[1]; - struct ggml_tensor * dst = const_cast(tensor); - GGMLQNN_LOG_DEBUG("op name:%s, tensor type:%s", ggml_op_name(tensor->op), - ggml_type_name(tensor->type)); - GGMLQNN_LOG_DEBUG("src0 type:%s", ggml_type_name(tensor->src[0]->type)); - GGMLQNN_LOG_DEBUG("src1 type:%s", ggml_type_name(tensor->src[1]->type)); + struct ggml_tensor * src1 = tensor->src[1]; + struct ggml_tensor * dst = const_cast(tensor); + GGMLQNN_LOG_DEBUG("op name:%s, tensor type:%s", ggml_op_name(tensor->op), ggml_type_name(tensor->type)); print_tensors_info(nullptr, nullptr, src0, src1, dst); } @@ -3008,8 +3000,13 @@ static void get_qnn_dimensions_from_ggml_dimensions(uint32_t * qnn_dimensions, u GGMLQNN_LOG_WARN("invalid params"); return; } - qnn_dimensions[0] = ggml_dimensions[1]; - qnn_dimensions[1] = ggml_dimensions[0]; + for (size_t idx = 0; idx < GGML_MAX_DIMS; idx++) + qnn_dimensions[idx] = ggml_dimensions[idx]; + + if (rank >= 2) { + qnn_dimensions[rank - 1] = ggml_dimensions[rank - 2]; + qnn_dimensions[rank - 2] = ggml_dimensions[rank - 1]; + } } // ================================================================================================= @@ -3060,9 +3057,16 @@ static bool ggml_qnn_can_handle_op(const struct ggml_tensor * tensor) { } if (tensor->op == GGML_OP_MUL_MAT) { - //dump_tensors_info(tensor); - if ((src0_rank != 2) || (src1_rank != 2)) //TODO: only support offload 2D matrix mulmat to QNN backend + dump_op_info(tensor); + if (src0_rank != src1_rank) // make QNN SDK happy + return false; + if (src0_rank < 2) // make QNN SDK happy + return false; + if (src0_rank > 3) //TODO: 4D matrix return false; + if ((src1->ne[2] != src0->ne[2]) || (src1->ne[3] != src0->ne[3])) // make QNN SDK happy + return false; + //TODO: support more data type in func ggml_qnn_mul_mat(...) //src0: q4_0, q6_k, ... //src1: f32 @@ -3073,8 +3077,8 @@ static bool ggml_qnn_can_handle_op(const struct ggml_tensor * tensor) { } if (tensor->op == GGML_OP_MUL) { - dump_tensors_info(tensor); - if ((src0_rank != 2) || (src1_rank != 2)) //TODO: only support offload 2D matrix mul to QNN backend + //dump_tensors_info(tensor); + if ((src0_rank != 2) || (src1_rank != 2)) //TODO: 3D and 4D matrix return false; return (src0->type == GGML_TYPE_F32) && (src1->type == GGML_TYPE_F32) @@ -3340,6 +3344,11 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { QNN_INTERFACE_VER_TYPE qnn_raw_interface = ctx->raw_interface; op_perf.start(); + uint32_t src0_rank = ggml_get_tensor_rank(src0); + uint32_t src1_rank = ggml_get_tensor_rank(src1); + GGML_ASSERT(src0_rank == src1_rank); + GGML_ASSERT(src0_rank >= 2); //QNN SDK's limitation + std::string graph_name; get_graph_key_from_op(op, graph_name); if (instance->_qnn_graph_map.find(graph_name) != instance->_qnn_graph_map.end()) { @@ -3353,12 +3362,12 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { p_param_tensor = tensors[3]; p_tensor2_transpose = tensors[4]; } else { - p_tensor0 = ggml_qnn_create_general_tensor(src0, nullptr, QNN_TENSOR_TYPE_APP_WRITE,QNN_DATATYPE_FLOAT_32, 2, nullptr, nullptr, 0); - p_tensor1 = ggml_qnn_create_general_tensor(src1, nullptr, QNN_TENSOR_TYPE_APP_WRITE,QNN_DATATYPE_FLOAT_32, 2, nullptr, nullptr, 0); - p_tensor2 = ggml_qnn_create_general_tensor(dst, nullptr, QNN_TENSOR_TYPE_APP_READ,QNN_DATATYPE_FLOAT_32, 2, nullptr, nullptr, 0); + p_tensor0 = ggml_qnn_create_general_tensor(src0, nullptr, QNN_TENSOR_TYPE_APP_WRITE,QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0); + p_tensor1 = ggml_qnn_create_general_tensor(src1, nullptr, QNN_TENSOR_TYPE_APP_WRITE,QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0); + p_tensor2 = ggml_qnn_create_general_tensor(dst, nullptr, QNN_TENSOR_TYPE_APP_READ,QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0); } - //print_tensors_info(__func__, ctx, src0, src1, dst); + print_tensors_info(__func__, ctx, src0, src1, dst); //ensure QNN tensor has correct tensor type QNN_VER_PTR(*p_tensor0)->type = QNN_TENSOR_TYPE_APP_WRITE; @@ -3403,9 +3412,16 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { return; } //step-2: create param tensor for mulmat of 2d matrix - uint32_t param_tensor_dims[] = {2}; - uint32_t param_tensor_data[2] = {1, 0}; - p_param_tensor = ggml_qnn_create_general_tensor(nullptr, "param", QNN_TENSOR_TYPE_STATIC,QNN_DATATYPE_UINT_32, 1, param_tensor_dims, param_tensor_data, 8); + const uint32_t param_tensor_data[GGML_MAX_DIMS][GGML_MAX_DIMS] = { + {0}, + {1, 0}, + {0, 2, 1}, + {0, 1, 3, 2}, + }; + uint32_t param_tensor_dims[1] = {src0_rank}; + p_param_tensor = ggml_qnn_create_general_tensor(nullptr, "param", QNN_TENSOR_TYPE_STATIC, QNN_DATATYPE_UINT_32, + 1, param_tensor_dims, + (void *) (param_tensor_data[src0_rank - 1]), src0_rank * sizeof(uint32_t)); CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_param_tensor)); //step-3: create compute tensor from ggml tensor @@ -3419,7 +3435,7 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { //step-4: create a transpose tensor uint32_t tensor2_transpose_dims[GGML_MAX_DIMS] = {}; - p_tensor2_transpose = ggml_qnn_create_general_tensor(dst, "transpose", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, 2, nullptr, nullptr, 0); + p_tensor2_transpose = ggml_qnn_create_general_tensor(dst, "transpose", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0); get_qnn_dimensions_from_ggml_dimensions(tensor2_transpose_dims, tensor_2_dimensions, ggml_get_tensor_rank(dst)); //save pointer because the dimensions of tensor p_tensor2_transpose will be changed later uint32_t * tensor2_dimensions_transpose = QNN_VER_PTR(*p_tensor2_transpose)->dimensions; @@ -3435,7 +3451,7 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { } }; - Qnn_Tensor_t out_0_inputs[] = {*p_tensor0,*p_tensor1}; + Qnn_Tensor_t out_0_inputs[] = {*p_tensor0,*p_tensor1}; Qnn_Tensor_t out_0_outputs[] = {*p_tensor2_transpose}; Qnn_OpConfig_t out_0 = { QNN_OPCONFIG_VERSION_1, .v1 = @@ -3455,7 +3471,7 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { "perm", .tensorParam = *p_param_tensor } }; - Qnn_Tensor_t out_trans1_0_inputs[] = {*p_tensor2_transpose}; + Qnn_Tensor_t out_trans1_0_inputs[] = {*p_tensor2_transpose}; Qnn_Tensor_t out_trans1_0_outputs[] = {*p_tensor2}; Qnn_OpConfig_t out_trans1_0 = { QNN_OPCONFIG_VERSION_1, @@ -3472,7 +3488,7 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { //step-6: finalize qnn graph and execute qnn graph CHECK_QNN_API(error, qnn_raw_interface.graphFinalize(graph_handle, NULL, NULL)); - Qnn_Tensor_t input_tensors_0[] = {*p_tensor0,*p_tensor1}; + Qnn_Tensor_t input_tensors_0[] = {*p_tensor0,*p_tensor1}; Qnn_Tensor_t output_tensors_0[] = {*p_tensor2}; CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, input_tensors_0, 2, @@ -3495,9 +3511,7 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { //restore pointer to avoid memory leak QNN_VER_PTR(*p_tensor2_transpose)->dimensions = tensor2_dimensions_transpose; //free_qnn_tensor(p_tensor2_transpose); - } else { - QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggml_get_tensor_data_size(src0)}; QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggml_get_tensor_data_size(src1)}; QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggml_get_tensor_data_size(dst)}; From 8e65331fcbb6c00b84d77225dd922a532041db70 Mon Sep 17 00:00:00 2001 From: zhouwg Date: Fri, 21 Feb 2025 17:43:25 +0800 Subject: [PATCH 19/76] ggml-qnn: sync from branch kantvai-ggmlqnn-npurpc --- ggml/src/ggml-qnn/ggml-qnn.cpp | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/ggml/src/ggml-qnn/ggml-qnn.cpp b/ggml/src/ggml-qnn/ggml-qnn.cpp index 9ef502421c051..e862b07a234eb 100644 --- a/ggml/src/ggml-qnn/ggml-qnn.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn.cpp @@ -1132,8 +1132,8 @@ struct ggml_backend_qnn_context { struct qcom_socinfo socinfo; } ; -//TODO: the following global vars and three helper funcs should be removed in the future -static int32_t g_ggmltensor_idx = 0; +//the following helper funcs are used to ensure every QNN tensor name is unique +static std::atomic g_ggmltensor_idx(0); static void reset_idx() { g_ggmltensor_idx = 0; } @@ -1143,7 +1143,7 @@ static void inc_idx() { } static int32_t get_idx() { - return g_ggmltensor_idx; + return g_ggmltensor_idx.load(); } // file:///opt/qcom/aistack/qairt/2.31.0.250130/docs/QNN/general/quantization.html @@ -1474,7 +1474,7 @@ static Qnn_Tensor_t * ggml_qnn_create_general_tensor(const ggml_tensor * tensor, Qnn_ErrorHandle_t error = QNN_SUCCESS; char tensor_name[GGML_MAX_NAME] = {0}; - //TODO:remove get_idx() and inc_idx() in the future but ensure the tensor name is unique + //ensure the tensor name is unique if (nullptr != name) { snprintf(tensor_name, GGML_MAX_NAME, "tensor_%-8d", get_idx()); } else { @@ -2762,7 +2762,6 @@ int qnn_instance::qnn_finalize() { Qnn_ErrorHandle_t error = QNN_SUCCESS; GGMLQNN_LOG_DEBUG("enter %s\n", __func__); - //TODO:should be removed in the future reset_idx(); free_rpcmem(); @@ -3451,7 +3450,7 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { } }; - Qnn_Tensor_t out_0_inputs[] = {*p_tensor0,*p_tensor1}; + Qnn_Tensor_t out_0_inputs[] = {*p_tensor0, *p_tensor1}; Qnn_Tensor_t out_0_outputs[] = {*p_tensor2_transpose}; Qnn_OpConfig_t out_0 = { QNN_OPCONFIG_VERSION_1, .v1 = @@ -3488,7 +3487,7 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { //step-6: finalize qnn graph and execute qnn graph CHECK_QNN_API(error, qnn_raw_interface.graphFinalize(graph_handle, NULL, NULL)); - Qnn_Tensor_t input_tensors_0[] = {*p_tensor0,*p_tensor1}; + Qnn_Tensor_t input_tensors_0[] = {*p_tensor0, *p_tensor1}; Qnn_Tensor_t output_tensors_0[] = {*p_tensor2}; CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, input_tensors_0, 2, From 1233defca3d757eb15621d1ee725c95cc35e05cc Mon Sep 17 00:00:00 2001 From: zhouwg Date: Sun, 23 Feb 2025 10:23:03 +0800 Subject: [PATCH 20/76] ggml-qnn: fix a minior typo in internal doc --- ggml/src/ggml-qnn/ggml-qnn.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ggml/src/ggml-qnn/ggml-qnn.cpp b/ggml/src/ggml-qnn/ggml-qnn.cpp index e862b07a234eb..effcd5d54648f 100644 --- a/ggml/src/ggml-qnn/ggml-qnn.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn.cpp @@ -3410,7 +3410,7 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { GGMLQNN_LOG_INFO("can't create qnn graph handle with graph name %s, error = %d\n", graph_name.c_str(), error); return; } - //step-2: create param tensor for mulmat of 2d matrix + //step-2: create param tensor for mulmat of 2d/3d/4d matrix const uint32_t param_tensor_data[GGML_MAX_DIMS][GGML_MAX_DIMS] = { {0}, {1, 0}, From d85b65fae00cd5a16152366f24d9c8d15436a27f Mon Sep 17 00:00:00 2001 From: zhouwg Date: Sun, 23 Feb 2025 22:41:31 +0800 Subject: [PATCH 21/76] ggml-qnn: refine function ggml_qnn_create_general_tensor() to avoid complex/redundant pointer operation --- ggml/src/ggml-qnn/ggml-qnn.cpp | 141 ++++++++++++++++++++------------- 1 file changed, 87 insertions(+), 54 deletions(-) diff --git a/ggml/src/ggml-qnn/ggml-qnn.cpp b/ggml/src/ggml-qnn/ggml-qnn.cpp index effcd5d54648f..1b1e280f09505 100644 --- a/ggml/src/ggml-qnn/ggml-qnn.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn.cpp @@ -104,6 +104,12 @@ struct ggml_backend_qnn_context; static int free_qnn_tensor(Qnn_Tensor_t * tensor); static enum ggml_status ggml_backend_qnn_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph); static void ggmlqnn_log_internal(ggml_log_level level, const char * file, const char * func, int line, const char * format, ...); +static Qnn_Tensor_t * ggml_qnn_create_general_tensor(const ggml_tensor * tensor, const char * name, + Qnn_TensorType_t qnn_tensor_type, + Qnn_DataType_t qnn_data_type, + uint32_t rank, uint32_t * dims, + void * data, uint32_t data_size, + bool b_transpose = false); // ================================================================================================= // section-2: ggml-qnn internal troubleshooting function @@ -163,6 +169,7 @@ static void ggmlqnn_log_internal(ggml_log_level level, const char * file, const #define GGMLQNN_MEM_ADD(alignment) (sizeof (size_t) + alignment) #define GGMLQNN_MEM_MASK(alignment) ((uintptr_t)alignment - 1) +#define GQCGT ggml_qnn_create_general_tensor static intptr_t ggmlqnn_align_to(size_t alignment, intptr_t offset) { return offset % alignment == 0 ? offset @@ -1013,6 +1020,20 @@ static const char * qnn_get_error_string(Qnn_ErrorHandle_t qnn_error_code) { } } +// helper function to create an operation config +static Qnn_OpConfig_t create_op_config(const char * name, const char * package, const char * type, + Qnn_Param_t * params, uint32_t num_params, + Qnn_Tensor_t * inputs, uint32_t num_inputs, + Qnn_Tensor_t * outputs, uint32_t num_outputs) { + Qnn_OpConfigV1_t v1 = {name, package, type, + num_params, params, + num_inputs, inputs, + num_outputs, outputs + }; + + return (Qnn_OpConfig_t){QNN_OPCONFIG_VERSION_1, .v1 = v1}; +} + // ================================================================================================= // section-5:ggml-qnn backend helper macro / data structure / function / class // ================================================================================================= @@ -1469,10 +1490,32 @@ static const char * qnn_opname_from_ggmlop(enum ggml_op ggmlop) { return nullptr; } -static Qnn_Tensor_t * ggml_qnn_create_general_tensor(const ggml_tensor * tensor, const char * name, Qnn_TensorType_t qnn_tensor_type, - Qnn_DataType_t qnn_data_type, uint32_t rank, uint32_t * dims, void * data, uint32_t data_size) { - Qnn_ErrorHandle_t error = QNN_SUCCESS; - char tensor_name[GGML_MAX_NAME] = {0}; +static void get_qnn_dimensions_from_ggml_dimensions(uint32_t * qnn_dimensions, const uint32_t * ggml_dimensions, uint32_t rank) { + if (rank > GGML_MAX_DIMS) { + GGMLQNN_LOG_WARN("invalid params"); + return; + } + if (nullptr == qnn_dimensions || nullptr == ggml_dimensions) { + GGMLQNN_LOG_WARN("invalid params"); + return; + } + for (size_t idx = 0; idx < GGML_MAX_DIMS; idx++) + qnn_dimensions[idx] = ggml_dimensions[idx]; + + if (rank >= 2) { + qnn_dimensions[rank - 1] = ggml_dimensions[rank - 2]; + qnn_dimensions[rank - 2] = ggml_dimensions[rank - 1]; + } +} + +static Qnn_Tensor_t * ggml_qnn_create_general_tensor(const ggml_tensor * tensor, const char * name, + Qnn_TensorType_t qnn_tensor_type, + Qnn_DataType_t qnn_data_type, + uint32_t rank, uint32_t * dims, + void * data, uint32_t data_size, + bool b_transpose) { + Qnn_ErrorHandle_t error = QNN_SUCCESS; + char tensor_name[GGML_MAX_NAME] = {}; //ensure the tensor name is unique if (nullptr != name) { @@ -1483,19 +1526,36 @@ static Qnn_Tensor_t * ggml_qnn_create_general_tensor(const ggml_tensor * tensor, GGMLQNN_LOG_DEBUG("init_tensor %d", get_idx()); inc_idx(); - uint32_t dimensions_transpose[GGML_MAX_DIMS] = {}; - uint32_t * tensor_dims = nullptr; + uint32_t reverse_dims[GGML_MAX_DIMS] = {}; + uint32_t transpose_dims[GGML_MAX_DIMS] = {}; + uint32_t * tensor_dims = nullptr; + //case 1:use dims info from ggml tensor if (nullptr != tensor) { //there are different dimension order between ggml tensor and qnn tensor for (size_t idx = 0; idx < rank; idx++) { - dimensions_transpose[idx] = (uint32_t)tensor->ne[rank - 1 - idx]; + reverse_dims[idx] = (uint32_t)tensor->ne[rank - 1 - idx]; } - tensor_dims = dimensions_transpose; + tensor_dims = reverse_dims; } - //re-assign tensor_dims + //case 2: use user's specified tensor_dims if (nullptr != dims) { tensor_dims = dims; } + //case 3: transpose for dst tensor + if (b_transpose) { + GGML_ASSERT(tensor != nullptr); //ensure ggml_tensor is not nullptr for this special case + + get_qnn_dimensions_from_ggml_dimensions(transpose_dims, reverse_dims, ggml_get_tensor_rank(tensor)); + tensor_dims = transpose_dims; +#if 0 + for (size_t idx = 0; idx < 4; idx++) { + GGMLQNN_LOG_DEBUG("origin dim[%d]=%d\n", idx, reverse_dims[idx]); + } + for (size_t idx = 0; idx < 4; idx++) { + GGMLQNN_LOG_DEBUG("trans dim[%d]=%d\n", idx, transpose_dims[idx]); + } +#endif + } Qnn_Tensor_t qnn_tensor = { .version= QNN_TENSOR_VERSION_1, @@ -2989,25 +3049,6 @@ static void dump_op_info(const struct ggml_tensor * tensor) { print_tensors_info(nullptr, nullptr, src0, src1, dst); } -//TODO: currently only support offloading 2D matrix to QNN backend -static void get_qnn_dimensions_from_ggml_dimensions(uint32_t * qnn_dimensions, uint32_t * ggml_dimensions, uint32_t rank) { - if (rank > GGML_MAX_DIMS) { - GGMLQNN_LOG_WARN("invalid params"); - return; - } - if (nullptr == qnn_dimensions || nullptr == ggml_dimensions) { - GGMLQNN_LOG_WARN("invalid params"); - return; - } - for (size_t idx = 0; idx < GGML_MAX_DIMS; idx++) - qnn_dimensions[idx] = ggml_dimensions[idx]; - - if (rank >= 2) { - qnn_dimensions[rank - 1] = ggml_dimensions[rank - 2]; - qnn_dimensions[rank - 2] = ggml_dimensions[rank - 1]; - } -} - // ================================================================================================= // section-6: implementation of ggml-qnn backend // ================================================================================================= @@ -3056,10 +3097,9 @@ static bool ggml_qnn_can_handle_op(const struct ggml_tensor * tensor) { } if (tensor->op == GGML_OP_MUL_MAT) { - dump_op_info(tensor); if (src0_rank != src1_rank) // make QNN SDK happy return false; - if (src0_rank < 2) // make QNN SDK happy + if (src0_rank < 2) // QNN's limitation, make QNN SDK happy return false; if (src0_rank > 3) //TODO: 4D matrix return false; @@ -3327,7 +3367,7 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { bool graph_initialized = false; qnn_perf op_perf = qnn_perf("ggml_qnn_mul_mat"); qnn_instance * instance = nullptr; - ggml_backend_qnn_context * ctx = (ggml_backend_qnn_context *) backend->context; + ggml_backend_qnn_context * ctx = (ggml_backend_qnn_context *)backend->context; Qnn_GraphHandle_t graph_handle = nullptr; Qnn_Tensor_t * p_tensor0 = nullptr; Qnn_Tensor_t * p_tensor1 = nullptr; @@ -3361,11 +3401,10 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { p_param_tensor = tensors[3]; p_tensor2_transpose = tensors[4]; } else { - p_tensor0 = ggml_qnn_create_general_tensor(src0, nullptr, QNN_TENSOR_TYPE_APP_WRITE,QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0); - p_tensor1 = ggml_qnn_create_general_tensor(src1, nullptr, QNN_TENSOR_TYPE_APP_WRITE,QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0); - p_tensor2 = ggml_qnn_create_general_tensor(dst, nullptr, QNN_TENSOR_TYPE_APP_READ,QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0); + p_tensor0 = GQCGT(src0, nullptr, QNN_TENSOR_TYPE_APP_WRITE,QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0); + p_tensor1 = GQCGT(src1, nullptr, QNN_TENSOR_TYPE_APP_WRITE,QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0); + p_tensor2 = GQCGT(dst, nullptr, QNN_TENSOR_TYPE_APP_READ,QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0); } - print_tensors_info(__func__, ctx, src0, src1, dst); //ensure QNN tensor has correct tensor type @@ -3418,9 +3457,7 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { {0, 1, 3, 2}, }; uint32_t param_tensor_dims[1] = {src0_rank}; - p_param_tensor = ggml_qnn_create_general_tensor(nullptr, "param", QNN_TENSOR_TYPE_STATIC, QNN_DATATYPE_UINT_32, - 1, param_tensor_dims, - (void *) (param_tensor_data[src0_rank - 1]), src0_rank * sizeof(uint32_t)); + p_param_tensor = GQCGT(nullptr, "param", QNN_TENSOR_TYPE_STATIC, QNN_DATATYPE_UINT_32, 1, param_tensor_dims, (void *)(param_tensor_data[src0_rank - 1]), src0_rank * sizeof(uint32_t)); CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_param_tensor)); //step-3: create compute tensor from ggml tensor @@ -3433,13 +3470,7 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggml_get_tensor_data_size(dst)}; //step-4: create a transpose tensor - uint32_t tensor2_transpose_dims[GGML_MAX_DIMS] = {}; - p_tensor2_transpose = ggml_qnn_create_general_tensor(dst, "transpose", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0); - get_qnn_dimensions_from_ggml_dimensions(tensor2_transpose_dims, tensor_2_dimensions, ggml_get_tensor_rank(dst)); - //save pointer because the dimensions of tensor p_tensor2_transpose will be changed later - uint32_t * tensor2_dimensions_transpose = QNN_VER_PTR(*p_tensor2_transpose)->dimensions; - //update dimensions of tensor p_tensor2_transpose to make QNN SDK happy - QNN_VER_PTR(*p_tensor2_transpose)->dimensions = tensor2_transpose_dims; + p_tensor2_transpose = GQCGT(dst, "transpose", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0, true); CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor2_transpose)); //step-5: compose qnn graph: add mat_mul node @@ -3452,6 +3483,7 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { Qnn_Tensor_t out_0_inputs[] = {*p_tensor0, *p_tensor1}; Qnn_Tensor_t out_0_outputs[] = {*p_tensor2_transpose}; +#if 0 Qnn_OpConfig_t out_0 = { QNN_OPCONFIG_VERSION_1, .v1 = {"ggmlqnn_mulmat_opconfig", QNN_OP_PACKAGE_NAME_QTI_AISW, QNN_OP_MAT_MUL, @@ -3462,6 +3494,10 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { 1, out_0_outputs} }; +#else + Qnn_OpConfig_t out_0 = create_op_config("ggmlqnn_mulmat_opconfig", QNN_OP_PACKAGE_NAME_QTI_AISW, QNN_OP_MAT_MUL, + out_0_params, 1, out_0_inputs, 2, out_0_outputs, 1); +#endif CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle,out_0)); //step-5: compose qnn graph: add transpose node @@ -3472,10 +3508,11 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { }; Qnn_Tensor_t out_trans1_0_inputs[] = {*p_tensor2_transpose}; Qnn_Tensor_t out_trans1_0_outputs[] = {*p_tensor2}; +#if 0 Qnn_OpConfig_t out_trans1_0 = { QNN_OPCONFIG_VERSION_1, .v1 = {"ggmlqnn_mulmat_transpose_opconfig", - "qti.aisw", + QNN_OP_PACKAGE_NAME_QTI_AISW, QNN_OP_TRANSPOSE, 1, out_trans1_0_params, 1, @@ -3483,6 +3520,10 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { 1, out_trans1_0_outputs} }; +#else + Qnn_OpConfig_t out_trans1_0 = create_op_config("ggmlqnn_mulmat_transpose_opconfig", QNN_OP_PACKAGE_NAME_QTI_AISW, QNN_OP_TRANSPOSE, + out_trans1_0_params, 1, out_trans1_0_inputs, 1, out_trans1_0_outputs, 1); +#endif CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle,out_trans1_0)); //step-6: finalize qnn graph and execute qnn graph @@ -3501,15 +3542,8 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { ggml_op_mulmat_tensors.push_back(p_tensor2); ggml_op_mulmat_tensors.push_back(p_param_tensor); ggml_op_mulmat_tensors.push_back(p_tensor2_transpose); - auto graph_item = std::make_tuple(graph_handle, ggml_op_mulmat_tensors); instance->_qnn_graph_map[graph_name] = graph_item; - - //avoid cleanup these resource to make test_backend_ops happy - //free_qnn_tensor(p_param_tensor); - //restore pointer to avoid memory leak - QNN_VER_PTR(*p_tensor2_transpose)->dimensions = tensor2_dimensions_transpose; - //free_qnn_tensor(p_tensor2_transpose); } else { QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggml_get_tensor_data_size(src0)}; QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggml_get_tensor_data_size(src1)}; @@ -3522,7 +3556,6 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { Qnn_Tensor_t tensor_outputs[] = { *p_tensor2 }; - //attention: // this is the second technical approach of "how to utilize the Hexagon NPU maximally" through // QNN SDK, details could be found at // https://github.com/kantv-ai/llama.cpp/wiki/mapping-ggml-compute-graph-to-QNN-compute-graph From 078fc4e0375701e59169954b22eb70f44d43a905 Mon Sep 17 00:00:00 2001 From: zhouwg Date: Mon, 24 Feb 2025 09:58:42 +0800 Subject: [PATCH 22/76] ggml-qnn: fix a minor typo in source code --- ggml/src/ggml-qnn/ggml-qnn.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ggml/src/ggml-qnn/ggml-qnn.cpp b/ggml/src/ggml-qnn/ggml-qnn.cpp index 1b1e280f09505..120cea777ea20 100644 --- a/ggml/src/ggml-qnn/ggml-qnn.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn.cpp @@ -2654,20 +2654,20 @@ int qnn_instance::qnn_init(const QnnSaver_Config_t ** saver_config) { GGMLQNN_LOG_DEBUG("load QNN system lib successfully\n"); } - std::string bakend_lib_path = _lib_path + _backend_name; - if (0 == _lib_path_to_backend_id.count(bakend_lib_path)) { - int is_load_ok = load_backend(bakend_lib_path, saver_config); + std::string backend_lib_path = _lib_path + _backend_name; + if (0 == _lib_path_to_backend_id.count(backend_lib_path)) { + int is_load_ok = load_backend(backend_lib_path, saver_config); if (0 != is_load_ok) { GGMLQNN_LOG_WARN("failed to load QNN backend\n"); return 2; } } - backend_id = _lib_path_to_backend_id[bakend_lib_path]; + backend_id = _lib_path_to_backend_id[backend_lib_path]; if (0 == _loaded_backend.count(backend_id) || 0 == _loaded_lib_handle.count(backend_id)) { GGMLQNN_LOG_WARN("library %s is loaded but loaded backend count=%zu, loaded lib_handle count=%zu\n", - bakend_lib_path.c_str(), + backend_lib_path.c_str(), _loaded_backend.count(backend_id), _loaded_lib_handle.count(backend_id)); return 3; From 62e27ceb069c277324a422387e218852eaf67352 Mon Sep 17 00:00:00 2001 From: zhouwg Date: Mon, 24 Feb 2025 16:59:12 +0800 Subject: [PATCH 23/76] build: avoid ggml-qnn backend breaking other backend's builds --- ggml/src/ggml-qnn/CMakeLists.txt | 35 -------------------------------- 1 file changed, 35 deletions(-) delete mode 100644 ggml/src/ggml-qnn/CMakeLists.txt diff --git a/ggml/src/ggml-qnn/CMakeLists.txt b/ggml/src/ggml-qnn/CMakeLists.txt deleted file mode 100644 index 1156c98fbc9d7..0000000000000 --- a/ggml/src/ggml-qnn/CMakeLists.txt +++ /dev/null @@ -1,35 +0,0 @@ -message(STATUS "Using QNN backend") - -if(CMAKE_SYSTEM_NAME STREQUAL "Android") - find_library(LOG_LIB log) - set(QNN_LINK_LIBRARIES ${LOG_LIB}) - set(QNN_DEFAULT_LIB_SEARCH_PATH "/data/local/tmp/" CACHE STRING "customized library search path for QNN backend") -elseif(CMAKE_SYSTEM_NAME STREQUAL "Windows") - set(QNN_DEFAULT_LIB_SEARCH_PATH "C:\\" CACHE STRING "customized library search path for QNN backend") -else() - message(FATAL_ERROR "QNN now only available on Android and Windows(Windows on ARM)") -endif() - -if(NOT DEFINED GGML_QNN_SDK_PATH) -# try read from environment variable - if(DEFINED ENV{QNN_SDK_PATH}) - set(GGML_QNN_SDK_PATH $ENV{QNN_SDK_PATH}) - else() - message(FATAL_ERROR "GGML_QNN_SDK_PATH not defined") - endif() -endif() - -message("QNN_SDK_PATH: ${GGML_QNN_SDK_PATH}") - -set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3") - -file(GLOB QNN_SOURCES "${CMAKE_CURRENT_LIST_DIR}/*.cpp") - ggml_add_backend_library(ggml-qnn - ${QNN_SOURCES} -) - -target_include_directories(ggml-qnn PRIVATE ${GGML_QNN_SDK_PATH}/include/QNN ${CMAKE_CURRENT_LIST_DIR}) -target_link_libraries(ggml-qnn PRIVATE ${QNN_LINK_LIBRARIES}) - -string(REGEX REPLACE "/$" "" GGML_QNN_DEFAULT_LIB_SEARCH_PATH "${QNN_DEFAULT_LIB_SEARCH_PATH}") -target_compile_definitions(ggml-qnn PRIVATE GGML_QNN_DEFAULT_LIB_SEARCH_PATH="${QNN_DEFAULT_LIB_SEARCH_PATH}/") From 21762d8ef4f4a0e637d9e1348770fb8e7a97e56a Mon Sep 17 00:00:00 2001 From: zhouwg Date: Tue, 25 Feb 2025 08:22:27 +0800 Subject: [PATCH 24/76] ggml-qnn: remove redundant codes to make PR reviewers happy --- ggml/src/ggml-qnn/ggml-qnn.cpp | 409 ++------------------------------- 1 file changed, 14 insertions(+), 395 deletions(-) diff --git a/ggml/src/ggml-qnn/ggml-qnn.cpp b/ggml/src/ggml-qnn/ggml-qnn.cpp index 120cea777ea20..8b33d346bd91a 100644 --- a/ggml/src/ggml-qnn/ggml-qnn.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn.cpp @@ -18,8 +18,6 @@ * - GGML_OP_MUL: this is a simple skeleton, can expand other ggml ops according to expertise * - GGML_OP_MUL_MAT:this is a complicated skeleton, can expand other complex ggml ops accordingly * - * of course, can porting ggml-qnn to Windows on ARM as need. - * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to * deal in the Software without restriction, including without limitation the @@ -38,7 +36,6 @@ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS * IN THE SOFTWARE. */ - #include #include #include @@ -144,11 +141,8 @@ static void ggmlqnn_log_internal(ggml_log_level level, const char * file, const #if (defined __ANDROID__) || (defined ANDROID) //for Android application(standard APP or command line tool) __android_log_print(ANDROID_LOG_INFO, "ggml-qnn", "%s\n", s_ggmlqnn_log_internal_buf); -#endif -#if (defined __ANDROID__) || (defined ANDROID) - //do nothing when running on Snapdragon based Android device #else - //for Snapdragon based WoA(Windows on ARM) device + //for Snapdragon based WoA(Windows on ARM) device or Linux printf("%s\n", s_ggmlqnn_log_internal_buf); #endif } @@ -167,8 +161,6 @@ static void ggmlqnn_log_internal(ggml_log_level level, const char * file, const class_name(class_name &&) = delete; \ void operator=(class_name &&) = delete -#define GGMLQNN_MEM_ADD(alignment) (sizeof (size_t) + alignment) -#define GGMLQNN_MEM_MASK(alignment) ((uintptr_t)alignment - 1) #define GQCGT ggml_qnn_create_general_tensor static intptr_t ggmlqnn_align_to(size_t alignment, intptr_t offset) { @@ -178,62 +170,36 @@ static intptr_t ggmlqnn_align_to(size_t alignment, intptr_t offset) { offset % static_cast(alignment)); } -static void * ggmlqnn_mallocz_aligned(size_t size, size_t alignment) { - uint8_t * buffer = NULL; - size_t * sp = NULL; - buffer = static_cast(calloc(1, size + GGMLQNN_MEM_ADD(alignment))); - if (!buffer) - return NULL; - sp = (size_t *)buffer; - *sp = size; - buffer = (uint8_t *)(((uintptr_t) buffer + GGMLQNN_MEM_ADD(alignment)) & ~GGMLQNN_MEM_MASK(alignment)); - buffer[-1] = buffer - (uint8_t *)sp; - return buffer; -} - -static void * ggmlqnn_malloc_aligned(size_t size, size_t alignment) { - uint8_t * buffer = NULL; - size_t * sp = NULL; - buffer = static_cast(malloc(size + GGMLQNN_MEM_ADD(alignment))); - if (!buffer) - return NULL; - sp = (size_t *)buffer; - *sp = size; - buffer = (uint8_t *)(((uintptr_t) buffer + GGMLQNN_MEM_ADD(alignment)) & ~GGMLQNN_MEM_MASK(alignment)); - buffer[-1] = buffer - (uint8_t *)sp; - return buffer; -} - -static void ggmqnn_free_aligned(void * ptr) { - uint8_t * old = (uint8_t *)ptr; - if (!old) - return; - old -= old[-1]; - free(old); -} - static size_t get_system_total_memory_in_bytes() { +#if defined(__ANDROID__) || defined(__linux__) struct sysinfo info = {}; - if (sysinfo(&info) == 0) { + if (0 == sysinfo(&info)) { return (info.totalram + info.totalswap) * info.mem_unit; } - auto pages = (size_t)sysconf(_SC_PHYS_PAGES); auto page_size = (size_t)sysconf(_SC_PAGE_SIZE); return pages * page_size; +#else + //TODO: Snapdragon based WoA(Windows on ARM) + return 0; +#endif } static size_t get_system_free_memory_in_bytes() { +#if defined(__ANDROID__) || defined(__linux__) struct sysinfo info = {}; - if (sysinfo(&info) == 0) { + if (0 == sysinfo(&info)) { return (info.freeram + info.freeswap) * info.mem_unit; } - auto avail_pages = (size_t)sysconf(_SC_AVPHYS_PAGES); auto page_size = (size_t)sysconf(_SC_PAGE_SIZE); return avail_pages * page_size; +#else + //TODO: Snapdragon based WoA(Windows on ARM) + return 0; +#endif } static size_t ggmlqnn_memscpy(void * dst, size_t dst_size, const void * src, size_t copy_size) { @@ -288,34 +254,7 @@ static void * ggmlqnn_host_malloc(size_t n) { #define VALIDATE_TENSOR_VERSION(tensor, err) VALIDATE(validate_tensor_version(tensor), err) -#define VALIDATE_OP_CONFIG_VERSION(op, err) VALIDATE(validate_op_config_version(op), err) - #define QNN_VER_PTR(x) (&((x).v1)) -#define QNN_OP_CFG_VALID(op_config) ((op_config).version == QNN_OPCONFIG_VERSION_1) - -#define QNN_OP_CFG_GET_NAME(op_config) get_qnn_oponfig_name(op_config) -#define QNN_OP_CFG_GET_PACKAGE_NAME(op_config) get_qnn_op_config_packagename(op_config) -#define QNN_OP_CFG_GET_TYPE_NAME(op_config) get_qnn_op_config_typename(op_config) -#define QNN_OP_CFG_GET_NUM_PARAMS(op_config) get_qnn_op_config_numparams(op_config) -#define QNN_OP_CFG_GET_PARAMS(op_config) get_qnn_op_config_params(op_config) -#define QNN_OP_CFG_GET_NUM_INPUTS(op_config) get_qnn_op_config_numinputs(op_config) -#define QNN_OP_CFG_GET_INPUTS(op_config) get_qnn_op_config_inputs(op_config) -#define QNN_OP_CFG_GET_NUM_OUTPUTS(op_config) get_qnn_op_config_numoutputs(op_config) -#define QNN_OP_CFG_GET_OUTPUTS(op_config) get_qnn_op_config_outputs(op_config) - -#define QNN_OP_CFG_SET_NAME(op_config, value) set_qnn_op_config_name(op_config, value) -#define QNN_OP_CFG_SET_PACKAGE_NAME(op_config, value) set_qnn_op_config_packagename(op_config, value) -#define QNN_OP_CFG_SET_TYPE_NAME(op_config, value) set_qnn_op_config_typename(op_config, value) - -#define QNN_OP_CFG_SET_PARAMS(op_config, num_of_params, params) \ - set_qnn_op_config_params(op_config, num_of_params, params) - -#define QNN_OP_CFG_SET_INPUTS(op_config, num_of_inputs, inputTensors) \ - set_qnn_op_config_inputs(op_config, num_of_inputs, inputTensors) - -#define QNN_OP_CFG_SET_OUTPUTS(op_config, num_of_outputs, output_tensors) \ - set_qnn_op_config_outputs(op_config, num_of_outputs, output_tensors) - #define QNN_TENSOR_GET_ID(tensor) get_qnn_tensorid(tensor) #define QNN_TENSOR_GET_NAME(tensor) get_qnn_tensorname(tensor) #define QNN_TENSOR_GET_TYPE(tensor) get_qnn_tensortype(tensor) @@ -350,190 +289,6 @@ static inline int validate_tensor_version(Qnn_Tensor_t tensor) { return 0; } -[[maybe_unused]] static inline int validate_op_config_version(Qnn_OpConfig_t op_config) { - if (op_config.version != QNN_OPCONFIG_VERSION_1) { - GGMLQNN_LOG_WARN("validate_op_config_version() op %s, got unsupported version %d\n", - op_config.v1.name, - op_config.version); - return 1; - } - return 0; -} - -static inline const char * get_qnn_oponfig_name(const Qnn_OpConfig_t & op_config) { - if (op_config.version == QNN_OPCONFIG_VERSION_1) { - return op_config.v1.name; - } - return nullptr; -} - -[[maybe_unused]] static inline const char * get_qnn_oponfig_name(const Qnn_OpConfig_t * op_config) { - return get_qnn_oponfig_name(*op_config); -} - -static inline const char * get_qnn_op_config_packagename(const Qnn_OpConfig_t & op_config) { - if (op_config.version == QNN_OPCONFIG_VERSION_1) { - return op_config.v1.packageName; - } - return nullptr; -} - -[[maybe_unused]] static inline const char * get_qnn_op_config_packagename(const Qnn_OpConfig_t * op_config) { - return get_qnn_op_config_packagename(*op_config); -} - -static inline const char * get_qnn_op_config_typename(const Qnn_OpConfig_t & op_config) { - if (op_config.version == QNN_OPCONFIG_VERSION_1) { - return op_config.v1.typeName; - } - return nullptr; -} - -[[maybe_unused]] static inline const char * get_qnn_op_config_typename(const Qnn_OpConfig_t * op_config) { - return get_qnn_op_config_typename(*op_config); -} - -static inline uint32_t get_qnn_op_config_numparams(const Qnn_OpConfig_t & op_config) { - if (op_config.version == QNN_OPCONFIG_VERSION_1) { - return op_config.v1.numOfParams; - } - return 0u; -} - -[[maybe_unused]] static inline uint32_t get_qnn_op_config_numparams(const Qnn_OpConfig_t * op_config) { - return get_qnn_op_config_numparams(*op_config); -} - -static inline const Qnn_Param_t * get_qnn_op_config_params(const Qnn_OpConfig_t & op_config) { - if (op_config.version == QNN_OPCONFIG_VERSION_1) { - return op_config.v1.params; - } - return nullptr; -} - -[[maybe_unused]] static inline const Qnn_Param_t * get_qnn_op_config_params(const Qnn_OpConfig_t * op_config) { - return get_qnn_op_config_params(*op_config); -} - -static inline uint32_t get_qnn_op_config_numinputs(const Qnn_OpConfig_t & op_config) { - if (op_config.version == QNN_OPCONFIG_VERSION_1) { - return op_config.v1.numOfInputs; - } - return 0u; -} - -[[maybe_unused]] static inline uint32_t get_qnn_op_config_numinputs(const Qnn_OpConfig_t * op_config) { - return get_qnn_op_config_numinputs(*op_config); -} - -static inline const Qnn_Tensor_t * get_qnn_op_config_inputs(const Qnn_OpConfig_t & op_config) { - if (op_config.version == QNN_OPCONFIG_VERSION_1) { - return op_config.v1.inputTensors; - } - return nullptr; -} - -[[maybe_unused]] static inline const Qnn_Tensor_t * get_qnn_op_config_inputs(const Qnn_OpConfig_t * op_config) { - return get_qnn_op_config_inputs(*op_config); -} - -static inline uint32_t get_qnn_op_config_numoutputs(const Qnn_OpConfig_t & op_config) { - if (op_config.version == QNN_OPCONFIG_VERSION_1) { - return op_config.v1.numOfOutputs; - } - return 0u; -} - -[[maybe_unused]] static inline uint32_t get_qnn_op_config_numoutputs(const Qnn_OpConfig_t * op_config) { - return get_qnn_op_config_numoutputs(*op_config); -} - -static inline const Qnn_Tensor_t * get_qnn_op_config_outputs(const Qnn_OpConfig_t & op_config) { - if (op_config.version == QNN_OPCONFIG_VERSION_1) { - return op_config.v1.outputTensors; - } - return nullptr; -} - -[[maybe_unused]] static inline const Qnn_Tensor_t * get_qnn_op_config_outputs(const Qnn_OpConfig_t * op_config) { - return get_qnn_op_config_outputs(*op_config); -} - -static inline void set_qnn_op_config_name(Qnn_OpConfig_t & op_config, const char * name) { - if (op_config.version == QNN_OPCONFIG_VERSION_1) { - op_config.v1.name = name; - } -} - -[[maybe_unused]] static inline void set_qnn_op_config_name(Qnn_OpConfig_t * op_config, const char * name) { - set_qnn_op_config_name(*op_config, name); -} - -static inline void set_qnn_op_config_packagename(Qnn_OpConfig_t & op_config, const char * package_name) { - if (op_config.version == QNN_OPCONFIG_VERSION_1) { - op_config.v1.packageName = package_name; - } -} - -[[maybe_unused]] static inline void set_qnn_op_config_packagename(Qnn_OpConfig_t * op_config, const char * package_name) { - set_qnn_op_config_packagename(*op_config, package_name); -} - -static inline void set_qnn_op_config_typename(Qnn_OpConfig_t & op_config, const char * type_name) { - if (op_config.version == QNN_OPCONFIG_VERSION_1) { - op_config.v1.typeName = type_name; - } -} - -[[maybe_unused]] static inline void set_qnn_op_config_typename(Qnn_OpConfig_t * op_config, const char * type_name) { - set_qnn_op_config_typename(*op_config, type_name); -} - -static inline void set_qnn_op_config_params(Qnn_OpConfig_t & op_config, - uint32_t num_of_params, - Qnn_Param_t * params) { - if (op_config.version == QNN_OPCONFIG_VERSION_1) { - op_config.v1.numOfParams = num_of_params; - op_config.v1.params = params; - } -} - -[[maybe_unused]] static inline void set_qnn_op_config_params(Qnn_OpConfig_t * op_config, - uint32_t num_of_params, - Qnn_Param_t * params) { - set_qnn_op_config_params(*op_config, num_of_params, params); -} - -static inline void set_qnn_op_config_inputs(Qnn_OpConfig_t & op_config, - uint32_t num_of_inputs, - Qnn_Tensor_t * input_tensors) { - if (op_config.version == QNN_OPCONFIG_VERSION_1) { - op_config.v1.numOfInputs = num_of_inputs; - op_config.v1.inputTensors = input_tensors; - } -} - -[[maybe_unused]] static inline void set_qnn_op_config_inputs(Qnn_OpConfig_t * op_config, - uint32_t num_of_inputs, - Qnn_Tensor_t * input_tensors) { - set_qnn_op_config_inputs(*op_config, num_of_inputs, input_tensors); -} - -static inline void set_qnn_op_config_outputs(Qnn_OpConfig_t & op_config, - uint32_t num_of_outputs, - Qnn_Tensor_t * output_tensors) { - if (op_config.version == QNN_OPCONFIG_VERSION_1) { - op_config.v1.numOfOutputs = num_of_outputs; - op_config.v1.outputTensors = output_tensors; - } -} - -[[maybe_unused]] static inline void set_qnn_op_config_outputs(Qnn_OpConfig_t * op_config, - uint32_t num_of_outputs, - Qnn_Tensor_t * output_tensors) { - set_qnn_op_config_outputs(*op_config, num_of_outputs, output_tensors); -} - static inline uint32_t get_qnn_tensorid(const Qnn_Tensor_t & tensor) { if (tensor.version == QNN_TENSOR_VERSION_1) { return tensor.v1.id; @@ -542,10 +297,6 @@ static inline uint32_t get_qnn_tensorid(const Qnn_Tensor_t & tensor) { return 0u; } -[[maybe_unused]] static inline uint32_t get_qnn_tensorid(const Qnn_Tensor_t * tensor) { - return get_qnn_tensorid(*tensor); -} - static inline const char * get_qnn_tensorname(const Qnn_Tensor_t & tensor) { if (tensor.version == QNN_TENSOR_VERSION_1) { return tensor.v1.name; @@ -553,10 +304,6 @@ static inline const char * get_qnn_tensorname(const Qnn_Tensor_t & tensor) { return nullptr; } -static inline const char * get_qnn_tensorname(const Qnn_Tensor_t * tensor) { - return get_qnn_tensorname(*tensor); -} - static inline Qnn_TensorType_t get_qnn_tensortype(const Qnn_Tensor_t & tensor) { if (tensor.version == QNN_TENSOR_VERSION_1) { return tensor.v1.type; @@ -564,10 +311,6 @@ static inline Qnn_TensorType_t get_qnn_tensortype(const Qnn_Tensor_t & tensor) { return QNN_TENSOR_TYPE_UNDEFINED; } -[[maybe_unused]] static inline Qnn_TensorType_t get_qnn_tensortype(const Qnn_Tensor_t * tensor) { - return get_qnn_tensortype(*tensor); -} - static inline Qnn_TensorDataFormat_t get_qnn_tensor_dataformat(const Qnn_Tensor_t & tensor) { if (tensor.version == QNN_TENSOR_VERSION_1) { return tensor.v1.dataFormat; @@ -575,10 +318,6 @@ static inline Qnn_TensorDataFormat_t get_qnn_tensor_dataformat(const Qnn_Tensor_ return QNN_TENSOR_DATA_FORMAT_FLAT_BUFFER; } -[[maybe_unused]] static inline Qnn_TensorDataFormat_t get_qnn_tensor_dataformat(const Qnn_Tensor_t * tensor) { - return get_qnn_tensor_dataformat(*tensor); -} - static inline Qnn_DataType_t get_qnn_tensor_datatype(const Qnn_Tensor_t & tensor) { if (tensor.version == QNN_TENSOR_VERSION_1) { return tensor.v1.dataType; @@ -586,10 +325,6 @@ static inline Qnn_DataType_t get_qnn_tensor_datatype(const Qnn_Tensor_t & tensor return QNN_DATATYPE_UNDEFINED; } -[[maybe_unused]] static inline Qnn_DataType_t get_qnn_tensor_datatype(const Qnn_Tensor_t * tensor) { - return get_qnn_tensor_datatype(*tensor); -} - static inline Qnn_QuantizeParams_t get_qnn_tensor_quantparams(const Qnn_Tensor_t & tensor) { if (tensor.version == QNN_TENSOR_VERSION_1) { return tensor.v1.quantizeParams; @@ -597,10 +332,6 @@ static inline Qnn_QuantizeParams_t get_qnn_tensor_quantparams(const Qnn_Tensor_t return QNN_QUANTIZE_PARAMS_INIT; } -[[maybe_unused]] static inline Qnn_QuantizeParams_t get_qnn_tensor_quantparams(const Qnn_Tensor_t * tensor) { - return get_qnn_tensor_quantparams(*tensor); -} - static inline uint32_t get_qnn_tensor_rank(const Qnn_Tensor_t & tensor) { if (tensor.version == QNN_TENSOR_VERSION_1) { return tensor.v1.rank; @@ -608,10 +339,6 @@ static inline uint32_t get_qnn_tensor_rank(const Qnn_Tensor_t & tensor) { return 0u; } -[[maybe_unused]] static inline uint32_t get_qnn_tensor_rank(const Qnn_Tensor_t * tensor) { - return get_qnn_tensor_rank(*tensor); -} - static inline uint32_t * get_qnn_tensor_dimensions(const Qnn_Tensor_t & tensor) { if (tensor.version == QNN_TENSOR_VERSION_1) { return tensor.v1.dimensions; @@ -619,10 +346,6 @@ static inline uint32_t * get_qnn_tensor_dimensions(const Qnn_Tensor_t & tensor) return nullptr; } -[[maybe_unused]] static inline uint32_t * get_qnn_tensor_dimensions(const Qnn_Tensor_t * tensor) { - return get_qnn_tensor_dimensions(*tensor); -} - static inline Qnn_TensorMemType_t get_qnn_tensor_memtype(const Qnn_Tensor_t & tensor) { if (tensor.version == QNN_TENSOR_VERSION_1) { return tensor.v1.memType; @@ -630,153 +353,72 @@ static inline Qnn_TensorMemType_t get_qnn_tensor_memtype(const Qnn_Tensor_t & te return QNN_TENSORMEMTYPE_UNDEFINED; } -[[maybe_unused]] static inline Qnn_TensorMemType_t get_qnn_tensor_memtype(const Qnn_Tensor_t * tensor) { - return get_qnn_tensor_memtype(*tensor); -} - -static inline Qnn_ClientBuffer_t get_qnn_tensor_clientbuf(const Qnn_Tensor_t & tensor) { - if (tensor.version == QNN_TENSOR_VERSION_1) { - return tensor.v1.clientBuf; - } - return QNN_CLIENT_BUFFER_INIT; -} - -[[maybe_unused]] static inline Qnn_ClientBuffer_t get_qnn_tensor_clientbuf(const Qnn_Tensor_t * tensor) { - return get_qnn_tensor_clientbuf(*tensor); -} - -static inline Qnn_MemHandle_t get_qnn_tensor_memhandle(const Qnn_Tensor_t & tensor) { - if (tensor.version == QNN_TENSOR_VERSION_1) { - return tensor.v1.memHandle; - } - return nullptr; -} - -[[maybe_unused]] static inline Qnn_MemHandle_t get_qnn_tensor_memhandle(const Qnn_Tensor_t * tensor) { - return get_qnn_tensor_memhandle(*tensor); -} - static inline void set_qnn_tensor_id(Qnn_Tensor_t & tensor, uint32_t id) { if (tensor.version == QNN_TENSOR_VERSION_1) { tensor.v1.id = id; } } -[[maybe_unused]] static inline void set_qnn_tensor_id(Qnn_Tensor_t * tensor, uint32_t id) { - set_qnn_tensor_id(*tensor, id); -} - static inline void set_qnn_tensor_name(Qnn_Tensor_t & tensor, const char * name) { if (tensor.version == QNN_TENSOR_VERSION_1) { tensor.v1.name = name; } } -[[maybe_unused]] static inline void set_qnn_tensor_name(Qnn_Tensor_t * tensor, const char * name) { - set_qnn_tensor_name(*tensor, name); -} - static inline void set_qnn_tensor_type(Qnn_Tensor_t & tensor, Qnn_TensorType_t type) { if (tensor.version == QNN_TENSOR_VERSION_1) { tensor.v1.type = type; } } -[[maybe_unused]] static inline void set_qnn_tensor_type(Qnn_Tensor_t * tensor, Qnn_TensorType_t type) { - set_qnn_tensor_type(*tensor, type); -} - static inline void set_qnn_tensor_dataformat(Qnn_Tensor_t & tensor, Qnn_TensorDataFormat_t format) { if (tensor.version == QNN_TENSOR_VERSION_1) { tensor.v1.dataFormat = format; } } -[[maybe_unused]] static inline void set_qnn_tensor_dataformat(Qnn_Tensor_t * tensor, Qnn_TensorDataFormat_t format) { - set_qnn_tensor_dataformat(*tensor, format); -} - static inline void set_qnn_tensor_datatype(Qnn_Tensor_t & tensor, Qnn_DataType_t dataType) { if (tensor.version == QNN_TENSOR_VERSION_1) { tensor.v1.dataType = dataType; } } -[[maybe_unused]] static inline void set_qnn_tensor_datatype(Qnn_Tensor_t * tensor, Qnn_DataType_t dataType) { - set_qnn_tensor_datatype(*tensor, dataType); -} - static inline void set_qnn_tensor_quantparams(Qnn_Tensor_t & tensor, Qnn_QuantizeParams_t params) { if (tensor.version == QNN_TENSOR_VERSION_1) { tensor.v1.quantizeParams = params; } } -[[maybe_unused]] static inline void set_qnn_tensor_quantparams(Qnn_Tensor_t * tensor, Qnn_QuantizeParams_t params) { - set_qnn_tensor_quantparams(*tensor, params); -} - static inline void set_qnn_tensor_rank(Qnn_Tensor_t & tensor, uint32_t rank) { if (tensor.version == QNN_TENSOR_VERSION_1) { tensor.v1.rank = rank; } } -[[maybe_unused]] static inline void set_qnn_tensor_rank(Qnn_Tensor_t * tensor, uint32_t rank) { - set_qnn_tensor_rank(*tensor, rank); -} - static inline void set_qnn_tensor_dimensions(Qnn_Tensor_t & tensor, uint32_t * dims) { if (tensor.version == QNN_TENSOR_VERSION_1) { tensor.v1.dimensions = dims; } } -[[maybe_unused]] static inline void set_qnn_tensor_dimensions(Qnn_Tensor_t * tensor, uint32_t * dims) { - set_qnn_tensor_dimensions(*tensor, dims); -} - static inline void set_qnn_tensor_memtype(Qnn_Tensor_t & tensor, Qnn_TensorMemType_t memType) { if (tensor.version == QNN_TENSOR_VERSION_1) { tensor.v1.memType = memType; } } -[[maybe_unused]] static inline void set_qnn_tensor_memtype(Qnn_Tensor_t * tensor, Qnn_TensorMemType_t memType) { - set_qnn_tensor_memtype(*tensor, memType); -} - static inline void set_qnn_tensor_clientbuf(Qnn_Tensor_t & tensor, Qnn_ClientBuffer_t clientBuf) { if (tensor.version == QNN_TENSOR_VERSION_1) { tensor.v1.clientBuf = clientBuf; } } -[[maybe_unused]] static inline void set_qnn_tensor_clientbuf(Qnn_Tensor_t * tensor, Qnn_ClientBuffer_t clientBuf) { - set_qnn_tensor_clientbuf(*tensor, clientBuf); -} - static inline void set_qnn_tensor_memhandle(Qnn_Tensor_t & tensor, Qnn_MemHandle_t handle) { if (tensor.version == QNN_TENSOR_VERSION_1) { tensor.v1.memHandle = handle; } } -[[maybe_unused]] static inline void set_qnn_tensor_memhandle(Qnn_Tensor_t * tensor, Qnn_MemHandle_t handle) { - set_qnn_tensor_memhandle(*tensor, handle); -} - -inline static Qnn_Tensor_t qnn_tensor_init(Qnn_TensorVersion_t version) { - Qnn_Tensor_t tensor; - tensor.version = version; - if (version == QNN_TENSOR_VERSION_1) { - tensor.v1 = QNN_TENSOR_V1_INIT; - } else if (version == QNN_TENSOR_VERSION_2) { - tensor.v2 = QNN_TENSOR_V2_INIT; - } - return tensor; -} - static int deep_copy_qnn_tensors(Qnn_Tensor_t & src, Qnn_Tensor_t & dst) { int err = 0; VALIDATE_TENSOR_VERSION(src, err); @@ -2445,22 +2087,7 @@ int qnn_instance::load_backend(std::string & lib_path, const QnnSaver_Config_t * } _loaded_lib_handle[backend_id] = lib_handle; _backend_id = backend_id; - -#if 0 // leave them here for further use - QnnSaver_Config_t outputdir_cfg; - outputdir_cfg.option = QNN_SAVER_CONFIG_OPTION_OUTPUT_DIRECTORY; - outputdir_cfg.outputDirectory = "/data/local/tmp/"; - QnnSaver_Config_t backendid_cfg; - backendid_cfg.option = QNN_SAVER_CONFIG_OPTION_BACKEND_ID; - backendid_cfg.backendId = _backend_id; - - const QnnSaver_Config_t * saver_cfg[] = {&outputdir_cfg, &backendid_cfg, nullptr}; - if (0 == QnnSaver_initialize(saver_cfg)) { - GGMLQNN_LOG_INFO("QnnSaver_initialize successfully"); - } else { - GGMLQNN_LOG_WARN("QnnSaver_initialize failure"); - } -#endif + auto saver_initialize = load_qnn_functionpointers<_pfn_QnnSaver_initialize *>( _loaded_lib_handle[backend_id], "QnnSaver_initialize"); @@ -3682,14 +3309,6 @@ static void ggml_backend_qnn_buffer_clear(ggml_backend_buffer_t buffer, uint8_t memset(ctx->buffer, value, ctx->buffer_size); } -[[maybe_unused]]static void ggml_backend_qnn_buffer_reset(ggml_backend_buffer_t buffer) { - ggml_backend_qnn_buffer_context * ctx = (ggml_backend_qnn_buffer_context *)buffer->context; - for (auto * sub_buffer : ctx->sub_buffers) { - free(sub_buffer); - } - ctx->sub_buffers.clear(); -} - static ggml_backend_buffer_i ggml_backend_qnn_buffer_interface = { /* .free_buffer = */ ggml_backend_qnn_buffer_free_buffer, /* .get_base = */ ggml_backend_qnn_buffer_get_base, From bda5d0fbadfbb482a357f734d7b0b0ff40acd16d Mon Sep 17 00:00:00 2001 From: zhouwg Date: Tue, 25 Feb 2025 13:49:04 +0800 Subject: [PATCH 25/76] ggml-qnn: refine code format --- ggml/src/ggml-qnn/ggml-qnn.cpp | 285 ++++++++++++++++++++------------- 1 file changed, 177 insertions(+), 108 deletions(-) diff --git a/ggml/src/ggml-qnn/ggml-qnn.cpp b/ggml/src/ggml-qnn/ggml-qnn.cpp index 8b33d346bd91a..aaf9fd694f8b9 100644 --- a/ggml/src/ggml-qnn/ggml-qnn.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn.cpp @@ -44,12 +44,14 @@ #include #include #include +#if defined(__ANDROID__) || defined(__linux__) #include #include #include #include #include #include +#endif #include #include @@ -77,6 +79,10 @@ #include "android/log.h" #endif +#if defined(_WIN32) || defined(_MSC_VER) +#include +#endif + #include "QnnTypes.h" #include "QnnCommon.h" #include "QnnContext.h" @@ -98,7 +104,7 @@ // ================================================================================================= class qnn_instance; struct ggml_backend_qnn_context; -static int free_qnn_tensor(Qnn_Tensor_t * tensor); +static int free_qnn_tensor(Qnn_Tensor_t * tensor); static enum ggml_status ggml_backend_qnn_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph); static void ggmlqnn_log_internal(ggml_log_level level, const char * file, const char * func, int line, const char * format, ...); static Qnn_Tensor_t * ggml_qnn_create_general_tensor(const ggml_tensor * tensor, const char * name, @@ -180,9 +186,11 @@ static size_t get_system_total_memory_in_bytes() { auto page_size = (size_t)sysconf(_SC_PAGE_SIZE); return pages * page_size; -#else +#elif defined(_WIN32) || defined(_MSC_VER) //TODO: Snapdragon based WoA(Windows on ARM) return 0; +#else +#error "ggml-qnn only support WoA, Android, Linux" #endif } @@ -196,9 +204,11 @@ static size_t get_system_free_memory_in_bytes() { auto page_size = (size_t)sysconf(_SC_PAGE_SIZE); return avail_pages * page_size; -#else +#elif defined(_WIN32) || defined(_MSC_VER) //TODO: Snapdragon based WoA(Windows on ARM) return 0; +#else +#error "ggml-qnn only support WoA, Android, Linux" #endif } @@ -218,12 +228,19 @@ static char * ggmlqnn_strndup(const char * source, size_t maxlen) { } static void * ggmlqnn_host_malloc(size_t n) { - void * data = NULL; - int result = posix_memalign((void **) &data, sysconf(_SC_PAGESIZE), n); +#if defined(__ANDROID__) || defined(__linux__) + void * data = nullptr; + int result = posix_memalign((void **)&data, sysconf(_SC_PAGESIZE), n); if (result != 0) { GGMLQNN_LOG_WARN("%s: error: posix_memalign failed\n", __func__); - return NULL; + return nullptr; } +#elif defined(_WIN32) || defined(_MSC_VER) + //TODO: Snapdragon based WoA(Windows on ARM) + return nullptr; +#else +#error "ggml-qnn only support WoA, Android, Linux" +#endif return data; } @@ -231,15 +248,6 @@ static void * ggmlqnn_host_malloc(size_t n) { // ================================================================================================= // section-4: QNN helper macro / data structure / function // ================================================================================================= -#define VALIDATE(value, status) \ - do { \ - status = value; \ - if (status != QNN_SUCCESS) { \ - GGMLQNN_LOG_WARN("%s expected QNN_SUCCESS\n", #value); \ - return status; \ - } \ - } while (0) - #define CHECK_QNN_API(error, result) \ do { \ error = (result); \ @@ -252,8 +260,6 @@ static void * ggmlqnn_host_malloc(size_t n) { } \ } while (0) -#define VALIDATE_TENSOR_VERSION(tensor, err) VALIDATE(validate_tensor_version(tensor), err) - #define QNN_VER_PTR(x) (&((x).v1)) #define QNN_TENSOR_GET_ID(tensor) get_qnn_tensorid(tensor) #define QNN_TENSOR_GET_NAME(tensor) get_qnn_tensorname(tensor) @@ -279,16 +285,6 @@ static void * ggmlqnn_host_malloc(size_t n) { #define QNN_TENSOR_SET_CLIENT_BUF(tensor, value) set_qnn_tensor_clientbuf(tensor, value) #define QNN_TENSOR_SET_MEM_HANDLE(tensor, value) set_qnn_tensor_memhandle(tensor, value) -static inline int validate_tensor_version(Qnn_Tensor_t tensor) { - if (tensor.version != QNN_TENSOR_VERSION_1) { - GGMLQNN_LOG_WARN("validate_tensor_version() tensor %s, got unsupported version %d\n", - tensor.v1.name, - tensor.version); - return 1; - } - return 0; -} - static inline uint32_t get_qnn_tensorid(const Qnn_Tensor_t & tensor) { if (tensor.version == QNN_TENSOR_VERSION_1) { return tensor.v1.id; @@ -421,7 +417,6 @@ static inline void set_qnn_tensor_memhandle(Qnn_Tensor_t & tensor, Qnn_MemHandle static int deep_copy_qnn_tensors(Qnn_Tensor_t & src, Qnn_Tensor_t & dst) { int err = 0; - VALIDATE_TENSOR_VERSION(src, err); dst.version = src.version; QNN_TENSOR_SET_NAME( @@ -492,7 +487,7 @@ static int deep_copy_qnn_tensors(Qnn_Tensor_t & src, Qnn_Tensor_t & dst) { static int free_qnn_tensor(Qnn_Tensor_t * tensor) { int err = 0; - VALIDATE_TENSOR_VERSION(*tensor, err); + free((void *) QNN_TENSOR_GET_NAME(*tensor)); Qnn_QuantizeParams_t src_qparam = QNN_TENSOR_GET_QUANT_PARAMS(*tensor); @@ -511,7 +506,6 @@ static int free_qnn_tensor(Qnn_Tensor_t * tensor) { return err; } - static size_t qnn_datatype_size(Qnn_DataType_t qnn_type) { switch (qnn_type) { case QNN_DATATYPE_FLOAT_32: @@ -720,6 +714,11 @@ enum qcom_chipset_soc_model { SM8550 = 43, // v73, SD 8 Gen 2 SM8650 = 57, // v75, SD 8 Gen 3 SM8750 = 69, // v79, SD 8 Gen 4 +#if defined(_WIN32) || defined(_MSC_VER) + SC7280X = 44, + SC8280X = 37, + SC8380XP = 60, +#endif }; struct qcom_socinfo { @@ -780,6 +779,29 @@ static struct qcom_socinfo g_qnn_soc_info_table[] = { .vtcm_size_in_mb = 8, .soc_desc = "Qualcomm SnapDragon 8 Gen 4"}, +#if defined(_WIN32) || defined(_MSC_VER) + /* Qualcomm SnapDragon 7c Gen 2 */ + [SC7280X] = { + .soc_model = SC7280X, + .htp_arch = V68, + .vtcm_size_in_mb = 8, + .soc_desc = "Qualcomm SnapDragon 7c Gen 2"}, + + /* Qualcomm SnapDragon 8cx Gen 3 */ + [SC8280X] = { + .soc_model = SC8280X, + .htp_arch = V68, + .vtcm_size_in_mb = 8, + .soc_desc = "Qualcomm SnapDragon 8cx Gen 3"}, + + /* Qualcomm SnapDragon 8cx Gen 4 */ + [SC8380XP] = { + .soc_model = SC8380XP, + .htp_arch = V73, + .vtcm_size_in_mb = 8, + .soc_desc = "Qualcomm SnapDragon 8cx Gen 4"}, +#endif + }; struct ggml_backend_qnn_context { @@ -820,7 +842,11 @@ static struct ggml_backend_qnn_context g_qnn_mgr[GGML_QNN_MAX_DEVICES] = { .threads = 1, .name = "qnn-cpu", .desc = "Qualcomm Kryo CPU", +#if defined(_WIN32) || defined(_MSC_VER) + .lib = "QnnCpu.dll", +#else .lib = "libQnnCpu.so", +#endif .instance = nullptr, .backend = nullptr, .raw_interface = {}, @@ -831,7 +857,11 @@ static struct ggml_backend_qnn_context g_qnn_mgr[GGML_QNN_MAX_DEVICES] = { .threads = 1, .name = "qnn-gpu", .desc = "Qualcomm Adreno GPU", +#if defined(_WIN32) || defined(_MSC_VER) + .lib = "QnnGpu.dll", +#else .lib = "libQnnGpu.so", +#endif .instance = nullptr, .backend = nullptr, .raw_interface = {}, @@ -842,7 +872,11 @@ static struct ggml_backend_qnn_context g_qnn_mgr[GGML_QNN_MAX_DEVICES] = { .threads = 1, .name = "qnn-npu", .desc = "Qualcomm NPU(Hexagon Tensor Processor)", +#if defined(_WIN32) || defined(_MSC_VER) + .lib = "QnnHtp.dll", +#else .lib = "libQnnHtp.so", +#endif .instance = nullptr, .backend = nullptr, .raw_interface = {}, @@ -1252,8 +1286,8 @@ static Qnn_Tensor_t * ggml_qnn_create_compute_tensor(const ggml_tensor * tensor) qnn_data_type = qnn_datatype_from_ggml_datatype(tensor->type); Qnn_Tensor_t * p_qnn_tensor = ggml_qnn_create_general_tensor(tensor, nullptr, qnn_tensor_type, qnn_data_type, - ggml_n_dims(tensor), dimensions, - nullptr, 0); + ggml_n_dims(tensor), dimensions, + nullptr, 0); return p_qnn_tensor; } @@ -1351,7 +1385,14 @@ class qnn_perf { template Fn load_qnn_functionpointers(void * handle, const char * function_name) { +#if defined(__ANDROID__) || defined(__linux__) return reinterpret_cast(dlsym(handle, function_name)); +#elif defined(_WIN32) || defined(_MSC_VER) + //TODO: Snapdragon based WoA(Windows on ARM) + return nullptr; +#else +#error "ggml-qnn only support WoA, Android, Linux" +#endif } class qnn_interface { @@ -1485,7 +1526,7 @@ class qnn_instance { using BackendIdType = decltype(QnnInterface_t{}.backendId); explicit qnn_instance(const std::string & lib_path, const std::string & backend_name, - const std::string & model_name) : + const std::string & model_name) : _lib_path(std::move(lib_path)), _backend_name(std::move(backend_name)), _model_name(std::move(model_name)) {}; @@ -1567,8 +1608,7 @@ class qnn_instance { if (_qnn_rpc_pollingtime > 0) { QnnHtpPerfInfrastructure_PowerConfig_t rpc_pollingtime; memset(&rpc_pollingtime, 0, sizeof(rpc_pollingtime)); - rpc_pollingtime.option = - QNN_HTP_PERF_INFRASTRUCTURE_POWER_CONFIGOPTION_RPC_POLLING_TIME; + rpc_pollingtime.option = QNN_HTP_PERF_INFRASTRUCTURE_POWER_CONFIGOPTION_RPC_POLLING_TIME; rpc_pollingtime.rpcPollingTimeConfig = _qnn_rpc_pollingtime; const QnnHtpPerfInfrastructure_PowerConfig_t * power_configs[] = {&rpc_pollingtime, nullptr}; if (_qnn_htp_perfinfra) { @@ -1653,16 +1693,15 @@ class qnn_instance { } void probe_device_meminfo() { - size_t candidate_size = 0; - uint8_t *rpc_buffer = nullptr; - const int SIZE_IN_MB = (1 << 20); - size_t probe_slots[] = {1024, 1536, 2048 - 48, 2048}; - size_t probe_counts = sizeof(probe_slots) / sizeof(size_t); + size_t candidate_size = 0; + uint8_t * rpc_buffer = nullptr; + const int SIZE_IN_MB = (1 << 20); + size_t probe_slots[] = {1024, 1536, 2048 - 48, 2048}; + size_t probe_counts = sizeof(probe_slots) / sizeof(size_t); for (size_t idx = 0; idx < probe_counts; idx++) { rpc_buffer = static_cast(alloc_rpcmem_internal(probe_slots[idx] * SIZE_IN_MB, 4)); if (nullptr == rpc_buffer) { - GGMLQNN_LOG_DEBUG("alloc rpcmem %d (MB) failure, %s\n", probe_slots[idx], - strerror(errno)); + GGMLQNN_LOG_DEBUG("alloc rpcmem %d (MB) failure, %s\n", probe_slots[idx], strerror(errno)); break; } else { candidate_size = probe_slots[idx]; @@ -1697,17 +1736,17 @@ class qnn_instance { void set_qnn_raw_system_interface(QNN_SYSTEM_INTERFACE_VER_TYPE & raw_interface) { _qnn_raw_system_interface = raw_interface; } - + void * alloc_rpcmem_internal(size_t bytes, size_t alignment); private: static constexpr const int _required_num_providers = 1; private: - std::string _lib_path; - std::string _backend_name; - std::string _model_name; // name of prebuilt QNN model, might be used in the future - BackendIdType _backend_id; + std::string _lib_path; + std::string _backend_name; + std::string _model_name; // name of prebuilt QNN model, might be used in the future + BackendIdType _backend_id; bool _debug_tensor = false; // flag to indicate if requested graph is to be run in debug mode bool _do_node_validations = true; // flag to indicate whether all add_node calls need to be validated @@ -1715,17 +1754,15 @@ class qnn_instance { ggml_qnn_profile_level _profile_level = ggml_qnn_profile_level::profile_detail; - qnn_interface _qnn_interface; - - void * _system_lib_handle = nullptr; + void * _system_lib_handle = nullptr; - Qnn_GraphHandle_t _qnn_graph_handle = nullptr; + Qnn_GraphHandle_t _qnn_graph_handle = nullptr; - Qnn_LogHandle_t _qnn_log_handle = nullptr; + Qnn_LogHandle_t _qnn_log_handle = nullptr; Qnn_ProfileHandle_t _qnn_profile_handle = nullptr; - Qnn_DeviceHandle_t _qnn_device_handle = nullptr; + Qnn_DeviceHandle_t _qnn_device_handle = nullptr; Qnn_BackendHandle_t _qnn_backend_handle = nullptr; @@ -1733,10 +1770,11 @@ class qnn_instance { QnnSystemContext_Handle_t _qnn_system_handle = nullptr; - QnnHtpDevice_PerfInfrastructure_t *_qnn_htp_perfinfra = nullptr; - uint32_t _qnn_power_configid = 1; - uint32_t _qnn_rpc_pollingtime = 9999; // 0-10000 us for high performing + QnnHtpDevice_PerfInfrastructure_t * _qnn_htp_perfinfra = nullptr; + uint32_t _qnn_power_configid = 1; + uint32_t _qnn_rpc_pollingtime = 9999; // 0-10000 us for high performing + qnn_interface _qnn_interface; QNN_INTERFACE_VER_TYPE _qnn_raw_interface; QNN_SYSTEM_INTERFACE_VER_TYPE _qnn_raw_system_interface; @@ -1748,7 +1786,6 @@ class qnn_instance { static std::unordered_map _lib_path_to_backend_id; static std::unordered_map _loaded_backend; - void * _rpc_lib_handle = nullptr; std::atomic_bool _rpcmem_initialized{false}; pfn_rpc_mem_alloc _pfn_rpc_mem_alloc; pfn_rpc_mem_free _pfn_rpc_mem_free; @@ -1757,12 +1794,13 @@ class qnn_instance { pfn_rpc_mem_deinit _pfn_rpc_mem_deinit; std::unordered_map _rpcmem_store_map; std::unordered_map _rpcmem_usage_map; - size_t _rpcmem_capacity = 512; // mempool size in Mbytes size_t _rpcmem_usage = 0; // mempool usage in Mbytes + size_t _rpcmem_capacity = 512; // mempool size in Mbytes std::string _graph_name; QNNBackend _device_id; - bool _enable_qnn_rpc = false; //TODO:unknown issue with QNN RPC feature + void * _rpc_lib_handle = nullptr; + bool _enable_qnn_rpc = false; //TODO:unknown issue with QNN RPC feature DISABLE_COPY(qnn_instance); DISABLE_MOVE(qnn_instance); @@ -1781,13 +1819,13 @@ void * qnn_instance::alloc_rpcmem_internal(size_t bytes, size_t alignment) { auto allocate_bytes = static_cast(bytes + alignment); void * buf = _pfn_rpc_mem_alloc(RPCMEM_HEAP_ID_SYSTEM, RPCMEM_DEFAULT_FLAGS, allocate_bytes); - if (buf == nullptr) { + if (nullptr == buf) { GGMLQNN_LOG_WARN("failed to allocate rpc memory\n"); return nullptr; } auto aligned_buf = reinterpret_cast(ggmlqnn_align_to(alignment, - reinterpret_cast(buf))); + reinterpret_cast(buf))); bool status = _rpcmem_store_map.insert(std::pair(aligned_buf, buf)).second; if (!status) { GGMLQNN_LOG_WARN("failed to allocate rpc memory\n"); @@ -1886,13 +1924,13 @@ int qnn_instance::register_rpcmem(void * p_data, Qnn_Tensor_t * p_tensor) { if (is_rpcmem_registered((QNN_VER_PTR(*p_tensor)->memHandle))) { GGMLQNN_LOG_WARN("tensor %s has been registered shared memory\n", (QNN_VER_PTR(*p_tensor)->name)); - return 4; + return 3; } int32_t mem_fd = rpcmem_to_fd(p_data); if (-1 == mem_fd) { GGMLQNN_LOG_WARN("failed to get file descriptor\n"); - return 5; + return 4; } GGMLQNN_LOG_DEBUG("mem_fd %d\n", mem_fd); Qnn_MemDescriptor_t descriptor = { @@ -1908,9 +1946,8 @@ int qnn_instance::register_rpcmem(void * p_data, Qnn_Tensor_t * p_tensor) { /*numDescriptors=*/1, &handle); if (error != QNN_SUCCESS) { - GGMLQNN_LOG_WARN("failed to register shared memory, error %d, %s\n", QNN_GET_ERROR_CODE(error), - strerror(error)); - return 6; + GGMLQNN_LOG_WARN("failed to register shared memory, error %d, %s\n", QNN_GET_ERROR_CODE(error), strerror(error)); + return 5; } else { GGMLQNN_LOG_INFO("tensor %s successfully register shared memory\n", (QNN_VER_PTR(*p_tensor)->name)); } @@ -1949,8 +1986,7 @@ Qnn_MemHandle_t qnn_instance::register_rpcmem(void * p_data, const uint32_t ran {{mem_fd}} }; Qnn_MemHandle_t handle = nullptr; - auto error = _qnn_interface.qnn_mem_register(_qnn_context_handle, &descriptor, - /*numDescriptors=*/1, &handle); + auto error = _qnn_interface.qnn_mem_register(_qnn_context_handle, &descriptor, /*numDescriptors=*/1, &handle); if (error != QNN_SUCCESS) { GGMLQNN_LOG_WARN("failed to register shared memory, error %d, %s", QNN_GET_ERROR_CODE(error), strerror(error)); return nullptr; @@ -1987,8 +2023,7 @@ void qnn_instance::unregister_rpcmem() { Qnn_MemHandle_t mem_handle = it->second; error = _qnn_interface.qnn_mem_de_register(&mem_handle, 1); if (error != QNN_SUCCESS) { - GGMLQNN_LOG_WARN("failed to unregister shared memory, error %d\n", - QNN_GET_ERROR_CODE(error)); + GGMLQNN_LOG_WARN("failed to unregister shared memory, error %d\n", QNN_GET_ERROR_CODE(error)); } else { GGMLQNN_LOG_DEBUG("unregister shared memory ok"); } @@ -2020,15 +2055,22 @@ int qnn_instance::load_backend(std::string & lib_path, const QnnSaver_Config_t * Qnn_ErrorHandle_t error = QNN_SUCCESS; GGMLQNN_LOG_DEBUG("lib_path:%s\n", lib_path.c_str()); +#if defined(__ANDROID__) || defined(__linux__) void * lib_handle = dlopen(lib_path.c_str(), RTLD_NOW | RTLD_GLOBAL); +#elif defined(_WIN32) || defined(_MSC_VER) + //TODO: Snapdragon based WoA(Windows on ARM) + void * lib_handle = nullptr; +#else +#error "ggml-qnn only support WoA, Android, Linux" +#endif if (nullptr == lib_handle) { GGMLQNN_LOG_WARN("can not open QNN library %s, with error: %s", lib_path.c_str(), dlerror()); return 1; } - auto get_providers = - load_qnn_functionpointers<_pfn_QnnInterface_getProviders *>(lib_handle, - "QnnInterface_getProviders"); + auto get_providers = load_qnn_functionpointers<_pfn_QnnInterface_getProviders *>( + lib_handle, + "QnnInterface_getProviders"); if (nullptr == get_providers) { GGMLQNN_LOG_WARN("can not load symbol QnnInterface_getProviders : %s", dlerror()); return 2; @@ -2087,7 +2129,7 @@ int qnn_instance::load_backend(std::string & lib_path, const QnnSaver_Config_t * } _loaded_lib_handle[backend_id] = lib_handle; _backend_id = backend_id; - + auto saver_initialize = load_qnn_functionpointers<_pfn_QnnSaver_initialize *>( _loaded_lib_handle[backend_id], "QnnSaver_initialize"); @@ -2106,7 +2148,7 @@ int qnn_instance::load_backend(std::string & lib_path, const QnnSaver_Config_t * int qnn_instance::unload_backend() { int dlclose_error = 0; - for (auto &it : _loaded_lib_handle) { + for (auto & it : _loaded_lib_handle) { dlclose_error = dlclose(it.second); if (dlclose_error != 0) { GGMLQNN_LOG_WARN("failed to close QNN backend %d, error %s\n", it.first, dlerror()); @@ -2126,13 +2168,27 @@ int qnn_instance::load_system() { std::string system_lib_path = _lib_path + "libQnnSystem.so"; GGMLQNN_LOG_DEBUG("system_lib_path:%s\n", system_lib_path.c_str()); +#if defined(__ANDROID__) || defined(__linux__) _system_lib_handle = dlopen(system_lib_path.c_str(), RTLD_NOW | RTLD_LOCAL); +#elif defined(_WIN32) || defined(_MSC_VER) + //TODO: Snapdragon based WoA(Windows on ARM) + _system_lib_handle = nullptr; +#else +#error "ggml-qnn only support WoA, Android, Linux" +#endif if (nullptr == _system_lib_handle) { GGMLQNN_LOG_WARN("can not open QNN library %s, error: %s\n", system_lib_path.c_str(), dlerror()); //re-try with default path of QNN binary runtime lib _lib_path = "/data/local/tmp/"; system_lib_path = _lib_path + "libQnnSystem.so"; +#if defined(__ANDROID__) || defined(__linux__) _system_lib_handle = dlopen(system_lib_path.c_str(), RTLD_NOW | RTLD_LOCAL); +#elif defined(_WIN32) || defined(_MSC_VER) + //TODO: Snapdragon based WoA(Windows on ARM) + _system_lib_handle = nullptr; +#else +#error "ggml-qnn only support WoA, Android, Linux" +#endif if (nullptr == _system_lib_handle) { GGMLQNN_LOG_WARN("can not open QNN library %s, error: %s\n", system_lib_path.c_str(), dlerror()); return 1; @@ -2348,7 +2404,7 @@ int qnn_instance::qnn_init(const QnnSaver_Config_t ** saver_config) { if (QNN_PROFILE_NO_ERROR != _qnn_raw_interface.profileCreate( _qnn_backend_handle, QNN_PROFILE_LEVEL_BASIC, &_qnn_profile_handle)) { GGMLQNN_LOG_WARN("unable to create profile handle in the backend\n"); - return 7; + return 6; } else { GGMLQNN_LOG_DEBUG("initialize qnn profile successfully\n"); } @@ -2364,10 +2420,17 @@ int qnn_instance::qnn_init(const QnnSaver_Config_t ** saver_config) { } } +#if defined(__ANDROID__) || defined(__linux__) _rpc_lib_handle = dlopen("libcdsprpc.so", RTLD_NOW | RTLD_LOCAL); +#elif defined(_WIN32) || defined(_MSC_VER) + //TODO: Snapdragon based WoA(Windows on ARM) + _rpc_lib_handle = nullptr; +#else +#error "ggml-qnn only support WoA, Android, Linux" +#endif if (nullptr == _rpc_lib_handle) { GGMLQNN_LOG_WARN("failed to load qualcomm's rpc lib, error:%s\n", dlerror()); - return 9; + return 8; } else { GGMLQNN_LOG_DEBUG("load rpcmem lib successfully\n"); set_rpcmem_initialized(true); @@ -2381,7 +2444,7 @@ int qnn_instance::qnn_init(const QnnSaver_Config_t ** saver_config) { || nullptr == _pfn_rpc_mem_to_fd) { GGMLQNN_LOG_WARN("unable to access symbols in QNN RPC lib. dlerror(): %s", dlerror()); dlclose(_rpc_lib_handle); - return 10; + return 9; } if (nullptr != _pfn_rpc_mem_init) // make Qualcomm's SoC based low-end phone happy @@ -2393,7 +2456,7 @@ int qnn_instance::qnn_init(const QnnSaver_Config_t ** saver_config) { &_qnn_context_handle); if (nullptr == _qnn_context_handle) { GGMLQNN_LOG_WARN("why failed to initialize qnn context, error:%s\n", strerror(errno)); - return 8; + return 10; } else { GGMLQNN_LOG_DEBUG("initialize qnn context successfully\n"); } @@ -2578,7 +2641,7 @@ int qnn_instance::init_qnn_graph(const std::string & graph_name, QNNBackend devi } int qnn_instance::init_qnn_graph(const char * graph_name, bool debug, uint8_t do_node_validation, - const QnnGraph_Config_t ** graph_configs) { + const QnnGraph_Config_t ** graph_configs) { int result = 0; if (nullptr == graph_name) { @@ -2685,14 +2748,18 @@ static bool ggml_qnn_can_handle_op(const struct ggml_tensor * tensor) { return true; } if (ggml_is_empty(tensor) || tensor->op == GGML_OP_RESHAPE - || tensor->op == GGML_OP_TRANSPOSE || tensor->op == GGML_OP_VIEW - || tensor->op == GGML_OP_PERMUTE) { + || tensor->op == GGML_OP_TRANSPOSE + || tensor->op == GGML_OP_VIEW + || tensor->op == GGML_OP_PERMUTE + ) { return false; } //TODO: support other op - bool supported_op = ((tensor->op == GGML_OP_ADD) || (tensor->op == GGML_OP_MUL_MAT) - || (tensor->op == GGML_OP_MUL)); + bool supported_op = ((tensor->op == GGML_OP_ADD) + || (tensor->op == GGML_OP_MUL_MAT) + || (tensor->op == GGML_OP_MUL) + ); if (!supported_op) { return false; } @@ -2700,14 +2767,14 @@ static bool ggml_qnn_can_handle_op(const struct ggml_tensor * tensor) { struct ggml_tensor * src0 = tensor->src[0]; struct ggml_tensor * src1 = tensor->src[1]; - const int64_t ne00 = tensor->src[0]->ne[0]; - const int64_t ne01 = tensor->src[0]->ne[1]; + const int64_t ne00 = tensor->src[0]->ne[0]; + const int64_t ne01 = tensor->src[0]->ne[1]; - const int64_t ne10 = tensor->src[1]->ne[0]; - const int64_t ne11 = tensor->src[1]->ne[1]; + const int64_t ne10 = tensor->src[1]->ne[0]; + const int64_t ne11 = tensor->src[1]->ne[1]; - const int64_t ne0 = tensor->ne[0]; - const int64_t ne1 = tensor->ne[1]; + const int64_t ne0 = tensor->ne[0]; + const int64_t ne1 = tensor->ne[1]; const uint32_t src0_rank = ggml_get_tensor_rank(src0); const uint32_t src1_rank = ggml_get_tensor_rank(src1); @@ -3104,7 +3171,7 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { Qnn_Param_t out_0_params[] = { {QNN_PARAMTYPE_SCALAR, QNN_OP_MAT_MUL_PARAM_TRANSPOSE_IN1, - .scalarParam = {QNN_DATATYPE_BOOL_8, .bool8Value = 1} + .scalarParam = {QNN_DATATYPE_BOOL_8, .bool8Value = 1} } }; @@ -3154,13 +3221,13 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle,out_trans1_0)); //step-6: finalize qnn graph and execute qnn graph - CHECK_QNN_API(error, qnn_raw_interface.graphFinalize(graph_handle, NULL, NULL)); + CHECK_QNN_API(error, qnn_raw_interface.graphFinalize(graph_handle, nullptr, nullptr)); Qnn_Tensor_t input_tensors_0[] = {*p_tensor0, *p_tensor1}; Qnn_Tensor_t output_tensors_0[] = {*p_tensor2}; CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, input_tensors_0, 2, output_tensors_0, 1, - NULL, NULL)); + nullptr, nullptr)); qnn_tensors_t ggml_op_mulmat_tensors; ggml_op_mulmat_tensors.reserve(5); @@ -3318,7 +3385,7 @@ static ggml_backend_buffer_i ggml_backend_qnn_buffer_interface = { /* .get_tensor = */ ggml_backend_qnn_buffer_get_tensor, /* .cpy_tensor = */ ggml_backend_qnn_buffer_cpy_tensor, /* .clear = */ ggml_backend_qnn_buffer_clear, - /* .reset = */ NULL, + /* .reset = */ nullptr, }; static const char * ggml_backend_qnn_buffer_type_name(ggml_backend_buffer_type_t buft) { @@ -3349,7 +3416,7 @@ static size_t ggml_backend_qnn_buffer_type_get_alignment(ggml_backend_buffer_typ return 32; } -//FIXME: this value is an experimental value on Xiaomi14 +//FIXME: this value is an experimental value on Snapdragon 8 Gen3 based phone static size_t ggml_backend_qnn_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) { GGML_UNUSED(buft); @@ -3429,8 +3496,6 @@ static const char * ggml_backend_qnn_device_get_name(ggml_backend_dev_t dev) { return "unknown"; } return ctx->name; - - GGML_UNUSED(dev); } static const char * ggml_backend_qnn_device_get_description(ggml_backend_dev_t dev) { @@ -3520,10 +3585,10 @@ ggml_backend_buffer_type_t ggml_backend_qnn_buffer_type(size_t device_index) { /* .alloc_buffer = */ ggml_backend_qnn_buffer_type_alloc_buffer, /* .get_alignment = */ ggml_backend_qnn_buffer_type_get_alignment, /* .get_max_size = */ ggml_backend_qnn_buffer_type_get_max_size, - /* .get_alloc_size = */ NULL,// defaults to ggml_nbytes + /* .get_alloc_size = */ nullptr,// defaults to ggml_nbytes /* .is_host = */ ggml_backend_qnn_buffer_is_host }, - /* .context = */ NULL, + /* .context = */ nullptr, }; return &ggml_backend_buffer_type_qnn; @@ -3561,14 +3626,14 @@ static struct ggml_backend_device_i ggml_backend_qnn_device_interface = { /* .get_props = */ ggml_backend_qnn_device_get_props, /* .init_backend = */ ggml_backend_qnn_device_init_backend, /* .get_buffer_type = */ ggml_backend_qnn_device_get_buffer_type, - /* .get_host_buffer_type = */ NULL, + /* .get_host_buffer_type = */ nullptr, /* .buffer_from_host_ptr = */ ggml_backend_qnn_device_buffer_from_host_ptr, /* .supports_op = */ ggml_backend_qnn_device_supports_op, /* .supports_buft = */ ggml_backend_qnn_device_supports_buft, - /* .offload_op = */ NULL, - /* .event_new = */ NULL, - /* .event_free = */ NULL, - /* .event_synchronize = */ NULL, + /* .offload_op = */ nullptr, + /* .event_new = */ nullptr, + /* .event_free = */ nullptr, + /* .event_synchronize = */ nullptr, }; static ggml_backend_i ggml_backend_qnn_interface = { @@ -3616,9 +3681,8 @@ struct ggml_backend_qnn_reg_context { }; static const char * ggml_backend_qnn_reg_get_name(ggml_backend_reg_t reg) { - return "ggml-qnn"; - GGML_UNUSED(reg); + return "ggml-qnn"; } static size_t ggml_backend_qnn_reg_get_device_count(ggml_backend_reg_t reg) { @@ -3639,10 +3703,15 @@ static ggml_backend_dev_t ggml_backend_qnn_reg_get_device(ggml_backend_reg_t reg static void * ggml_backend_qnn_reg_get_proc_address(ggml_backend_reg_t reg, const char * name) { GGML_UNUSED(reg); - if (std::strcmp(name, "ggml_backend_set_n_threads") == 0) { + if (nullptr == name) + return nullptr; + + const char * slot_name = "ggml_backend_set_n_threads"; + //avoid buffer attack rather than strcmp + if (0 == std::memcmp(name, slot_name, strlen(slot_name))) { return (void *)ggml_backend_qnn_set_n_threads; } - return NULL; + return nullptr; } static const ggml_backend_reg_i ggml_backend_qnn_reg_interface = { From aa78a6db2c57e9a6d2f67509beb367bb56b25719 Mon Sep 17 00:00:00 2001 From: zhouwg Date: Wed, 26 Feb 2025 13:38:12 +0800 Subject: [PATCH 26/76] ggml-qnn: offload quantized type mulmat to QNN backend --- ggml/src/ggml-qnn/ggml-qnn.cpp | 172 +++++++++++++++++++++++++-------- 1 file changed, 130 insertions(+), 42 deletions(-) diff --git a/ggml/src/ggml-qnn/ggml-qnn.cpp b/ggml/src/ggml-qnn/ggml-qnn.cpp index aaf9fd694f8b9..3a474f1bffee5 100644 --- a/ggml/src/ggml-qnn/ggml-qnn.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn.cpp @@ -75,6 +75,7 @@ #include #include #include +#include #if (defined __ANDROID__) || (defined ANDROID) #include "android/log.h" #endif @@ -815,6 +816,11 @@ struct ggml_backend_qnn_context { QNN_INTERFACE_VER_TYPE raw_interface; QNN_SYSTEM_INTERFACE_VER_TYPE raw_system_interface; struct qcom_socinfo socinfo; + + std::unique_ptr work_data; + std::vector> tasks; + size_t work_size = 0; + int n_threads = GGML_DEFAULT_N_THREADS; } ; //the following helper funcs are used to ensure every QNN tensor name is unique @@ -2780,7 +2786,7 @@ static bool ggml_qnn_can_handle_op(const struct ggml_tensor * tensor) { const uint32_t src1_rank = ggml_get_tensor_rank(src1); if (tensor->op == GGML_OP_ADD) { - //dump_tensors_info(tensor); + //dump_op_info(tensor); if (!ggml_are_same_shape(src0, src1)) { return false; } @@ -2791,6 +2797,7 @@ static bool ggml_qnn_can_handle_op(const struct ggml_tensor * tensor) { } if (tensor->op == GGML_OP_MUL_MAT) { + dump_op_info(tensor); if (src0_rank != src1_rank) // make QNN SDK happy return false; if (src0_rank < 2) // QNN's limitation, make QNN SDK happy @@ -2800,17 +2807,18 @@ static bool ggml_qnn_can_handle_op(const struct ggml_tensor * tensor) { if ((src1->ne[2] != src0->ne[2]) || (src1->ne[3] != src0->ne[3])) // make QNN SDK happy return false; - //TODO: support more data type in func ggml_qnn_mul_mat(...) - //src0: q4_0, q6_k, ... - //src1: f32 - //dst : f32 - return (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16) - && (src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16) - && (src0->type == src1->type) && (src0->type == tensor->type); + if (2 != src0_rank) { //TODO: quantize src0 for 3D & 4D matrix + return (src0->type == GGML_TYPE_F32) + && (src1->type == GGML_TYPE_F32) + && (tensor->type == GGML_TYPE_F32); + } else { + return (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_Q8_0 || src0->type == GGML_TYPE_Q4_0 || src0->type == GGML_TYPE_Q6_K) + && (src1->type == GGML_TYPE_F32) && (tensor->type == GGML_TYPE_F32); + } } if (tensor->op == GGML_OP_MUL) { - //dump_tensors_info(tensor); + //dump_op_info(tensor); if ((src0_rank != 2) || (src1_rank != 2)) //TODO: 3D and 4D matrix return false; return (src0->type == GGML_TYPE_F32) @@ -2870,7 +2878,9 @@ static void ggml_qnn_general_node(ggml_backend_t backend, ggml_tensor * op) { p_tensor1 = ggml_qnn_create_compute_tensor(src1); p_tensor2 = ggml_qnn_create_compute_tensor(dst); } +#if GGMLQNN_PRINT_OP_ADD_LOG print_tensors_info(__func__, ctx, src0, src1, dst); +#endif //ensure QNN tensor has correct tensor type QNN_VER_PTR(*p_tensor0)->type = QNN_TENSOR_TYPE_APP_WRITE; @@ -2966,7 +2976,6 @@ static void ggml_qnn_general_node(ggml_backend_t backend, ggml_tensor * op) { auto graph_item = std::make_tuple(graph_handle, ggml_op_add_tensors); instance->_qnn_graph_map[graph_name] = graph_item; - } else { Qnn_DataType_t src0_qnn_type = QNN_DATATYPE_FLOAT_32; Qnn_DataType_t src1_qnn_type = QNN_DATATYPE_FLOAT_32; @@ -3039,22 +3048,31 @@ static void ggml_qnn_general_node(ggml_backend_t backend, ggml_tensor * op) { QNN_VER_PTR(*p_tensor0)->dimensions = tensor_0_dimensions; QNN_VER_PTR(*p_tensor1)->dimensions = tensor_1_dimensions; QNN_VER_PTR(*p_tensor2)->dimensions = tensor_2_dimensions; + +#if GGMLQNN_PRINT_OP_ADD_LOG op_perf.info(); +#endif } /* - * the logic of ggml_qnn_mul_mat is similar to ggml_qnn_general_node but much more complicated - * than ggml_qnn_general_node. - * matrix transpose and type trait are required for offload mulmat to QNN backend, - * so it's a standalone function. accordingly, this is another typical skeleton for offload other - * ggml ops to QNN backend + * @brief performs matrix multiplication with FP32 & quantized weights and floating-point inputs + * using the QNN backend. this function performs matrix multiplication of the input tensor + * `src1` and the weight tensor `src0`, handling transposing, and quantization as needed, + * and stores the result in the destination tensor `dst`. * - * MUL_MAT take most of the compute time (about 95%).so to speed up llama inference, should focus on MUL_MAT. + * @param backend the context which got through (ggml_backend_qnn_context *)backend->context for the + * QNN backend operations. + * @param op the destination tensor where the result of the matrix multiplication will be stored. * - * have three kinds of MUL_MAT to compute: - * mul_mat_f32: both src0 and src1 are F32, this will be naturally handled in QNN backend - * mul_mat_f16_f32: src0 is F16 and src1 is F32, f16 in src0 -> f32 in src0', then src0' * src1 - * mul_mat_q_f32: src0 is quantized (Q4_0, Q4_1, ...) and src1 is F32, src0 -> f32 in src0', then src0' * src1 + * @note the logic of ggml_qnn_mul_mat is similar to ggml_qnn_general_node but much more complicated + * than ggml_qnn_general_node. so it's a standalone function. accordingly, this is another + * typical skeleton for offload other ggml ops to QNN backend. MUL_MAT take most of the compute + * time (about 95%).so to speed up llama inference, should focus on this func. there are three kinds + * of MUL_MAT to compute: + * mul_mat_f32: both src0 and src1 are F32, this will be naturally handled in QNN backend + * mul_mat_f16_f32: src0 is F16 and src1 is F32, f16 in src0 -> f32 in src0', then src0' * src1 + * mul_mat_q_f32: src0 is quantized (Q4_0, Q4_1, Q6_K...) + * and src1 is F32, src0 -> f32 in src0', then src0' * src1 */ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { Qnn_ErrorHandle_t error = QNN_SUCCESS; @@ -3077,10 +3095,72 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { QNN_INTERFACE_VER_TYPE qnn_raw_interface = ctx->raw_interface; op_perf.start(); - uint32_t src0_rank = ggml_get_tensor_rank(src0); - uint32_t src1_rank = ggml_get_tensor_rank(src1); + const enum ggml_type type = src0->type; + const uint32_t src0_rank = ggml_get_tensor_rank(src0); + const uint32_t src1_rank = ggml_get_tensor_rank(src1); + + GGML_TENSOR_BINARY_OP_LOCALS + GGML_ASSERT(ne0 == ne01); + GGML_ASSERT(ne1 == ne11); + GGML_ASSERT(ne2 == ne12); + GGML_ASSERT(ne3 == ne13); + GGML_ASSERT(nb00 == ggml_type_size(type)); + GGML_ASSERT(nb10 == ggml_type_size(src1->type)); + GGML_ASSERT(src0_rank == src1_rank); - GGML_ASSERT(src0_rank >= 2); //QNN SDK's limitation + GGML_ASSERT(src0_rank >= 2); //QNN SDK's limitation, make QNN SDK happy + + // broadcast factors + const int64_t r2 = ne12 / ne02; + const int64_t r3 = ne13 / ne03; + const int64_t ne_plane = ne01 * ne00; + const size_t desired_size = ((GGML_TYPE_F32 == type) ? 0 : ne03 * ne02 * ne_plane * sizeof(float)); + if (ctx->work_size < desired_size) { + ctx->work_data.reset(new char[desired_size]); + ctx->work_size = desired_size; + } + void * wdata = ctx->work_data.get(); + // convert src0 to float + if (type != GGML_TYPE_F32) { + const auto * type_traits = ggml_get_type_traits(type); + ggml_to_float_t const to_float = type_traits->to_float; + + for (int64_t i03 = 0; i03 < ne03; i03++) { + for (int64_t i02 = 0; i02 < ne02; i02++) { + const void * x = (char *)src0->data + i02 * nb02 + i03 * nb03; + float * const wplane = (float *)wdata + i02 * ne_plane + i03 * ne02 * ne_plane; + + const int min_cols_per_thread = 4096; + const int min_rows_per_thread = std::max((int)(min_cols_per_thread / ne00), 1); + const int n_threads = std::max(std::min(ctx->n_threads, (int)(ne01 / min_rows_per_thread)), 1); + for (int i = 1; i < n_threads; i++) { + const int64_t start = i * ne01 / n_threads; + const int64_t end = (i + 1) * ne01 / n_threads; + if (start < end) { + ctx->tasks.push_back(std::async(std::launch::async, [=]() { + for (int64_t i01 = start; i01 < end; i01++) { + to_float((const char *)x + i01 * nb01, wplane + i01 * ne00, ne00); + } + })); + } + } + { + // reuse the current thread for the first task + const int64_t start = 0; + const int64_t end = ne01 / n_threads; + for (int64_t i01 = start; i01 < end; i01++) { + to_float((const char *)x + i01 * nb01, wplane + i01 * ne00, ne00); + } + } + } + } + + // wait for all tasks to finish + for (auto & task : ctx->tasks) { + task.get(); + } + ctx->tasks.clear(); + } std::string graph_name; get_graph_key_from_op(op, graph_name); @@ -3133,9 +3213,10 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { 2. QNN's MatMul can only support input tensors with rank >= 2 - there is gap between ggml mulmat and QNN mulmat,we need to perform a transpose operation when offloading mulmat to QNN backend. + in the all, there is gap between ggml mulmat and QNN mulmat,we need to perform a transpose + operation when offloading mulmat to QNN backend. this concise implementation will handle + transpose in func ggml_qnn_create_general_tensor() */ - //step-1: create qnn graph error = qnn_raw_interface.graphCreate(instance->get_qnn_context_handle(), graph_name.c_str(), nullptr, &graph_handle); @@ -3158,8 +3239,11 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor0)); CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor1)); CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor2)); - - QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggml_get_tensor_data_size(src0)}; + if (type != GGML_TYPE_F32) { + QNN_VER_PTR(*p_tensor0)->clientBuf = {wdata, static_cast(desired_size)}; + } else { + QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggml_get_tensor_data_size(src0)}; + } QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggml_get_tensor_data_size(src1)}; QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggml_get_tensor_data_size(dst)}; @@ -3170,14 +3254,14 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { //step-5: compose qnn graph: add mat_mul node Qnn_Param_t out_0_params[] = { {QNN_PARAMTYPE_SCALAR, - QNN_OP_MAT_MUL_PARAM_TRANSPOSE_IN1, - .scalarParam = {QNN_DATATYPE_BOOL_8, .bool8Value = 1} + QNN_OP_MAT_MUL_PARAM_TRANSPOSE_IN1, + .scalarParam = {QNN_DATATYPE_BOOL_8, .bool8Value = 1} } }; Qnn_Tensor_t out_0_inputs[] = {*p_tensor0, *p_tensor1}; Qnn_Tensor_t out_0_outputs[] = {*p_tensor2_transpose}; -#if 0 +#if 0 //leave here for easily understand code, can be removed in the future Qnn_OpConfig_t out_0 = { QNN_OPCONFIG_VERSION_1, .v1 = {"ggmlqnn_mulmat_opconfig", QNN_OP_PACKAGE_NAME_QTI_AISW, QNN_OP_MAT_MUL, @@ -3202,7 +3286,7 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { }; Qnn_Tensor_t out_trans1_0_inputs[] = {*p_tensor2_transpose}; Qnn_Tensor_t out_trans1_0_outputs[] = {*p_tensor2}; -#if 0 +#if 0 //leave here for easily understand code, can be removed in the future Qnn_OpConfig_t out_trans1_0 = { QNN_OPCONFIG_VERSION_1, .v1 = {"ggmlqnn_mulmat_transpose_opconfig", @@ -3216,7 +3300,7 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { }; #else Qnn_OpConfig_t out_trans1_0 = create_op_config("ggmlqnn_mulmat_transpose_opconfig", QNN_OP_PACKAGE_NAME_QTI_AISW, QNN_OP_TRANSPOSE, - out_trans1_0_params, 1, out_trans1_0_inputs, 1, out_trans1_0_outputs, 1); + out_trans1_0_params, 1, out_trans1_0_inputs, 1, out_trans1_0_outputs, 1); #endif CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle,out_trans1_0)); @@ -3225,9 +3309,9 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { Qnn_Tensor_t input_tensors_0[] = {*p_tensor0, *p_tensor1}; Qnn_Tensor_t output_tensors_0[] = {*p_tensor2}; CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, - input_tensors_0, 2, - output_tensors_0, 1, - nullptr, nullptr)); + input_tensors_0, 2, + output_tensors_0, 1, + nullptr, nullptr)); qnn_tensors_t ggml_op_mulmat_tensors; ggml_op_mulmat_tensors.reserve(5); @@ -3239,7 +3323,11 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { auto graph_item = std::make_tuple(graph_handle, ggml_op_mulmat_tensors); instance->_qnn_graph_map[graph_name] = graph_item; } else { - QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggml_get_tensor_data_size(src0)}; + if (type != GGML_TYPE_F32) { + QNN_VER_PTR(*p_tensor0)->clientBuf = {wdata, static_cast(desired_size)}; + } else { + QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggml_get_tensor_data_size(src0)}; + } QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggml_get_tensor_data_size(src1)}; QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggml_get_tensor_data_size(dst)}; @@ -3250,13 +3338,13 @@ static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { Qnn_Tensor_t tensor_outputs[] = { *p_tensor2 }; - // this is the second technical approach of "how to utilize the Hexagon NPU maximally" through - // QNN SDK, details could be found at - // https://github.com/kantv-ai/llama.cpp/wiki/mapping-ggml-compute-graph-to-QNN-compute-graph + // this is the second technical approach or another pipeline of "how to utilize the Hexagon + // NPU maximally" through QNN SDK, details could be found at + // https://github.com/ggml-org/llama.cpp/pull/12049#issuecomment-2678308360 CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, - tensor_inputs, 2, - tensor_outputs, 1, - nullptr, nullptr)); + tensor_inputs, 2, + tensor_outputs, 1, + nullptr, nullptr)); } // restore the original dimensions of qnn tensors to avoid memory leak in func free_qnn_tensor From b54adefbaad878ef28513eb4fa3f8827cc0e77b0 Mon Sep 17 00:00:00 2001 From: zhouwg Date: Thu, 27 Feb 2025 14:34:10 +0800 Subject: [PATCH 27/76] ggml-qnn: refine source code structure to make code more clearly --- ggml/src/ggml-qnn/ggml-qnn-impl.h | 138 +++ ggml/src/ggml-qnn/ggml-qnn-ops.cpp | 140 +++ ggml/src/ggml-qnn/ggml-qnn-ops.h | 5 + ggml/src/ggml-qnn/ggml-qnn.cpp | 1662 ++++++---------------------- scripts/build-run-android.sh | 282 ----- 5 files changed, 630 insertions(+), 1597 deletions(-) delete mode 100755 scripts/build-run-android.sh diff --git a/ggml/src/ggml-qnn/ggml-qnn-impl.h b/ggml/src/ggml-qnn/ggml-qnn-impl.h index 5a2fe5752a097..974755955f9d2 100644 --- a/ggml/src/ggml-qnn/ggml-qnn-impl.h +++ b/ggml/src/ggml-qnn/ggml-qnn-impl.h @@ -64,8 +64,12 @@ #include "android/log.h" #endif +<<<<<<< HEAD #if defined(_WIN32) #include +======= +#if defined(_WIN32) || defined(_MSC_VER) +>>>>>>> ggml-qnn: refine source code structure to make code more clearly #include #endif @@ -90,7 +94,11 @@ class qnn_instance; struct ggml_backend_qnn_context; void ggmlqnn_log_internal(ggml_log_level level, const char * file, const char * func, int line, const char * format, ...); +<<<<<<< HEAD #if 0//def NDEBUG +======= +#ifdef NDEBUG +>>>>>>> ggml-qnn: refine source code structure to make code more clearly #define GGMLQNN_DEBUG 0 #define ENABLE_QNNBACKEND_PERF 0 // enable/disable op's perf info #define GGMLQNN_PRINT_QNN_INTERNAL_LOG 0 // enable/disable QNN's internal log @@ -105,9 +113,15 @@ void ggmlqnn_log_internal(ggml_log_level level, const char * file, const char #endif #define GGML_QNN_LOGBUF_LEN 4096 +<<<<<<< HEAD #define GGMLQNN_LOG_ERROR(...) ggmlqnn_log_internal(GGML_LOG_LEVEL_ERROR, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) #define GGMLQNN_LOG_WARN(...) ggmlqnn_log_internal(GGML_LOG_LEVEL_WARN , __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) #define GGMLQNN_LOG_INFO(...) ggmlqnn_log_internal(GGML_LOG_LEVEL_INFO , __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) +======= +#define GGMLQNN_LOG_ERROR(...) ggmlqnn_log_internal(GGML_LOG_LEVEL_DEBUG, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) +#define GGMLQNN_LOG_WARN(...) ggmlqnn_log_internal(GGML_LOG_LEVEL_DEBUG , __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) +#define GGMLQNN_LOG_INFO(...) ggmlqnn_log_internal(GGML_LOG_LEVEL_DEBUG , __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) +>>>>>>> ggml-qnn: refine source code structure to make code more clearly #if GGMLQNN_DEBUG #define GGMLQNN_LOG_DEBUG(...) ggmlqnn_log_internal(GGML_LOG_LEVEL_DEBUG, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) @@ -141,6 +155,7 @@ void ggmlqnn_log_internal(ggml_log_level level, const char * file, const char #define GQCGT ggmlqnn_create_general_tensor +<<<<<<< HEAD #if defined(_WIN32) #define RTLD_GLOBAL 0x100 #define RTLD_LOCAL 0x000 @@ -152,6 +167,8 @@ void * dlsym(void* handle, const char* name); const char * dlerror(void); #endif +======= +>>>>>>> ggml-qnn: refine source code structure to make code more clearly using pfn_rpc_mem_init = void (*)(void); using pfn_rpc_mem_deinit = void (*)(void); using pfn_rpc_mem_alloc = void *(*)(int, uint32_t, int); @@ -226,7 +243,11 @@ struct qnn_op_caps_t { const size_t input_param_count = 0; const char * qnn_param_name = nullptr; }; +<<<<<<< HEAD extern const qnn_op_caps_t ggmlqnn_k_op_caps[]; +======= +extern const qnn_op_caps_t k_op_caps[]; +>>>>>>> ggml-qnn: refine source code structure to make code more clearly #if ENABLE_QNNBACKEND_PERF class qnn_perf { @@ -255,9 +276,13 @@ class qnn_perf { #else class qnn_perf { public: +<<<<<<< HEAD qnn_perf(const std::string & perf_name) { GGML_UNUSED(perf_name); } +======= + qnn_perf(const std::string & perf_name) {} +>>>>>>> ggml-qnn: refine source code structure to make code more clearly qnn_perf() = delete; qnn_perf(const qnn_perf & ) = delete; qnn_perf & operator= (const qnn_perf & ) = delete; @@ -289,6 +314,7 @@ class qnn_interface { qnn_interface() = default; // QnnBackend +<<<<<<< HEAD DEFINE_SHIM_FUNCTION_INTERFACE(backend_create, backendCreate) DEFINE_SHIM_FUNCTION_INTERFACE(backend_free, backendFree) @@ -369,6 +395,88 @@ class qnn_interface { DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_get_binary_info, systemContextGetBinaryInfo) DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_free, systemContextFree) +======= + DEFINE_SHIM_FUNCTION_INTERFACE(backend_create, backendCreate); + + DEFINE_SHIM_FUNCTION_INTERFACE(backend_free, backendFree); + + DEFINE_SHIM_FUNCTION_INTERFACE(backend_register_op_package, backendRegisterOpPackage); + + DEFINE_SHIM_FUNCTION_INTERFACE(backend_validate_op_config, backendValidateOpConfig); + + DEFINE_SHIM_FUNCTION_INTERFACE(backend_get_api_version, backendGetApiVersion); + + // QnnDevice + DEFINE_SHIM_FUNCTION_INTERFACE(device_create, deviceCreate); + + DEFINE_SHIM_FUNCTION_INTERFACE(device_free, deviceFree); + + DEFINE_SHIM_FUNCTION_INTERFACE(device_get_infrastructure, deviceGetInfrastructure); + + DEFINE_SHIM_FUNCTION_INTERFACE(device_get_platform_info, deviceGetPlatformInfo); + + DEFINE_SHIM_FUNCTION_INTERFACE(device_get_info, deviceGetInfo); + + // QnnContext + DEFINE_SHIM_FUNCTION_INTERFACE(context_create, contextCreate); + + DEFINE_SHIM_FUNCTION_INTERFACE(context_get_binary_size, contextGetBinarySize); + + DEFINE_SHIM_FUNCTION_INTERFACE(context_get_binary, contextGetBinary); + + DEFINE_SHIM_FUNCTION_INTERFACE(context_create_from_binary, contextCreateFromBinary); + + DEFINE_SHIM_FUNCTION_INTERFACE(context_free, contextFree); + + // QnnGraph + DEFINE_SHIM_FUNCTION_INTERFACE(graph_create, graphCreate); + + DEFINE_SHIM_FUNCTION_INTERFACE(graph_add_node, graphAddNode); + + DEFINE_SHIM_FUNCTION_INTERFACE(graph_finalize, graphFinalize); + + DEFINE_SHIM_FUNCTION_INTERFACE(graph_execute, graphExecute); + + DEFINE_SHIM_FUNCTION_INTERFACE(graph_retrieve, graphRetrieve); + + // QnnLog + DEFINE_SHIM_FUNCTION_INTERFACE(log_create, logCreate); + + DEFINE_SHIM_FUNCTION_INTERFACE(log_free, logFree); + + DEFINE_SHIM_FUNCTION_INTERFACE(log_set_log_level, logSetLogLevel); + + // QnnProfile + DEFINE_SHIM_FUNCTION_INTERFACE(profile_create, profileCreate); + + DEFINE_SHIM_FUNCTION_INTERFACE(profile_get_events, profileGetEvents); + + DEFINE_SHIM_FUNCTION_INTERFACE(profile_get_sub_events, profileGetSubEvents); + + DEFINE_SHIM_FUNCTION_INTERFACE(profile_get_event_data, profileGetEventData); + + DEFINE_SHIM_FUNCTION_INTERFACE(profile_free, profileFree); + + // QnnMem + DEFINE_SHIM_FUNCTION_INTERFACE(mem_register, memRegister); + + DEFINE_SHIM_FUNCTION_INTERFACE(mem_de_register, memDeRegister); + + // QnnProperty + DEFINE_SHIM_FUNCTION_INTERFACE(property_has_capability, propertyHasCapability); + + // QnnTensor + DEFINE_SHIM_FUNCTION_INTERFACE(tensor_create_context_tensor, tensorCreateContextTensor); + + DEFINE_SHIM_FUNCTION_INTERFACE(tensor_create_graph_tensor, tensorCreateGraphTensor); + + // QnnSystem + DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_create, systemContextCreate); + + DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_get_binary_info, systemContextGetBinaryInfo); + + DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_free, systemContextFree); +>>>>>>> ggml-qnn: refine source code structure to make code more clearly void set_qnn_interface(const QnnInterface_t * qnn_interface) { _qnn_interface = qnn_interface; @@ -387,9 +495,15 @@ class qnn_interface { } private: +<<<<<<< HEAD const QnnInterface_t * _qnn_interface = nullptr; const QnnSystemInterface_t * _qnn_sys_interface = nullptr; +======= + const QnnInterface_t *_qnn_interface = nullptr; + + const QnnSystemInterface_t *_qnn_sys_interface = nullptr; +>>>>>>> ggml-qnn: refine source code structure to make code more clearly }; class qnn_instance { @@ -400,7 +514,11 @@ class qnn_instance { const std::string & model_name) : _lib_path(std::move(lib_path)), _backend_name(std::move(backend_name)), +<<<<<<< HEAD _model_name(std::move(model_name)) {} +======= + _model_name(std::move(model_name)) {}; +>>>>>>> ggml-qnn: refine source code structure to make code more clearly ~qnn_instance() { } @@ -430,6 +548,7 @@ class qnn_instance { return _qnn_raw_system_interface; } +<<<<<<< HEAD Qnn_LogHandle_t get_qnn_log_handle() { return _qnn_log_handle; } Qnn_ProfileHandle_t get_qnn_profile_handle() { return _qnn_profile_handle; } @@ -443,6 +562,21 @@ class qnn_instance { QnnSystemContext_Handle_t get_qnn_system_handle() { return _qnn_system_handle; } Qnn_GraphHandle_t get_qnn_graph_handle() { return _qnn_graph_handle; } +======= + const Qnn_LogHandle_t get_qnn_log_handle() { return _qnn_log_handle; } + + const Qnn_ProfileHandle_t get_qnn_profile_handle() { return _qnn_profile_handle; } + + const Qnn_DeviceHandle_t get_qnn_device_handle() { return _qnn_device_handle; } + + const Qnn_BackendHandle_t get_qnn_backend_handle() { return _qnn_backend_handle; } + + const Qnn_ContextHandle_t get_qnn_context_handle() { return _qnn_context_handle; } + + const QnnSystemContext_Handle_t get_qnn_system_handle() { return _qnn_system_handle; } + + const Qnn_GraphHandle_t get_qnn_graph_handle() { return _qnn_graph_handle; } +>>>>>>> ggml-qnn: refine source code structure to make code more clearly int init_qnn_graph(const char * graph_name, bool debug, @@ -596,6 +730,10 @@ const char * ggmlqnn_get_error_string(Qnn_ErrorHandle_t qnn_error_code); Qnn_DataType_t ggmlqnn_datatype_from_ggml_datatype(enum ggml_type ggmltype); void * ggmlqnn_type_trait(ggml_backend_qnn_context * ctx, ggml_tensor * op); void ggmlqnn_get_graphkey_from_op(const ggml_tensor * op, std::string & output); +<<<<<<< HEAD +======= +bool ggmlqnn_is_valid_params(ggml_backend_qnn_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); +>>>>>>> ggml-qnn: refine source code structure to make code more clearly uint8_t * ggmlqnn_create_rpc_buffer(qnn_instance * instance, const ggml_tensor * ggml_tensor, Qnn_Tensor_t * qnn_tensor, bool b_copydata); void ggmlqnn_print_tensors_info(const char * func_name, ggml_backend_qnn_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); diff --git a/ggml/src/ggml-qnn/ggml-qnn-ops.cpp b/ggml/src/ggml-qnn/ggml-qnn-ops.cpp index 00cb7da32c183..6614a1b90f6fd 100644 --- a/ggml/src/ggml-qnn/ggml-qnn-ops.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn-ops.cpp @@ -23,6 +23,7 @@ #include "ggml-common.h" #include "ggml-qnn-ops.h" +<<<<<<< HEAD static inline uint32_t ggmlqnn_get_tensor_data_size(const ggml_tensor * tensor) { /* size_t data_size = ggml_row_size(tensor->type, tensor->ne[0]); @@ -52,6 +53,8 @@ static inline bool ggmlqnn_is_valid_params(ggml_backend_qnn_context * ctx, const return true; } +======= +>>>>>>> ggml-qnn: refine source code structure to make code more clearly #define GGMLQNN_CHECK_PARAMS(ctx, src0, src1, dst) \ do { \ if (!ggmlqnn_is_valid_params((ctx), (src0), (src1), (dst))) { \ @@ -82,7 +85,11 @@ void ggml_qnn_general_node(ggml_backend_qnn_context * ctx, ggml_tensor * op) { QNN_INTERFACE_VER_TYPE qnn_raw_interface = ctx->raw_interface; size_t qnn_op_index = ggmlqnn_get_op_index(op); GGML_ASSERT(qnn_op_index < ggmlqnn_get_opcaps_size()); +<<<<<<< HEAD const char * qnn_op_name = ggmlqnn_k_op_caps[qnn_op_index].qnn_op_name; +======= + const char * qnn_op_name = k_op_caps[qnn_op_index].qnn_op_name; +>>>>>>> ggml-qnn: refine source code structure to make code more clearly std::string ggml_op_name_string = std::string("ggml_") + ggml_op_name(op->op); const char * ggml_op_name = ggml_op_name_string.c_str(); @@ -104,7 +111,13 @@ void ggml_qnn_general_node(ggml_backend_qnn_context * ctx, ggml_tensor * op) { p_tensor1 = ggmlqnn_create_compute_tensor(src1); p_tensor2 = ggmlqnn_create_compute_tensor(dst); } +<<<<<<< HEAD //ggmlqnn_print_tensors_info(__func__, ctx, src0, src1, dst); +======= +#if GGMLQNN_PRINT_OP_ADD_LOG + print_tensors_info(__func__, ctx, src0, src1, dst); +#endif +>>>>>>> ggml-qnn: refine source code structure to make code more clearly //ensure QNN tensor has correct tensor type QNN_VER_PTR(*p_tensor0)->type = QNN_TENSOR_TYPE_APP_WRITE; @@ -152,9 +165,15 @@ void ggml_qnn_general_node(ggml_backend_qnn_context * ctx, ggml_tensor * op) { return; } } else { +<<<<<<< HEAD QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggmlqnn_get_tensor_data_size(src0)}; QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggmlqnn_get_tensor_data_size(src1)}; QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggmlqnn_get_tensor_data_size(dst)}; +======= + QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, static_cast(ggml_nbytes(src0))}; + QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, static_cast(ggml_nbytes(src1))}; + QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, static_cast(ggml_nbytes(dst))}; +>>>>>>> ggml-qnn: refine source code structure to make code more clearly } Qnn_Tensor_t tensor_inputs[] = { @@ -242,9 +261,15 @@ void ggml_qnn_general_node(ggml_backend_qnn_context * ctx, ggml_tensor * op) { memcpy(qnn_buffer_1, src1->data, ggml_nbytes(src1)); } } else { +<<<<<<< HEAD QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggmlqnn_get_tensor_data_size(src0)}; QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggmlqnn_get_tensor_data_size(src1)}; QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggmlqnn_get_tensor_data_size(dst)}; +======= + QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, static_cast(ggml_nbytes(src0))}; + QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, static_cast(ggml_nbytes(src1))}; + QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, static_cast(ggml_nbytes(dst))}; +>>>>>>> ggml-qnn: refine source code structure to make code more clearly } Qnn_Tensor_t tensor_inputs[] = { @@ -279,6 +304,7 @@ void ggml_qnn_general_node(ggml_backend_qnn_context * ctx, ggml_tensor * op) { } /* +<<<<<<< HEAD * this function is AI-assisted code from Grok 3 for purpose of offload 4d matrix mulmat to QNN backend * UT in ggml-qnn-ut.cpp passed: * ./scripts/build-run-android.sh run_ut_mulmat 0 @@ -474,6 +500,8 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) } /* +======= +>>>>>>> ggml-qnn: refine source code structure to make code more clearly * @brief performs matrix multiplication with FP32 & quantized weights and floating-point inputs * using the QNN backend. this function performs matrix multiplication of the input tensor * `src1` and the weight tensor `src0`, handling transposing, and quantization as needed, @@ -518,9 +546,13 @@ void ggml_qnn_mul_mat(ggml_backend_qnn_context * ctx, ggml_tensor * op) { const uint32_t src1_rank = ggml_n_dims(src1); GGML_ASSERT(src0_rank == src1_rank); GGML_ASSERT(src0_rank >= 2); //QNN SDK's limitation, make QNN SDK happy +<<<<<<< HEAD if (4 == src0_rank) { return ggml_qnn_mul_mat_4d(ctx, op); } +======= + GGML_ASSERT(src0_rank != 4); //TODO: 4D matrix mulmat +>>>>>>> ggml-qnn: refine source code structure to make code more clearly void * wdata = ggmlqnn_type_trait(ctx, op); const size_t desired_size = ctx->desired_size; @@ -604,10 +636,17 @@ void ggml_qnn_mul_mat(ggml_backend_qnn_context * ctx, ggml_tensor * op) { if (src0_type != GGML_TYPE_F32) { QNN_VER_PTR(*p_tensor0)->clientBuf = {wdata, static_cast(desired_size)}; } else { +<<<<<<< HEAD QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggmlqnn_get_tensor_data_size(src0)}; } QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggmlqnn_get_tensor_data_size(src1)}; QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggmlqnn_get_tensor_data_size(dst)}; +======= + QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, static_cast(ggml_nbytes(src0))}; + } + QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, static_cast(ggml_nbytes(src1))}; + QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, static_cast(ggml_nbytes(dst))}; +>>>>>>> ggml-qnn: refine source code structure to make code more clearly //step-4: create a transpose tensor p_tensor2_transpose = GQCGT(dst, "transpose", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0, true); @@ -636,13 +675,21 @@ void ggml_qnn_mul_mat(ggml_backend_qnn_context * ctx, ggml_tensor * op) { }; #else Qnn_OpConfig_t out_0 = ggmlqnn_create_op_config("ggmlqnn_mulmat_opconfig", QNN_OP_PACKAGE_NAME_QTI_AISW, QNN_OP_MAT_MUL, +<<<<<<< HEAD out_0_params, 1, out_0_inputs, 2, out_0_outputs, 1); +======= + out_0_params, 1, out_0_inputs, 2, out_0_outputs, 1); +>>>>>>> ggml-qnn: refine source code structure to make code more clearly #endif CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle,out_0)); //step-5: compose qnn graph: add transpose node Qnn_Param_t out_trans1_0_params[] = { +<<<<<<< HEAD {QNN_PARAMTYPE_TENSOR, +======= + {(Qnn_ParamType_t) 1, +>>>>>>> ggml-qnn: refine source code structure to make code more clearly "perm", .tensorParam = *p_param_tensor } }; @@ -662,7 +709,11 @@ void ggml_qnn_mul_mat(ggml_backend_qnn_context * ctx, ggml_tensor * op) { }; #else Qnn_OpConfig_t out_trans1_0 = ggmlqnn_create_op_config("ggmlqnn_mulmat_transpose_opconfig", QNN_OP_PACKAGE_NAME_QTI_AISW, QNN_OP_TRANSPOSE, +<<<<<<< HEAD out_trans1_0_params, 1, out_trans1_0_inputs, 1, out_trans1_0_outputs, 1); +======= + out_trans1_0_params, 1, out_trans1_0_inputs, 1, out_trans1_0_outputs, 1); +>>>>>>> ggml-qnn: refine source code structure to make code more clearly #endif CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle,out_trans1_0)); @@ -688,10 +739,17 @@ void ggml_qnn_mul_mat(ggml_backend_qnn_context * ctx, ggml_tensor * op) { if (src0_type != GGML_TYPE_F32) { QNN_VER_PTR(*p_tensor0)->clientBuf = {wdata, static_cast(desired_size)}; } else { +<<<<<<< HEAD QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggmlqnn_get_tensor_data_size(src0)}; } QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggmlqnn_get_tensor_data_size(src1)}; QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggmlqnn_get_tensor_data_size(dst)}; +======= + QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, static_cast(ggml_nbytes(src0))}; + } + QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, static_cast(ggml_nbytes(src1))}; + QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, static_cast(ggml_nbytes(dst))}; +>>>>>>> ggml-qnn: refine source code structure to make code more clearly Qnn_Tensor_t tensor_inputs[] = { *p_tensor0, @@ -715,6 +773,7 @@ void ggml_qnn_mul_mat(ggml_backend_qnn_context * ctx, ggml_tensor * op) { QNN_VER_PTR(*p_tensor2)->dimensions = tensor_2_dimensions; op_perf.info(); } +<<<<<<< HEAD void ggml_qnn_repeat(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { GGML_UNUSED(ctx); @@ -820,6 +879,78 @@ void ggml_qnn_im2col(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { void ggml_qnn_timestep_embedding(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { GGML_UNUSED(ctx); GGML_UNUSED(dst); +======= +void ggml_qnn_repeat(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +} + +void ggml_qnn_add(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +} + +void ggml_qnn_div(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +} + +void ggml_qnn_leaky_relu(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +} + +void ggml_qnn_concat(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +} + +void ggml_qnn_arange(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +} + +void ggml_qnn_sqr(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +} + +void ggml_qnn_clamp(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +} + +void ggml_qnn_scale(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +} + +void ggml_qnn_argsort(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +} + +void ggml_qnn_norm(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +} + +void ggml_qnn_group_norm(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +} + +void ggml_qnn_acc(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +} + +void ggml_qnn_sum_rows(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +} + +void ggml_qnn_upsample_nearest2d(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +} + +void ggml_qnn_pad(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +} + +static void ggml_qnn_avg_pool2d(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +} + +static void ggml_qnn_max_pool2d(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +} + +void ggml_qnn_pool2d(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +} + +void ggml_qnn_dup(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +} + +void ggml_qnn_rms_norm(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +} + +void ggml_qnn_diag_mask(ggml_backend_qnn_context * ctx, ggml_tensor * dst, float value) { +} + +void ggml_qnn_im2col(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +} + +void ggml_qnn_timestep_embedding(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +>>>>>>> ggml-qnn: refine source code structure to make code more clearly } void ggml_qnn_cpy(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { @@ -827,6 +958,7 @@ void ggml_qnn_cpy(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { } void ggml_qnn_softmax(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +<<<<<<< HEAD GGML_UNUSED(ctx); GGML_UNUSED(dst); } @@ -839,4 +971,12 @@ void ggml_qnn_get_rows(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { void ggml_qnn_rope(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { GGML_UNUSED(ctx); GGML_UNUSED(dst); +======= +} + +void ggml_qnn_get_rows(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +} + +void ggml_qnn_rope(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +>>>>>>> ggml-qnn: refine source code structure to make code more clearly } diff --git a/ggml/src/ggml-qnn/ggml-qnn-ops.h b/ggml/src/ggml-qnn/ggml-qnn-ops.h index b1c388a32a87a..c25638a9397c6 100644 --- a/ggml/src/ggml-qnn/ggml-qnn-ops.h +++ b/ggml/src/ggml-qnn/ggml-qnn-ops.h @@ -24,8 +24,13 @@ #include "ggml-qnn-impl.h" void ggml_qnn_general_node(ggml_backend_qnn_context * ctx, ggml_tensor * dst); void ggml_qnn_mul_mat(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +<<<<<<< HEAD void ggml_qnn_repeat(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +======= +void ggml_qnn_repeat(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +void ggml_qnn_add(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +>>>>>>> ggml-qnn: refine source code structure to make code more clearly void ggml_qnn_div(ggml_backend_qnn_context * ctx, ggml_tensor * dst); void ggml_qnn_leaky_relu(ggml_backend_qnn_context * ctx, ggml_tensor * dst); void ggml_qnn_concat(ggml_backend_qnn_context * ctx, ggml_tensor * dst); diff --git a/ggml/src/ggml-qnn/ggml-qnn.cpp b/ggml/src/ggml-qnn/ggml-qnn.cpp index 3a474f1bffee5..ff1a8a0f39506 100644 --- a/ggml/src/ggml-qnn/ggml-qnn.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn.cpp @@ -13,7 +13,7 @@ * section-5 does ggml-qnn backend helper macro / data structure / function / class * section-6 does implementation of ggml-qnn backend according to ggml's backend subsystem * - * currently provide following ggml ops' QNN backend implementation: + * currently provide following ggml ops' QNN backend implementation in ggml-qnn-ops.cpp: * - GGML_OP_ADD: this is a simple skeleton, can expand other ggml ops according to expertise * - GGML_OP_MUL: this is a simple skeleton, can expand other ggml ops according to expertise * - GGML_OP_MUL_MAT:this is a complicated skeleton, can expand other complex ggml ops accordingly @@ -36,105 +36,19 @@ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS * IN THE SOFTWARE. */ -#include -#include -#include -#include -#include -#include -#include -#include -#if defined(__ANDROID__) || defined(__linux__) -#include -#include -#include -#include -#include -#include -#endif - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#if (defined __ANDROID__) || (defined ANDROID) -#include "android/log.h" -#endif - -#if defined(_WIN32) || defined(_MSC_VER) -#include -#endif - -#include "QnnTypes.h" -#include "QnnCommon.h" -#include "QnnContext.h" -#include "QnnBackend.h" -#include "QnnGraph.h" -#include "QnnProperty.h" -#include "QnnTensor.h" -#include "QnnInterface.h" -#include "Saver/QnnSaver.h" -#include "System/QnnSystemInterface.h" -#include "HTP/QnnHtpDevice.h" -#include "HTP/QnnHtpGraph.h" - -#include "ggml-qnn.h" -#include "ggml-impl.h" -#include "ggml-backend-impl.h" +#include "ggml-qnn-impl.h" +#include "ggml-qnn-ops.h" // ================================================================================================= // section-1: forward/external declaration // ================================================================================================= -class qnn_instance; -struct ggml_backend_qnn_context; static int free_qnn_tensor(Qnn_Tensor_t * tensor); static enum ggml_status ggml_backend_qnn_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph); -static void ggmlqnn_log_internal(ggml_log_level level, const char * file, const char * func, int line, const char * format, ...); -static Qnn_Tensor_t * ggml_qnn_create_general_tensor(const ggml_tensor * tensor, const char * name, - Qnn_TensorType_t qnn_tensor_type, - Qnn_DataType_t qnn_data_type, - uint32_t rank, uint32_t * dims, - void * data, uint32_t data_size, - bool b_transpose = false); +typedef void (* ggmlqnn_op_func_t)(ggml_backend_qnn_context * ctx, ggml_tensor * op); // ================================================================================================= // section-2: ggml-qnn internal troubleshooting function // ================================================================================================= -#define GGMLQNN_DEBUG 1 // for troubleshooting QNN backend -#define GGML_QNN_LOGBUF_LEN 4096 -#define ENABLE_QNNBACKEND_PERF 0 // enable/disable op's perf info -#define GGMLQNN_PRINT_QNN_INTERNAL_LOG 0 // enable/disable QNN's internal log -#define GGMLQNN_PRINT_OP_ADD_LOG 0 // GGML_OP_ADD already verified with QNN-CPU / QNN-GPU / QNN-NPU -#define GGMLQNN_PRINT_OP_MUL_MAT_LOG 1 - -#define GGMLQNN_LOG_ERROR(...) ggmlqnn_log_internal(GGML_LOG_LEVEL_DEBUG, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) -#define GGMLQNN_LOG_WARN(...) ggmlqnn_log_internal(GGML_LOG_LEVEL_DEBUG , __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) -#define GGMLQNN_LOG_INFO(...) ggmlqnn_log_internal(GGML_LOG_LEVEL_DEBUG , __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) - -#if GGMLQNN_DEBUG -#define GGMLQNN_LOG_DEBUG(...) ggmlqnn_log_internal(GGML_LOG_LEVEL_DEBUG, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) -#else -#define GGMLQNN_LOG_DEBUG(...) -#endif -static void ggmlqnn_log_internal(ggml_log_level level, const char * file, const char * func, int line, const char * format, ...) { +void ggmlqnn_log_internal(ggml_log_level level, const char * file, const char * func, int line, const char * format, ...) { static std::mutex ggmlqnn_log_internal_mutex; static char s_ggmlqnn_log_internal_buf[GGML_QNN_LOGBUF_LEN]; @@ -160,16 +74,6 @@ static void ggmlqnn_log_internal(ggml_log_level level, const char * file, const // ================================================================================================= // section-3: general helper macro / data structure / function // ================================================================================================= -#define DISABLE_COPY(class_name) \ - class_name(const class_name &) = delete; \ - void operator=(const class_name &) = delete - -#define DISABLE_MOVE(class_name) \ - class_name(class_name &&) = delete; \ - void operator=(class_name &&) = delete - -#define GQCGT ggml_qnn_create_general_tensor - static intptr_t ggmlqnn_align_to(size_t alignment, intptr_t offset) { return offset % alignment == 0 ? offset : offset + @@ -249,19 +153,6 @@ static void * ggmlqnn_host_malloc(size_t n) { // ================================================================================================= // section-4: QNN helper macro / data structure / function // ================================================================================================= -#define CHECK_QNN_API(error, result) \ - do { \ - error = (result); \ - if (QNN_SUCCESS != error) { \ - if (error == QNN_COMMON_ERROR_NOT_SUPPORTED) { \ - GGMLQNN_LOG_WARN("WARNING: QNN feature/API not supported\n"); \ - } else { \ - GGMLQNN_LOG_INFO("QNN API error = %d(%s)\n", error, qnn_get_error_string(error)); \ - } \ - } \ - } while (0) - -#define QNN_VER_PTR(x) (&((x).v1)) #define QNN_TENSOR_GET_ID(tensor) get_qnn_tensorid(tensor) #define QNN_TENSOR_GET_NAME(tensor) get_qnn_tensorname(tensor) #define QNN_TENSOR_GET_TYPE(tensor) get_qnn_tensortype(tensor) @@ -420,9 +311,8 @@ static int deep_copy_qnn_tensors(Qnn_Tensor_t & src, Qnn_Tensor_t & dst) { int err = 0; dst.version = src.version; - QNN_TENSOR_SET_NAME( - dst, ggmlqnn_strndup(QNN_TENSOR_GET_NAME(src), std::string(QNN_TENSOR_GET_NAME(src)).size())); - if (QNN_TENSOR_GET_NAME(dst) == nullptr) { + QNN_TENSOR_SET_NAME(dst, ggmlqnn_strndup(QNN_TENSOR_GET_NAME(src), std::string(QNN_TENSOR_GET_NAME(src)).size())); + if (nullptr == QNN_TENSOR_GET_NAME(dst)) { return 1; } QNN_TENSOR_SET_ID(dst, QNN_TENSOR_GET_ID(src)); @@ -441,20 +331,20 @@ static int deep_copy_qnn_tensors(Qnn_Tensor_t & src, Qnn_Tensor_t & dst) { } Qnn_QuantizeParams_t src_qparam = QNN_TENSOR_GET_QUANT_PARAMS(src); - Qnn_QuantizationEncoding_t encoding = src_qparam.quantizationEncoding; + Qnn_QuantizationEncoding_t encoding = src_qparam.quantizationEncoding; if (encoding == QNN_QUANTIZATION_ENCODING_AXIS_SCALE_OFFSET) { - Qnn_QuantizeParams_t src_qparam_cpy = src_qparam; + Qnn_QuantizeParams_t src_qparam_cpy = src_qparam; Qnn_AxisScaleOffset_t & axis_scale_offset = src_qparam_cpy.axisScaleOffsetEncoding; - Qnn_ScaleOffset_t ** scale_offset = &axis_scale_offset.scaleOffset; + Qnn_ScaleOffset_t ** scale_offset = &axis_scale_offset.scaleOffset; size_t scale_offset_size = axis_scale_offset.numScaleOffsets * sizeof(Qnn_ScaleOffset_t); - *scale_offset = (Qnn_ScaleOffset_t *)malloc(scale_offset_size); + *scale_offset = (Qnn_ScaleOffset_t *)malloc(scale_offset_size); ggmlqnn_memscpy(*scale_offset, scale_offset_size, src_qparam.axisScaleOffsetEncoding.scaleOffset, scale_offset_size); QNN_TENSOR_SET_QUANT_PARAMS(dst, src_qparam_cpy); } else if (encoding == QNN_QUANTIZATION_ENCODING_BW_AXIS_SCALE_OFFSET) { - Qnn_QuantizeParams_t src_qparam_cpy = src_qparam; + Qnn_QuantizeParams_t src_qparam_cpy = src_qparam; Qnn_BwAxisScaleOffset_t & bwaxis_scale_offset = src_qparam_cpy.bwAxisScaleOffsetEncoding; size_t scale_size = bwaxis_scale_offset.numElements * sizeof(float); float ** scales = &bwaxis_scale_offset.scales; @@ -476,7 +366,7 @@ static int deep_copy_qnn_tensors(Qnn_Tensor_t & src, Qnn_Tensor_t & dst) { QNN_TENSOR_SET_RANK(dst, rank); size_t dim_size = GGML_MAX_DIMS * sizeof(uint32_t); uint32_t * dimensions = (uint32_t *)malloc(dim_size); - if (dimensions == nullptr) { + if (nullptr == dimensions) { GGMLQNN_LOG_WARN("deep_copy_qnn_tensors() allocation error while copying tensor %s\n", QNN_TENSOR_GET_NAME(src)); return 1; } @@ -488,10 +378,8 @@ static int deep_copy_qnn_tensors(Qnn_Tensor_t & src, Qnn_Tensor_t & dst) { static int free_qnn_tensor(Qnn_Tensor_t * tensor) { int err = 0; - free((void *) QNN_TENSOR_GET_NAME(*tensor)); - - Qnn_QuantizeParams_t src_qparam = QNN_TENSOR_GET_QUANT_PARAMS(*tensor); + Qnn_QuantizeParams_t src_qparam = QNN_TENSOR_GET_QUANT_PARAMS(*tensor); Qnn_QuantizationEncoding_t encoding = src_qparam.quantizationEncoding; if (encoding == QNN_QUANTIZATION_ENCODING_AXIS_SCALE_OFFSET) { free(src_qparam.axisScaleOffsetEncoding.scaleOffset); @@ -507,54 +395,7 @@ static int free_qnn_tensor(Qnn_Tensor_t * tensor) { return err; } -static size_t qnn_datatype_size(Qnn_DataType_t qnn_type) { - switch (qnn_type) { - case QNN_DATATYPE_FLOAT_32: - return sizeof(float); - case QNN_DATATYPE_FLOAT_16: - return sizeof(uint16_t); - case QNN_DATATYPE_UINT_32: - case QNN_DATATYPE_INT_32: - return sizeof(int32_t); - case QNN_DATATYPE_INT_16: - return sizeof(int16_t); - case QNN_DATATYPE_INT_8: - return sizeof(int8_t); - case QNN_DATATYPE_SFIXED_POINT_8: - return sizeof(int8_t); - case QNN_DATATYPE_SFIXED_POINT_4: - return sizeof(int8_t); - default: - break; - } - return 0; -} - -static const char * qnn_datatype_to_string(Qnn_DataType_t qnn_type) { - switch (qnn_type) { - case QNN_DATATYPE_FLOAT_32: - return "QNN_DATATYPE_FLOAT_32"; - case QNN_DATATYPE_FLOAT_16: - return "QNN_DATATYPE_FLOAT_16"; - case QNN_DATATYPE_UINT_32: - return "QNN_DATATYPE_UINT_32"; - case QNN_DATATYPE_INT_32: - return "QNN_DATATYPE_INT_32"; - case QNN_DATATYPE_INT_16: - return "QNN_DATATYPE_INT_16"; - case QNN_DATATYPE_INT_8: - return "QNN_DATATYPE_INT_8"; - case QNN_DATATYPE_SFIXED_POINT_8: - return "QNN_DATATYPE_SFIXED_POINT_8"; - case QNN_DATATYPE_SFIXED_POINT_4: - return "QNN_DATATYPE_SFIXED_POINT_4"; - default: - break; - } - return "QNN_DATATYPE_UNDEFINED"; -} - -static const char * qnn_get_error_string(Qnn_ErrorHandle_t qnn_error_code) { +const char * ggmlqnn_get_error_string(Qnn_ErrorHandle_t qnn_error_code) { // file:///opt/qcom/aistack/qairt/2.31.0.250130/docs/QNN/general/api_error_codes.html switch (qnn_error_code) { case QNN_SUCCESS: @@ -658,7 +499,7 @@ static const char * qnn_get_error_string(Qnn_ErrorHandle_t qnn_error_code) { } // helper function to create an operation config -static Qnn_OpConfig_t create_op_config(const char * name, const char * package, const char * type, +Qnn_OpConfig_t ggmlqnn_create_op_config(const char * name, const char * package, const char * type, Qnn_Param_t * params, uint32_t num_params, Qnn_Tensor_t * inputs, uint32_t num_inputs, Qnn_Tensor_t * outputs, uint32_t num_outputs) { @@ -674,61 +515,6 @@ static Qnn_OpConfig_t create_op_config(const char * name, const char * package, // ================================================================================================= // section-5:ggml-qnn backend helper macro / data structure / function / class // ================================================================================================= -#define RPCMEM_DEFAULT_FLAGS 1 -#define RPCMEM_HEAP_ID_SYSTEM 25 - -typedef void (* ggmlqnn_op_func_t)(ggml_backend_t backend, ggml_tensor * op); - -using pfn_rpc_mem_init = void (*)(void); -using pfn_rpc_mem_deinit = void (*)(void); -using pfn_rpc_mem_alloc = void *(*)(int, uint32_t, int); -using pfn_rpc_mem_free = void (*)(void *); -using pfn_rpc_mem_to_fd = int (*)(void *); -using _pfn_QnnSaver_initialize = decltype(QnnSaver_initialize); -using _pfn_QnnInterface_getProviders = decltype(QnnInterface_getProviders); -using _pfn_QnnSystemInterface_getProviders = decltype(QnnSystemInterface_getProviders); - -using qnn_res_t = std::tuple>; -using qnn_tensors_t = std::vector< Qnn_Tensor_t *>; - -enum class ggml_qnn_profile_level { - profile_off = 0, - profile_basic = 1, - profile_detail = 2 -}; - -enum qcom_htp_arch { - NONE = 0, - V68 = 68, - V69 = 69, - V73 = 73, - V75 = 75, - V79 = 79, -}; - -enum qcom_chipset_soc_model { - UNKNOWN_SM = 0, - SM7450 = 41, // v69, 7 Gen1 - SM8350 = 30, // v68, 888 - SM8450 = 36, // v69, SD 8 Gen 1 - SM8475 = 42, // v69, SD 8+ Gen 1 - SM8550 = 43, // v73, SD 8 Gen 2 - SM8650 = 57, // v75, SD 8 Gen 3 - SM8750 = 69, // v79, SD 8 Gen 4 -#if defined(_WIN32) || defined(_MSC_VER) - SC7280X = 44, - SC8280X = 37, - SC8380XP = 60, -#endif -}; - -struct qcom_socinfo { - uint32_t soc_model; - size_t htp_arch; - size_t vtcm_size_in_mb; - char soc_desc[GGML_MAX_NAME]; -}; - //file:///opt/qcom/aistack/qairt/2.31.0.250130/docs/QNN/general/overview.html#tbl-supported-snapdragon-devices static struct qcom_socinfo g_qnn_soc_info_table[] = { /* Qualcomm SnapDragon 7 Gen 1 */ @@ -780,7 +566,7 @@ static struct qcom_socinfo g_qnn_soc_info_table[] = { .vtcm_size_in_mb = 8, .soc_desc = "Qualcomm SnapDragon 8 Gen 4"}, -#if defined(_WIN32) || defined(_MSC_VER) +#if defined(_MSC_VER) /* Qualcomm SnapDragon 7c Gen 2 */ [SC7280X] = { .soc_model = SC7280X, @@ -805,24 +591,6 @@ static struct qcom_socinfo g_qnn_soc_info_table[] = { }; -struct ggml_backend_qnn_context { - int device; - int threads; - char name[GGML_MAX_NAME]; - char desc[GGML_MAX_NAME]; - char lib[GGML_MAX_NAME]; - qnn_instance * instance; - struct ggml_backend * backend; - QNN_INTERFACE_VER_TYPE raw_interface; - QNN_SYSTEM_INTERFACE_VER_TYPE raw_system_interface; - struct qcom_socinfo socinfo; - - std::unique_ptr work_data; - std::vector> tasks; - size_t work_size = 0; - int n_threads = GGML_DEFAULT_N_THREADS; -} ; - //the following helper funcs are used to ensure every QNN tensor name is unique static std::atomic g_ggmltensor_idx(0); static void reset_idx() { @@ -848,7 +616,7 @@ static struct ggml_backend_qnn_context g_qnn_mgr[GGML_QNN_MAX_DEVICES] = { .threads = 1, .name = "qnn-cpu", .desc = "Qualcomm Kryo CPU", -#if defined(_WIN32) || defined(_MSC_VER) +#if defined(_MSC_VER) .lib = "QnnCpu.dll", #else .lib = "libQnnCpu.so", @@ -863,7 +631,7 @@ static struct ggml_backend_qnn_context g_qnn_mgr[GGML_QNN_MAX_DEVICES] = { .threads = 1, .name = "qnn-gpu", .desc = "Qualcomm Adreno GPU", -#if defined(_WIN32) || defined(_MSC_VER) +#if defined(_MSC_VER) .lib = "QnnGpu.dll", #else .lib = "libQnnGpu.so", @@ -878,7 +646,7 @@ static struct ggml_backend_qnn_context g_qnn_mgr[GGML_QNN_MAX_DEVICES] = { .threads = 1, .name = "qnn-npu", .desc = "Qualcomm NPU(Hexagon Tensor Processor)", -#if defined(_WIN32) || defined(_MSC_VER) +#if defined(_MSC_VER) .lib = "QnnHtp.dll", #else .lib = "libQnnHtp.so", @@ -890,13 +658,7 @@ static struct ggml_backend_qnn_context g_qnn_mgr[GGML_QNN_MAX_DEVICES] = { .socinfo = {}}, }; -struct qnn_op_caps_t { - const char * qnn_op_name = nullptr; - const size_t input_param_count = 0; - const char * qnn_param_name = nullptr; -}; - -static const qnn_op_caps_t k_op_caps[] = { +const qnn_op_caps_t k_op_caps[] = { {}, // GGML_OP_NONE {}, // GGML_OP_DUP { @@ -1056,54 +818,6 @@ static struct qcom_socinfo * qnn_get_socinfo_from_socmodel(uint32_t soc_model) { return nullptr; } -static bool ggmlqnn_is_valid_params(ggml_backend_qnn_context * ctx, const ggml_tensor * src0, - const ggml_tensor * src1, ggml_tensor * dst) { - if ((nullptr == ctx) || (nullptr == src0) || (nullptr == src1) || (nullptr == dst)) { - GGMLQNN_LOG_WARN("invalid params\n"); - return false; - } - - qnn_instance * instance = ctx->instance; - if (nullptr == instance) { - GGMLQNN_LOG_WARN("invalid params\n"); - return false; - } - - return true; -} - -#define GGMLQNN_CHECK_PARAMS(ctx, src0, src1, dst) \ - do { \ - if (!ggmlqnn_is_valid_params((ctx), (src0), (src1), (dst))) { \ - return; \ - } \ - } while (0) - -static uint32_t ggml_get_tensor_rank(const ggml_tensor * tensor) { - /* - uint32_t rank = 0; - for (int i = 0; i < GGML_MAX_DIMS; i++) { - if ((0 != tensor->ne[i]) && (1 != tensor->ne[i])) { - rank++; - } - } - return rank; - */ - return ggml_n_dims(tensor); -} - -static uint32_t ggml_get_tensor_data_size(const ggml_tensor * tensor) { - /* - size_t data_size = ggml_row_size(tensor->type, tensor->ne[0]); - size_t n_dims = ggml_get_tensor_rank(tensor); - for (int i = 1; i < n_dims; i++) { - data_size *= tensor->ne[i]; - } - - return data_size; - */ - return ggml_nbytes(tensor); -} static const char * ggml_get_type_name(ggml_type type) { const struct ggml_type_traits * traits = ggml_get_type_traits(type); @@ -1115,9 +829,8 @@ static const char * get_ggml_type_name(ggml_type type) { return traits->type_name; } -//TODO: // ref:explanation of k-quants, https://github.com/ggerganov/llama.cpp/pull/1684 -static Qnn_DataType_t qnn_datatype_from_ggml_datatype(enum ggml_type ggmltype) { +Qnn_DataType_t ggmlqnn_datatype_from_ggml_datatype(enum ggml_type ggmltype) { switch (ggmltype) { case GGML_TYPE_F16: return QNN_DATATYPE_FLOAT_16; @@ -1135,7 +848,6 @@ static Qnn_DataType_t qnn_datatype_from_ggml_datatype(enum ggml_type ggmltype) { return QNN_DATATYPE_UNDEFINED; } -//TODO: static ggml_type ggml_datatype_from_qnn_datatype(Qnn_DataType_t qnn_type) { switch (qnn_type) { case QNN_DATATYPE_FLOAT_32: @@ -1190,7 +902,7 @@ static void get_qnn_dimensions_from_ggml_dimensions(uint32_t * qnn_dimensions, c } } -static Qnn_Tensor_t * ggml_qnn_create_general_tensor(const ggml_tensor * tensor, const char * name, +Qnn_Tensor_t * ggmlqnn_create_general_tensor(const ggml_tensor * tensor, const char * name, Qnn_TensorType_t qnn_tensor_type, Qnn_DataType_t qnn_data_type, uint32_t rank, uint32_t * dims, @@ -1227,7 +939,7 @@ static Qnn_Tensor_t * ggml_qnn_create_general_tensor(const ggml_tensor * tensor, if (b_transpose) { GGML_ASSERT(tensor != nullptr); //ensure ggml_tensor is not nullptr for this special case - get_qnn_dimensions_from_ggml_dimensions(transpose_dims, reverse_dims, ggml_get_tensor_rank(tensor)); + get_qnn_dimensions_from_ggml_dimensions(transpose_dims, reverse_dims, ggml_n_dims(tensor)); tensor_dims = transpose_dims; #if 0 for (size_t idx = 0; idx < 4; idx++) { @@ -1277,7 +989,7 @@ static Qnn_Tensor_t * ggml_qnn_create_general_tensor(const ggml_tensor * tensor, return p_qnn_tensor; } -static Qnn_Tensor_t * ggml_qnn_create_compute_tensor(const ggml_tensor * tensor) { +Qnn_Tensor_t * ggmlqnn_create_compute_tensor(const ggml_tensor * tensor) { uint32_t dimensions[] = {(uint32_t) tensor->ne[0], (uint32_t) tensor->ne[1], (uint32_t) tensor->ne[2], (uint32_t) tensor->ne[3]}; Qnn_DataType_t qnn_data_type = QNN_DATATYPE_FLOAT_32; @@ -1289,8 +1001,8 @@ static Qnn_Tensor_t * ggml_qnn_create_compute_tensor(const ggml_tensor * tensor) qnn_tensor_type = QNN_TENSOR_TYPE_APP_READ; } - qnn_data_type = qnn_datatype_from_ggml_datatype(tensor->type); - Qnn_Tensor_t * p_qnn_tensor = ggml_qnn_create_general_tensor(tensor, nullptr, + qnn_data_type = ggmlqnn_datatype_from_ggml_datatype(tensor->type); + Qnn_Tensor_t * p_qnn_tensor = ggmlqnn_create_general_tensor(tensor, nullptr, qnn_tensor_type, qnn_data_type, ggml_n_dims(tensor), dimensions, nullptr, 0); @@ -1298,6 +1010,77 @@ static Qnn_Tensor_t * ggml_qnn_create_compute_tensor(const ggml_tensor * tensor) return p_qnn_tensor; } +void * ggmlqnn_type_trait(ggml_backend_qnn_context * ctx, ggml_tensor * op) { + const ggml_tensor * src0 = op->src[0]; + const ggml_tensor * src1 = op->src[1]; + ggml_tensor * dst = op; + const enum ggml_type src0_type = src0->type; + + GGML_TENSOR_BINARY_OP_LOCALS + GGML_ASSERT(ne0 == ne01); + GGML_ASSERT(ne1 == ne11); + GGML_ASSERT(ne2 == ne12); + GGML_ASSERT(ne3 == ne13); + GGML_ASSERT(nb00 == ggml_type_size(src0_type)); + GGML_ASSERT(nb10 == ggml_type_size(src1->type)); + + // broadcast factors + const int64_t r2 = ne12 / ne02; + const int64_t r3 = ne13 / ne03; + const int64_t ne_plane = ne01 * ne00; + const size_t desired_size = ((GGML_TYPE_F32 == src0_type) ? 0 : ne03 * ne02 * ne_plane * sizeof(float)); + ctx->desired_size = desired_size; + if (ctx->work_size < desired_size) { + ctx->work_data.reset(new char[desired_size]); + ctx->work_size = desired_size; + } + ctx->n_threads = std::thread::hardware_concurrency(); + void * wdata = ctx->work_data.get(); + // convert src0 to float + if (src0_type != GGML_TYPE_F32) { + const auto *type_traits = ggml_get_type_traits(src0_type); + ggml_to_float_t const to_float = type_traits->to_float; + + for (int64_t i03 = 0; i03 < ne03; i03++) { + for (int64_t i02 = 0; i02 < ne02; i02++) { + const void *x = (char *) src0->data + i02 * nb02 + i03 * nb03; + float *const wplane = (float *) wdata + i02 * ne_plane + i03 * ne02 * ne_plane; + + const int min_cols_per_thread = 4096; + const int min_rows_per_thread = std::max((int) (min_cols_per_thread / ne00), 1); + const int n_threads = std::max( + std::min(ctx->n_threads, (int) (ne01 / min_rows_per_thread)), 1); + for (int i = 1; i < n_threads; i++) { + const int64_t start = i * ne01 / n_threads; + const int64_t end = (i + 1) * ne01 / n_threads; + if (start < end) { + ctx->tasks.push_back(std::async(std::launch::async, [=]() { + for (int64_t i01 = start; i01 < end; i01++) { + to_float((const char *) x + i01 * nb01, wplane + i01 * ne00, ne00); + } + })); + } + } + { + // reuse the current thread for the first task + const int64_t start = 0; + const int64_t end = ne01 / n_threads; + for (int64_t i01 = start; i01 < end; i01++) { + to_float((const char *) x + i01 * nb01, wplane + i01 * ne00, ne00); + } + } + } + } + + // wait for all tasks to finish + for (auto &task: ctx->tasks) { + task.get(); + } + ctx->tasks.clear(); + } + return wdata; +} + static void append_tensor_dimensions(const ggml_tensor * tensor, std::string & output) { char buffer[256] = {}; const char * type_name = get_ggml_type_name(tensor->type); @@ -1323,7 +1106,11 @@ static void append_tensor_dimensions(const ggml_tensor * tensor, std::string & o output.append(buffer, len); } -static size_t get_qnn_op_index(const ggml_tensor * tensor) { +size_t ggmlqnn_get_opcaps_size() { + return std::size(k_op_caps); +} + +size_t ggmlqnn_get_op_index(const ggml_tensor * tensor) { if (tensor->op == GGML_OP_UNARY) { return GGML_OP_COUNT + ggml_get_unary_op(tensor); } @@ -1331,17 +1118,17 @@ static size_t get_qnn_op_index(const ggml_tensor * tensor) { return tensor->op; } -static size_t get_qnn_op_input_param_count(const ggml_tensor * op) { - auto op_index = get_qnn_op_index(op); +static size_t ggmlqnn_get_op_input_param_count(const ggml_tensor * op) { + auto op_index = ggmlqnn_get_op_index(op); GGML_ASSERT(op_index < std::size(k_op_caps)); return k_op_caps[op_index].input_param_count; } -static void get_graph_key_from_op(const ggml_tensor * op, std::string & output) { +void ggmlqnn_get_graphkey_from_op(const ggml_tensor * op, std::string & output) { GGML_ASSERT(op->op != GGML_OP_NONE); output += ggml_op_desc(op); output += get_ggml_type_name(op->type); - size_t param_count = get_qnn_op_input_param_count(op); + size_t param_count = ggmlqnn_get_op_input_param_count(op); for (size_t i = 0; i < param_count; ++i) { auto * input = op->src[i]; if (!input) { @@ -1352,42 +1139,21 @@ static void get_graph_key_from_op(const ggml_tensor * op, std::string & output) } } -#if ENABLE_QNNBACKEND_PERF -class qnn_perf { -public: - qnn_perf(const std::string & perf_name) : _perf_name(std::move(perf_name)) {}; - qnn_perf() = delete; - qnn_perf(const qnn_perf & ) = delete; - qnn_perf & operator= (const qnn_perf & ) = delete; - - void start() { - _begin_time = ggml_time_us(); +bool ggmlqnn_is_valid_params(ggml_backend_qnn_context * ctx, const ggml_tensor * src0, + const ggml_tensor * src1, ggml_tensor * dst) { + if ((nullptr == ctx) || (nullptr == src0) || (nullptr == src1) || (nullptr == dst)) { + GGMLQNN_LOG_WARN("invalid params\n"); + return false; } - void info() { - _end_time = ggml_time_us(); - _duration = (_end_time - _begin_time); - GGMLQNN_LOG_DEBUG("duration of %s : %lld microseconds\n", _perf_name.c_str(), _duration); + qnn_instance * instance = ctx->instance; + if (nullptr == instance) { + GGMLQNN_LOG_WARN("invalid params\n"); + return false; } -private: - int64_t _begin_time = 0LL; - int64_t _end_time = 0LL; - int64_t _duration = 0LL; - std::string _perf_name; -}; -#else -class qnn_perf { -public: - qnn_perf(const std::string & perf_name) {} - qnn_perf() = delete; - qnn_perf(const qnn_perf & ) = delete; - qnn_perf & operator= (const qnn_perf & ) = delete; - - void start() {} - void info() {} -}; -#endif + return true; +} template Fn load_qnn_functionpointers(void * handle, const char * function_name) { @@ -1401,417 +1167,6 @@ Fn load_qnn_functionpointers(void * handle, const char * function_name) { #endif } -class qnn_interface { - -#define DEFINE_SHIM_FUNCTION_INTERFACE(F, pointer_name) \ - template \ - inline auto qnn_##F(Args... args) const { \ - return (_qnn_interface->QNN_INTERFACE_VER_NAME.pointer_name)( \ - std::forward(args)...); \ - } - - -#define DEFINE_SHIM_FUNCTION_SYS_INTERFACE(F, pointer_name) \ - template \ - inline auto qnn_##F(Args... args) const { \ - return (_qnn_sys_interface->QNN_SYSTEM_INTERFACE_VER_NAME.pointer_name)( \ - std::forward(args)...); \ - } - - friend class qnn_instance; - -public: - qnn_interface() = default; - - // QnnBackend - DEFINE_SHIM_FUNCTION_INTERFACE(backend_create, backendCreate); - - DEFINE_SHIM_FUNCTION_INTERFACE(backend_free, backendFree); - - DEFINE_SHIM_FUNCTION_INTERFACE(backend_register_op_package, backendRegisterOpPackage); - - DEFINE_SHIM_FUNCTION_INTERFACE(backend_validate_op_config, backendValidateOpConfig); - - DEFINE_SHIM_FUNCTION_INTERFACE(backend_get_api_version, backendGetApiVersion); - - // QnnDevice - DEFINE_SHIM_FUNCTION_INTERFACE(device_create, deviceCreate); - - DEFINE_SHIM_FUNCTION_INTERFACE(device_free, deviceFree); - - DEFINE_SHIM_FUNCTION_INTERFACE(device_get_infrastructure, deviceGetInfrastructure); - - DEFINE_SHIM_FUNCTION_INTERFACE(device_get_platform_info, deviceGetPlatformInfo); - - DEFINE_SHIM_FUNCTION_INTERFACE(device_get_info, deviceGetInfo); - - // QnnContext - DEFINE_SHIM_FUNCTION_INTERFACE(context_create, contextCreate); - - DEFINE_SHIM_FUNCTION_INTERFACE(context_get_binary_size, contextGetBinarySize); - - DEFINE_SHIM_FUNCTION_INTERFACE(context_get_binary, contextGetBinary); - - DEFINE_SHIM_FUNCTION_INTERFACE(context_create_from_binary, contextCreateFromBinary); - - DEFINE_SHIM_FUNCTION_INTERFACE(context_free, contextFree); - - // QnnGraph - DEFINE_SHIM_FUNCTION_INTERFACE(graph_create, graphCreate); - - DEFINE_SHIM_FUNCTION_INTERFACE(graph_add_node, graphAddNode); - - DEFINE_SHIM_FUNCTION_INTERFACE(graph_finalize, graphFinalize); - - DEFINE_SHIM_FUNCTION_INTERFACE(graph_execute, graphExecute); - - DEFINE_SHIM_FUNCTION_INTERFACE(graph_retrieve, graphRetrieve); - - // QnnLog - DEFINE_SHIM_FUNCTION_INTERFACE(log_create, logCreate); - - DEFINE_SHIM_FUNCTION_INTERFACE(log_free, logFree); - - DEFINE_SHIM_FUNCTION_INTERFACE(log_set_log_level, logSetLogLevel); - - // QnnProfile - DEFINE_SHIM_FUNCTION_INTERFACE(profile_create, profileCreate); - - DEFINE_SHIM_FUNCTION_INTERFACE(profile_get_events, profileGetEvents); - - DEFINE_SHIM_FUNCTION_INTERFACE(profile_get_sub_events, profileGetSubEvents); - - DEFINE_SHIM_FUNCTION_INTERFACE(profile_get_event_data, profileGetEventData); - - DEFINE_SHIM_FUNCTION_INTERFACE(profile_free, profileFree); - - // QnnMem - DEFINE_SHIM_FUNCTION_INTERFACE(mem_register, memRegister); - - DEFINE_SHIM_FUNCTION_INTERFACE(mem_de_register, memDeRegister); - - // QnnProperty - DEFINE_SHIM_FUNCTION_INTERFACE(property_has_capability, propertyHasCapability); - - // QnnTensor - DEFINE_SHIM_FUNCTION_INTERFACE(tensor_create_context_tensor, tensorCreateContextTensor); - - DEFINE_SHIM_FUNCTION_INTERFACE(tensor_create_graph_tensor, tensorCreateGraphTensor); - - // QnnSystem - DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_create, systemContextCreate); - - DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_get_binary_info, systemContextGetBinaryInfo); - - DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_free, systemContextFree); - - void set_qnn_interface(const QnnInterface_t * qnn_interface) { - _qnn_interface = qnn_interface; - } - - void set_qnn_system_interface(const QnnSystemInterface_t * qnn_sys_interface) { - _qnn_sys_interface = qnn_sys_interface; - } - - uint32_t get_backend_id() const { - return _qnn_interface->backendId; - } - - bool is_loaded() const { - return ((_qnn_sys_interface != nullptr) && (_qnn_interface != nullptr)); - } - -private: - const QnnInterface_t *_qnn_interface = nullptr; - - const QnnSystemInterface_t *_qnn_sys_interface = nullptr; -}; - -class qnn_instance { -public: - using BackendIdType = decltype(QnnInterface_t{}.backendId); - - explicit qnn_instance(const std::string & lib_path, const std::string & backend_name, - const std::string & model_name) : - _lib_path(std::move(lib_path)), - _backend_name(std::move(backend_name)), - _model_name(std::move(model_name)) {}; - - ~qnn_instance() { - } - - int qnn_init(const QnnSaver_Config_t ** saver_config); - - int qnn_finalize(); - - const qnn_interface & get_qnn_interface() { - if (!_qnn_interface.is_loaded()) { - GGMLQNN_LOG_WARN("pls check why _qnn_interface is not loaded\n"); - } - return _qnn_interface; - } - - const QNN_INTERFACE_VER_TYPE & get_qnn_raw_interface() { - if (!_qnn_interface.is_loaded()) { - GGMLQNN_LOG_WARN("pls check why _qnn_interface is not loaded\n"); - } - return _qnn_raw_interface; - } - - const QNN_SYSTEM_INTERFACE_VER_TYPE & get_qnn_raw_system_interface() { - if (!_qnn_interface.is_loaded()) { - GGMLQNN_LOG_WARN("pls check why _qnn_interface is not loaded\n"); - } - return _qnn_raw_system_interface; - } - - const Qnn_LogHandle_t get_qnn_log_handle() { return _qnn_log_handle; } - - const Qnn_ProfileHandle_t get_qnn_profile_handle() { return _qnn_profile_handle; } - - const Qnn_DeviceHandle_t get_qnn_device_handle() { return _qnn_device_handle; } - - const Qnn_BackendHandle_t get_qnn_backend_handle() { return _qnn_backend_handle; } - - const Qnn_ContextHandle_t get_qnn_context_handle() { return _qnn_context_handle; } - - const QnnSystemContext_Handle_t get_qnn_system_handle() { return _qnn_system_handle; } - - const Qnn_GraphHandle_t get_qnn_graph_handle() { return _qnn_graph_handle; } - - int init_qnn_graph(const char * graph_name, - bool debug, - uint8_t do_node_validation = 1, - const QnnGraph_Config_t ** graph_configs = nullptr - ); - int init_qnn_graph(const std::string & graph_name, QNNBackend device, size_t vtcm_size_in_mb = 8, size_t hvx_threads = 8); - - int finalize_qnn_graph(); - - bool is_valid_graph() const { return _qnn_graph_handle != nullptr; } - - int init_htp_perfinfra() { - QnnDevice_Infrastructure_t device_infra = nullptr; - int error = _qnn_raw_interface.deviceGetInfrastructure(&device_infra); - if (error != QNN_SUCCESS) { - GGMLQNN_LOG_WARN("failed to get qnn device infra\n"); - return 1; - } - - QnnHtpDevice_Infrastructure_t * htp_infra = static_cast(device_infra); - QnnHtpDevice_PerfInfrastructure_t * htp_perfinfra = &htp_infra->perfInfra; - uint32_t power_configid = 1; - uint32_t device_id = 0; - uint32_t core_id = 0; - htp_perfinfra->createPowerConfigId(device_id, core_id, &power_configid); - _qnn_htp_perfinfra = htp_perfinfra; - _qnn_power_configid = power_configid; - - return 0; - } - - int set_rpc_polling() { - if (_qnn_rpc_pollingtime > 0) { - QnnHtpPerfInfrastructure_PowerConfig_t rpc_pollingtime; - memset(&rpc_pollingtime, 0, sizeof(rpc_pollingtime)); - rpc_pollingtime.option = QNN_HTP_PERF_INFRASTRUCTURE_POWER_CONFIGOPTION_RPC_POLLING_TIME; - rpc_pollingtime.rpcPollingTimeConfig = _qnn_rpc_pollingtime; - const QnnHtpPerfInfrastructure_PowerConfig_t * power_configs[] = {&rpc_pollingtime, nullptr}; - if (_qnn_htp_perfinfra) { - _qnn_htp_perfinfra->setPowerConfig(_qnn_power_configid, power_configs); - } - } - return 0; - } - - int set_high_performance_mode() { - if (nullptr == _qnn_htp_perfinfra) { - GGMLQNN_LOG_DEBUG("perf intra is null\n"); - return 1; - } - - QnnHtpPerfInfrastructure_PowerConfig_t power_config; - memset(&power_config, 0, sizeof(power_config)); - power_config.option = QNN_HTP_PERF_INFRASTRUCTURE_POWER_CONFIGOPTION_DCVS_V3; - power_config.dcvsV3Config.dcvsEnable = 0; - power_config.dcvsV3Config.setDcvsEnable = 1; - power_config.dcvsV3Config.contextId = _qnn_power_configid; - power_config.dcvsV3Config.powerMode = QNN_HTP_PERF_INFRASTRUCTURE_POWERMODE_PERFORMANCE_MODE; - power_config.dcvsV3Config.setSleepLatency = 1; // True to consider Latency parameter otherwise False - power_config.dcvsV3Config.setBusParams = 1; // True to consider Bus parameter otherwise False - power_config.dcvsV3Config.setCoreParams = 1; // True to consider Core parameter otherwise False - power_config.dcvsV3Config.sleepDisable = 0; // True to consider sleep/LPM modes, False to enable - power_config.dcvsV3Config.setSleepDisable = 0; // True to consider sleep disable/enable parameter otherwise False - // set Sleep latency parameter - uint32_t latencyValue = 40; - power_config.dcvsV3Config.sleepLatency = latencyValue; // range 40-2000 micro sec - // set Bus Clock Parameters (refer QnnHtpPerfInfrastructure_VoltageCorner_t enum) - power_config.dcvsV3Config.busVoltageCornerMin = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; - power_config.dcvsV3Config.busVoltageCornerTarget = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; - power_config.dcvsV3Config.busVoltageCornerMax = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; - // set Core Clock Parameters (refer QnnHtpPerfInfrastructure_VoltageCorner_t enum) - power_config.dcvsV3Config.coreVoltageCornerMin = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; - power_config.dcvsV3Config.coreVoltageCornerTarget = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; - power_config.dcvsV3Config.coreVoltageCornerMax = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; - // set power config with different performance parameters - const QnnHtpPerfInfrastructure_PowerConfig_t * power_configs[] = {&power_config, nullptr}; - - _qnn_htp_perfinfra->setPowerConfig(_qnn_power_configid, power_configs); - - return 0; - } - - std::string & get_qnn_graph_name() { return _graph_name; } - - bool is_rpcmem_initialized() { - return _rpcmem_initialized; - } - - void set_rpcmem_initialized(bool initialized) { - _rpcmem_initialized = initialized; - } - - size_t get_rpcmem_capacity() { return _rpcmem_capacity; } - size_t get_rpcmem_usage() { return _rpcmem_usage; } - - int32_t rpcmem_to_fd(void * buf); - - int register_rpcmem(void * p_data, Qnn_Tensor_t * p_tensor); - Qnn_MemHandle_t register_rpcmem(void * p_data, const uint32_t rank, uint32_t * dimensions, Qnn_DataType_t data_type); - - void unregister_rpcmem(); - void unregister_rpcmem(Qnn_MemHandle_t mem_handle); - - void * alloc_rpcmem(size_t bytes, size_t alignment); - void * get_rpcmem_from_memhandle(Qnn_MemHandle_t mem_handle); - - void free_rpcmem(void * buf); - void free_rpcmem(); - - bool is_rpcmem_allocated(void * buf); - - bool is_rpcmem_registered(Qnn_MemHandle_t handle) { - return _qnn_mem_set.count(handle) != 0U; - } - - bool enable_qnn_rpc() { - return _enable_qnn_rpc; - } - - void probe_device_meminfo() { - size_t candidate_size = 0; - uint8_t * rpc_buffer = nullptr; - const int SIZE_IN_MB = (1 << 20); - size_t probe_slots[] = {1024, 1536, 2048 - 48, 2048}; - size_t probe_counts = sizeof(probe_slots) / sizeof(size_t); - for (size_t idx = 0; idx < probe_counts; idx++) { - rpc_buffer = static_cast(alloc_rpcmem_internal(probe_slots[idx] * SIZE_IN_MB, 4)); - if (nullptr == rpc_buffer) { - GGMLQNN_LOG_DEBUG("alloc rpcmem %d (MB) failure, %s\n", probe_slots[idx], strerror(errno)); - break; - } else { - candidate_size = probe_slots[idx]; - free_rpcmem(rpc_buffer); - rpc_buffer = nullptr; - } - } - if (candidate_size > _rpcmem_capacity) - _rpcmem_capacity = candidate_size; - - free_rpcmem(); - _rpcmem_usage = 0; - GGMLQNN_LOG_INFO("capacity of rpc ion memory %d MB\n", _rpcmem_capacity); - } - -public: - std::map>> _qnn_graph_map; - -private: - int load_system(); - - int unload_system(); - - int load_backend(std::string & lib_path, const QnnSaver_Config_t ** saver_config); - - int unload_backend(); - - void set_qnn_raw_interface(QNN_INTERFACE_VER_TYPE & raw_interface) { - _qnn_raw_interface = raw_interface; - } - - void set_qnn_raw_system_interface(QNN_SYSTEM_INTERFACE_VER_TYPE & raw_interface) { - _qnn_raw_system_interface = raw_interface; - } - - void * alloc_rpcmem_internal(size_t bytes, size_t alignment); - -private: - static constexpr const int _required_num_providers = 1; - -private: - std::string _lib_path; - std::string _backend_name; - std::string _model_name; // name of prebuilt QNN model, might be used in the future - BackendIdType _backend_id; - - bool _debug_tensor = false; // flag to indicate if requested graph is to be run in debug mode - bool _do_node_validations = true; // flag to indicate whether all add_node calls need to be validated - QnnLog_Level_t _qnn_log_level = QNN_LOG_LEVEL_DEBUG; - - ggml_qnn_profile_level _profile_level = ggml_qnn_profile_level::profile_detail; - - void * _system_lib_handle = nullptr; - - Qnn_GraphHandle_t _qnn_graph_handle = nullptr; - - Qnn_LogHandle_t _qnn_log_handle = nullptr; - - Qnn_ProfileHandle_t _qnn_profile_handle = nullptr; - - Qnn_DeviceHandle_t _qnn_device_handle = nullptr; - - Qnn_BackendHandle_t _qnn_backend_handle = nullptr; - - Qnn_ContextHandle_t _qnn_context_handle = nullptr; - - QnnSystemContext_Handle_t _qnn_system_handle = nullptr; - - QnnHtpDevice_PerfInfrastructure_t * _qnn_htp_perfinfra = nullptr; - uint32_t _qnn_power_configid = 1; - uint32_t _qnn_rpc_pollingtime = 9999; // 0-10000 us for high performing - - qnn_interface _qnn_interface; - QNN_INTERFACE_VER_TYPE _qnn_raw_interface; - QNN_SYSTEM_INTERFACE_VER_TYPE _qnn_raw_system_interface; - - std::unordered_map _qnn_mem_set; - std::unordered_map _qnn_rpc_buffer_to_handles; - - static std::mutex _init_mutex; - static std::unordered_map _loaded_lib_handle; - static std::unordered_map _lib_path_to_backend_id; - static std::unordered_map _loaded_backend; - - std::atomic_bool _rpcmem_initialized{false}; - pfn_rpc_mem_alloc _pfn_rpc_mem_alloc; - pfn_rpc_mem_free _pfn_rpc_mem_free; - pfn_rpc_mem_to_fd _pfn_rpc_mem_to_fd; - pfn_rpc_mem_init _pfn_rpc_mem_init; - pfn_rpc_mem_deinit _pfn_rpc_mem_deinit; - std::unordered_map _rpcmem_store_map; - std::unordered_map _rpcmem_usage_map; - size_t _rpcmem_usage = 0; // mempool usage in Mbytes - size_t _rpcmem_capacity = 512; // mempool size in Mbytes - - std::string _graph_name; - QNNBackend _device_id; - void * _rpc_lib_handle = nullptr; - bool _enable_qnn_rpc = false; //TODO:unknown issue with QNN RPC feature - - DISABLE_COPY(qnn_instance); - DISABLE_MOVE(qnn_instance); -}; - std::mutex qnn_instance::_init_mutex; std::unordered_map qnn_instance::_loaded_lib_handle; std::unordered_map qnn_instance::_lib_path_to_backend_id; @@ -2637,7 +1992,7 @@ int qnn_instance::init_qnn_graph(const std::string & graph_name, QNNBackend devi if (error != QNN_SUCCESS) { GGMLQNN_LOG_ERROR("[%s][%s]failed to create qnn graph, error: %s", ggml_backend_qnn_get_devname(device), graph_name.c_str(), - qnn_get_error_string(error)); + ggmlqnn_get_error_string(error)); return error; } @@ -2697,7 +2052,103 @@ int qnn_instance::finalize_qnn_graph() { return 0; } -static uint8_t * create_rpc_buffer(qnn_instance * instance, const ggml_tensor * ggml_tensor, Qnn_Tensor_t * qnn_tensor, bool b_copydata) { +int qnn_instance::init_htp_perfinfra() { + QnnDevice_Infrastructure_t device_infra = nullptr; + int error = _qnn_raw_interface.deviceGetInfrastructure(&device_infra); + if (error != QNN_SUCCESS) { + GGMLQNN_LOG_WARN("failed to get qnn device infra\n"); + return 1; + } + + QnnHtpDevice_Infrastructure_t * htp_infra = static_cast(device_infra); + QnnHtpDevice_PerfInfrastructure_t * htp_perfinfra = &htp_infra->perfInfra; + uint32_t power_configid = 1; + uint32_t device_id = 0; + uint32_t core_id = 0; + htp_perfinfra->createPowerConfigId(device_id, core_id, &power_configid); + _qnn_htp_perfinfra = htp_perfinfra; + _qnn_power_configid = power_configid; + + return 0; +} + +int qnn_instance::set_rpc_polling() { + if (_qnn_rpc_pollingtime > 0) { + QnnHtpPerfInfrastructure_PowerConfig_t rpc_pollingtime; + memset(&rpc_pollingtime, 0, sizeof(rpc_pollingtime)); + rpc_pollingtime.option = QNN_HTP_PERF_INFRASTRUCTURE_POWER_CONFIGOPTION_RPC_POLLING_TIME; + rpc_pollingtime.rpcPollingTimeConfig = _qnn_rpc_pollingtime; + const QnnHtpPerfInfrastructure_PowerConfig_t * power_configs[] = {&rpc_pollingtime, nullptr}; + if (_qnn_htp_perfinfra) { + _qnn_htp_perfinfra->setPowerConfig(_qnn_power_configid, power_configs); + } + } + return 0; +} + +int qnn_instance::set_high_performance_mode() { + if (nullptr == _qnn_htp_perfinfra) { + GGMLQNN_LOG_DEBUG("perf intra is null\n"); + return 1; + } + + QnnHtpPerfInfrastructure_PowerConfig_t power_config; + memset(&power_config, 0, sizeof(power_config)); + power_config.option = QNN_HTP_PERF_INFRASTRUCTURE_POWER_CONFIGOPTION_DCVS_V3; + power_config.dcvsV3Config.dcvsEnable = 0; + power_config.dcvsV3Config.setDcvsEnable = 1; + power_config.dcvsV3Config.contextId = _qnn_power_configid; + power_config.dcvsV3Config.powerMode = QNN_HTP_PERF_INFRASTRUCTURE_POWERMODE_PERFORMANCE_MODE; + power_config.dcvsV3Config.setSleepLatency = 1; // True to consider Latency parameter otherwise False + power_config.dcvsV3Config.setBusParams = 1; // True to consider Bus parameter otherwise False + power_config.dcvsV3Config.setCoreParams = 1; // True to consider Core parameter otherwise False + power_config.dcvsV3Config.sleepDisable = 0; // True to consider sleep/LPM modes, False to enable + power_config.dcvsV3Config.setSleepDisable = 0; // True to consider sleep disable/enable parameter otherwise False + // set Sleep latency parameter + uint32_t latencyValue = 40; + power_config.dcvsV3Config.sleepLatency = latencyValue; // range 40-2000 micro sec + // set Bus Clock Parameters (refer QnnHtpPerfInfrastructure_VoltageCorner_t enum) + power_config.dcvsV3Config.busVoltageCornerMin = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; + power_config.dcvsV3Config.busVoltageCornerTarget = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; + power_config.dcvsV3Config.busVoltageCornerMax = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; + // set Core Clock Parameters (refer QnnHtpPerfInfrastructure_VoltageCorner_t enum) + power_config.dcvsV3Config.coreVoltageCornerMin = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; + power_config.dcvsV3Config.coreVoltageCornerTarget = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; + power_config.dcvsV3Config.coreVoltageCornerMax = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; + // set power config with different performance parameters + const QnnHtpPerfInfrastructure_PowerConfig_t * power_configs[] = {&power_config, nullptr}; + + _qnn_htp_perfinfra->setPowerConfig(_qnn_power_configid, power_configs); + + return 0; +} + +void qnn_instance::probe_device_meminfo() { + size_t candidate_size = 0; + uint8_t * rpc_buffer = nullptr; + const int SIZE_IN_MB = (1 << 20); + size_t probe_slots[] = {1024, 1536, 2048 - 48, 2048}; + size_t probe_counts = sizeof(probe_slots) / sizeof(size_t); + for (size_t idx = 0; idx < probe_counts; idx++) { + rpc_buffer = static_cast(alloc_rpcmem_internal(probe_slots[idx] * SIZE_IN_MB, 4)); + if (nullptr == rpc_buffer) { + GGMLQNN_LOG_DEBUG("alloc rpcmem %d (MB) failure, %s\n", probe_slots[idx], strerror(errno)); + break; + } else { + candidate_size = probe_slots[idx]; + free_rpcmem(rpc_buffer); + rpc_buffer = nullptr; + } + } + if (candidate_size > _rpcmem_capacity) + _rpcmem_capacity = candidate_size; + + free_rpcmem(); + _rpcmem_usage = 0; + GGMLQNN_LOG_INFO("capacity of rpc ion memory %d MB\n", _rpcmem_capacity); +} + +uint8_t * ggmlqnn_create_rpc_buffer(qnn_instance * instance, const ggml_tensor * ggml_tensor, Qnn_Tensor_t * qnn_tensor, bool b_copydata) { if (nullptr == instance || nullptr == ggml_tensor || nullptr == qnn_tensor) { GGMLQNN_LOG_WARN("invalid params\n"); return nullptr; @@ -2716,7 +2167,7 @@ static uint8_t * create_rpc_buffer(qnn_instance * instance, const ggml_tensor * return qnn_rpcbuffer; } -static void print_tensors_info(const char * func_name, ggml_backend_qnn_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +void ggmlqnn_print_tensors_info(const char * func_name, ggml_backend_qnn_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { //skip sanity check of params if (nullptr != func_name && nullptr != ctx) { GGMLQNN_LOG_DEBUG("call %s in dev %s\n", func_name, ctx->name); @@ -2742,14 +2193,14 @@ static void dump_op_info(const struct ggml_tensor * tensor) { struct ggml_tensor * src1 = tensor->src[1]; struct ggml_tensor * dst = const_cast(tensor); GGMLQNN_LOG_DEBUG("op name:%s, tensor type:%s", ggml_op_name(tensor->op), ggml_type_name(tensor->type)); - print_tensors_info(nullptr, nullptr, src0, src1, dst); + ggmlqnn_print_tensors_info(nullptr, nullptr, src0, src1, dst); } // ================================================================================================= // section-6: implementation of ggml-qnn backend // ================================================================================================= //TODO: refine this function as it is a performance hotspot/bottleneck function -static bool ggml_qnn_can_handle_op(const struct ggml_tensor * tensor) { +static bool ggml_qnn_can_handle_op(const ggml_backend_qnn_context * ctx, const struct ggml_tensor * tensor) { if (tensor->op == GGML_OP_NONE) { return true; } @@ -2761,7 +2212,7 @@ static bool ggml_qnn_can_handle_op(const struct ggml_tensor * tensor) { return false; } - //TODO: support other op + //TODO: add other op here bool supported_op = ((tensor->op == GGML_OP_ADD) || (tensor->op == GGML_OP_MUL_MAT) || (tensor->op == GGML_OP_MUL) @@ -2782,8 +2233,8 @@ static bool ggml_qnn_can_handle_op(const struct ggml_tensor * tensor) { const int64_t ne0 = tensor->ne[0]; const int64_t ne1 = tensor->ne[1]; - const uint32_t src0_rank = ggml_get_tensor_rank(src0); - const uint32_t src1_rank = ggml_get_tensor_rank(src1); + const uint32_t src0_rank = ggml_n_dims(src0); + const uint32_t src1_rank = ggml_n_dims(src1); if (tensor->op == GGML_OP_ADD) { //dump_op_info(tensor); @@ -2802,19 +2253,22 @@ static bool ggml_qnn_can_handle_op(const struct ggml_tensor * tensor) { return false; if (src0_rank < 2) // QNN's limitation, make QNN SDK happy return false; - if (src0_rank > 3) //TODO: 4D matrix + if (4 == src0_rank) //TODO: 4D matrix mulmat return false; if ((src1->ne[2] != src0->ne[2]) || (src1->ne[3] != src0->ne[3])) // make QNN SDK happy return false; - if (2 != src0_rank) { //TODO: quantize src0 for 3D & 4D matrix - return (src0->type == GGML_TYPE_F32) - && (src1->type == GGML_TYPE_F32) - && (tensor->type == GGML_TYPE_F32); - } else { - return (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_Q8_0 || src0->type == GGML_TYPE_Q4_0 || src0->type == GGML_TYPE_Q6_K) + if (ctx->device == QNN_BACKEND_NPU) + if (2 == src0_rank) + return (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 + || src0->type == GGML_TYPE_Q4_0 || src0->type == GGML_TYPE_Q8_0 + || src0->type == GGML_TYPE_Q6_K || src0->type == GGML_TYPE_Q8_K + ) && (src1->type == GGML_TYPE_F32) && (tensor->type == GGML_TYPE_F32); + else + return (src0->type == GGML_TYPE_F32) && (src1->type == GGML_TYPE_F32) && (tensor->type == GGML_TYPE_F32); + else + return (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || ggml_is_quantized(src0->type)) && (src1->type == GGML_TYPE_F32) && (tensor->type == GGML_TYPE_F32); - } } if (tensor->op == GGML_OP_MUL) { @@ -2826,556 +2280,135 @@ static bool ggml_qnn_can_handle_op(const struct ggml_tensor * tensor) { && (tensor->type == src1->type); } - return (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16) - && (src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16) - && (src0->type == src1->type) && (src0->type == tensor->type); -} - -/* - * provide a general skeleton to offload ggml op to QNN backend: a single node contains 2 input - * tensor and 1 output tensor -*/ -static void ggml_qnn_general_node(ggml_backend_t backend, ggml_tensor * op) { - Qnn_ErrorHandle_t error = QNN_SUCCESS; - enum ggml_status result = GGML_STATUS_SUCCESS; - bool graph_initialized = false; - qnn_instance * instance = nullptr; - ggml_backend_qnn_context * ctx = (ggml_backend_qnn_context *)backend->context; - Qnn_GraphHandle_t graph_handle = nullptr; - Qnn_Tensor_t * p_tensor0 = nullptr; - Qnn_Tensor_t * p_tensor1 = nullptr; - Qnn_Tensor_t * p_tensor2 = nullptr; - Qnn_Param_t qnn_params[] = {}; - const ggml_tensor * src0 = op->src[0]; - const ggml_tensor * src1 = op->src[1]; - ggml_tensor * dst = op; - - GGMLQNN_CHECK_PARAMS(ctx, src0, src1, dst); - instance = ctx->instance; - QNN_INTERFACE_VER_TYPE qnn_raw_interface = ctx->raw_interface; - - size_t qnn_op_index = get_qnn_op_index(op); - GGML_ASSERT(qnn_op_index < std::size(k_op_caps)); - const char * qnn_op_name = k_op_caps[qnn_op_index].qnn_op_name; - std::string ggml_op_name_string = std::string("ggml_") + ggml_op_name(op->op); - const char * ggml_op_name = ggml_op_name_string.c_str(); - - qnn_perf op_perf = qnn_perf(ggml_op_name); - op_perf.start(); - - std::string graph_name; - get_graph_key_from_op(op, graph_name); - if (instance->_qnn_graph_map.find(graph_name) != instance->_qnn_graph_map.end()) { - graph_initialized = true; - qnn_res_t & graph_item = instance->_qnn_graph_map[graph_name]; - graph_handle = std::get<0>(graph_item); - qnn_tensors_t & tensor = std::get<1>(graph_item); - p_tensor0 = tensor[0]; - p_tensor1 = tensor[1]; - p_tensor2 = tensor[2]; - } else { - p_tensor0 = ggml_qnn_create_compute_tensor(src0); - p_tensor1 = ggml_qnn_create_compute_tensor(src1); - p_tensor2 = ggml_qnn_create_compute_tensor(dst); - } -#if GGMLQNN_PRINT_OP_ADD_LOG - print_tensors_info(__func__, ctx, src0, src1, dst); -#endif - - //ensure QNN tensor has correct tensor type - QNN_VER_PTR(*p_tensor0)->type = QNN_TENSOR_TYPE_APP_WRITE; - QNN_VER_PTR(*p_tensor1)->type = QNN_TENSOR_TYPE_APP_WRITE; - QNN_VER_PTR(*p_tensor2)->type = QNN_TENSOR_TYPE_APP_READ; - - //save the original dimensions of qnn tensors - uint32_t * tensor_0_dimensions = QNN_VER_PTR(*p_tensor0)->dimensions; - uint32_t * tensor_1_dimensions = QNN_VER_PTR(*p_tensor1)->dimensions; - uint32_t * tensor_2_dimensions = QNN_VER_PTR(*p_tensor2)->dimensions; - - bool enable_npu_rpc = instance->enable_qnn_rpc() && ctx->device == QNN_BACKEND_NPU; - - if (!graph_initialized) { - GGMLQNN_LOG_DEBUG("graph name %s", graph_name.c_str()); - error = instance->init_qnn_graph(graph_name, static_cast(ctx->device), 8); - if (QNN_SUCCESS != error) { - GGMLQNN_LOG_INFO("can't create qnn graph handle with graph name %s, error = %d\n", graph_name.c_str(), error); - return; - } - graph_handle = instance->get_qnn_graph_handle(); - - if (enable_npu_rpc) { - QNN_VER_PTR(*p_tensor0)->memType = QNN_TENSORMEMTYPE_MEMHANDLE; - QNN_VER_PTR(*p_tensor0)->clientBuf = {.data=nullptr, .dataSize=0}; - - QNN_VER_PTR(*p_tensor1)->memType = QNN_TENSORMEMTYPE_MEMHANDLE; - QNN_VER_PTR(*p_tensor1)->clientBuf = {.data=nullptr, .dataSize=0}; - - QNN_VER_PTR(*p_tensor2)->memType = QNN_TENSORMEMTYPE_MEMHANDLE; - QNN_VER_PTR(*p_tensor2)->clientBuf = {.data=nullptr, .dataSize=0}; - } - - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor0)); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor1)); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor2)); - - if (enable_npu_rpc) { - uint8_t * qnn_rpcbuffer_0 = create_rpc_buffer(instance, src0, p_tensor0, true); - uint8_t * qnn_rpcbuffer_1 = create_rpc_buffer(instance, src1, p_tensor1, true); - uint8_t * qnn_rpcbuffer_2 = create_rpc_buffer(instance, dst, p_tensor2, false); - if (nullptr == qnn_rpcbuffer_0 || nullptr == qnn_rpcbuffer_1 || nullptr == qnn_rpcbuffer_2) { - GGMLQNN_LOG_INFO("create rpc buffer failure\n"); - //TODO: potential memory leak although it shouldn't happen - return; - } - } else { - QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggml_get_tensor_data_size(src0)}; - QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggml_get_tensor_data_size(src1)}; - QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggml_get_tensor_data_size(dst)}; - } - - Qnn_Tensor_t tensor_inputs[] = { - *p_tensor0, - *p_tensor1 - }; - Qnn_Tensor_t tensor_outputs[] = { - *p_tensor2 - }; - Qnn_OpConfig_t op_config = { - QNN_OPCONFIG_VERSION_1, .v1 = { - ggml_op_name, - QNN_OP_PACKAGE_NAME_QTI_AISW, - qnn_op_name, - 0, - qnn_params, - 2, - tensor_inputs, - 1, - tensor_outputs - } - }; - CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, op_config)); - CHECK_QNN_API(error, qnn_raw_interface.graphFinalize(graph_handle, nullptr, nullptr)); - CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, - tensor_inputs, 2, - tensor_outputs, 1, - nullptr, nullptr)); - - if (enable_npu_rpc) { - uint8_t * qnn_rpcbuffer = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*p_tensor2)->memHandle)); - GGMLQNN_LOG_INFO("qnn_rpcbuffer = %p\n", qnn_rpcbuffer); - if (nullptr != qnn_rpcbuffer) { - memcpy(dst->data, qnn_rpcbuffer, ggml_nbytes(dst)); - } - } - - qnn_tensors_t ggml_op_add_tensors; - ggml_op_add_tensors.reserve(3); - ggml_op_add_tensors.push_back(p_tensor0); - ggml_op_add_tensors.push_back(p_tensor1); - ggml_op_add_tensors.push_back(p_tensor2); - - auto graph_item = std::make_tuple(graph_handle, ggml_op_add_tensors); - instance->_qnn_graph_map[graph_name] = graph_item; - } else { - Qnn_DataType_t src0_qnn_type = QNN_DATATYPE_FLOAT_32; - Qnn_DataType_t src1_qnn_type = QNN_DATATYPE_FLOAT_32; - Qnn_DataType_t dst_qnn_type = QNN_DATATYPE_FLOAT_32; - - src0_qnn_type = qnn_datatype_from_ggml_datatype(src0->type); - src1_qnn_type = qnn_datatype_from_ggml_datatype(src1->type); - dst_qnn_type = qnn_datatype_from_ggml_datatype(dst->type); - - uint32_t dimensions_input_0[] = {(uint32_t) src0->ne[0], (uint32_t) src0->ne[1], - (uint32_t) src0->ne[2], (uint32_t) src0->ne[3]}; - uint32_t dimensions_input_1[] = {(uint32_t) src1->ne[0], (uint32_t) src1->ne[1], - (uint32_t) src1->ne[2], (uint32_t) src1->ne[3]}; - uint32_t dimensions_output[] = {(uint32_t) dst->ne[0], (uint32_t) dst->ne[1], - (uint32_t) dst->ne[2], (uint32_t) dst->ne[3]}; - - QNN_VER_PTR(*p_tensor0)->dimensions = dimensions_input_0; - QNN_VER_PTR(*p_tensor0)->rank = ggml_get_tensor_rank(src0); - QNN_VER_PTR(*p_tensor0)->dataType = src0_qnn_type; - - QNN_VER_PTR(*p_tensor1)->dimensions = dimensions_input_1; - QNN_VER_PTR(*p_tensor1)->rank = ggml_get_tensor_rank(src1); - QNN_VER_PTR(*p_tensor1)->dataType = src1_qnn_type; - - QNN_VER_PTR(*p_tensor2)->dimensions = dimensions_output; - QNN_VER_PTR(*p_tensor2)->rank = ggml_get_tensor_rank(dst); - QNN_VER_PTR(*p_tensor2)->dataType = dst_qnn_type; - - if (enable_npu_rpc) { - //TODO: NPU RPC feature will failed with test-backend-ops - uint8_t * qnn_buffer_0 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*p_tensor0)->memHandle)); - GGMLQNN_LOG_INFO("qnn_rpcbuffer_0 = %p\n", qnn_buffer_0); - if (nullptr != qnn_buffer_0) { - memcpy(qnn_buffer_0, src0->data, ggml_nbytes(src0)); - } - - uint8_t * qnn_buffer_1 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*p_tensor1)->memHandle)); - GGMLQNN_LOG_INFO("qnn_rpcbuffer_1 = %p\n", qnn_buffer_1); - if (nullptr != qnn_buffer_1) { - memcpy(qnn_buffer_1, src1->data, ggml_nbytes(src1)); - } - } else { - QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggml_get_tensor_data_size(src0)}; - QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggml_get_tensor_data_size(src1)}; - QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggml_get_tensor_data_size(dst)}; - } - - Qnn_Tensor_t tensor_inputs[] = { - *p_tensor0, - *p_tensor1 - }; - Qnn_Tensor_t tensor_outputs[] = { - *p_tensor2 - }; - CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, - tensor_inputs, 2, - tensor_outputs, 1, - nullptr, nullptr)); - - if (enable_npu_rpc) { - //TODO:NPU RPC feature will failed with test-backend-ops - uint8_t * qnn_buffer_2 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*p_tensor2)->memHandle)); - if (nullptr != qnn_buffer_2) { - memcpy(dst->data, qnn_buffer_2, ggml_nbytes(dst)); - } - } - } - - // restore the original dimensions of qnn tensors to avoid memory leak in func free_qnn_tensor - QNN_VER_PTR(*p_tensor0)->dimensions = tensor_0_dimensions; - QNN_VER_PTR(*p_tensor1)->dimensions = tensor_1_dimensions; - QNN_VER_PTR(*p_tensor2)->dimensions = tensor_2_dimensions; - -#if GGMLQNN_PRINT_OP_ADD_LOG - op_perf.info(); -#endif + return false; } -/* - * @brief performs matrix multiplication with FP32 & quantized weights and floating-point inputs - * using the QNN backend. this function performs matrix multiplication of the input tensor - * `src1` and the weight tensor `src0`, handling transposing, and quantization as needed, - * and stores the result in the destination tensor `dst`. - * - * @param backend the context which got through (ggml_backend_qnn_context *)backend->context for the - * QNN backend operations. - * @param op the destination tensor where the result of the matrix multiplication will be stored. - * - * @note the logic of ggml_qnn_mul_mat is similar to ggml_qnn_general_node but much more complicated - * than ggml_qnn_general_node. so it's a standalone function. accordingly, this is another - * typical skeleton for offload other ggml ops to QNN backend. MUL_MAT take most of the compute - * time (about 95%).so to speed up llama inference, should focus on this func. there are three kinds - * of MUL_MAT to compute: - * mul_mat_f32: both src0 and src1 are F32, this will be naturally handled in QNN backend - * mul_mat_f16_f32: src0 is F16 and src1 is F32, f16 in src0 -> f32 in src0', then src0' * src1 - * mul_mat_q_f32: src0 is quantized (Q4_0, Q4_1, Q6_K...) - * and src1 is F32, src0 -> f32 in src0', then src0' * src1 -*/ -static void ggml_qnn_mul_mat(ggml_backend_t backend, ggml_tensor * op) { - Qnn_ErrorHandle_t error = QNN_SUCCESS; - bool graph_initialized = false; - qnn_perf op_perf = qnn_perf("ggml_qnn_mul_mat"); - qnn_instance * instance = nullptr; - ggml_backend_qnn_context * ctx = (ggml_backend_qnn_context *)backend->context; - Qnn_GraphHandle_t graph_handle = nullptr; - Qnn_Tensor_t * p_tensor0 = nullptr; - Qnn_Tensor_t * p_tensor1 = nullptr; - Qnn_Tensor_t * p_tensor2 = nullptr; - Qnn_Tensor_t * p_param_tensor = nullptr; - Qnn_Tensor_t * p_tensor2_transpose = nullptr; - const ggml_tensor * src0 = op->src[0]; - const ggml_tensor * src1 = op->src[1]; - ggml_tensor * dst = op; - - GGMLQNN_CHECK_PARAMS(ctx, src0, src1, dst); - instance = ctx->instance; - QNN_INTERFACE_VER_TYPE qnn_raw_interface = ctx->raw_interface; - op_perf.start(); - - const enum ggml_type type = src0->type; - const uint32_t src0_rank = ggml_get_tensor_rank(src0); - const uint32_t src1_rank = ggml_get_tensor_rank(src1); - - GGML_TENSOR_BINARY_OP_LOCALS - GGML_ASSERT(ne0 == ne01); - GGML_ASSERT(ne1 == ne11); - GGML_ASSERT(ne2 == ne12); - GGML_ASSERT(ne3 == ne13); - GGML_ASSERT(nb00 == ggml_type_size(type)); - GGML_ASSERT(nb10 == ggml_type_size(src1->type)); - - GGML_ASSERT(src0_rank == src1_rank); - GGML_ASSERT(src0_rank >= 2); //QNN SDK's limitation, make QNN SDK happy - - // broadcast factors - const int64_t r2 = ne12 / ne02; - const int64_t r3 = ne13 / ne03; - const int64_t ne_plane = ne01 * ne00; - const size_t desired_size = ((GGML_TYPE_F32 == type) ? 0 : ne03 * ne02 * ne_plane * sizeof(float)); - if (ctx->work_size < desired_size) { - ctx->work_data.reset(new char[desired_size]); - ctx->work_size = desired_size; - } - void * wdata = ctx->work_data.get(); - // convert src0 to float - if (type != GGML_TYPE_F32) { - const auto * type_traits = ggml_get_type_traits(type); - ggml_to_float_t const to_float = type_traits->to_float; - - for (int64_t i03 = 0; i03 < ne03; i03++) { - for (int64_t i02 = 0; i02 < ne02; i02++) { - const void * x = (char *)src0->data + i02 * nb02 + i03 * nb03; - float * const wplane = (float *)wdata + i02 * ne_plane + i03 * ne02 * ne_plane; - - const int min_cols_per_thread = 4096; - const int min_rows_per_thread = std::max((int)(min_cols_per_thread / ne00), 1); - const int n_threads = std::max(std::min(ctx->n_threads, (int)(ne01 / min_rows_per_thread)), 1); - for (int i = 1; i < n_threads; i++) { - const int64_t start = i * ne01 / n_threads; - const int64_t end = (i + 1) * ne01 / n_threads; - if (start < end) { - ctx->tasks.push_back(std::async(std::launch::async, [=]() { - for (int64_t i01 = start; i01 < end; i01++) { - to_float((const char *)x + i01 * nb01, wplane + i01 * ne00, ne00); - } - })); - } - } - { - // reuse the current thread for the first task - const int64_t start = 0; - const int64_t end = ne01 / n_threads; - for (int64_t i01 = start; i01 < end; i01++) { - to_float((const char *)x + i01 * nb01, wplane + i01 * ne00, ne00); - } - } - } - } - - // wait for all tasks to finish - for (auto & task : ctx->tasks) { - task.get(); - } - ctx->tasks.clear(); - } - - std::string graph_name; - get_graph_key_from_op(op, graph_name); - if (instance->_qnn_graph_map.find(graph_name) != instance->_qnn_graph_map.end()) { - graph_initialized = true; - qnn_res_t & graph_item = instance->_qnn_graph_map[graph_name]; - graph_handle = std::get<0>(graph_item); - qnn_tensors_t & tensors = std::get<1>(graph_item); - p_tensor0 = tensors[0]; - p_tensor1 = tensors[1]; - p_tensor2 = tensors[2]; - p_param_tensor = tensors[3]; - p_tensor2_transpose = tensors[4]; - } else { - p_tensor0 = GQCGT(src0, nullptr, QNN_TENSOR_TYPE_APP_WRITE,QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0); - p_tensor1 = GQCGT(src1, nullptr, QNN_TENSOR_TYPE_APP_WRITE,QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0); - p_tensor2 = GQCGT(dst, nullptr, QNN_TENSOR_TYPE_APP_READ,QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0); - } - print_tensors_info(__func__, ctx, src0, src1, dst); - - //ensure QNN tensor has correct tensor type - QNN_VER_PTR(*p_tensor0)->type = QNN_TENSOR_TYPE_APP_WRITE; - QNN_VER_PTR(*p_tensor1)->type = QNN_TENSOR_TYPE_APP_WRITE; - QNN_VER_PTR(*p_tensor2)->type = QNN_TENSOR_TYPE_APP_READ; - - //save the original dimensions of qnn tensors - uint32_t * tensor_0_dimensions = QNN_VER_PTR(*p_tensor0)->dimensions; - uint32_t * tensor_1_dimensions = QNN_VER_PTR(*p_tensor1)->dimensions; - uint32_t * tensor_2_dimensions = QNN_VER_PTR(*p_tensor2)->dimensions; - - if (!graph_initialized) { - GGMLQNN_LOG_DEBUG("graph name %s", graph_name.c_str()); - /* - there are two key-points in properly handling how to offload mulmat to the QNN backend in ggml-qnn - 1. transpose - a 3x2 f32 matrix which means 3 rows and 2 columns. in ggml, it could be created from: - struct ggml_tensor* matrix = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 2, 3); - which like this: - +---+---+ - | 0 | 1 | - +---+---+ - | 2 | 3 | - +---+---+ - | 4 | 5 | - +---+---+ - with - ne[0] = 2 - ne[1] = 3 - there are different dimension order between ggml tensor and qnn tensor - - 2. QNN's MatMul can only support input tensors with rank >= 2 - - in the all, there is gap between ggml mulmat and QNN mulmat,we need to perform a transpose - operation when offloading mulmat to QNN backend. this concise implementation will handle - transpose in func ggml_qnn_create_general_tensor() - */ - //step-1: create qnn graph - error = qnn_raw_interface.graphCreate(instance->get_qnn_context_handle(), - graph_name.c_str(), nullptr, &graph_handle); - if (QNN_SUCCESS != error) { - GGMLQNN_LOG_INFO("can't create qnn graph handle with graph name %s, error = %d\n", graph_name.c_str(), error); - return; - } - //step-2: create param tensor for mulmat of 2d/3d/4d matrix - const uint32_t param_tensor_data[GGML_MAX_DIMS][GGML_MAX_DIMS] = { - {0}, - {1, 0}, - {0, 2, 1}, - {0, 1, 3, 2}, - }; - uint32_t param_tensor_dims[1] = {src0_rank}; - p_param_tensor = GQCGT(nullptr, "param", QNN_TENSOR_TYPE_STATIC, QNN_DATATYPE_UINT_32, 1, param_tensor_dims, (void *)(param_tensor_data[src0_rank - 1]), src0_rank * sizeof(uint32_t)); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_param_tensor)); - - //step-3: create compute tensor from ggml tensor - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor0)); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor1)); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor2)); - if (type != GGML_TYPE_F32) { - QNN_VER_PTR(*p_tensor0)->clientBuf = {wdata, static_cast(desired_size)}; - } else { - QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggml_get_tensor_data_size(src0)}; - } - QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggml_get_tensor_data_size(src1)}; - QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggml_get_tensor_data_size(dst)}; - - //step-4: create a transpose tensor - p_tensor2_transpose = GQCGT(dst, "transpose", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0, true); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor2_transpose)); - - //step-5: compose qnn graph: add mat_mul node - Qnn_Param_t out_0_params[] = { - {QNN_PARAMTYPE_SCALAR, - QNN_OP_MAT_MUL_PARAM_TRANSPOSE_IN1, - .scalarParam = {QNN_DATATYPE_BOOL_8, .bool8Value = 1} - } - }; - - Qnn_Tensor_t out_0_inputs[] = {*p_tensor0, *p_tensor1}; - Qnn_Tensor_t out_0_outputs[] = {*p_tensor2_transpose}; -#if 0 //leave here for easily understand code, can be removed in the future - Qnn_OpConfig_t out_0 = { - QNN_OPCONFIG_VERSION_1, .v1 = - {"ggmlqnn_mulmat_opconfig", QNN_OP_PACKAGE_NAME_QTI_AISW, QNN_OP_MAT_MUL, - 1, - out_0_params, - 2, - out_0_inputs, - 1, - out_0_outputs} - }; -#else - Qnn_OpConfig_t out_0 = create_op_config("ggmlqnn_mulmat_opconfig", QNN_OP_PACKAGE_NAME_QTI_AISW, QNN_OP_MAT_MUL, - out_0_params, 1, out_0_inputs, 2, out_0_outputs, 1); -#endif - CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle,out_0)); - - //step-5: compose qnn graph: add transpose node - Qnn_Param_t out_trans1_0_params[] = { - {(Qnn_ParamType_t) 1, - "perm", .tensorParam = *p_param_tensor - } - }; - Qnn_Tensor_t out_trans1_0_inputs[] = {*p_tensor2_transpose}; - Qnn_Tensor_t out_trans1_0_outputs[] = {*p_tensor2}; -#if 0 //leave here for easily understand code, can be removed in the future - Qnn_OpConfig_t out_trans1_0 = { - QNN_OPCONFIG_VERSION_1, - .v1 = {"ggmlqnn_mulmat_transpose_opconfig", - QNN_OP_PACKAGE_NAME_QTI_AISW, - QNN_OP_TRANSPOSE, 1, - out_trans1_0_params, - 1, - out_trans1_0_inputs, - 1, - out_trans1_0_outputs} - }; -#else - Qnn_OpConfig_t out_trans1_0 = create_op_config("ggmlqnn_mulmat_transpose_opconfig", QNN_OP_PACKAGE_NAME_QTI_AISW, QNN_OP_TRANSPOSE, - out_trans1_0_params, 1, out_trans1_0_inputs, 1, out_trans1_0_outputs, 1); -#endif - CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle,out_trans1_0)); - - //step-6: finalize qnn graph and execute qnn graph - CHECK_QNN_API(error, qnn_raw_interface.graphFinalize(graph_handle, nullptr, nullptr)); - Qnn_Tensor_t input_tensors_0[] = {*p_tensor0, *p_tensor1}; - Qnn_Tensor_t output_tensors_0[] = {*p_tensor2}; - CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, - input_tensors_0, 2, - output_tensors_0, 1, - nullptr, nullptr)); - - qnn_tensors_t ggml_op_mulmat_tensors; - ggml_op_mulmat_tensors.reserve(5); - ggml_op_mulmat_tensors.push_back(p_tensor0); - ggml_op_mulmat_tensors.push_back(p_tensor1); - ggml_op_mulmat_tensors.push_back(p_tensor2); - ggml_op_mulmat_tensors.push_back(p_param_tensor); - ggml_op_mulmat_tensors.push_back(p_tensor2_transpose); - auto graph_item = std::make_tuple(graph_handle, ggml_op_mulmat_tensors); - instance->_qnn_graph_map[graph_name] = graph_item; - } else { - if (type != GGML_TYPE_F32) { - QNN_VER_PTR(*p_tensor0)->clientBuf = {wdata, static_cast(desired_size)}; - } else { - QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggml_get_tensor_data_size(src0)}; - } - QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggml_get_tensor_data_size(src1)}; - QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggml_get_tensor_data_size(dst)}; - - Qnn_Tensor_t tensor_inputs[] = { - *p_tensor0, - *p_tensor1 - }; - Qnn_Tensor_t tensor_outputs[] = { - *p_tensor2 - }; - // this is the second technical approach or another pipeline of "how to utilize the Hexagon - // NPU maximally" through QNN SDK, details could be found at - // https://github.com/ggml-org/llama.cpp/pull/12049#issuecomment-2678308360 - CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, - tensor_inputs, 2, - tensor_outputs, 1, - nullptr, nullptr)); - } - - // restore the original dimensions of qnn tensors to avoid memory leak in func free_qnn_tensor - QNN_VER_PTR(*p_tensor0)->dimensions = tensor_0_dimensions; - QNN_VER_PTR(*p_tensor1)->dimensions = tensor_1_dimensions; - QNN_VER_PTR(*p_tensor2)->dimensions = tensor_2_dimensions; - op_perf.info(); -} - -static bool ggml_qnn_compute_forward(ggml_backend_t backend, struct ggml_tensor * tensor) { +static bool ggml_qnn_compute_forward(ggml_backend_t backend, struct ggml_tensor * dst) { ggmlqnn_op_func_t func = nullptr; + ggml_backend_qnn_context * ctx = (ggml_backend_qnn_context *)backend->context; - switch (tensor->op) { + switch (dst->op) { + case GGML_OP_REPEAT: + ggml_qnn_repeat(ctx, dst); + break; + case GGML_OP_GET_ROWS: + ggml_qnn_get_rows(ctx, dst); + break; + case GGML_OP_DUP: + ggml_qnn_dup(ctx, dst); + break; case GGML_OP_ADD: func = ggml_qnn_general_node; break; - - case GGML_OP_MUL_MAT: - func = ggml_qnn_mul_mat; + case GGML_OP_ACC: + ggml_qnn_acc(ctx, dst); break; - case GGML_OP_MUL: func = ggml_qnn_general_node; break; - + case GGML_OP_DIV: + ggml_qnn_div(ctx, dst); + break; + case GGML_OP_UNARY: + switch (ggml_get_unary_op(dst)) { + case GGML_UNARY_OP_GELU: + break; + case GGML_UNARY_OP_SILU: + break; + case GGML_UNARY_OP_GELU_QUICK: + break; + case GGML_UNARY_OP_TANH: + break; + case GGML_UNARY_OP_RELU: + break; + case GGML_UNARY_OP_HARDSIGMOID: + break; + case GGML_UNARY_OP_HARDSWISH: + break; + default: + return false; + } + break; + case GGML_OP_NORM: + ggml_qnn_norm(ctx, dst); + break; + case GGML_OP_GROUP_NORM: + ggml_qnn_group_norm(ctx, dst); + break; + case GGML_OP_CONCAT: + ggml_qnn_concat(ctx, dst); + break; + case GGML_OP_UPSCALE: + ggml_qnn_upsample_nearest2d(ctx, dst); + break; + case GGML_OP_PAD: + ggml_qnn_pad(ctx, dst); + break; + case GGML_OP_ARANGE: + ggml_qnn_arange(ctx, dst); + break; + case GGML_OP_TIMESTEP_EMBEDDING: + ggml_qnn_timestep_embedding(ctx, dst); + break; + case GGML_OP_LEAKY_RELU: + ggml_qnn_leaky_relu(ctx, dst); + break; + case GGML_OP_RMS_NORM: + ggml_qnn_rms_norm(ctx, dst); + break; + case GGML_OP_MUL_MAT: + ggml_qnn_mul_mat(ctx, dst); + break; + case GGML_OP_MUL_MAT_ID: + return false; + case GGML_OP_SCALE: + ggml_qnn_scale(ctx, dst); + break; + case GGML_OP_SQR: + ggml_qnn_sqr(ctx, dst); + break; + case GGML_OP_CLAMP: + ggml_qnn_clamp(ctx, dst); + break; + case GGML_OP_CPY: + ggml_qnn_cpy(ctx, dst); + break; + case GGML_OP_CONT: + ggml_qnn_dup(ctx, dst); + break; + case GGML_OP_NONE: + case GGML_OP_RESHAPE: + case GGML_OP_VIEW: + case GGML_OP_PERMUTE: + case GGML_OP_TRANSPOSE: + break; + case GGML_OP_DIAG_MASK_INF: + ggml_qnn_diag_mask(ctx, dst, -INFINITY); + break; + case GGML_OP_SOFT_MAX: + ggml_qnn_softmax(ctx, dst); + break; + case GGML_OP_ROPE: + ggml_qnn_rope(ctx, dst); + break; + case GGML_OP_IM2COL: + ggml_qnn_im2col(ctx, dst); + break; + case GGML_OP_POOL_2D: + ggml_qnn_pool2d(ctx, dst); + break; + case GGML_OP_SUM_ROWS: + ggml_qnn_sum_rows(ctx, dst); + break; + case GGML_OP_ARGSORT: + ggml_qnn_argsort(ctx, dst); + break; default: return false; } if (nullptr != func) - func(backend, tensor); + func(ctx, dst); return true; } @@ -3695,10 +2728,9 @@ static ggml_backend_buffer_t ggml_backend_qnn_device_buffer_from_host_ptr(ggml_b GGML_UNUSED(max_tensor_size); } - static bool ggml_backend_qnn_device_supports_op(ggml_backend_dev_t dev, const struct ggml_tensor * op) { ggml_backend_qnn_context * ctx = (ggml_backend_qnn_context *) dev->context; - return (ggml_qnn_can_handle_op(op)); + return (ggml_qnn_can_handle_op(ctx,op)); } static bool ggml_backend_qnn_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) { diff --git a/scripts/build-run-android.sh b/scripts/build-run-android.sh deleted file mode 100755 index 3d239510b8d63..0000000000000 --- a/scripts/build-run-android.sh +++ /dev/null @@ -1,282 +0,0 @@ -#!/bin/bash - -set -e - -PWD=`pwd` -ANDROID_PLATFORM=android-34 -ANDROID_NDK=${PWD}/android-ndk-r26c -REMOTE_PATH=/data/local/tmp/ -GGUF_MODEL_NAME=/sdcard/deepseek-r1-distill-qwen-1.5b-q4_0.gguf -GGUF_MODEL_NAME=/sdcard/qwen1_5-1_8b-chat-q4_0.gguf - -#QNN SDK could be found at: -#https://www.qualcomm.com/developer/software/qualcomm-ai-engine-direct-sdk -#https://developer.qualcomm.com/software/hexagon-dsp-sdk/tools -QNN_SDK_URL=https://www.qualcomm.com/developer/software/qualcomm-ai-engine-direct-sdk -QNN_SDK_PATH=/opt/qcom/aistack/qairt/2.31.0.250130/ - -#default is QNN NPU -qnnbackend=2 - -function dump_vars() -{ - echo -e "ANDROID_NDK: ${ANDROID_NDK}" - echo -e "QNN_SDK_PATH: ${QNN_SDK_PATH}" -} - - -function show_pwd() -{ - echo -e "current working path:$(pwd)\n" -} - - -function check_qnn_sdk() -{ - if [ ! -d ${QNN_SDK_PATH} ]; then - echo -e "QNN_SDK_PATH ${QNN_SDK_PATH} not exist, pls check or download it from ${QNN_SDK_URL}...\n" - exit 1 - fi -} - - -function check_and_download_ndk() -{ - is_android_ndk_exist=1 - - if [ ! -d ${ANDROID_NDK} ]; then - is_android_ndk_exist=0 - fi - - if [ ! -f ${ANDROID_NDK}/build/cmake/android.toolchain.cmake ]; then - is_android_ndk_exist=0 - fi - - if [ ${is_android_ndk_exist} -eq 0 ]; then - - if [ ! -f android-ndk-r26c-linux.zip ]; then - wget --no-config --quiet --show-progress -O android-ndk-r26c-linux.zip https://dl.google.com/android/repository/android-ndk-r26c-linux.zip - fi - - unzip android-ndk-r26c-linux.zip - - if [ $? -ne 0 ]; then - printf "failed to download android ndk to %s \n" "${ANDROID_NDK}" - exit 1 - fi - - printf "android ndk saved to ${ANDROID_NDK} \n\n" - else - printf "android ndk already exist:${ANDROID_NDK} \n\n" - fi -} - - -function build_arm64 -{ - cmake -H. -B./out/android -DCMAKE_BUILD_TYPE=Release -DGGML_USE_QNN=ON -DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK}/build/cmake/android.toolchain.cmake -DANDROID_ABI=arm64-v8a -DANDROID_PLATFORM=latest -DCMAKE_C_FLAGS=-march=armv8.7-a -DGGML_QNN=ON -DGGML_QNN_SDK_PATH=${QNN_SDK_PATH} - cd out/android - make -j16 - show_pwd - - cd - -} - - -function remove_temp_dir() -{ - if [ -d out ]; then - echo "remove out directory in `pwd`" - rm -rf out - fi -} - - -function check_qnn_libs() -{ - #reuse the cached qnn libs on Android phone - adb shell ls ${REMOTE_PATH}/libQnnCpu.so - if [ $? -eq 0 ]; then - printf "QNN libs already exist on Android phone\n" - else - update_qnn_libs - fi -} - - -function update_qnn_libs() -{ - adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnSystem.so ${REMOTE_PATH}/ - adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnCpu.so ${REMOTE_PATH}/ - adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnGpu.so ${REMOTE_PATH}/ - - adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnHtp.so ${REMOTE_PATH}/ - adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnHtpNetRunExtensions.so ${REMOTE_PATH}/ - adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnHtpPrepare.so ${REMOTE_PATH}/ - adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnHtpV75Stub.so ${REMOTE_PATH}/ - adb push ${QNN_SDK_PATH}/lib/hexagon-v75/unsigned/libQnnHtpV75Skel.so ${REMOTE_PATH}/ -} - - -function build_ggml_qnn() -{ - show_pwd - check_and_download_ndk - check_qnn_sdk - dump_vars - remove_temp_dir - build_arm64 -} - - -function prepare_run_on_phone() -{ - if [ $# != 1 ]; then - print "invalid param" - return - fi - program=$1 - - check_qnn_libs - - if [ -f ./out/android/bin/libggml-qnn.so ]; then - adb push ./out/android/bin/*.so ${REMOTE_PATH}/ - fi - adb push ./out/android/bin/${program} ${REMOTE_PATH}/ - adb shell chmod +x ${REMOTE_PATH}/${program} -} - -function run_llamacli() -{ - prepare_run_on_phone llama-cli - - adb shell "cd ${REMOTE_PATH} \ - && export LD_LIBRARY_PATH=${REMOTE_PATH} \ - && ${REMOTE_PATH}/llama-cli -mg ${qnnbackend} -no-cnv -m ${GGUF_MODEL_NAME} -p \"introduce the movie Once Upon a Time in America briefly.\n\"" - -} - - -function run_llamabench() -{ - prepare_run_on_phone llama-bench - - adb shell "cd ${REMOTE_PATH} \ - && export LD_LIBRARY_PATH=${REMOTE_PATH} \ - && ${REMOTE_PATH}/llama-bench -mg ${qnnbackend} -m ${GGUF_MODEL_NAME}" - -} - - -function run_test-backend-ops() -{ - prepare_run_on_phone test-backend-ops - - adb shell "cd ${REMOTE_PATH} \ - && export LD_LIBRARY_PATH=${REMOTE_PATH} \ - && ${REMOTE_PATH}/test-backend-ops test" - -} - -function run_ut_add() -{ - prepare_run_on_phone ggml-qnn-ut - - adb shell "cd ${REMOTE_PATH} \ - && export LD_LIBRARY_PATH=${REMOTE_PATH} \ - && ${REMOTE_PATH}/ggml-qnn-ut -t GGML_OP_ADD -b $qnnbackend" - -} - -function run_ut_mulmat() -{ - prepare_run_on_phone ggml-qnn-ut - - adb shell "cd ${REMOTE_PATH} \ - && export LD_LIBRARY_PATH=${REMOTE_PATH} \ - && ${REMOTE_PATH}/ggml-qnn-ut -t GGML_OP_MUL_MAT -b $qnnbackend" - -} - -function run_ut_mul() -{ - prepare_run_on_phone ggml-qnn-ut - - adb shell "cd ${REMOTE_PATH} \ - && export LD_LIBRARY_PATH=${REMOTE_PATH} \ - && ${REMOTE_PATH}/ggml-qnn-ut -t GGML_OP_MUL -b $qnnbackend" - -} - - -function show_usage() -{ - echo "Usage:" - echo " $0 build" - echo " $0 updateqnnlib" - echo " $0 run_testop" - echo " $0 run_ut_add 0 (QNN_CPU) / 1 (QNN_GPU) / 2 (QNN_NPU) / 3 (ggml)" - echo " $0 run_ut_mulmat 0 (QNN_CPU) / 1 (QNN_GPU) / 2 (QNN_NPU) / 3 (ggml)" - echo " $0 run_ut_mul 0 (QNN_CPU) / 1 (QNN_GPU) / 2 (QNN_NPU) / 3 (ggml)" - echo " $0 run_llamacli 0 (QNN_CPU) / 1 (QNN_GPU) / 2 (QNN_NPU) / 3 (ggml)" - echo " $0 run_llamabench 0 (QNN_CPU) / 1 (QNN_GPU) / 2 (QNN_NPU) / 3 (ggml)" - echo -e "\n\n\n" -} - - -show_pwd - -check_qnn_sdk - -if [ $# == 0 ]; then - show_usage - exit 1 -elif [ $# == 1 ]; then - if [ "$1" == "-h" ]; then - show_usage - exit 1 - elif [ "$1" == "help" ]; then - show_usage - exit 1 - elif [ "$1" == "build" ]; then - build_ggml_qnn - exit 0 - - elif [ "$1" == "run_testop" ]; then - run_test-backend-ops - exit 0 - - elif [ "$1" == "updateqnnlib" ]; then - update_qnn_libs - exit 0 - else - show_usage - exit 1 - fi -elif [ $# == 2 ]; then - qnnbackend=$2 - if [ ${qnnbackend} -gt 3 ]; then - show_usage - exit 1 - fi - - if [ "$1" == "run_llamacli" ]; then - run_llamacli - exit 0 - elif [ "$1" == "run_llamabench" ]; then - run_llamabench - exit 0 - elif [ "$1" == "run_ut_add" ]; then - run_ut_add - exit 0 - elif [ "$1" == "run_ut_mulmat" ]; then - run_ut_mulmat - exit 0 - elif [ "$1" == "run_ut_mul" ]; then - run_ut_mul - exit 0 - fi -else - show_usage - exit 1 -fi From 64b7698897a65514a7ab252703b0933635d80025 Mon Sep 17 00:00:00 2001 From: zhouwg Date: Thu, 27 Feb 2025 16:51:25 +0800 Subject: [PATCH 28/76] ggml-qnn: enable release build with necessary logs to make reviewers happy --- ggml/src/ggml-qnn/ggml-qnn-impl.h | 4 + ggml/src/ggml-qnn/ggml-qnn-ops.cpp | 4 + ggml/src/ggml-qnn/ggml-qnn.cpp | 9 +- scripts/build-run-android.sh | 240 +++++++++++++++++++++++++++++ 4 files changed, 253 insertions(+), 4 deletions(-) create mode 100755 scripts/build-run-android.sh diff --git a/ggml/src/ggml-qnn/ggml-qnn-impl.h b/ggml/src/ggml-qnn/ggml-qnn-impl.h index 974755955f9d2..a4e00e0b7bbd7 100644 --- a/ggml/src/ggml-qnn/ggml-qnn-impl.h +++ b/ggml/src/ggml-qnn/ggml-qnn-impl.h @@ -94,11 +94,15 @@ class qnn_instance; struct ggml_backend_qnn_context; void ggmlqnn_log_internal(ggml_log_level level, const char * file, const char * func, int line, const char * format, ...); +<<<<<<< HEAD <<<<<<< HEAD #if 0//def NDEBUG ======= #ifdef NDEBUG >>>>>>> ggml-qnn: refine source code structure to make code more clearly +======= +#if 0//def NDEBUG +>>>>>>> ggml-qnn: enable release build with necessary logs to make reviewers happy #define GGMLQNN_DEBUG 0 #define ENABLE_QNNBACKEND_PERF 0 // enable/disable op's perf info #define GGMLQNN_PRINT_QNN_INTERNAL_LOG 0 // enable/disable QNN's internal log diff --git a/ggml/src/ggml-qnn/ggml-qnn-ops.cpp b/ggml/src/ggml-qnn/ggml-qnn-ops.cpp index 6614a1b90f6fd..b6a8f020bbeb7 100644 --- a/ggml/src/ggml-qnn/ggml-qnn-ops.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn-ops.cpp @@ -111,6 +111,7 @@ void ggml_qnn_general_node(ggml_backend_qnn_context * ctx, ggml_tensor * op) { p_tensor1 = ggmlqnn_create_compute_tensor(src1); p_tensor2 = ggmlqnn_create_compute_tensor(dst); } +<<<<<<< HEAD <<<<<<< HEAD //ggmlqnn_print_tensors_info(__func__, ctx, src0, src1, dst); ======= @@ -118,6 +119,9 @@ void ggml_qnn_general_node(ggml_backend_qnn_context * ctx, ggml_tensor * op) { print_tensors_info(__func__, ctx, src0, src1, dst); #endif >>>>>>> ggml-qnn: refine source code structure to make code more clearly +======= + ggmlqnn_print_tensors_info(__func__, ctx, src0, src1, dst); +>>>>>>> ggml-qnn: enable release build with necessary logs to make reviewers happy //ensure QNN tensor has correct tensor type QNN_VER_PTR(*p_tensor0)->type = QNN_TENSOR_TYPE_APP_WRITE; diff --git a/ggml/src/ggml-qnn/ggml-qnn.cpp b/ggml/src/ggml-qnn/ggml-qnn.cpp index ff1a8a0f39506..5276001a8523b 100644 --- a/ggml/src/ggml-qnn/ggml-qnn.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn.cpp @@ -2248,7 +2248,7 @@ static bool ggml_qnn_can_handle_op(const ggml_backend_qnn_context * ctx, const s } if (tensor->op == GGML_OP_MUL_MAT) { - dump_op_info(tensor); + //dump_op_info(tensor); if (src0_rank != src1_rank) // make QNN SDK happy return false; if (src0_rank < 2) // QNN's limitation, make QNN SDK happy @@ -2260,15 +2260,16 @@ static bool ggml_qnn_can_handle_op(const ggml_backend_qnn_context * ctx, const s if (ctx->device == QNN_BACKEND_NPU) if (2 == src0_rank) - return (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 + return (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_Q4_0 || src0->type == GGML_TYPE_Q8_0 || src0->type == GGML_TYPE_Q6_K || src0->type == GGML_TYPE_Q8_K ) && (src1->type == GGML_TYPE_F32) && (tensor->type == GGML_TYPE_F32); else return (src0->type == GGML_TYPE_F32) && (src1->type == GGML_TYPE_F32) && (tensor->type == GGML_TYPE_F32); else - return (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || ggml_is_quantized(src0->type)) - && (src1->type == GGML_TYPE_F32) && (tensor->type == GGML_TYPE_F32); + return (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_Q4_0 + || src0->type == GGML_TYPE_Q8_0 || src0->type == GGML_TYPE_Q6_K || src0->type == GGML_TYPE_Q8_K) + && (src1->type == GGML_TYPE_F32) && (tensor->type == GGML_TYPE_F32); } if (tensor->op == GGML_OP_MUL) { diff --git a/scripts/build-run-android.sh b/scripts/build-run-android.sh new file mode 100755 index 0000000000000..1a5f362fe2083 --- /dev/null +++ b/scripts/build-run-android.sh @@ -0,0 +1,240 @@ +#!/bin/bash + +set -e + +PWD=`pwd` +ANDROID_PLATFORM=android-34 +ANDROID_NDK=${PWD}/android-ndk-r26c +REMOTE_PATH=/data/local/tmp/ +GGUF_MODEL_NAME=/sdcard/deepseek-r1-distill-qwen-1.5b-q4_0.gguf +GGUF_MODEL_NAME=/sdcard/qwen1_5-1_8b-chat-q4_0.gguf + +#QNN SDK could be found at: +#https://www.qualcomm.com/developer/software/qualcomm-ai-engine-direct-sdk +#https://developer.qualcomm.com/software/hexagon-dsp-sdk/tools +QNN_SDK_URL=https://www.qualcomm.com/developer/software/qualcomm-ai-engine-direct-sdk +QNN_SDK_PATH=/opt/qcom/aistack/qairt/2.31.0.250130/ + +#default is QNN NPU +qnnbackend=2 + +function dump_vars() +{ + echo -e "ANDROID_NDK: ${ANDROID_NDK}" + echo -e "QNN_SDK_PATH: ${QNN_SDK_PATH}" +} + + +function show_pwd() +{ + echo -e "current working path:$(pwd)\n" +} + + +function check_qnn_sdk() +{ + if [ ! -d ${QNN_SDK_PATH} ]; then + echo -e "QNN_SDK_PATH ${QNN_SDK_PATH} not exist, pls check or download it from ${QNN_SDK_URL}...\n" + exit 1 + fi +} + + +function check_and_download_ndk() +{ + is_android_ndk_exist=1 + + if [ ! -d ${ANDROID_NDK} ]; then + is_android_ndk_exist=0 + fi + + if [ ! -f ${ANDROID_NDK}/build/cmake/android.toolchain.cmake ]; then + is_android_ndk_exist=0 + fi + + if [ ${is_android_ndk_exist} -eq 0 ]; then + + if [ ! -f android-ndk-r26c-linux.zip ]; then + wget --no-config --quiet --show-progress -O android-ndk-r26c-linux.zip https://dl.google.com/android/repository/android-ndk-r26c-linux.zip + fi + + unzip android-ndk-r26c-linux.zip + + if [ $? -ne 0 ]; then + printf "failed to download android ndk to %s \n" "${ANDROID_NDK}" + exit 1 + fi + + printf "android ndk saved to ${ANDROID_NDK} \n\n" + else + printf "android ndk already exist:${ANDROID_NDK} \n\n" + fi +} + + +function build_arm64 +{ + cmake -H. -B./out/android -DCMAKE_BUILD_TYPE=Release -DGGML_USE_QNN=ON -DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK}/build/cmake/android.toolchain.cmake -DANDROID_ABI=arm64-v8a -DANDROID_PLATFORM=latest -DCMAKE_C_FLAGS=-march=armv8.7-a -DGGML_QNN=ON -DGGML_QNN_SDK_PATH=${QNN_SDK_PATH} + cd out/android + make -j16 + show_pwd + + cd - +} + + +function remove_temp_dir() +{ + if [ -d out ]; then + echo "remove out directory in `pwd`" + rm -rf out + fi +} + + +function check_qnn_libs() +{ + #reuse the cached qnn libs on Android phone + adb shell ls ${REMOTE_PATH}/libQnnCpu.so + if [ $? -eq 0 ]; then + printf "QNN libs already exist on Android phone\n" + else + update_qnn_libs + fi +} + + +function update_qnn_libs() +{ + adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnSystem.so ${REMOTE_PATH}/ + adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnCpu.so ${REMOTE_PATH}/ + adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnGpu.so ${REMOTE_PATH}/ + + adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnHtp.so ${REMOTE_PATH}/ + adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnHtpNetRunExtensions.so ${REMOTE_PATH}/ + adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnHtpPrepare.so ${REMOTE_PATH}/ + adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnHtpV75Stub.so ${REMOTE_PATH}/ + adb push ${QNN_SDK_PATH}/lib/hexagon-v75/unsigned/libQnnHtpV75Skel.so ${REMOTE_PATH}/ +} + + +function build_ggml_qnn() +{ + show_pwd + check_and_download_ndk + check_qnn_sdk + dump_vars + remove_temp_dir + build_arm64 +} + + +function run_llamacli() +{ + check_qnn_libs + + if [ -f ./out/android/bin/libggml-qnn.so ]; then + adb push ./out/android/bin/*.so ${REMOTE_PATH}/ + fi + adb push ./out/android/bin/llama-cli ${REMOTE_PATH}/ + adb shell chmod +x ${REMOTE_PATH}/llama-cli + + adb shell "cd ${REMOTE_PATH} \ + && export LD_LIBRARY_PATH=${REMOTE_PATH} \ + && ${REMOTE_PATH}/llama-cli -mg ${qnnbackend} -no-cnv -m ${GGUF_MODEL_NAME} -p \"introduce the movie Once Upon a Time in America briefly.\n\"" + +} + + +function run_llamabench() +{ + check_qnn_libs + + if [ -f ./out/android/bin/libggml-qnn.so ]; then + adb push ./out/android/bin/*.so ${REMOTE_PATH}/ + fi + adb push ./out/android/bin/llama-bench ${REMOTE_PATH}/ + adb shell chmod +x ${REMOTE_PATH}/llama-bench + + adb shell "cd ${REMOTE_PATH} \ + && export LD_LIBRARY_PATH=${REMOTE_PATH} \ + && ${REMOTE_PATH}/llama-bench -mg ${qnnbackend} -m ${GGUF_MODEL_NAME}" + +} + + +function run_test-backend-ops() +{ + check_qnn_libs + + if [ -f ./out/android/bin/libggml-qnn.so ]; then + adb push ./out/android/bin/*.so ${REMOTE_PATH}/ + fi + adb push ./out/android/bin/test-backend-ops ${REMOTE_PATH}/ + adb shell chmod +x ${REMOTE_PATH}/test-backend-ops + + adb shell "cd ${REMOTE_PATH} \ + && export LD_LIBRARY_PATH=${REMOTE_PATH} \ + && ${REMOTE_PATH}/test-backend-ops test" + +} + + +function show_usage() +{ + echo "Usage:" + echo " $0 build" + echo " $0 updateqnnlib" + echo " $0 run_testop" + echo " $0 run_llamacli 0 (QNN_CPU) / 1 (QNN_GPU) / 2 (QNN_NPU) / 3 (ggml)" + echo " $0 run_llamabench 0 (QNN_CPU) / 1 (QNN_GPU) / 2 (QNN_NPU) / 3 (ggml)" + echo -e "\n\n\n" +} + + +show_pwd + +check_qnn_sdk + +if [ $# == 0 ]; then + show_usage + exit 1 +elif [ $# == 1 ]; then + if [ "$1" == "-h" ]; then + show_usage + exit 1 + elif [ "$1" == "help" ]; then + show_usage + exit 1 + elif [ "$1" == "build" ]; then + build_ggml_qnn + exit 0 + + elif [ "$1" == "run_testop" ]; then + run_test-backend-ops + exit 0 + elif [ "$1" == "updateqnnlib" ]; then + update_qnn_libs + exit 0 + else + show_usage + exit 1 + fi +elif [ $# == 2 ]; then + qnnbackend=$2 + if [ ${qnnbackend} -gt 3 ]; then + show_usage + exit 1 + fi + + if [ "$1" == "run_llamacli" ]; then + run_llamacli + exit 0 + elif [ "$1" == "run_llamabench" ]; then + run_llamabench + exit 0 + fi +else + show_usage + exit 1 +fi From 8e27c12aeb87b116decb56050c9e7c82adbeb010 Mon Sep 17 00:00:00 2001 From: zhouwg Date: Thu, 27 Feb 2025 17:14:47 +0800 Subject: [PATCH 29/76] ggml-qnn: enable all quantize type with 2d mulmat --- ggml/src/ggml-qnn/ggml-qnn-ops.cpp | 986 ----------------------------- ggml/src/ggml-qnn/ggml-qnn.cpp | 3 +- 2 files changed, 1 insertion(+), 988 deletions(-) delete mode 100644 ggml/src/ggml-qnn/ggml-qnn-ops.cpp diff --git a/ggml/src/ggml-qnn/ggml-qnn-ops.cpp b/ggml/src/ggml-qnn/ggml-qnn-ops.cpp deleted file mode 100644 index b6a8f020bbeb7..0000000000000 --- a/ggml/src/ggml-qnn/ggml-qnn-ops.cpp +++ /dev/null @@ -1,986 +0,0 @@ -/* - * Copyright (c) 2023-2024 The ggml authors - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ -#include "ggml-impl.h" -#include "ggml-common.h" -#include "ggml-qnn-ops.h" - -<<<<<<< HEAD -static inline uint32_t ggmlqnn_get_tensor_data_size(const ggml_tensor * tensor) { - /* - size_t data_size = ggml_row_size(tensor->type, tensor->ne[0]); - size_t n_dims = ggml_get_tensor_rank(tensor); - for (int i = 1; i < n_dims; i++) { - data_size *= tensor->ne[i]; - } - - return data_size; - */ - return ggml_nbytes(tensor); -} - -static inline bool ggmlqnn_is_valid_params(ggml_backend_qnn_context * ctx, const ggml_tensor * src0, - const ggml_tensor * src1, ggml_tensor * dst) { - if ((nullptr == ctx) || (nullptr == src0) || (nullptr == src1) || (nullptr == dst)) { - GGMLQNN_LOG_WARN("invalid params\n"); - return false; - } - - qnn_instance * instance = ctx->instance; - if (nullptr == instance) { - GGMLQNN_LOG_WARN("invalid params\n"); - return false; - } - - return true; -} - -======= ->>>>>>> ggml-qnn: refine source code structure to make code more clearly -#define GGMLQNN_CHECK_PARAMS(ctx, src0, src1, dst) \ - do { \ - if (!ggmlqnn_is_valid_params((ctx), (src0), (src1), (dst))) { \ - return; \ - } \ - } while (0) - -/* - * provide a general skeleton to offload ggml op to QNN backend: a single node contains 2 input - * tensor and 1 output tensor -*/ -void ggml_qnn_general_node(ggml_backend_qnn_context * ctx, ggml_tensor * op) { - Qnn_ErrorHandle_t error = QNN_SUCCESS; - enum ggml_status result = GGML_STATUS_SUCCESS; - bool graph_initialized = false; - qnn_instance * instance = nullptr; - Qnn_GraphHandle_t graph_handle = nullptr; - Qnn_Tensor_t * p_tensor0 = nullptr; - Qnn_Tensor_t * p_tensor1 = nullptr; - Qnn_Tensor_t * p_tensor2 = nullptr; - Qnn_Param_t qnn_params[] = {}; - const ggml_tensor * src0 = op->src[0]; - const ggml_tensor * src1 = op->src[1]; - ggml_tensor * dst = op; - - GGMLQNN_CHECK_PARAMS(ctx, src0, src1, dst); - instance = ctx->instance; - QNN_INTERFACE_VER_TYPE qnn_raw_interface = ctx->raw_interface; - size_t qnn_op_index = ggmlqnn_get_op_index(op); - GGML_ASSERT(qnn_op_index < ggmlqnn_get_opcaps_size()); -<<<<<<< HEAD - const char * qnn_op_name = ggmlqnn_k_op_caps[qnn_op_index].qnn_op_name; -======= - const char * qnn_op_name = k_op_caps[qnn_op_index].qnn_op_name; ->>>>>>> ggml-qnn: refine source code structure to make code more clearly - std::string ggml_op_name_string = std::string("ggml_") + ggml_op_name(op->op); - const char * ggml_op_name = ggml_op_name_string.c_str(); - - qnn_perf op_perf = qnn_perf(ggml_op_name); - op_perf.start(); - - std::string graph_name; - ggmlqnn_get_graphkey_from_op(op, graph_name); - if (instance->_qnn_graph_map.find(graph_name) != instance->_qnn_graph_map.end()) { - graph_initialized = true; - qnn_res_t & graph_item = instance->_qnn_graph_map[graph_name]; - graph_handle = std::get<0>(graph_item); - qnn_tensors_t & tensor = std::get<1>(graph_item); - p_tensor0 = tensor[0]; - p_tensor1 = tensor[1]; - p_tensor2 = tensor[2]; - } else { - p_tensor0 = ggmlqnn_create_compute_tensor(src0); - p_tensor1 = ggmlqnn_create_compute_tensor(src1); - p_tensor2 = ggmlqnn_create_compute_tensor(dst); - } -<<<<<<< HEAD -<<<<<<< HEAD - //ggmlqnn_print_tensors_info(__func__, ctx, src0, src1, dst); -======= -#if GGMLQNN_PRINT_OP_ADD_LOG - print_tensors_info(__func__, ctx, src0, src1, dst); -#endif ->>>>>>> ggml-qnn: refine source code structure to make code more clearly -======= - ggmlqnn_print_tensors_info(__func__, ctx, src0, src1, dst); ->>>>>>> ggml-qnn: enable release build with necessary logs to make reviewers happy - - //ensure QNN tensor has correct tensor type - QNN_VER_PTR(*p_tensor0)->type = QNN_TENSOR_TYPE_APP_WRITE; - QNN_VER_PTR(*p_tensor1)->type = QNN_TENSOR_TYPE_APP_WRITE; - QNN_VER_PTR(*p_tensor2)->type = QNN_TENSOR_TYPE_APP_READ; - - //save the original dimensions of qnn tensors - uint32_t * tensor_0_dimensions = QNN_VER_PTR(*p_tensor0)->dimensions; - uint32_t * tensor_1_dimensions = QNN_VER_PTR(*p_tensor1)->dimensions; - uint32_t * tensor_2_dimensions = QNN_VER_PTR(*p_tensor2)->dimensions; - - bool enable_npu_rpc = instance->enable_qnn_rpc() && ctx->device == QNN_BACKEND_NPU; - - if (!graph_initialized) { - GGMLQNN_LOG_DEBUG("graph name %s", graph_name.c_str()); - error = instance->init_qnn_graph(graph_name, static_cast(ctx->device), 8); - if (QNN_SUCCESS != error) { - GGMLQNN_LOG_INFO("can't create qnn graph handle with graph name %s, error = %d\n", graph_name.c_str(), error); - return; - } - graph_handle = instance->get_qnn_graph_handle(); - - if (enable_npu_rpc) { - QNN_VER_PTR(*p_tensor0)->memType = QNN_TENSORMEMTYPE_MEMHANDLE; - QNN_VER_PTR(*p_tensor0)->clientBuf = {.data=nullptr, .dataSize=0}; - - QNN_VER_PTR(*p_tensor1)->memType = QNN_TENSORMEMTYPE_MEMHANDLE; - QNN_VER_PTR(*p_tensor1)->clientBuf = {.data=nullptr, .dataSize=0}; - - QNN_VER_PTR(*p_tensor2)->memType = QNN_TENSORMEMTYPE_MEMHANDLE; - QNN_VER_PTR(*p_tensor2)->clientBuf = {.data=nullptr, .dataSize=0}; - } - - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor0)); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor1)); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor2)); - - if (enable_npu_rpc) { - uint8_t * qnn_rpcbuffer_0 = ggmlqnn_create_rpc_buffer(instance, src0, p_tensor0, true); - uint8_t * qnn_rpcbuffer_1 = ggmlqnn_create_rpc_buffer(instance, src1, p_tensor1, true); - uint8_t * qnn_rpcbuffer_2 = ggmlqnn_create_rpc_buffer(instance, dst, p_tensor2, false); - if (nullptr == qnn_rpcbuffer_0 || nullptr == qnn_rpcbuffer_1 || nullptr == qnn_rpcbuffer_2) { - GGMLQNN_LOG_INFO("create rpc buffer failure\n"); - //TODO: potential memory leak although it shouldn't happen - return; - } - } else { -<<<<<<< HEAD - QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggmlqnn_get_tensor_data_size(src0)}; - QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggmlqnn_get_tensor_data_size(src1)}; - QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggmlqnn_get_tensor_data_size(dst)}; -======= - QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, static_cast(ggml_nbytes(src0))}; - QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, static_cast(ggml_nbytes(src1))}; - QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, static_cast(ggml_nbytes(dst))}; ->>>>>>> ggml-qnn: refine source code structure to make code more clearly - } - - Qnn_Tensor_t tensor_inputs[] = { - *p_tensor0, - *p_tensor1 - }; - Qnn_Tensor_t tensor_outputs[] = { - *p_tensor2 - }; - Qnn_OpConfig_t op_config = { - QNN_OPCONFIG_VERSION_1, .v1 = { - ggml_op_name, - QNN_OP_PACKAGE_NAME_QTI_AISW, - qnn_op_name, - 0, - qnn_params, - 2, - tensor_inputs, - 1, - tensor_outputs - } - }; - CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, op_config)); - CHECK_QNN_API(error, qnn_raw_interface.graphFinalize(graph_handle, nullptr, nullptr)); - CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, - tensor_inputs, 2, - tensor_outputs, 1, - nullptr, nullptr)); - - if (enable_npu_rpc) { - uint8_t * qnn_rpcbuffer = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*p_tensor2)->memHandle)); - GGMLQNN_LOG_INFO("qnn_rpcbuffer = %p\n", qnn_rpcbuffer); - if (nullptr != qnn_rpcbuffer) { - memcpy(dst->data, qnn_rpcbuffer, ggml_nbytes(dst)); - } - } - - qnn_tensors_t ggml_op_add_tensors; - ggml_op_add_tensors.reserve(3); - ggml_op_add_tensors.push_back(p_tensor0); - ggml_op_add_tensors.push_back(p_tensor1); - ggml_op_add_tensors.push_back(p_tensor2); - - auto graph_item = std::make_tuple(graph_handle, ggml_op_add_tensors); - instance->_qnn_graph_map[graph_name] = graph_item; - } else { - Qnn_DataType_t src0_qnn_type = QNN_DATATYPE_FLOAT_32; - Qnn_DataType_t src1_qnn_type = QNN_DATATYPE_FLOAT_32; - Qnn_DataType_t dst_qnn_type = QNN_DATATYPE_FLOAT_32; - - src0_qnn_type = ggmlqnn_datatype_from_ggml_datatype(src0->type); - src1_qnn_type = ggmlqnn_datatype_from_ggml_datatype(src1->type); - dst_qnn_type = ggmlqnn_datatype_from_ggml_datatype(dst->type); - - uint32_t dimensions_input_0[] = {(uint32_t) src0->ne[0], (uint32_t) src0->ne[1], - (uint32_t) src0->ne[2], (uint32_t) src0->ne[3]}; - uint32_t dimensions_input_1[] = {(uint32_t) src1->ne[0], (uint32_t) src1->ne[1], - (uint32_t) src1->ne[2], (uint32_t) src1->ne[3]}; - uint32_t dimensions_output[] = {(uint32_t) dst->ne[0], (uint32_t) dst->ne[1], - (uint32_t) dst->ne[2], (uint32_t) dst->ne[3]}; - - QNN_VER_PTR(*p_tensor0)->dimensions = dimensions_input_0; - QNN_VER_PTR(*p_tensor0)->rank = ggml_n_dims(src0); - QNN_VER_PTR(*p_tensor0)->dataType = src0_qnn_type; - - QNN_VER_PTR(*p_tensor1)->dimensions = dimensions_input_1; - QNN_VER_PTR(*p_tensor1)->rank = ggml_n_dims(src1); - QNN_VER_PTR(*p_tensor1)->dataType = src1_qnn_type; - - QNN_VER_PTR(*p_tensor2)->dimensions = dimensions_output; - QNN_VER_PTR(*p_tensor2)->rank = ggml_n_dims(dst); - QNN_VER_PTR(*p_tensor2)->dataType = dst_qnn_type; - - if (enable_npu_rpc) { - //TODO: NPU RPC feature will failed with test-backend-ops - uint8_t * qnn_buffer_0 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*p_tensor0)->memHandle)); - GGMLQNN_LOG_INFO("qnn_rpcbuffer_0 = %p\n", qnn_buffer_0); - if (nullptr != qnn_buffer_0) { - memcpy(qnn_buffer_0, src0->data, ggml_nbytes(src0)); - } - - uint8_t * qnn_buffer_1 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*p_tensor1)->memHandle)); - GGMLQNN_LOG_INFO("qnn_rpcbuffer_1 = %p\n", qnn_buffer_1); - if (nullptr != qnn_buffer_1) { - memcpy(qnn_buffer_1, src1->data, ggml_nbytes(src1)); - } - } else { -<<<<<<< HEAD - QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggmlqnn_get_tensor_data_size(src0)}; - QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggmlqnn_get_tensor_data_size(src1)}; - QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggmlqnn_get_tensor_data_size(dst)}; -======= - QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, static_cast(ggml_nbytes(src0))}; - QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, static_cast(ggml_nbytes(src1))}; - QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, static_cast(ggml_nbytes(dst))}; ->>>>>>> ggml-qnn: refine source code structure to make code more clearly - } - - Qnn_Tensor_t tensor_inputs[] = { - *p_tensor0, - *p_tensor1 - }; - Qnn_Tensor_t tensor_outputs[] = { - *p_tensor2 - }; - CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, - tensor_inputs, 2, - tensor_outputs, 1, - nullptr, nullptr)); - - if (enable_npu_rpc) { - //TODO:NPU RPC feature will failed with test-backend-ops - uint8_t * qnn_buffer_2 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*p_tensor2)->memHandle)); - if (nullptr != qnn_buffer_2) { - memcpy(dst->data, qnn_buffer_2, ggml_nbytes(dst)); - } - } - } - - // restore the original dimensions of qnn tensors to avoid memory leak in func free_qnn_tensor - QNN_VER_PTR(*p_tensor0)->dimensions = tensor_0_dimensions; - QNN_VER_PTR(*p_tensor1)->dimensions = tensor_1_dimensions; - QNN_VER_PTR(*p_tensor2)->dimensions = tensor_2_dimensions; - -#if GGMLQNN_PRINT_OP_ADD_LOG - op_perf.info(); -#endif -} - -/* -<<<<<<< HEAD - * this function is AI-assisted code from Grok 3 for purpose of offload 4d matrix mulmat to QNN backend - * UT in ggml-qnn-ut.cpp passed: - * ./scripts/build-run-android.sh run_ut_mulmat 0 - * ./scripts/build-run-android.sh run_ut_mulmat 1 - * ./scripts/build-run-android.sh run_ut_mulmat 2 - * - * the logic of ggml_qnn_mul_mat_4d is similar to ggml_qnn_mul_mat but much more complicated - * than ggml_qnn_mul_mat, so it's a standalone function. - * it will be combined with ggml_qnn_mul_mat in the future - */ -static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) { - Qnn_ErrorHandle_t error = QNN_SUCCESS; - bool graph_initialized = false; - qnn_perf op_perf = qnn_perf("ggml_qnn_mul_mat_4d"); - qnn_instance *instance = ctx->instance; - QNN_INTERFACE_VER_TYPE qnn_raw_interface = ctx->raw_interface; - - const ggml_tensor *src0 = op->src[0]; - const ggml_tensor *src1 = op->src[1]; - ggml_tensor *dst = op; - - GGMLQNN_CHECK_PARAMS(ctx, src0, src1, dst); - GGML_ASSERT(ggml_n_dims(src0) == 4 && ggml_n_dims(src1) == 4); - op_perf.start(); - - std::string graph_name; - ggmlqnn_get_graphkey_from_op(op, graph_name); - GGMLQNN_LOG_DEBUG("graph name %s\n", graph_name.c_str()); - - ggmlqnn_print_tensors_info(__func__, ctx, src0, src1, dst); - - Qnn_GraphHandle_t graph_handle = nullptr; - Qnn_Tensor_t *p_tensor0 = nullptr; - Qnn_Tensor_t *p_reshape0_out = nullptr; - Qnn_Tensor_t *p_tile0_out = nullptr; - Qnn_Tensor_t *p_tensor1 = nullptr; - Qnn_Tensor_t *p_permute1_out = nullptr; - Qnn_Tensor_t *p_reshape1_out = nullptr; - Qnn_Tensor_t *p_matmul_out = nullptr; - Qnn_Tensor_t *p_reshape2_out = nullptr; - - if (instance->_qnn_graph_map.find(graph_name) != instance->_qnn_graph_map.end()) { - graph_initialized = true; - qnn_res_t &graph_item = instance->_qnn_graph_map[graph_name]; - graph_handle = std::get<0>(graph_item); - qnn_tensors_t &tensors = std::get<1>(graph_item); - p_tensor0 = tensors[0]; - p_reshape0_out = tensors[1]; - p_tile0_out = tensors[2]; - p_tensor1 = tensors[3]; - p_permute1_out = tensors[4]; - p_reshape1_out = tensors[5]; - p_matmul_out = tensors[6]; - p_reshape2_out = tensors[7]; - } else { - CHECK_QNN_API(error, qnn_raw_interface.graphCreate(instance->get_qnn_context_handle(), - graph_name.c_str(), NULL, &graph_handle)); - - // Define dimensions - uint32_t K = src0->ne[0]; // Inner dimension - uint32_t M = src0->ne[1]; // Rows of src0 - uint32_t N = src1->ne[1]; // Columns of src1 - uint32_t B0 = src0->ne[2] * src0->ne[3]; // src0 batch - uint32_t B1 = src1->ne[2] * src1->ne[3]; // src1 batch (drives output) - - // Validate K only - GGML_ASSERT(src0->ne[0] == src1->ne[0]); // K must match - - // src0: [K, M, H0, B0] -> QNN: [B0, H0, M, K] - uint32_t src0_dims[] = {static_cast(src0->ne[3]), static_cast(src0->ne[2]), static_cast(src0->ne[1]), static_cast(src0->ne[0])}; - p_tensor0 = GQCGT(src0, "input0", QNN_TENSOR_TYPE_APP_WRITE, QNN_DATATYPE_FLOAT_32, 4, - src0_dims, nullptr, 0); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor0)); - - // Reshape src0 to [B0, M, K] - uint32_t reshape0_out_dims[] = {B0, M, K}; - p_reshape0_out = GQCGT(nullptr, "reshape0_out", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, 3, - reshape0_out_dims, nullptr, 0); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_reshape0_out)); - Qnn_Tensor_t reshape0_inputs[] = {*p_tensor0}; - Qnn_Tensor_t reshape0_outputs[] = {*p_reshape0_out}; - Qnn_OpConfig_t reshape0_op = ggmlqnn_create_op_config("reshape0", QNN_OP_PACKAGE_NAME_QTI_AISW, - QNN_OP_RESHAPE, nullptr, 0, - reshape0_inputs, 1, reshape0_outputs, 1); - CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, reshape0_op)); - - // Tile src0 to match B1: [B0, M, K] -> [B1, M, K] - uint32_t tile0_out_dims[] = {B1, M, K}; - p_tile0_out = GQCGT(nullptr, "tile0_out", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, 3, - tile0_out_dims, nullptr, 0); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tile0_out)); - uint32_t tile_multiples[] = {B1 / B0, 1, 1}; - uint32_t tile_dims[] = {3}; - Qnn_Tensor_t *p_tile_multiples = GQCGT(nullptr, "tile_multiples", QNN_TENSOR_TYPE_STATIC, QNN_DATATYPE_UINT_32, 1, - tile_dims, tile_multiples, sizeof(tile_multiples)); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tile_multiples)); - Qnn_Param_t tile_params[] = {{QNN_PARAMTYPE_TENSOR, "multiples", .tensorParam = *p_tile_multiples}}; - Qnn_Tensor_t tile0_inputs[] = {*p_reshape0_out}; - Qnn_Tensor_t tile0_outputs[] = {*p_tile0_out}; - Qnn_OpConfig_t tile0_op = ggmlqnn_create_op_config("tile0", QNN_OP_PACKAGE_NAME_QTI_AISW, - QNN_OP_TILE, tile_params, 1, - tile0_inputs, 1, tile0_outputs, 1); - CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, tile0_op)); - - // src1: [N, K, H1, B1] -> QNN: [B1, H1, N, K] - uint32_t src1_dims[] = {static_cast(src1->ne[3]), static_cast(src1->ne[2]), static_cast(src1->ne[1]), static_cast(src1->ne[0])}; - p_tensor1 = GQCGT(src1, "input1", QNN_TENSOR_TYPE_APP_WRITE, QNN_DATATYPE_FLOAT_32, 4, - src1_dims, nullptr, 0); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor1)); - - // Permute src1 to [B1, H1, K, N] - uint32_t perm_data[] = {0, 1, 3, 2}; - uint32_t perm_dims[] = {4}; - Qnn_Tensor_t *p_perm = GQCGT(nullptr, "perm", QNN_TENSOR_TYPE_STATIC, QNN_DATATYPE_UINT_32, 1, - perm_dims, perm_data, sizeof(perm_data)); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_perm)); - uint32_t permute1_out_dims[] = {static_cast(src1->ne[3]), static_cast(src1->ne[2]), static_cast(src1->ne[0]), static_cast(src1->ne[1])}; - p_permute1_out = GQCGT(nullptr, "permute1_out", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, 4, - permute1_out_dims, nullptr, 0); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_permute1_out)); - Qnn_Param_t permute1_params[] = {{QNN_PARAMTYPE_TENSOR, "perm", .tensorParam = *p_perm}}; - Qnn_Tensor_t permute1_inputs[] = {*p_tensor1}; - Qnn_Tensor_t permute1_outputs[] = {*p_permute1_out}; - Qnn_OpConfig_t permute1_op = ggmlqnn_create_op_config("permute1", QNN_OP_PACKAGE_NAME_QTI_AISW, - QNN_OP_TRANSPOSE, permute1_params, 1, - permute1_inputs, 1, permute1_outputs, 1); - CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, permute1_op)); - - // Reshape src1 to [B1, K, N] - uint32_t reshape1_out_dims[] = {B1, K, N}; - p_reshape1_out = GQCGT(nullptr, "reshape1_out", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, 3, - reshape1_out_dims, nullptr, 0); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_reshape1_out)); - Qnn_Tensor_t reshape1_inputs[] = {*p_permute1_out}; - Qnn_Tensor_t reshape1_outputs[] = {*p_reshape1_out}; - Qnn_OpConfig_t reshape1_op = ggmlqnn_create_op_config("reshape1", QNN_OP_PACKAGE_NAME_QTI_AISW, - QNN_OP_RESHAPE, nullptr, 0, - reshape1_inputs, 1, reshape1_outputs, 1); - CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, reshape1_op)); - - // MatMul: [B1, M, K] x [B1, K, N] -> [B1, M, N] - uint32_t matmul_out_dims[] = {B1, M, N}; - p_matmul_out = GQCGT(nullptr, "matmul_out", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, 3, - matmul_out_dims, nullptr, 0); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_matmul_out)); - Qnn_Tensor_t matmul_inputs[] = {*p_tile0_out, *p_reshape1_out}; - Qnn_Tensor_t matmul_outputs[] = {*p_matmul_out}; - Qnn_OpConfig_t matmul_op = ggmlqnn_create_op_config("matmul", QNN_OP_PACKAGE_NAME_QTI_AISW, - QNN_OP_MAT_MUL, nullptr, 0, - matmul_inputs, 2, matmul_outputs, 1); - CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, matmul_op)); - - // Output: [N, M, H1, B1] -> QNN: [B1, H1, M, N] - uint32_t reshape2_out_dims[] = {static_cast(dst->ne[3]), static_cast(dst->ne[2]), static_cast(dst->ne[1]), static_cast(dst->ne[0])}; - p_reshape2_out = GQCGT(dst, "output", QNN_TENSOR_TYPE_APP_READ, QNN_DATATYPE_FLOAT_32, 4, - reshape2_out_dims, nullptr, 0); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_reshape2_out)); - Qnn_Tensor_t reshape2_inputs[] = {*p_matmul_out}; - Qnn_Tensor_t reshape2_outputs[] = {*p_reshape2_out}; - Qnn_OpConfig_t reshape2_op = ggmlqnn_create_op_config("reshape2", QNN_OP_PACKAGE_NAME_QTI_AISW, - QNN_OP_RESHAPE, nullptr, 0, - reshape2_inputs, 1, reshape2_outputs, 1); - CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, reshape2_op)); - - // Finalize - CHECK_QNN_API(error, qnn_raw_interface.graphFinalize(graph_handle, NULL, NULL)); - - // Cache - qnn_tensors_t ggml_op_mulmat_tensors = {p_tensor0, p_reshape0_out, p_tile0_out, p_tensor1, p_permute1_out, p_reshape1_out, p_matmul_out, p_reshape2_out}; - instance->_qnn_graph_map[graph_name] = std::make_tuple(graph_handle, ggml_op_mulmat_tensors); - } - - // Execute - QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, static_cast(ggml_nbytes(src0))}; - QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, static_cast(ggml_nbytes(src1))}; - QNN_VER_PTR(*p_reshape2_out)->clientBuf = {dst->data, static_cast(ggml_nbytes(dst))}; - - Qnn_Tensor_t input_tensors[] = {*p_tensor0, *p_tensor1}; - Qnn_Tensor_t output_tensors[] = {*p_reshape2_out}; - CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, input_tensors, 2, - output_tensors, 1, NULL, NULL)); - -#if 0 - // Log dst for debugging - float *dst_data = (float *)dst->data; - GGMLQNN_LOG_DEBUG("dst shape: [%d, %d, %d, %d]\n", dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]); - for (int i = 0; i < dst->ne[0] * dst->ne[1] * dst->ne[2] * dst->ne[3]; i++) { - GGMLQNN_LOG_DEBUG("dst[%d] = %f\n", i, dst_data[i]); - } -#endif - - op_perf.info(); -} - -/* -======= ->>>>>>> ggml-qnn: refine source code structure to make code more clearly - * @brief performs matrix multiplication with FP32 & quantized weights and floating-point inputs - * using the QNN backend. this function performs matrix multiplication of the input tensor - * `src1` and the weight tensor `src0`, handling transposing, and quantization as needed, - * and stores the result in the destination tensor `dst`. - * - * @param backend the context which got through (ggml_backend_qnn_context *)backend->context for the - * QNN backend operations. - * @param op the destination tensor where the result of the matrix multiplication will be stored. - * - * @note the logic of ggml_qnn_mul_mat is similar to ggml_qnn_general_node but much more complicated - * than ggml_qnn_general_node. so it's a standalone function. accordingly, this is another - * typical skeleton for offload other ggml ops to QNN backend. MUL_MAT take most of the compute - * time (about 95%).so to speed up llama inference, should focus on this func. there are three kinds - * of MUL_MAT to compute: - * mul_mat_f32: both src0 and src1 are F32, this will be naturally handled in QNN backend - * mul_mat_f16_f32: src0 is F16 and src1 is F32, f16 in src0 -> f32 in src0', then src0' * src1 - * mul_mat_q_f32: src0 is quantized (Q4_0, Q4_1, Q6_K...) - * and src1 is F32, src0 -> f32 in src0', then src0' * src1 -*/ -void ggml_qnn_mul_mat(ggml_backend_qnn_context * ctx, ggml_tensor * op) { - Qnn_ErrorHandle_t error = QNN_SUCCESS; - bool graph_initialized = false; - qnn_perf op_perf = qnn_perf("ggml_qnn_mul_mat"); - qnn_instance * instance = nullptr; - Qnn_GraphHandle_t graph_handle = nullptr; - Qnn_Tensor_t * p_tensor0 = nullptr; - Qnn_Tensor_t * p_tensor1 = nullptr; - Qnn_Tensor_t * p_tensor2 = nullptr; - Qnn_Tensor_t * p_param_tensor = nullptr; - Qnn_Tensor_t * p_tensor2_transpose = nullptr; - const ggml_tensor * src0 = op->src[0]; - const ggml_tensor * src1 = op->src[1]; - ggml_tensor * dst = op; - - GGMLQNN_CHECK_PARAMS(ctx, src0, src1, dst); - instance = ctx->instance; - QNN_INTERFACE_VER_TYPE qnn_raw_interface = ctx->raw_interface; - op_perf.start(); - - const enum ggml_type src0_type = src0->type; - const uint32_t src0_rank = ggml_n_dims(src0); - const uint32_t src1_rank = ggml_n_dims(src1); - GGML_ASSERT(src0_rank == src1_rank); - GGML_ASSERT(src0_rank >= 2); //QNN SDK's limitation, make QNN SDK happy -<<<<<<< HEAD - if (4 == src0_rank) { - return ggml_qnn_mul_mat_4d(ctx, op); - } -======= - GGML_ASSERT(src0_rank != 4); //TODO: 4D matrix mulmat ->>>>>>> ggml-qnn: refine source code structure to make code more clearly - void * wdata = ggmlqnn_type_trait(ctx, op); - const size_t desired_size = ctx->desired_size; - - std::string graph_name; - ggmlqnn_get_graphkey_from_op(op, graph_name); - if (instance->_qnn_graph_map.find(graph_name) != instance->_qnn_graph_map.end()) { - graph_initialized = true; - qnn_res_t & graph_item = instance->_qnn_graph_map[graph_name]; - graph_handle = std::get<0>(graph_item); - qnn_tensors_t & tensors = std::get<1>(graph_item); - p_tensor0 = tensors[0]; - p_tensor1 = tensors[1]; - p_tensor2 = tensors[2]; - p_param_tensor = tensors[3]; - p_tensor2_transpose = tensors[4]; - } else { - p_tensor0 = GQCGT(src0, nullptr, QNN_TENSOR_TYPE_APP_WRITE,QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0); - p_tensor1 = GQCGT(src1, nullptr, QNN_TENSOR_TYPE_APP_WRITE,QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0); - p_tensor2 = GQCGT(dst, nullptr, QNN_TENSOR_TYPE_APP_READ,QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0); - } - ggmlqnn_print_tensors_info(__func__, ctx, src0, src1, dst); - - //ensure QNN tensor has correct tensor type - QNN_VER_PTR(*p_tensor0)->type = QNN_TENSOR_TYPE_APP_WRITE; - QNN_VER_PTR(*p_tensor1)->type = QNN_TENSOR_TYPE_APP_WRITE; - QNN_VER_PTR(*p_tensor2)->type = QNN_TENSOR_TYPE_APP_READ; - - //save the original dimensions of qnn tensors - uint32_t * tensor_0_dimensions = QNN_VER_PTR(*p_tensor0)->dimensions; - uint32_t * tensor_1_dimensions = QNN_VER_PTR(*p_tensor1)->dimensions; - uint32_t * tensor_2_dimensions = QNN_VER_PTR(*p_tensor2)->dimensions; - - if (!graph_initialized) { - GGMLQNN_LOG_DEBUG("graph name %s", graph_name.c_str()); - /* - there are two key-points in properly handling how to offload mulmat to the QNN backend in ggml-qnn - 1. transpose - a 3x2 f32 matrix which means 3 rows and 2 columns. in ggml, it could be created from: - struct ggml_tensor* matrix = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 2, 3); - which like this: - +---+---+ - | 0 | 1 | - +---+---+ - | 2 | 3 | - +---+---+ - | 4 | 5 | - +---+---+ - with - ne[0] = 2 - ne[1] = 3 - there are different dimension order between ggml tensor and qnn tensor - - 2. QNN's MatMul can only support input tensors with rank >= 2 - - in the all, there is gap between ggml mulmat and QNN mulmat,we need to perform a transpose - operation when offloading mulmat to QNN backend. this concise implementation will handle - transpose in func ggml_qnn_create_general_tensor() - */ - //step-1: create qnn graph - error = qnn_raw_interface.graphCreate(instance->get_qnn_context_handle(), - graph_name.c_str(), nullptr, &graph_handle); - if (QNN_SUCCESS != error) { - GGMLQNN_LOG_INFO("can't create qnn graph handle with graph name %s, error = %d\n", graph_name.c_str(), error); - return; - } - //step-2: create param tensor for mulmat of 2d/3d/4d matrix - const uint32_t param_tensor_data[GGML_MAX_DIMS][GGML_MAX_DIMS] = { - {0}, - {1, 0}, - {0, 2, 1}, - {0, 1, 3, 2}, - }; - uint32_t param_tensor_dims[1] = {src0_rank}; - p_param_tensor = GQCGT(nullptr, "param", QNN_TENSOR_TYPE_STATIC, QNN_DATATYPE_UINT_32, 1, param_tensor_dims, (void *)(param_tensor_data[src0_rank - 1]), src0_rank * sizeof(uint32_t)); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_param_tensor)); - - //step-3: create compute tensor from ggml tensor - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor0)); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor1)); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor2)); - if (src0_type != GGML_TYPE_F32) { - QNN_VER_PTR(*p_tensor0)->clientBuf = {wdata, static_cast(desired_size)}; - } else { -<<<<<<< HEAD - QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggmlqnn_get_tensor_data_size(src0)}; - } - QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggmlqnn_get_tensor_data_size(src1)}; - QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggmlqnn_get_tensor_data_size(dst)}; -======= - QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, static_cast(ggml_nbytes(src0))}; - } - QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, static_cast(ggml_nbytes(src1))}; - QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, static_cast(ggml_nbytes(dst))}; ->>>>>>> ggml-qnn: refine source code structure to make code more clearly - - //step-4: create a transpose tensor - p_tensor2_transpose = GQCGT(dst, "transpose", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0, true); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor2_transpose)); - - //step-5: compose qnn graph: add mat_mul node - Qnn_Param_t out_0_params[] = { - {QNN_PARAMTYPE_SCALAR, - QNN_OP_MAT_MUL_PARAM_TRANSPOSE_IN1, - .scalarParam = {QNN_DATATYPE_BOOL_8, .bool8Value = 1} - } - }; - - Qnn_Tensor_t out_0_inputs[] = {*p_tensor0, *p_tensor1}; - Qnn_Tensor_t out_0_outputs[] = {*p_tensor2_transpose}; -#if 0 //leave here for easily understand code, can be removed in the future - Qnn_OpConfig_t out_0 = { - QNN_OPCONFIG_VERSION_1, .v1 = - {"ggmlqnn_mulmat_opconfig", QNN_OP_PACKAGE_NAME_QTI_AISW, QNN_OP_MAT_MUL, - 1, - out_0_params, - 2, - out_0_inputs, - 1, - out_0_outputs} - }; -#else - Qnn_OpConfig_t out_0 = ggmlqnn_create_op_config("ggmlqnn_mulmat_opconfig", QNN_OP_PACKAGE_NAME_QTI_AISW, QNN_OP_MAT_MUL, -<<<<<<< HEAD - out_0_params, 1, out_0_inputs, 2, out_0_outputs, 1); -======= - out_0_params, 1, out_0_inputs, 2, out_0_outputs, 1); ->>>>>>> ggml-qnn: refine source code structure to make code more clearly -#endif - CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle,out_0)); - - //step-5: compose qnn graph: add transpose node - Qnn_Param_t out_trans1_0_params[] = { -<<<<<<< HEAD - {QNN_PARAMTYPE_TENSOR, -======= - {(Qnn_ParamType_t) 1, ->>>>>>> ggml-qnn: refine source code structure to make code more clearly - "perm", .tensorParam = *p_param_tensor - } - }; - Qnn_Tensor_t out_trans1_0_inputs[] = {*p_tensor2_transpose}; - Qnn_Tensor_t out_trans1_0_outputs[] = {*p_tensor2}; -#if 0 //leave here for easily understand code, can be removed in the future - Qnn_OpConfig_t out_trans1_0 = { - QNN_OPCONFIG_VERSION_1, - .v1 = {"ggmlqnn_mulmat_transpose_opconfig", - QNN_OP_PACKAGE_NAME_QTI_AISW, - QNN_OP_TRANSPOSE, 1, - out_trans1_0_params, - 1, - out_trans1_0_inputs, - 1, - out_trans1_0_outputs} - }; -#else - Qnn_OpConfig_t out_trans1_0 = ggmlqnn_create_op_config("ggmlqnn_mulmat_transpose_opconfig", QNN_OP_PACKAGE_NAME_QTI_AISW, QNN_OP_TRANSPOSE, -<<<<<<< HEAD - out_trans1_0_params, 1, out_trans1_0_inputs, 1, out_trans1_0_outputs, 1); -======= - out_trans1_0_params, 1, out_trans1_0_inputs, 1, out_trans1_0_outputs, 1); ->>>>>>> ggml-qnn: refine source code structure to make code more clearly -#endif - CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle,out_trans1_0)); - - //step-6: finalize qnn graph and execute qnn graph - CHECK_QNN_API(error, qnn_raw_interface.graphFinalize(graph_handle, nullptr, nullptr)); - Qnn_Tensor_t input_tensors_0[] = {*p_tensor0, *p_tensor1}; - Qnn_Tensor_t output_tensors_0[] = {*p_tensor2}; - CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, - input_tensors_0, 2, - output_tensors_0, 1, - nullptr, nullptr)); - - qnn_tensors_t ggml_op_mulmat_tensors; - ggml_op_mulmat_tensors.reserve(5); - ggml_op_mulmat_tensors.push_back(p_tensor0); - ggml_op_mulmat_tensors.push_back(p_tensor1); - ggml_op_mulmat_tensors.push_back(p_tensor2); - ggml_op_mulmat_tensors.push_back(p_param_tensor); - ggml_op_mulmat_tensors.push_back(p_tensor2_transpose); - auto graph_item = std::make_tuple(graph_handle, ggml_op_mulmat_tensors); - instance->_qnn_graph_map[graph_name] = graph_item; - } else { - if (src0_type != GGML_TYPE_F32) { - QNN_VER_PTR(*p_tensor0)->clientBuf = {wdata, static_cast(desired_size)}; - } else { -<<<<<<< HEAD - QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggmlqnn_get_tensor_data_size(src0)}; - } - QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggmlqnn_get_tensor_data_size(src1)}; - QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggmlqnn_get_tensor_data_size(dst)}; -======= - QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, static_cast(ggml_nbytes(src0))}; - } - QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, static_cast(ggml_nbytes(src1))}; - QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, static_cast(ggml_nbytes(dst))}; ->>>>>>> ggml-qnn: refine source code structure to make code more clearly - - Qnn_Tensor_t tensor_inputs[] = { - *p_tensor0, - *p_tensor1 - }; - Qnn_Tensor_t tensor_outputs[] = { - *p_tensor2 - }; - // this is the second technical approach or another pipeline of "how to utilize the Hexagon - // NPU maximally" through QNN SDK, details could be found at - // https://github.com/ggml-org/llama.cpp/pull/12049#issuecomment-2678308360 - CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, - tensor_inputs, 2, - tensor_outputs, 1, - nullptr, nullptr)); - } - - // restore the original dimensions of qnn tensors to avoid memory leak in func free_qnn_tensor - QNN_VER_PTR(*p_tensor0)->dimensions = tensor_0_dimensions; - QNN_VER_PTR(*p_tensor1)->dimensions = tensor_1_dimensions; - QNN_VER_PTR(*p_tensor2)->dimensions = tensor_2_dimensions; - op_perf.info(); -} -<<<<<<< HEAD - -void ggml_qnn_repeat(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { - GGML_UNUSED(ctx); - GGML_UNUSED(dst); -} - -void ggml_qnn_div(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { - GGML_UNUSED(ctx); - GGML_UNUSED(dst); -} - -void ggml_qnn_leaky_relu(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { - GGML_UNUSED(ctx); - GGML_UNUSED(dst); -} - -void ggml_qnn_concat(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { - GGML_UNUSED(ctx); - GGML_UNUSED(dst); -} - -void ggml_qnn_arange(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { - GGML_UNUSED(ctx); - GGML_UNUSED(dst); -} - -void ggml_qnn_sqr(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { - GGML_UNUSED(ctx); - GGML_UNUSED(dst); -} - -void ggml_qnn_clamp(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { - GGML_UNUSED(ctx); - GGML_UNUSED(dst); -} - -void ggml_qnn_scale(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { - GGML_UNUSED(ctx); - GGML_UNUSED(dst); -} - -void ggml_qnn_argsort(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { - GGML_UNUSED(ctx); - GGML_UNUSED(dst); -} - -void ggml_qnn_norm(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { - GGML_UNUSED(ctx); - GGML_UNUSED(dst); -} - -void ggml_qnn_group_norm(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { - GGML_UNUSED(ctx); - GGML_UNUSED(dst); -} - -void ggml_qnn_acc(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { - GGML_UNUSED(ctx); - GGML_UNUSED(dst); -} - -void ggml_qnn_sum_rows(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { - GGML_UNUSED(ctx); - GGML_UNUSED(dst); -} - -void ggml_qnn_upsample_nearest2d(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { - GGML_UNUSED(ctx); - GGML_UNUSED(dst); -} - -void ggml_qnn_pad(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { - GGML_UNUSED(ctx); - GGML_UNUSED(dst); -} - -void ggml_qnn_pool2d(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { - GGML_UNUSED(ctx); - GGML_UNUSED(dst); -} - -void ggml_qnn_dup(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { - GGML_UNUSED(ctx); - GGML_UNUSED(dst); -} - -void ggml_qnn_rms_norm(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { - GGML_UNUSED(ctx); - GGML_UNUSED(dst); -} - -void ggml_qnn_diag_mask(ggml_backend_qnn_context * ctx, ggml_tensor * dst, float value) { - GGML_UNUSED(ctx); - GGML_UNUSED(dst); - GGML_UNUSED(value); -} - -void ggml_qnn_im2col(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { - GGML_UNUSED(ctx); - GGML_UNUSED(dst); -} - -void ggml_qnn_timestep_embedding(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { - GGML_UNUSED(ctx); - GGML_UNUSED(dst); -======= -void ggml_qnn_repeat(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { -} - -void ggml_qnn_add(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { -} - -void ggml_qnn_div(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { -} - -void ggml_qnn_leaky_relu(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { -} - -void ggml_qnn_concat(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { -} - -void ggml_qnn_arange(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { -} - -void ggml_qnn_sqr(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { -} - -void ggml_qnn_clamp(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { -} - -void ggml_qnn_scale(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { -} - -void ggml_qnn_argsort(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { -} - -void ggml_qnn_norm(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { -} - -void ggml_qnn_group_norm(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { -} - -void ggml_qnn_acc(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { -} - -void ggml_qnn_sum_rows(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { -} - -void ggml_qnn_upsample_nearest2d(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { -} - -void ggml_qnn_pad(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { -} - -static void ggml_qnn_avg_pool2d(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { -} - -static void ggml_qnn_max_pool2d(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { -} - -void ggml_qnn_pool2d(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { -} - -void ggml_qnn_dup(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { -} - -void ggml_qnn_rms_norm(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { -} - -void ggml_qnn_diag_mask(ggml_backend_qnn_context * ctx, ggml_tensor * dst, float value) { -} - -void ggml_qnn_im2col(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { -} - -void ggml_qnn_timestep_embedding(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { ->>>>>>> ggml-qnn: refine source code structure to make code more clearly -} - -void ggml_qnn_cpy(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { - ggml_qnn_dup(ctx, dst); -} - -void ggml_qnn_softmax(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { -<<<<<<< HEAD - GGML_UNUSED(ctx); - GGML_UNUSED(dst); -} - -void ggml_qnn_get_rows(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { - GGML_UNUSED(ctx); - GGML_UNUSED(dst); -} - -void ggml_qnn_rope(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { - GGML_UNUSED(ctx); - GGML_UNUSED(dst); -======= -} - -void ggml_qnn_get_rows(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { -} - -void ggml_qnn_rope(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { ->>>>>>> ggml-qnn: refine source code structure to make code more clearly -} diff --git a/ggml/src/ggml-qnn/ggml-qnn.cpp b/ggml/src/ggml-qnn/ggml-qnn.cpp index 5276001a8523b..7704a4ad038f7 100644 --- a/ggml/src/ggml-qnn/ggml-qnn.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn.cpp @@ -2267,8 +2267,7 @@ static bool ggml_qnn_can_handle_op(const ggml_backend_qnn_context * ctx, const s else return (src0->type == GGML_TYPE_F32) && (src1->type == GGML_TYPE_F32) && (tensor->type == GGML_TYPE_F32); else - return (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_Q4_0 - || src0->type == GGML_TYPE_Q8_0 || src0->type == GGML_TYPE_Q6_K || src0->type == GGML_TYPE_Q8_K) + return (src0->type == GGML_TYPE_F32 || ggml_is_quantized(src0->type)) && (src1->type == GGML_TYPE_F32) && (tensor->type == GGML_TYPE_F32); } From e866c6ebccd19ca94f82e154c7ee3bacc8ee4748 Mon Sep 17 00:00:00 2001 From: zhouwg Date: Fri, 28 Feb 2025 12:23:31 +0800 Subject: [PATCH 30/76] ggml-qnn: enable log output of GGMLQNN_LOG_INFO in command line mode for benchmark more conveniently --- ggml/src/ggml-qnn/ggml-qnn-impl.h | 753 ------------------------------ ggml/src/ggml-qnn/ggml-qnn.cpp | 21 +- 2 files changed, 12 insertions(+), 762 deletions(-) delete mode 100644 ggml/src/ggml-qnn/ggml-qnn-impl.h diff --git a/ggml/src/ggml-qnn/ggml-qnn-impl.h b/ggml/src/ggml-qnn/ggml-qnn-impl.h deleted file mode 100644 index a4e00e0b7bbd7..0000000000000 --- a/ggml/src/ggml-qnn/ggml-qnn-impl.h +++ /dev/null @@ -1,753 +0,0 @@ -/* -* Copyright (c) 2023-2024 The ggml authors -* -* Permission is hereby granted, free of charge, to any person obtaining a copy -* of this software and associated documentation files (the "Software"), to -* deal in the Software without restriction, including without limitation the -* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -* sell copies of the Software, and to permit persons to whom the Software is -* furnished to do so, subject to the following conditions: -* -* The above copyright notice and this permission notice shall be included in -* all copies or substantial portions of the Software. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -* IN THE SOFTWARE. -*/ -#pragma once -#include -#include -#include -#include -#include -#include -#include -#include -#if defined(__ANDROID__) || defined(__linux__) -#include -#include -#include -#include -#include -#include -#endif - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#if (defined __ANDROID__) || (defined ANDROID) -#include "android/log.h" -#endif - -<<<<<<< HEAD -#if defined(_WIN32) -#include -======= -#if defined(_WIN32) || defined(_MSC_VER) ->>>>>>> ggml-qnn: refine source code structure to make code more clearly -#include -#endif - -#include "QnnTypes.h" -#include "QnnCommon.h" -#include "QnnContext.h" -#include "QnnBackend.h" -#include "QnnGraph.h" -#include "QnnProperty.h" -#include "QnnTensor.h" -#include "QnnInterface.h" -#include "Saver/QnnSaver.h" -#include "System/QnnSystemInterface.h" -#include "HTP/QnnHtpDevice.h" -#include "HTP/QnnHtpGraph.h" - -#include "ggml-qnn.h" -#include "ggml-impl.h" -#include "ggml-backend-impl.h" - -class qnn_instance; -struct ggml_backend_qnn_context; -void ggmlqnn_log_internal(ggml_log_level level, const char * file, const char * func, int line, const char * format, ...); - -<<<<<<< HEAD -<<<<<<< HEAD -#if 0//def NDEBUG -======= -#ifdef NDEBUG ->>>>>>> ggml-qnn: refine source code structure to make code more clearly -======= -#if 0//def NDEBUG ->>>>>>> ggml-qnn: enable release build with necessary logs to make reviewers happy -#define GGMLQNN_DEBUG 0 -#define ENABLE_QNNBACKEND_PERF 0 // enable/disable op's perf info -#define GGMLQNN_PRINT_QNN_INTERNAL_LOG 0 // enable/disable QNN's internal log -#define GGMLQNN_PRINT_OP_ADD_LOG 0 // GGML_OP_ADD already verified with QNN-CPU / QNN-GPU / QNN-NPU -#define GGMLQNN_PRINT_OP_MUL_MAT_LOG 0 -#else -#define GGMLQNN_DEBUG 1 // for troubleshooting QNN backend -#define ENABLE_QNNBACKEND_PERF 0 // enable/disable op's perf info -#define GGMLQNN_PRINT_QNN_INTERNAL_LOG 0 // enable/disable QNN's internal log -#define GGMLQNN_PRINT_OP_ADD_LOG 0 // GGML_OP_ADD already verified with QNN-CPU / QNN-GPU / QNN-NPU -#define GGMLQNN_PRINT_OP_MUL_MAT_LOG 1 -#endif -#define GGML_QNN_LOGBUF_LEN 4096 - -<<<<<<< HEAD -#define GGMLQNN_LOG_ERROR(...) ggmlqnn_log_internal(GGML_LOG_LEVEL_ERROR, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) -#define GGMLQNN_LOG_WARN(...) ggmlqnn_log_internal(GGML_LOG_LEVEL_WARN , __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) -#define GGMLQNN_LOG_INFO(...) ggmlqnn_log_internal(GGML_LOG_LEVEL_INFO , __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) -======= -#define GGMLQNN_LOG_ERROR(...) ggmlqnn_log_internal(GGML_LOG_LEVEL_DEBUG, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) -#define GGMLQNN_LOG_WARN(...) ggmlqnn_log_internal(GGML_LOG_LEVEL_DEBUG , __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) -#define GGMLQNN_LOG_INFO(...) ggmlqnn_log_internal(GGML_LOG_LEVEL_DEBUG , __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) ->>>>>>> ggml-qnn: refine source code structure to make code more clearly - -#if GGMLQNN_DEBUG -#define GGMLQNN_LOG_DEBUG(...) ggmlqnn_log_internal(GGML_LOG_LEVEL_DEBUG, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) -#else -#define GGMLQNN_LOG_DEBUG(...) -#endif - -#define CHECK_QNN_API(error, result) \ - do { \ - error = (result); \ - if (QNN_SUCCESS != error) { \ - if (error == QNN_COMMON_ERROR_NOT_SUPPORTED) { \ - GGMLQNN_LOG_WARN("WARNING: QNN feature/API not supported\n"); \ - } else { \ - GGMLQNN_LOG_INFO("QNN API error = %d(%s)\n", error, ggmlqnn_get_error_string(error)); \ - } \ - } \ - } while (0) - -#define QNN_VER_PTR(x) (&((x).v1)) -#define RPCMEM_DEFAULT_FLAGS 1 -#define RPCMEM_HEAP_ID_SYSTEM 25 - -#define DISABLE_COPY(class_name) \ - class_name(const class_name &) = delete; \ - void operator=(const class_name &) = delete - -#define DISABLE_MOVE(class_name) \ - class_name(class_name &&) = delete; \ - void operator=(class_name &&) = delete - -#define GQCGT ggmlqnn_create_general_tensor - -<<<<<<< HEAD -#if defined(_WIN32) -#define RTLD_GLOBAL 0x100 -#define RTLD_LOCAL 0x000 -#define RTLD_LAZY 0x000 -#define RTLD_NOW 0x001 -void * dlopen(const char * filename, int flag); -int dlclose(void * handle); -void * dlsym(void* handle, const char* name); -const char * dlerror(void); -#endif - -======= ->>>>>>> ggml-qnn: refine source code structure to make code more clearly -using pfn_rpc_mem_init = void (*)(void); -using pfn_rpc_mem_deinit = void (*)(void); -using pfn_rpc_mem_alloc = void *(*)(int, uint32_t, int); -using pfn_rpc_mem_free = void (*)(void *); -using pfn_rpc_mem_to_fd = int (*)(void *); -using _pfn_QnnSaver_initialize = decltype(QnnSaver_initialize); -using _pfn_QnnInterface_getProviders = decltype(QnnInterface_getProviders); -using _pfn_QnnSystemInterface_getProviders = decltype(QnnSystemInterface_getProviders); - -using qnn_res_t = std::tuple>; -using qnn_tensors_t = std::vector< Qnn_Tensor_t *>; - -enum class ggml_qnn_profile_level { - profile_off = 0, - profile_basic = 1, - profile_detail = 2 -}; - -enum qcom_htp_arch { - NONE = 0, - V68 = 68, - V69 = 69, - V73 = 73, - V75 = 75, - V79 = 79, -}; - -enum qcom_chipset_soc_model { - UNKNOWN_SM = 0, - SM7450 = 41, // v69, 7 Gen1 - SM8350 = 30, // v68, 888 - SM8450 = 36, // v69, SD 8 Gen 1 - SM8475 = 42, // v69, SD 8+ Gen 1 - SM8550 = 43, // v73, SD 8 Gen 2 - SM8650 = 57, // v75, SD 8 Gen 3 - SM8750 = 69, // v79, SD 8 Gen 4 -#if defined(_MSC_VER) - SC7280X = 44, - SC8280X = 37, - SC8380XP = 60, -#endif -}; - -struct qcom_socinfo { - uint32_t soc_model; - size_t htp_arch; - size_t vtcm_size_in_mb; - char soc_desc[GGML_MAX_NAME]; -}; - -struct ggml_backend_qnn_context { - int device; - int threads; - char name[GGML_MAX_NAME]; - char desc[GGML_MAX_NAME]; - char lib[GGML_MAX_NAME]; - qnn_instance * instance; - struct ggml_backend * backend; - QNN_INTERFACE_VER_TYPE raw_interface; - QNN_SYSTEM_INTERFACE_VER_TYPE raw_system_interface; - struct qcom_socinfo socinfo; - - std::unique_ptr work_data; - std::vector> tasks; - size_t work_size = 0; - size_t desired_size = 0; - int n_threads = GGML_DEFAULT_N_THREADS; -}; - -struct qnn_op_caps_t { - const char * qnn_op_name = nullptr; - const size_t input_param_count = 0; - const char * qnn_param_name = nullptr; -}; -<<<<<<< HEAD -extern const qnn_op_caps_t ggmlqnn_k_op_caps[]; -======= -extern const qnn_op_caps_t k_op_caps[]; ->>>>>>> ggml-qnn: refine source code structure to make code more clearly - -#if ENABLE_QNNBACKEND_PERF -class qnn_perf { -public: - qnn_perf(const std::string & perf_name) : _perf_name(std::move(perf_name)) {}; - qnn_perf() = delete; - qnn_perf(const qnn_perf & ) = delete; - qnn_perf & operator= (const qnn_perf & ) = delete; - - void start() { - _begin_time = ggml_time_us(); - } - - void info() { - _end_time = ggml_time_us(); - _duration = (_end_time - _begin_time); - GGMLQNN_LOG_DEBUG("duration of %s : %lld microseconds\n", _perf_name.c_str(), _duration); - } - -private: - int64_t _begin_time = 0LL; - int64_t _end_time = 0LL; - int64_t _duration = 0LL; - std::string _perf_name; -}; -#else -class qnn_perf { -public: -<<<<<<< HEAD - qnn_perf(const std::string & perf_name) { - GGML_UNUSED(perf_name); - } -======= - qnn_perf(const std::string & perf_name) {} ->>>>>>> ggml-qnn: refine source code structure to make code more clearly - qnn_perf() = delete; - qnn_perf(const qnn_perf & ) = delete; - qnn_perf & operator= (const qnn_perf & ) = delete; - - void start() {} - void info() {} -}; -#endif - -class qnn_interface { -#define DEFINE_SHIM_FUNCTION_INTERFACE(F, pointer_name) \ - template \ - inline auto qnn_##F(Args... args) const { \ - return (_qnn_interface->QNN_INTERFACE_VER_NAME.pointer_name)( \ - std::forward(args)...); \ - } - - -#define DEFINE_SHIM_FUNCTION_SYS_INTERFACE(F, pointer_name) \ - template \ - inline auto qnn_##F(Args... args) const { \ - return (_qnn_sys_interface->QNN_SYSTEM_INTERFACE_VER_NAME.pointer_name)( \ - std::forward(args)...); \ - } - - friend class qnn_instance; - -public: - qnn_interface() = default; - - // QnnBackend -<<<<<<< HEAD - DEFINE_SHIM_FUNCTION_INTERFACE(backend_create, backendCreate) - - DEFINE_SHIM_FUNCTION_INTERFACE(backend_free, backendFree) - - DEFINE_SHIM_FUNCTION_INTERFACE(backend_register_op_package, backendRegisterOpPackage) - - DEFINE_SHIM_FUNCTION_INTERFACE(backend_validate_op_config, backendValidateOpConfig) - - DEFINE_SHIM_FUNCTION_INTERFACE(backend_get_api_version, backendGetApiVersion) - - // QnnDevice - DEFINE_SHIM_FUNCTION_INTERFACE(device_create, deviceCreate) - - DEFINE_SHIM_FUNCTION_INTERFACE(device_free, deviceFree) - - DEFINE_SHIM_FUNCTION_INTERFACE(device_get_infrastructure, deviceGetInfrastructure) - - DEFINE_SHIM_FUNCTION_INTERFACE(device_get_platform_info, deviceGetPlatformInfo) - - DEFINE_SHIM_FUNCTION_INTERFACE(device_get_info, deviceGetInfo) - - // QnnContext - DEFINE_SHIM_FUNCTION_INTERFACE(context_create, contextCreate) - - DEFINE_SHIM_FUNCTION_INTERFACE(context_get_binary_size, contextGetBinarySize) - - DEFINE_SHIM_FUNCTION_INTERFACE(context_get_binary, contextGetBinary) - - DEFINE_SHIM_FUNCTION_INTERFACE(context_create_from_binary, contextCreateFromBinary) - - DEFINE_SHIM_FUNCTION_INTERFACE(context_free, contextFree) - - // QnnGraph - DEFINE_SHIM_FUNCTION_INTERFACE(graph_create, graphCreate) - - DEFINE_SHIM_FUNCTION_INTERFACE(graph_add_node, graphAddNode) - - DEFINE_SHIM_FUNCTION_INTERFACE(graph_finalize, graphFinalize) - - DEFINE_SHIM_FUNCTION_INTERFACE(graph_execute, graphExecute) - - DEFINE_SHIM_FUNCTION_INTERFACE(graph_retrieve, graphRetrieve) - - // QnnLog - DEFINE_SHIM_FUNCTION_INTERFACE(log_create, logCreate) - - DEFINE_SHIM_FUNCTION_INTERFACE(log_free, logFree) - - DEFINE_SHIM_FUNCTION_INTERFACE(log_set_log_level, logSetLogLevel) - - // QnnProfile - DEFINE_SHIM_FUNCTION_INTERFACE(profile_create, profileCreate) - - DEFINE_SHIM_FUNCTION_INTERFACE(profile_get_events, profileGetEvents) - - DEFINE_SHIM_FUNCTION_INTERFACE(profile_get_sub_events, profileGetSubEvents) - - DEFINE_SHIM_FUNCTION_INTERFACE(profile_get_event_data, profileGetEventData) - - DEFINE_SHIM_FUNCTION_INTERFACE(profile_free, profileFree) - - // QnnMem - DEFINE_SHIM_FUNCTION_INTERFACE(mem_register, memRegister) - - DEFINE_SHIM_FUNCTION_INTERFACE(mem_de_register, memDeRegister) - - // QnnProperty - DEFINE_SHIM_FUNCTION_INTERFACE(property_has_capability, propertyHasCapability) - - // QnnTensor - DEFINE_SHIM_FUNCTION_INTERFACE(tensor_create_context_tensor, tensorCreateContextTensor) - - DEFINE_SHIM_FUNCTION_INTERFACE(tensor_create_graph_tensor, tensorCreateGraphTensor) - - // QnnSystem - DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_create, systemContextCreate) - - DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_get_binary_info, systemContextGetBinaryInfo) - - DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_free, systemContextFree) -======= - DEFINE_SHIM_FUNCTION_INTERFACE(backend_create, backendCreate); - - DEFINE_SHIM_FUNCTION_INTERFACE(backend_free, backendFree); - - DEFINE_SHIM_FUNCTION_INTERFACE(backend_register_op_package, backendRegisterOpPackage); - - DEFINE_SHIM_FUNCTION_INTERFACE(backend_validate_op_config, backendValidateOpConfig); - - DEFINE_SHIM_FUNCTION_INTERFACE(backend_get_api_version, backendGetApiVersion); - - // QnnDevice - DEFINE_SHIM_FUNCTION_INTERFACE(device_create, deviceCreate); - - DEFINE_SHIM_FUNCTION_INTERFACE(device_free, deviceFree); - - DEFINE_SHIM_FUNCTION_INTERFACE(device_get_infrastructure, deviceGetInfrastructure); - - DEFINE_SHIM_FUNCTION_INTERFACE(device_get_platform_info, deviceGetPlatformInfo); - - DEFINE_SHIM_FUNCTION_INTERFACE(device_get_info, deviceGetInfo); - - // QnnContext - DEFINE_SHIM_FUNCTION_INTERFACE(context_create, contextCreate); - - DEFINE_SHIM_FUNCTION_INTERFACE(context_get_binary_size, contextGetBinarySize); - - DEFINE_SHIM_FUNCTION_INTERFACE(context_get_binary, contextGetBinary); - - DEFINE_SHIM_FUNCTION_INTERFACE(context_create_from_binary, contextCreateFromBinary); - - DEFINE_SHIM_FUNCTION_INTERFACE(context_free, contextFree); - - // QnnGraph - DEFINE_SHIM_FUNCTION_INTERFACE(graph_create, graphCreate); - - DEFINE_SHIM_FUNCTION_INTERFACE(graph_add_node, graphAddNode); - - DEFINE_SHIM_FUNCTION_INTERFACE(graph_finalize, graphFinalize); - - DEFINE_SHIM_FUNCTION_INTERFACE(graph_execute, graphExecute); - - DEFINE_SHIM_FUNCTION_INTERFACE(graph_retrieve, graphRetrieve); - - // QnnLog - DEFINE_SHIM_FUNCTION_INTERFACE(log_create, logCreate); - - DEFINE_SHIM_FUNCTION_INTERFACE(log_free, logFree); - - DEFINE_SHIM_FUNCTION_INTERFACE(log_set_log_level, logSetLogLevel); - - // QnnProfile - DEFINE_SHIM_FUNCTION_INTERFACE(profile_create, profileCreate); - - DEFINE_SHIM_FUNCTION_INTERFACE(profile_get_events, profileGetEvents); - - DEFINE_SHIM_FUNCTION_INTERFACE(profile_get_sub_events, profileGetSubEvents); - - DEFINE_SHIM_FUNCTION_INTERFACE(profile_get_event_data, profileGetEventData); - - DEFINE_SHIM_FUNCTION_INTERFACE(profile_free, profileFree); - - // QnnMem - DEFINE_SHIM_FUNCTION_INTERFACE(mem_register, memRegister); - - DEFINE_SHIM_FUNCTION_INTERFACE(mem_de_register, memDeRegister); - - // QnnProperty - DEFINE_SHIM_FUNCTION_INTERFACE(property_has_capability, propertyHasCapability); - - // QnnTensor - DEFINE_SHIM_FUNCTION_INTERFACE(tensor_create_context_tensor, tensorCreateContextTensor); - - DEFINE_SHIM_FUNCTION_INTERFACE(tensor_create_graph_tensor, tensorCreateGraphTensor); - - // QnnSystem - DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_create, systemContextCreate); - - DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_get_binary_info, systemContextGetBinaryInfo); - - DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_free, systemContextFree); ->>>>>>> ggml-qnn: refine source code structure to make code more clearly - - void set_qnn_interface(const QnnInterface_t * qnn_interface) { - _qnn_interface = qnn_interface; - } - - void set_qnn_system_interface(const QnnSystemInterface_t * qnn_sys_interface) { - _qnn_sys_interface = qnn_sys_interface; - } - - uint32_t get_backend_id() const { - return _qnn_interface->backendId; - } - - bool is_loaded() const { - return ((_qnn_sys_interface != nullptr) && (_qnn_interface != nullptr)); - } - -private: -<<<<<<< HEAD - const QnnInterface_t * _qnn_interface = nullptr; - - const QnnSystemInterface_t * _qnn_sys_interface = nullptr; -======= - const QnnInterface_t *_qnn_interface = nullptr; - - const QnnSystemInterface_t *_qnn_sys_interface = nullptr; ->>>>>>> ggml-qnn: refine source code structure to make code more clearly -}; - -class qnn_instance { -public: - using BackendIdType = decltype(QnnInterface_t{}.backendId); - - explicit qnn_instance(const std::string & lib_path, const std::string & backend_name, - const std::string & model_name) : - _lib_path(std::move(lib_path)), - _backend_name(std::move(backend_name)), -<<<<<<< HEAD - _model_name(std::move(model_name)) {} -======= - _model_name(std::move(model_name)) {}; ->>>>>>> ggml-qnn: refine source code structure to make code more clearly - - ~qnn_instance() { - } - - int qnn_init(const QnnSaver_Config_t ** saver_config); - - int qnn_finalize(); - - const qnn_interface & get_qnn_interface() { - if (!_qnn_interface.is_loaded()) { - GGMLQNN_LOG_WARN("pls check why _qnn_interface is not loaded\n"); - } - return _qnn_interface; - } - - const QNN_INTERFACE_VER_TYPE & get_qnn_raw_interface() { - if (!_qnn_interface.is_loaded()) { - GGMLQNN_LOG_WARN("pls check why _qnn_interface is not loaded\n"); - } - return _qnn_raw_interface; - } - - const QNN_SYSTEM_INTERFACE_VER_TYPE & get_qnn_raw_system_interface() { - if (!_qnn_interface.is_loaded()) { - GGMLQNN_LOG_WARN("pls check why _qnn_interface is not loaded\n"); - } - return _qnn_raw_system_interface; - } - -<<<<<<< HEAD - Qnn_LogHandle_t get_qnn_log_handle() { return _qnn_log_handle; } - - Qnn_ProfileHandle_t get_qnn_profile_handle() { return _qnn_profile_handle; } - - Qnn_DeviceHandle_t get_qnn_device_handle() { return _qnn_device_handle; } - - Qnn_BackendHandle_t get_qnn_backend_handle() { return _qnn_backend_handle; } - - Qnn_ContextHandle_t get_qnn_context_handle() { return _qnn_context_handle; } - - QnnSystemContext_Handle_t get_qnn_system_handle() { return _qnn_system_handle; } - - Qnn_GraphHandle_t get_qnn_graph_handle() { return _qnn_graph_handle; } -======= - const Qnn_LogHandle_t get_qnn_log_handle() { return _qnn_log_handle; } - - const Qnn_ProfileHandle_t get_qnn_profile_handle() { return _qnn_profile_handle; } - - const Qnn_DeviceHandle_t get_qnn_device_handle() { return _qnn_device_handle; } - - const Qnn_BackendHandle_t get_qnn_backend_handle() { return _qnn_backend_handle; } - - const Qnn_ContextHandle_t get_qnn_context_handle() { return _qnn_context_handle; } - - const QnnSystemContext_Handle_t get_qnn_system_handle() { return _qnn_system_handle; } - - const Qnn_GraphHandle_t get_qnn_graph_handle() { return _qnn_graph_handle; } ->>>>>>> ggml-qnn: refine source code structure to make code more clearly - - int init_qnn_graph(const char * graph_name, - bool debug, - uint8_t do_node_validation = 1, - const QnnGraph_Config_t ** graph_configs = nullptr - ); - int init_qnn_graph(const std::string & graph_name, QNNBackend device, size_t vtcm_size_in_mb = 8, size_t hvx_threads = 8); - - int finalize_qnn_graph(); - - bool is_valid_graph() const { return _qnn_graph_handle != nullptr; } - - int init_htp_perfinfra(); - - int set_rpc_polling(); - - int set_high_performance_mode(); - - std::string & get_qnn_graph_name() { return _graph_name; } - - bool is_rpcmem_initialized() { - return _rpcmem_initialized; - } - - void set_rpcmem_initialized(bool initialized) { - _rpcmem_initialized = initialized; - } - - size_t get_rpcmem_capacity() { return _rpcmem_capacity; } - size_t get_rpcmem_usage() { return _rpcmem_usage; } - - int32_t rpcmem_to_fd(void * buf); - - int register_rpcmem(void * p_data, Qnn_Tensor_t * p_tensor); - Qnn_MemHandle_t register_rpcmem(void * p_data, const uint32_t rank, uint32_t * dimensions, Qnn_DataType_t data_type); - - void unregister_rpcmem(); - void unregister_rpcmem(Qnn_MemHandle_t mem_handle); - - void * alloc_rpcmem(size_t bytes, size_t alignment); - void * get_rpcmem_from_memhandle(Qnn_MemHandle_t mem_handle); - - void free_rpcmem(void * buf); - void free_rpcmem(); - - bool is_rpcmem_allocated(void * buf); - - bool is_rpcmem_registered(Qnn_MemHandle_t handle) { - return _qnn_mem_set.count(handle) != 0U; - } - - bool enable_qnn_rpc() { - return _enable_qnn_rpc; - } - -public: - std::map>> _qnn_graph_map; - -private: - int load_system(); - - int unload_system(); - - int load_backend(std::string & lib_path, const QnnSaver_Config_t ** saver_config); - - int unload_backend(); - - void set_qnn_raw_interface(QNN_INTERFACE_VER_TYPE & raw_interface) { - _qnn_raw_interface = raw_interface; - } - - void set_qnn_raw_system_interface(QNN_SYSTEM_INTERFACE_VER_TYPE & raw_interface) { - _qnn_raw_system_interface = raw_interface; - } - - void * alloc_rpcmem_internal(size_t bytes, size_t alignment); - - void probe_device_meminfo(); - -private: - static constexpr const int _required_num_providers = 1; - -private: - std::string _lib_path; - std::string _backend_name; - std::string _model_name; // name of prebuilt QNN model, might be used in the future - BackendIdType _backend_id; - - bool _debug_tensor = false; // flag to indicate if requested graph is to be run in debug mode - bool _do_node_validations = true; // flag to indicate whether all add_node calls need to be validated - QnnLog_Level_t _qnn_log_level = QNN_LOG_LEVEL_DEBUG; - - ggml_qnn_profile_level _profile_level = ggml_qnn_profile_level::profile_detail; - - void * _system_lib_handle = nullptr; - - Qnn_GraphHandle_t _qnn_graph_handle = nullptr; - - Qnn_LogHandle_t _qnn_log_handle = nullptr; - - Qnn_ProfileHandle_t _qnn_profile_handle = nullptr; - - Qnn_DeviceHandle_t _qnn_device_handle = nullptr; - - Qnn_BackendHandle_t _qnn_backend_handle = nullptr; - - Qnn_ContextHandle_t _qnn_context_handle = nullptr; - - QnnSystemContext_Handle_t _qnn_system_handle = nullptr; - - QnnHtpDevice_PerfInfrastructure_t * _qnn_htp_perfinfra = nullptr; - uint32_t _qnn_power_configid = 1; - uint32_t _qnn_rpc_pollingtime = 9999; // 0-10000 us for high performing - - qnn_interface _qnn_interface; - QNN_INTERFACE_VER_TYPE _qnn_raw_interface; - QNN_SYSTEM_INTERFACE_VER_TYPE _qnn_raw_system_interface; - - std::unordered_map _qnn_mem_set; - std::unordered_map _qnn_rpc_buffer_to_handles; - - static std::mutex _init_mutex; - static std::unordered_map _loaded_lib_handle; - static std::unordered_map _lib_path_to_backend_id; - static std::unordered_map _loaded_backend; - - std::atomic_bool _rpcmem_initialized{false}; - pfn_rpc_mem_alloc _pfn_rpc_mem_alloc; - pfn_rpc_mem_free _pfn_rpc_mem_free; - pfn_rpc_mem_to_fd _pfn_rpc_mem_to_fd; - pfn_rpc_mem_init _pfn_rpc_mem_init; - pfn_rpc_mem_deinit _pfn_rpc_mem_deinit; - std::unordered_map _rpcmem_store_map; - std::unordered_map _rpcmem_usage_map; - size_t _rpcmem_usage = 0; // mempool usage in Mbytes - size_t _rpcmem_capacity = 512; // mempool size in Mbytes - - std::string _graph_name; - QNNBackend _device_id; - void * _rpc_lib_handle = nullptr; - bool _enable_qnn_rpc = false; //TODO:unknown issue with QNN RPC feature - - DISABLE_COPY(qnn_instance); - DISABLE_MOVE(qnn_instance); -}; - -size_t ggmlqnn_get_opcaps_size(void); -size_t ggmlqnn_get_op_index(const ggml_tensor * tensor); -Qnn_Tensor_t * ggmlqnn_create_compute_tensor(const ggml_tensor * tensor); -const char * ggmlqnn_get_error_string(Qnn_ErrorHandle_t qnn_error_code); -Qnn_DataType_t ggmlqnn_datatype_from_ggml_datatype(enum ggml_type ggmltype); -void * ggmlqnn_type_trait(ggml_backend_qnn_context * ctx, ggml_tensor * op); -void ggmlqnn_get_graphkey_from_op(const ggml_tensor * op, std::string & output); -<<<<<<< HEAD -======= -bool ggmlqnn_is_valid_params(ggml_backend_qnn_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); ->>>>>>> ggml-qnn: refine source code structure to make code more clearly -uint8_t * ggmlqnn_create_rpc_buffer(qnn_instance * instance, const ggml_tensor * ggml_tensor, Qnn_Tensor_t * qnn_tensor, bool b_copydata); -void ggmlqnn_print_tensors_info(const char * func_name, ggml_backend_qnn_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); - -Qnn_OpConfig_t ggmlqnn_create_op_config(const char * name, const char * package, const char * type, - Qnn_Param_t * params, uint32_t num_params, - Qnn_Tensor_t * inputs, uint32_t num_inputs, - Qnn_Tensor_t * outputs, uint32_t num_outputs); -Qnn_Tensor_t * ggmlqnn_create_general_tensor(const ggml_tensor * tensor, const char * name, - Qnn_TensorType_t qnn_tensor_type, - Qnn_DataType_t qnn_data_type, - uint32_t rank, uint32_t * dims, - void * data, uint32_t data_size, - bool b_transpose = false); diff --git a/ggml/src/ggml-qnn/ggml-qnn.cpp b/ggml/src/ggml-qnn/ggml-qnn.cpp index 7704a4ad038f7..2818a5e3e10d2 100644 --- a/ggml/src/ggml-qnn/ggml-qnn.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn.cpp @@ -62,6 +62,9 @@ void ggmlqnn_log_internal(ggml_log_level level, const char * file, const char * #if (defined __ANDROID__) || (defined ANDROID) //for Android application(standard APP or command line tool) __android_log_print(ANDROID_LOG_INFO, "ggml-qnn", "%s\n", s_ggmlqnn_log_internal_buf); + if (GGML_LOG_LEVEL_INFO == level) { + printf("%s\n", s_ggmlqnn_log_internal_buf); + } #else //for Snapdragon based WoA(Windows on ARM) device or Linux printf("%s\n", s_ggmlqnn_log_internal_buf); @@ -1038,25 +1041,25 @@ void * ggmlqnn_type_trait(ggml_backend_qnn_context * ctx, ggml_tensor * op) { void * wdata = ctx->work_data.get(); // convert src0 to float if (src0_type != GGML_TYPE_F32) { - const auto *type_traits = ggml_get_type_traits(src0_type); - ggml_to_float_t const to_float = type_traits->to_float; + const auto * type_traits = ggml_get_type_traits(src0_type); + ggml_to_float_t const to_float = type_traits->to_float; for (int64_t i03 = 0; i03 < ne03; i03++) { for (int64_t i02 = 0; i02 < ne02; i02++) { - const void *x = (char *) src0->data + i02 * nb02 + i03 * nb03; - float *const wplane = (float *) wdata + i02 * ne_plane + i03 * ne02 * ne_plane; + const void * x = (char *)src0->data + i02 * nb02 + i03 * nb03; + float * const wplane = (float *)wdata + i02 * ne_plane + i03 * ne02 * ne_plane; const int min_cols_per_thread = 4096; - const int min_rows_per_thread = std::max((int) (min_cols_per_thread / ne00), 1); + const int min_rows_per_thread = std::max((int)(min_cols_per_thread / ne00), 1); const int n_threads = std::max( - std::min(ctx->n_threads, (int) (ne01 / min_rows_per_thread)), 1); + std::min(ctx->n_threads, (int)(ne01 / min_rows_per_thread)), 1); for (int i = 1; i < n_threads; i++) { const int64_t start = i * ne01 / n_threads; - const int64_t end = (i + 1) * ne01 / n_threads; + const int64_t end = (i + 1) * ne01 / n_threads; if (start < end) { ctx->tasks.push_back(std::async(std::launch::async, [=]() { for (int64_t i01 = start; i01 < end; i01++) { - to_float((const char *) x + i01 * nb01, wplane + i01 * ne00, ne00); + to_float((const char *)x + i01 * nb01, wplane + i01 * ne00, ne00); } })); } @@ -1996,7 +1999,7 @@ int qnn_instance::init_qnn_graph(const std::string & graph_name, QNNBackend devi return error; } - GGMLQNN_LOG_INFO("[%s]create graph %s succeed", ggml_backend_qnn_get_devname(device), graph_name.c_str()); + GGMLQNN_LOG_DEBUG("[%s]create graph %s succeed", ggml_backend_qnn_get_devname(device), graph_name.c_str()); _qnn_graph_handle = graph_handle; return QNN_SUCCESS; } From cd2f21118dbe927b00bed1eff8c9c31251435edd Mon Sep 17 00:00:00 2001 From: zhouwg Date: Fri, 28 Feb 2025 22:34:46 +0800 Subject: [PATCH 31/76] ggml-qnn: Windows port --- step2 --- ggml/src/ggml-qnn/ggml-qnn-impl.h | 609 ++++++++++++++++++++++++++++++ ggml/src/ggml-qnn/ggml-qnn.cpp | 99 +++-- 2 files changed, 670 insertions(+), 38 deletions(-) create mode 100644 ggml/src/ggml-qnn/ggml-qnn-impl.h diff --git a/ggml/src/ggml-qnn/ggml-qnn-impl.h b/ggml/src/ggml-qnn/ggml-qnn-impl.h new file mode 100644 index 0000000000000..0f0daba6e1a93 --- /dev/null +++ b/ggml/src/ggml-qnn/ggml-qnn-impl.h @@ -0,0 +1,609 @@ +/* +* Copyright (c) 2023-2024 The ggml authors +* +* Permission is hereby granted, free of charge, to any person obtaining a copy +* of this software and associated documentation files (the "Software"), to +* deal in the Software without restriction, including without limitation the +* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +* sell copies of the Software, and to permit persons to whom the Software is +* furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +* IN THE SOFTWARE. +*/ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#if defined(__ANDROID__) || defined(__linux__) +#include +#include +#include +#include +#include +#include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if (defined __ANDROID__) || (defined ANDROID) +#include "android/log.h" +#endif + +#if defined(_WIN32) +#include +#include +#endif + +#include "QnnTypes.h" +#include "QnnCommon.h" +#include "QnnContext.h" +#include "QnnBackend.h" +#include "QnnGraph.h" +#include "QnnProperty.h" +#include "QnnTensor.h" +#include "QnnInterface.h" +#include "Saver/QnnSaver.h" +#include "System/QnnSystemInterface.h" +#include "HTP/QnnHtpDevice.h" +#include "HTP/QnnHtpGraph.h" + +#include "ggml-qnn.h" +#include "ggml-impl.h" +#include "ggml-backend-impl.h" + +class qnn_instance; +struct ggml_backend_qnn_context; +void ggmlqnn_log_internal(ggml_log_level level, const char * file, const char * func, int line, const char * format, ...); + +#if 0//def NDEBUG +#define GGMLQNN_DEBUG 0 +#define ENABLE_QNNBACKEND_PERF 0 // enable/disable op's perf info +#define GGMLQNN_PRINT_QNN_INTERNAL_LOG 0 // enable/disable QNN's internal log +#define GGMLQNN_PRINT_OP_ADD_LOG 0 // GGML_OP_ADD already verified with QNN-CPU / QNN-GPU / QNN-NPU +#define GGMLQNN_PRINT_OP_MUL_MAT_LOG 0 +#else +#define GGMLQNN_DEBUG 1 // for troubleshooting QNN backend +#define ENABLE_QNNBACKEND_PERF 0 // enable/disable op's perf info +#define GGMLQNN_PRINT_QNN_INTERNAL_LOG 0 // enable/disable QNN's internal log +#define GGMLQNN_PRINT_OP_ADD_LOG 0 // GGML_OP_ADD already verified with QNN-CPU / QNN-GPU / QNN-NPU +#define GGMLQNN_PRINT_OP_MUL_MAT_LOG 1 +#endif +#define GGML_QNN_LOGBUF_LEN 4096 + +#define GGMLQNN_LOG_ERROR(...) ggmlqnn_log_internal(GGML_LOG_LEVEL_ERROR, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) +#define GGMLQNN_LOG_WARN(...) ggmlqnn_log_internal(GGML_LOG_LEVEL_WARN , __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) +#define GGMLQNN_LOG_INFO(...) ggmlqnn_log_internal(GGML_LOG_LEVEL_INFO , __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) + +#if GGMLQNN_DEBUG +#define GGMLQNN_LOG_DEBUG(...) ggmlqnn_log_internal(GGML_LOG_LEVEL_DEBUG, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) +#else +#define GGMLQNN_LOG_DEBUG(...) +#endif + +#define CHECK_QNN_API(error, result) \ + do { \ + error = (result); \ + if (QNN_SUCCESS != error) { \ + if (error == QNN_COMMON_ERROR_NOT_SUPPORTED) { \ + GGMLQNN_LOG_WARN("WARNING: QNN feature/API not supported\n"); \ + } else { \ + GGMLQNN_LOG_INFO("QNN API error = %d(%s)\n", error, ggmlqnn_get_error_string(error)); \ + } \ + } \ + } while (0) + +#define QNN_VER_PTR(x) (&((x).v1)) +#define RPCMEM_DEFAULT_FLAGS 1 +#define RPCMEM_HEAP_ID_SYSTEM 25 + +#define DISABLE_COPY(class_name) \ + class_name(const class_name &) = delete; \ + void operator=(const class_name &) = delete + +#define DISABLE_MOVE(class_name) \ + class_name(class_name &&) = delete; \ + void operator=(class_name &&) = delete + +#define GQCGT ggmlqnn_create_general_tensor + +#if defined(_WIN32) +#define RTLD_GLOBAL 0x100 +#define RTLD_LOCAL 0x000 +#define RTLD_LAZY 0x000 +#define RTLD_NOW 0x001 +void * dlopen(const char * filename, int flag); +int dlclose(void * handle); +void * dlsym(void* handle, const char* name); +const char * dlerror(void); +#endif + +using pfn_rpc_mem_init = void (*)(void); +using pfn_rpc_mem_deinit = void (*)(void); +using pfn_rpc_mem_alloc = void *(*)(int, uint32_t, int); +using pfn_rpc_mem_free = void (*)(void *); +using pfn_rpc_mem_to_fd = int (*)(void *); +using _pfn_QnnSaver_initialize = decltype(QnnSaver_initialize); +using _pfn_QnnInterface_getProviders = decltype(QnnInterface_getProviders); +using _pfn_QnnSystemInterface_getProviders = decltype(QnnSystemInterface_getProviders); + +using qnn_res_t = std::tuple>; +using qnn_tensors_t = std::vector< Qnn_Tensor_t *>; + +enum class ggml_qnn_profile_level { + profile_off = 0, + profile_basic = 1, + profile_detail = 2 +}; + +enum qcom_htp_arch { + NONE = 0, + V68 = 68, + V69 = 69, + V73 = 73, + V75 = 75, + V79 = 79, +}; + +enum qcom_chipset_soc_model { + UNKNOWN_SM = 0, + SM7450 = 41, // v69, 7 Gen1 + SM8350 = 30, // v68, 888 + SM8450 = 36, // v69, SD 8 Gen 1 + SM8475 = 42, // v69, SD 8+ Gen 1 + SM8550 = 43, // v73, SD 8 Gen 2 + SM8650 = 57, // v75, SD 8 Gen 3 + SM8750 = 69, // v79, SD 8 Gen 4 +#if defined(_MSC_VER) + SC7280X = 44, + SC8280X = 37, + SC8380XP = 60, +#endif +}; + +struct qcom_socinfo { + uint32_t soc_model; + size_t htp_arch; + size_t vtcm_size_in_mb; + char soc_desc[GGML_MAX_NAME]; +}; + +struct ggml_backend_qnn_context { + int device; + int threads; + char name[GGML_MAX_NAME]; + char desc[GGML_MAX_NAME]; + char lib[GGML_MAX_NAME]; + qnn_instance * instance; + struct ggml_backend * backend; + QNN_INTERFACE_VER_TYPE raw_interface; + QNN_SYSTEM_INTERFACE_VER_TYPE raw_system_interface; + struct qcom_socinfo socinfo; + + std::unique_ptr work_data; + std::vector> tasks; + size_t work_size = 0; + size_t desired_size = 0; + int n_threads = GGML_DEFAULT_N_THREADS; +}; + +struct qnn_op_caps_t { + const char * qnn_op_name = nullptr; + const size_t input_param_count = 0; + const char * qnn_param_name = nullptr; +}; +extern const qnn_op_caps_t k_op_caps[]; + +#if ENABLE_QNNBACKEND_PERF +class qnn_perf { +public: + qnn_perf(const std::string & perf_name) : _perf_name(std::move(perf_name)) {}; + qnn_perf() = delete; + qnn_perf(const qnn_perf & ) = delete; + qnn_perf & operator= (const qnn_perf & ) = delete; + + void start() { + _begin_time = ggml_time_us(); + } + + void info() { + _end_time = ggml_time_us(); + _duration = (_end_time - _begin_time); + GGMLQNN_LOG_DEBUG("duration of %s : %lld microseconds\n", _perf_name.c_str(), _duration); + } + +private: + int64_t _begin_time = 0LL; + int64_t _end_time = 0LL; + int64_t _duration = 0LL; + std::string _perf_name; +}; +#else +class qnn_perf { +public: + qnn_perf(const std::string & perf_name) {} + qnn_perf() = delete; + qnn_perf(const qnn_perf & ) = delete; + qnn_perf & operator= (const qnn_perf & ) = delete; + + void start() {} + void info() {} +}; +#endif + +class qnn_interface { +#define DEFINE_SHIM_FUNCTION_INTERFACE(F, pointer_name) \ + template \ + inline auto qnn_##F(Args... args) const { \ + return (_qnn_interface->QNN_INTERFACE_VER_NAME.pointer_name)( \ + std::forward(args)...); \ + } + + +#define DEFINE_SHIM_FUNCTION_SYS_INTERFACE(F, pointer_name) \ + template \ + inline auto qnn_##F(Args... args) const { \ + return (_qnn_sys_interface->QNN_SYSTEM_INTERFACE_VER_NAME.pointer_name)( \ + std::forward(args)...); \ + } + + friend class qnn_instance; + +public: + qnn_interface() = default; + + // QnnBackend + DEFINE_SHIM_FUNCTION_INTERFACE(backend_create, backendCreate); + + DEFINE_SHIM_FUNCTION_INTERFACE(backend_free, backendFree); + + DEFINE_SHIM_FUNCTION_INTERFACE(backend_register_op_package, backendRegisterOpPackage); + + DEFINE_SHIM_FUNCTION_INTERFACE(backend_validate_op_config, backendValidateOpConfig); + + DEFINE_SHIM_FUNCTION_INTERFACE(backend_get_api_version, backendGetApiVersion); + + // QnnDevice + DEFINE_SHIM_FUNCTION_INTERFACE(device_create, deviceCreate); + + DEFINE_SHIM_FUNCTION_INTERFACE(device_free, deviceFree); + + DEFINE_SHIM_FUNCTION_INTERFACE(device_get_infrastructure, deviceGetInfrastructure); + + DEFINE_SHIM_FUNCTION_INTERFACE(device_get_platform_info, deviceGetPlatformInfo); + + DEFINE_SHIM_FUNCTION_INTERFACE(device_get_info, deviceGetInfo); + + // QnnContext + DEFINE_SHIM_FUNCTION_INTERFACE(context_create, contextCreate); + + DEFINE_SHIM_FUNCTION_INTERFACE(context_get_binary_size, contextGetBinarySize); + + DEFINE_SHIM_FUNCTION_INTERFACE(context_get_binary, contextGetBinary); + + DEFINE_SHIM_FUNCTION_INTERFACE(context_create_from_binary, contextCreateFromBinary); + + DEFINE_SHIM_FUNCTION_INTERFACE(context_free, contextFree); + + // QnnGraph + DEFINE_SHIM_FUNCTION_INTERFACE(graph_create, graphCreate); + + DEFINE_SHIM_FUNCTION_INTERFACE(graph_add_node, graphAddNode); + + DEFINE_SHIM_FUNCTION_INTERFACE(graph_finalize, graphFinalize); + + DEFINE_SHIM_FUNCTION_INTERFACE(graph_execute, graphExecute); + + DEFINE_SHIM_FUNCTION_INTERFACE(graph_retrieve, graphRetrieve); + + // QnnLog + DEFINE_SHIM_FUNCTION_INTERFACE(log_create, logCreate); + + DEFINE_SHIM_FUNCTION_INTERFACE(log_free, logFree); + + DEFINE_SHIM_FUNCTION_INTERFACE(log_set_log_level, logSetLogLevel); + + // QnnProfile + DEFINE_SHIM_FUNCTION_INTERFACE(profile_create, profileCreate); + + DEFINE_SHIM_FUNCTION_INTERFACE(profile_get_events, profileGetEvents); + + DEFINE_SHIM_FUNCTION_INTERFACE(profile_get_sub_events, profileGetSubEvents); + + DEFINE_SHIM_FUNCTION_INTERFACE(profile_get_event_data, profileGetEventData); + + DEFINE_SHIM_FUNCTION_INTERFACE(profile_free, profileFree); + + // QnnMem + DEFINE_SHIM_FUNCTION_INTERFACE(mem_register, memRegister); + + DEFINE_SHIM_FUNCTION_INTERFACE(mem_de_register, memDeRegister); + + // QnnProperty + DEFINE_SHIM_FUNCTION_INTERFACE(property_has_capability, propertyHasCapability); + + // QnnTensor + DEFINE_SHIM_FUNCTION_INTERFACE(tensor_create_context_tensor, tensorCreateContextTensor); + + DEFINE_SHIM_FUNCTION_INTERFACE(tensor_create_graph_tensor, tensorCreateGraphTensor); + + // QnnSystem + DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_create, systemContextCreate); + + DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_get_binary_info, systemContextGetBinaryInfo); + + DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_free, systemContextFree); + + void set_qnn_interface(const QnnInterface_t * qnn_interface) { + _qnn_interface = qnn_interface; + } + + void set_qnn_system_interface(const QnnSystemInterface_t * qnn_sys_interface) { + _qnn_sys_interface = qnn_sys_interface; + } + + uint32_t get_backend_id() const { + return _qnn_interface->backendId; + } + + bool is_loaded() const { + return ((_qnn_sys_interface != nullptr) && (_qnn_interface != nullptr)); + } + +private: + const QnnInterface_t * _qnn_interface = nullptr; + + const QnnSystemInterface_t * _qnn_sys_interface = nullptr; +}; + +class qnn_instance { +public: + using BackendIdType = decltype(QnnInterface_t{}.backendId); + + explicit qnn_instance(const std::string & lib_path, const std::string & backend_name, + const std::string & model_name) : + _lib_path(std::move(lib_path)), + _backend_name(std::move(backend_name)), + _model_name(std::move(model_name)) {}; + + ~qnn_instance() { + } + + int qnn_init(const QnnSaver_Config_t ** saver_config); + + int qnn_finalize(); + + const qnn_interface & get_qnn_interface() { + if (!_qnn_interface.is_loaded()) { + GGMLQNN_LOG_WARN("pls check why _qnn_interface is not loaded\n"); + } + return _qnn_interface; + } + + const QNN_INTERFACE_VER_TYPE & get_qnn_raw_interface() { + if (!_qnn_interface.is_loaded()) { + GGMLQNN_LOG_WARN("pls check why _qnn_interface is not loaded\n"); + } + return _qnn_raw_interface; + } + + const QNN_SYSTEM_INTERFACE_VER_TYPE & get_qnn_raw_system_interface() { + if (!_qnn_interface.is_loaded()) { + GGMLQNN_LOG_WARN("pls check why _qnn_interface is not loaded\n"); + } + return _qnn_raw_system_interface; + } + + const Qnn_LogHandle_t get_qnn_log_handle() { return _qnn_log_handle; } + + const Qnn_ProfileHandle_t get_qnn_profile_handle() { return _qnn_profile_handle; } + + const Qnn_DeviceHandle_t get_qnn_device_handle() { return _qnn_device_handle; } + + const Qnn_BackendHandle_t get_qnn_backend_handle() { return _qnn_backend_handle; } + + const Qnn_ContextHandle_t get_qnn_context_handle() { return _qnn_context_handle; } + + const QnnSystemContext_Handle_t get_qnn_system_handle() { return _qnn_system_handle; } + + const Qnn_GraphHandle_t get_qnn_graph_handle() { return _qnn_graph_handle; } + + int init_qnn_graph(const char * graph_name, + bool debug, + uint8_t do_node_validation = 1, + const QnnGraph_Config_t ** graph_configs = nullptr + ); + int init_qnn_graph(const std::string & graph_name, QNNBackend device, size_t vtcm_size_in_mb = 8, size_t hvx_threads = 8); + + int finalize_qnn_graph(); + + bool is_valid_graph() const { return _qnn_graph_handle != nullptr; } + + int init_htp_perfinfra(); + + int set_rpc_polling(); + + int set_high_performance_mode(); + + std::string & get_qnn_graph_name() { return _graph_name; } + + bool is_rpcmem_initialized() { + return _rpcmem_initialized; + } + + void set_rpcmem_initialized(bool initialized) { + _rpcmem_initialized = initialized; + } + + size_t get_rpcmem_capacity() { return _rpcmem_capacity; } + size_t get_rpcmem_usage() { return _rpcmem_usage; } + + int32_t rpcmem_to_fd(void * buf); + + int register_rpcmem(void * p_data, Qnn_Tensor_t * p_tensor); + Qnn_MemHandle_t register_rpcmem(void * p_data, const uint32_t rank, uint32_t * dimensions, Qnn_DataType_t data_type); + + void unregister_rpcmem(); + void unregister_rpcmem(Qnn_MemHandle_t mem_handle); + + void * alloc_rpcmem(size_t bytes, size_t alignment); + void * get_rpcmem_from_memhandle(Qnn_MemHandle_t mem_handle); + + void free_rpcmem(void * buf); + void free_rpcmem(); + + bool is_rpcmem_allocated(void * buf); + + bool is_rpcmem_registered(Qnn_MemHandle_t handle) { + return _qnn_mem_set.count(handle) != 0U; + } + + bool enable_qnn_rpc() { + return _enable_qnn_rpc; + } + +public: + std::map>> _qnn_graph_map; + +private: + int load_system(); + + int unload_system(); + + int load_backend(std::string & lib_path, const QnnSaver_Config_t ** saver_config); + + int unload_backend(); + + void set_qnn_raw_interface(QNN_INTERFACE_VER_TYPE & raw_interface) { + _qnn_raw_interface = raw_interface; + } + + void set_qnn_raw_system_interface(QNN_SYSTEM_INTERFACE_VER_TYPE & raw_interface) { + _qnn_raw_system_interface = raw_interface; + } + + void * alloc_rpcmem_internal(size_t bytes, size_t alignment); + + void probe_device_meminfo(); + +private: + static constexpr const int _required_num_providers = 1; + +private: + std::string _lib_path; + std::string _backend_name; + std::string _model_name; // name of prebuilt QNN model, might be used in the future + BackendIdType _backend_id; + + bool _debug_tensor = false; // flag to indicate if requested graph is to be run in debug mode + bool _do_node_validations = true; // flag to indicate whether all add_node calls need to be validated + QnnLog_Level_t _qnn_log_level = QNN_LOG_LEVEL_DEBUG; + + ggml_qnn_profile_level _profile_level = ggml_qnn_profile_level::profile_detail; + + void * _system_lib_handle = nullptr; + + Qnn_GraphHandle_t _qnn_graph_handle = nullptr; + + Qnn_LogHandle_t _qnn_log_handle = nullptr; + + Qnn_ProfileHandle_t _qnn_profile_handle = nullptr; + + Qnn_DeviceHandle_t _qnn_device_handle = nullptr; + + Qnn_BackendHandle_t _qnn_backend_handle = nullptr; + + Qnn_ContextHandle_t _qnn_context_handle = nullptr; + + QnnSystemContext_Handle_t _qnn_system_handle = nullptr; + + QnnHtpDevice_PerfInfrastructure_t * _qnn_htp_perfinfra = nullptr; + uint32_t _qnn_power_configid = 1; + uint32_t _qnn_rpc_pollingtime = 9999; // 0-10000 us for high performing + + qnn_interface _qnn_interface; + QNN_INTERFACE_VER_TYPE _qnn_raw_interface; + QNN_SYSTEM_INTERFACE_VER_TYPE _qnn_raw_system_interface; + + std::unordered_map _qnn_mem_set; + std::unordered_map _qnn_rpc_buffer_to_handles; + + static std::mutex _init_mutex; + static std::unordered_map _loaded_lib_handle; + static std::unordered_map _lib_path_to_backend_id; + static std::unordered_map _loaded_backend; + + std::atomic_bool _rpcmem_initialized{false}; + pfn_rpc_mem_alloc _pfn_rpc_mem_alloc; + pfn_rpc_mem_free _pfn_rpc_mem_free; + pfn_rpc_mem_to_fd _pfn_rpc_mem_to_fd; + pfn_rpc_mem_init _pfn_rpc_mem_init; + pfn_rpc_mem_deinit _pfn_rpc_mem_deinit; + std::unordered_map _rpcmem_store_map; + std::unordered_map _rpcmem_usage_map; + size_t _rpcmem_usage = 0; // mempool usage in Mbytes + size_t _rpcmem_capacity = 512; // mempool size in Mbytes + + std::string _graph_name; + QNNBackend _device_id; + void * _rpc_lib_handle = nullptr; + bool _enable_qnn_rpc = false; //TODO:unknown issue with QNN RPC feature + + DISABLE_COPY(qnn_instance); + DISABLE_MOVE(qnn_instance); +}; + +size_t ggmlqnn_get_opcaps_size(void); +size_t ggmlqnn_get_op_index(const ggml_tensor * tensor); +Qnn_Tensor_t * ggmlqnn_create_compute_tensor(const ggml_tensor * tensor); +const char * ggmlqnn_get_error_string(Qnn_ErrorHandle_t qnn_error_code); +Qnn_DataType_t ggmlqnn_datatype_from_ggml_datatype(enum ggml_type ggmltype); +void * ggmlqnn_type_trait(ggml_backend_qnn_context * ctx, ggml_tensor * op); +void ggmlqnn_get_graphkey_from_op(const ggml_tensor * op, std::string & output); +uint8_t * ggmlqnn_create_rpc_buffer(qnn_instance * instance, const ggml_tensor * ggml_tensor, Qnn_Tensor_t * qnn_tensor, bool b_copydata); +void ggmlqnn_print_tensors_info(const char * func_name, ggml_backend_qnn_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); + +Qnn_OpConfig_t ggmlqnn_create_op_config(const char * name, const char * package, const char * type, + Qnn_Param_t * params, uint32_t num_params, + Qnn_Tensor_t * inputs, uint32_t num_inputs, + Qnn_Tensor_t * outputs, uint32_t num_outputs); +Qnn_Tensor_t * ggmlqnn_create_general_tensor(const ggml_tensor * tensor, const char * name, + Qnn_TensorType_t qnn_tensor_type, + Qnn_DataType_t qnn_data_type, + uint32_t rank, uint32_t * dims, + void * data, uint32_t data_size, + bool b_transpose = false); diff --git a/ggml/src/ggml-qnn/ggml-qnn.cpp b/ggml/src/ggml-qnn/ggml-qnn.cpp index 2818a5e3e10d2..51678f7b51ca3 100644 --- a/ggml/src/ggml-qnn/ggml-qnn.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn.cpp @@ -77,6 +77,48 @@ void ggmlqnn_log_internal(ggml_log_level level, const char * file, const char * // ================================================================================================= // section-3: general helper macro / data structure / function // ================================================================================================= +#if defined(_WIN32) +static const char * last_func = nullptr; +static long last_err; +void * dlopen(const char * dll, int flags) { + HINSTANCE h = LoadLibraryA(dll); + if (h == NULL) { + last_err = GetLastError(); + last_func = "dlopen"; + } + return h; +} + +int dlclose(void * h) { + if (!FreeLibrary((HINSTANCE)h)) { + last_err = GetLastError(); + last_func = "dlclose"; + return -1; + } + return 0; +} + +void * dlsym(void * h, const char * name) { + FARPROC p = GetProcAddress((HINSTANCE)h, name); + if (!p) { + last_err = GetLastError(); + last_func = "dlsym"; + } + return (void*)(intptr_t)p; +} + +const char * dlerror(void) { + static char str[512]; + if (!last_err) return nullptr; + + snprintf(str, 512, "%s error #%ld", last_func, last_err); + last_err = 0; + last_func = NULL; + + return str; +} +#endif + static intptr_t ggmlqnn_align_to(size_t alignment, intptr_t offset) { return offset % alignment == 0 ? offset : offset + @@ -94,7 +136,7 @@ static size_t get_system_total_memory_in_bytes() { auto page_size = (size_t)sysconf(_SC_PAGE_SIZE); return pages * page_size; -#elif defined(_WIN32) || defined(_MSC_VER) +#elif defined(_WIN32) //TODO: Snapdragon based WoA(Windows on ARM) return 0; #else @@ -112,7 +154,7 @@ static size_t get_system_free_memory_in_bytes() { auto page_size = (size_t)sysconf(_SC_PAGE_SIZE); return avail_pages * page_size; -#elif defined(_WIN32) || defined(_MSC_VER) +#elif defined(_WIN32) //TODO: Snapdragon based WoA(Windows on ARM) return 0; #else @@ -143,7 +185,7 @@ static void * ggmlqnn_host_malloc(size_t n) { GGMLQNN_LOG_WARN("%s: error: posix_memalign failed\n", __func__); return nullptr; } -#elif defined(_WIN32) || defined(_MSC_VER) +#elif defined(_WIN32) //TODO: Snapdragon based WoA(Windows on ARM) return nullptr; #else @@ -569,7 +611,7 @@ static struct qcom_socinfo g_qnn_soc_info_table[] = { .vtcm_size_in_mb = 8, .soc_desc = "Qualcomm SnapDragon 8 Gen 4"}, -#if defined(_MSC_VER) +#if defined(_WIN32) /* Qualcomm SnapDragon 7c Gen 2 */ [SC7280X] = { .soc_model = SC7280X, @@ -619,7 +661,7 @@ static struct ggml_backend_qnn_context g_qnn_mgr[GGML_QNN_MAX_DEVICES] = { .threads = 1, .name = "qnn-cpu", .desc = "Qualcomm Kryo CPU", -#if defined(_MSC_VER) +#if defined(_WIN32) .lib = "QnnCpu.dll", #else .lib = "libQnnCpu.so", @@ -634,7 +676,7 @@ static struct ggml_backend_qnn_context g_qnn_mgr[GGML_QNN_MAX_DEVICES] = { .threads = 1, .name = "qnn-gpu", .desc = "Qualcomm Adreno GPU", -#if defined(_MSC_VER) +#if defined(_WIN32) .lib = "QnnGpu.dll", #else .lib = "libQnnGpu.so", @@ -649,7 +691,7 @@ static struct ggml_backend_qnn_context g_qnn_mgr[GGML_QNN_MAX_DEVICES] = { .threads = 1, .name = "qnn-npu", .desc = "Qualcomm NPU(Hexagon Tensor Processor)", -#if defined(_MSC_VER) +#if defined(_WIN32) .lib = "QnnHtp.dll", #else .lib = "libQnnHtp.so", @@ -1160,14 +1202,7 @@ bool ggmlqnn_is_valid_params(ggml_backend_qnn_context * ctx, const ggml_tensor * template Fn load_qnn_functionpointers(void * handle, const char * function_name) { -#if defined(__ANDROID__) || defined(__linux__) return reinterpret_cast(dlsym(handle, function_name)); -#elif defined(_WIN32) || defined(_MSC_VER) - //TODO: Snapdragon based WoA(Windows on ARM) - return nullptr; -#else -#error "ggml-qnn only support WoA, Android, Linux" -#endif } std::mutex qnn_instance::_init_mutex; @@ -1419,14 +1454,7 @@ int qnn_instance::load_backend(std::string & lib_path, const QnnSaver_Config_t * Qnn_ErrorHandle_t error = QNN_SUCCESS; GGMLQNN_LOG_DEBUG("lib_path:%s\n", lib_path.c_str()); -#if defined(__ANDROID__) || defined(__linux__) void * lib_handle = dlopen(lib_path.c_str(), RTLD_NOW | RTLD_GLOBAL); -#elif defined(_WIN32) || defined(_MSC_VER) - //TODO: Snapdragon based WoA(Windows on ARM) - void * lib_handle = nullptr; -#else -#error "ggml-qnn only support WoA, Android, Linux" -#endif if (nullptr == lib_handle) { GGMLQNN_LOG_WARN("can not open QNN library %s, with error: %s", lib_path.c_str(), dlerror()); return 1; @@ -1529,30 +1557,24 @@ int qnn_instance::unload_backend() { int qnn_instance::load_system() { Qnn_ErrorHandle_t error = QNN_SUCCESS; +#ifdef _WIN32 + std::string system_lib_path = _lib_path + "QnnSystem.dll"; +#else std::string system_lib_path = _lib_path + "libQnnSystem.so"; +#endif GGMLQNN_LOG_DEBUG("system_lib_path:%s\n", system_lib_path.c_str()); -#if defined(__ANDROID__) || defined(__linux__) _system_lib_handle = dlopen(system_lib_path.c_str(), RTLD_NOW | RTLD_LOCAL); -#elif defined(_WIN32) || defined(_MSC_VER) - //TODO: Snapdragon based WoA(Windows on ARM) - _system_lib_handle = nullptr; -#else -#error "ggml-qnn only support WoA, Android, Linux" -#endif if (nullptr == _system_lib_handle) { GGMLQNN_LOG_WARN("can not open QNN library %s, error: %s\n", system_lib_path.c_str(), dlerror()); //re-try with default path of QNN binary runtime lib _lib_path = "/data/local/tmp/"; - system_lib_path = _lib_path + "libQnnSystem.so"; -#if defined(__ANDROID__) || defined(__linux__) - _system_lib_handle = dlopen(system_lib_path.c_str(), RTLD_NOW | RTLD_LOCAL); -#elif defined(_WIN32) || defined(_MSC_VER) - //TODO: Snapdragon based WoA(Windows on ARM) - _system_lib_handle = nullptr; +#ifdef _WIN32 + system_lib_path = _lib_path + "QnnSystem.dll"; #else -#error "ggml-qnn only support WoA, Android, Linux" + system_lib_path = _lib_path + "libQnnSystem.so"; #endif + _system_lib_handle = dlopen(system_lib_path.c_str(), RTLD_NOW | RTLD_LOCAL); if (nullptr == _system_lib_handle) { GGMLQNN_LOG_WARN("can not open QNN library %s, error: %s\n", system_lib_path.c_str(), dlerror()); return 1; @@ -1786,9 +1808,8 @@ int qnn_instance::qnn_init(const QnnSaver_Config_t ** saver_config) { #if defined(__ANDROID__) || defined(__linux__) _rpc_lib_handle = dlopen("libcdsprpc.so", RTLD_NOW | RTLD_LOCAL); -#elif defined(_WIN32) || defined(_MSC_VER) - //TODO: Snapdragon based WoA(Windows on ARM) - _rpc_lib_handle = nullptr; +#elif defined(_WIN32) + _rpc_lib_handle = dlopen("libcdsprpc.dll", RTLD_NOW | RTLD_LOCAL); #else #error "ggml-qnn only support WoA, Android, Linux" #endif @@ -2901,6 +2922,7 @@ ggml_backend_t ggml_backend_qnn_init(size_t device, const char * qnn_lib_path) { return g_qnn_mgr[device].backend; } +#if defined(__ANDROID__) std::string path = qnn_lib_path; if (QNN_BACKEND_NPU == device) { if (0 == setenv("LD_LIBRARY_PATH", @@ -2929,6 +2951,7 @@ ggml_backend_t ggml_backend_qnn_init(size_t device, const char * qnn_lib_path) { GGMLQNN_LOG_ERROR("%s backend setenv failure\n", ggml_backend_qnn_get_devname(device)); } } +#endif qnn_instance * instance = nullptr; instance = new qnn_instance(qnn_lib_path, g_qnn_mgr[device].lib, ""); From 8de38620945da573845d70f738b659179e627db8 Mon Sep 17 00:00:00 2001 From: zhouwg Date: Sun, 2 Mar 2025 08:37:25 +0800 Subject: [PATCH 32/76] ggml-qnn: merge UT code and corresponding script from local dev branch to make workflow easily --- scripts/build-run-android-minimal.sh | 240 ++++++++++++ scripts/build-run-android.sh | 78 +++- tests/CMakeLists.txt | 1 + tests/ggml-qnn-ut.cpp | 550 +++++++++++++++++++++++++++ 4 files changed, 851 insertions(+), 18 deletions(-) create mode 100755 scripts/build-run-android-minimal.sh create mode 100644 tests/ggml-qnn-ut.cpp diff --git a/scripts/build-run-android-minimal.sh b/scripts/build-run-android-minimal.sh new file mode 100755 index 0000000000000..1a5f362fe2083 --- /dev/null +++ b/scripts/build-run-android-minimal.sh @@ -0,0 +1,240 @@ +#!/bin/bash + +set -e + +PWD=`pwd` +ANDROID_PLATFORM=android-34 +ANDROID_NDK=${PWD}/android-ndk-r26c +REMOTE_PATH=/data/local/tmp/ +GGUF_MODEL_NAME=/sdcard/deepseek-r1-distill-qwen-1.5b-q4_0.gguf +GGUF_MODEL_NAME=/sdcard/qwen1_5-1_8b-chat-q4_0.gguf + +#QNN SDK could be found at: +#https://www.qualcomm.com/developer/software/qualcomm-ai-engine-direct-sdk +#https://developer.qualcomm.com/software/hexagon-dsp-sdk/tools +QNN_SDK_URL=https://www.qualcomm.com/developer/software/qualcomm-ai-engine-direct-sdk +QNN_SDK_PATH=/opt/qcom/aistack/qairt/2.31.0.250130/ + +#default is QNN NPU +qnnbackend=2 + +function dump_vars() +{ + echo -e "ANDROID_NDK: ${ANDROID_NDK}" + echo -e "QNN_SDK_PATH: ${QNN_SDK_PATH}" +} + + +function show_pwd() +{ + echo -e "current working path:$(pwd)\n" +} + + +function check_qnn_sdk() +{ + if [ ! -d ${QNN_SDK_PATH} ]; then + echo -e "QNN_SDK_PATH ${QNN_SDK_PATH} not exist, pls check or download it from ${QNN_SDK_URL}...\n" + exit 1 + fi +} + + +function check_and_download_ndk() +{ + is_android_ndk_exist=1 + + if [ ! -d ${ANDROID_NDK} ]; then + is_android_ndk_exist=0 + fi + + if [ ! -f ${ANDROID_NDK}/build/cmake/android.toolchain.cmake ]; then + is_android_ndk_exist=0 + fi + + if [ ${is_android_ndk_exist} -eq 0 ]; then + + if [ ! -f android-ndk-r26c-linux.zip ]; then + wget --no-config --quiet --show-progress -O android-ndk-r26c-linux.zip https://dl.google.com/android/repository/android-ndk-r26c-linux.zip + fi + + unzip android-ndk-r26c-linux.zip + + if [ $? -ne 0 ]; then + printf "failed to download android ndk to %s \n" "${ANDROID_NDK}" + exit 1 + fi + + printf "android ndk saved to ${ANDROID_NDK} \n\n" + else + printf "android ndk already exist:${ANDROID_NDK} \n\n" + fi +} + + +function build_arm64 +{ + cmake -H. -B./out/android -DCMAKE_BUILD_TYPE=Release -DGGML_USE_QNN=ON -DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK}/build/cmake/android.toolchain.cmake -DANDROID_ABI=arm64-v8a -DANDROID_PLATFORM=latest -DCMAKE_C_FLAGS=-march=armv8.7-a -DGGML_QNN=ON -DGGML_QNN_SDK_PATH=${QNN_SDK_PATH} + cd out/android + make -j16 + show_pwd + + cd - +} + + +function remove_temp_dir() +{ + if [ -d out ]; then + echo "remove out directory in `pwd`" + rm -rf out + fi +} + + +function check_qnn_libs() +{ + #reuse the cached qnn libs on Android phone + adb shell ls ${REMOTE_PATH}/libQnnCpu.so + if [ $? -eq 0 ]; then + printf "QNN libs already exist on Android phone\n" + else + update_qnn_libs + fi +} + + +function update_qnn_libs() +{ + adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnSystem.so ${REMOTE_PATH}/ + adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnCpu.so ${REMOTE_PATH}/ + adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnGpu.so ${REMOTE_PATH}/ + + adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnHtp.so ${REMOTE_PATH}/ + adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnHtpNetRunExtensions.so ${REMOTE_PATH}/ + adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnHtpPrepare.so ${REMOTE_PATH}/ + adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnHtpV75Stub.so ${REMOTE_PATH}/ + adb push ${QNN_SDK_PATH}/lib/hexagon-v75/unsigned/libQnnHtpV75Skel.so ${REMOTE_PATH}/ +} + + +function build_ggml_qnn() +{ + show_pwd + check_and_download_ndk + check_qnn_sdk + dump_vars + remove_temp_dir + build_arm64 +} + + +function run_llamacli() +{ + check_qnn_libs + + if [ -f ./out/android/bin/libggml-qnn.so ]; then + adb push ./out/android/bin/*.so ${REMOTE_PATH}/ + fi + adb push ./out/android/bin/llama-cli ${REMOTE_PATH}/ + adb shell chmod +x ${REMOTE_PATH}/llama-cli + + adb shell "cd ${REMOTE_PATH} \ + && export LD_LIBRARY_PATH=${REMOTE_PATH} \ + && ${REMOTE_PATH}/llama-cli -mg ${qnnbackend} -no-cnv -m ${GGUF_MODEL_NAME} -p \"introduce the movie Once Upon a Time in America briefly.\n\"" + +} + + +function run_llamabench() +{ + check_qnn_libs + + if [ -f ./out/android/bin/libggml-qnn.so ]; then + adb push ./out/android/bin/*.so ${REMOTE_PATH}/ + fi + adb push ./out/android/bin/llama-bench ${REMOTE_PATH}/ + adb shell chmod +x ${REMOTE_PATH}/llama-bench + + adb shell "cd ${REMOTE_PATH} \ + && export LD_LIBRARY_PATH=${REMOTE_PATH} \ + && ${REMOTE_PATH}/llama-bench -mg ${qnnbackend} -m ${GGUF_MODEL_NAME}" + +} + + +function run_test-backend-ops() +{ + check_qnn_libs + + if [ -f ./out/android/bin/libggml-qnn.so ]; then + adb push ./out/android/bin/*.so ${REMOTE_PATH}/ + fi + adb push ./out/android/bin/test-backend-ops ${REMOTE_PATH}/ + adb shell chmod +x ${REMOTE_PATH}/test-backend-ops + + adb shell "cd ${REMOTE_PATH} \ + && export LD_LIBRARY_PATH=${REMOTE_PATH} \ + && ${REMOTE_PATH}/test-backend-ops test" + +} + + +function show_usage() +{ + echo "Usage:" + echo " $0 build" + echo " $0 updateqnnlib" + echo " $0 run_testop" + echo " $0 run_llamacli 0 (QNN_CPU) / 1 (QNN_GPU) / 2 (QNN_NPU) / 3 (ggml)" + echo " $0 run_llamabench 0 (QNN_CPU) / 1 (QNN_GPU) / 2 (QNN_NPU) / 3 (ggml)" + echo -e "\n\n\n" +} + + +show_pwd + +check_qnn_sdk + +if [ $# == 0 ]; then + show_usage + exit 1 +elif [ $# == 1 ]; then + if [ "$1" == "-h" ]; then + show_usage + exit 1 + elif [ "$1" == "help" ]; then + show_usage + exit 1 + elif [ "$1" == "build" ]; then + build_ggml_qnn + exit 0 + + elif [ "$1" == "run_testop" ]; then + run_test-backend-ops + exit 0 + elif [ "$1" == "updateqnnlib" ]; then + update_qnn_libs + exit 0 + else + show_usage + exit 1 + fi +elif [ $# == 2 ]; then + qnnbackend=$2 + if [ ${qnnbackend} -gt 3 ]; then + show_usage + exit 1 + fi + + if [ "$1" == "run_llamacli" ]; then + run_llamacli + exit 0 + elif [ "$1" == "run_llamabench" ]; then + run_llamabench + exit 0 + fi +else + show_usage + exit 1 +fi diff --git a/scripts/build-run-android.sh b/scripts/build-run-android.sh index 1a5f362fe2083..49079c9132769 100755 --- a/scripts/build-run-android.sh +++ b/scripts/build-run-android.sh @@ -129,32 +129,37 @@ function build_ggml_qnn() } -function run_llamacli() +function prepare_run_on_phone() { + if [ $# != 1 ]; then + print "invalid param" + return + fi + program=$1 + check_qnn_libs if [ -f ./out/android/bin/libggml-qnn.so ]; then adb push ./out/android/bin/*.so ${REMOTE_PATH}/ fi - adb push ./out/android/bin/llama-cli ${REMOTE_PATH}/ - adb shell chmod +x ${REMOTE_PATH}/llama-cli + adb push ./out/android/bin/${program} ${REMOTE_PATH}/ + adb shell chmod +x ${REMOTE_PATH}/${program} +} + +function run_llamacli() +{ + prepare_run_on_phone llama-cli adb shell "cd ${REMOTE_PATH} \ && export LD_LIBRARY_PATH=${REMOTE_PATH} \ - && ${REMOTE_PATH}/llama-cli -mg ${qnnbackend} -no-cnv -m ${GGUF_MODEL_NAME} -p \"introduce the movie Once Upon a Time in America briefly.\n\"" + && ${REMOTE_PATH}/llama-cli -mg ${qnnbackend} -no-ncv -m ${GGUF_MODEL_NAME} -p \"introduce the movie Once Upon a Time in America briefly.\n\"" } function run_llamabench() { - check_qnn_libs - - if [ -f ./out/android/bin/libggml-qnn.so ]; then - adb push ./out/android/bin/*.so ${REMOTE_PATH}/ - fi - adb push ./out/android/bin/llama-bench ${REMOTE_PATH}/ - adb shell chmod +x ${REMOTE_PATH}/llama-bench + prepare_run_on_phone llama-bench adb shell "cd ${REMOTE_PATH} \ && export LD_LIBRARY_PATH=${REMOTE_PATH} \ @@ -165,13 +170,7 @@ function run_llamabench() function run_test-backend-ops() { - check_qnn_libs - - if [ -f ./out/android/bin/libggml-qnn.so ]; then - adb push ./out/android/bin/*.so ${REMOTE_PATH}/ - fi - adb push ./out/android/bin/test-backend-ops ${REMOTE_PATH}/ - adb shell chmod +x ${REMOTE_PATH}/test-backend-ops + prepare_run_on_phone test-backend-ops adb shell "cd ${REMOTE_PATH} \ && export LD_LIBRARY_PATH=${REMOTE_PATH} \ @@ -179,6 +178,36 @@ function run_test-backend-ops() } +function run_ut_add() +{ + prepare_run_on_phone ggml-qnn-ut + + adb shell "cd ${REMOTE_PATH} \ + && export LD_LIBRARY_PATH=${REMOTE_PATH} \ + && ${REMOTE_PATH}/ggml-qnn-ut -t GGML_OP_ADD -b $qnnbackend" + +} + +function run_ut_mulmat() +{ + prepare_run_on_phone ggml-qnn-ut + + adb shell "cd ${REMOTE_PATH} \ + && export LD_LIBRARY_PATH=${REMOTE_PATH} \ + && ${REMOTE_PATH}/ggml-qnn-ut -t GGML_OP_MUL_MAT -b $qnnbackend" + +} + +function run_ut_mul() +{ + prepare_run_on_phone ggml-qnn-ut + + adb shell "cd ${REMOTE_PATH} \ + && export LD_LIBRARY_PATH=${REMOTE_PATH} \ + && ${REMOTE_PATH}/ggml-qnn-ut -t GGML_OP_MUL -b $qnnbackend" + +} + function show_usage() { @@ -186,6 +215,9 @@ function show_usage() echo " $0 build" echo " $0 updateqnnlib" echo " $0 run_testop" + echo " $0 run_ut_add 0 (QNN_CPU) / 1 (QNN_GPU) / 2 (QNN_NPU) / 3 (ggml)" + echo " $0 run_ut_mulmat 0 (QNN_CPU) / 1 (QNN_GPU) / 2 (QNN_NPU) / 3 (ggml)" + echo " $0 run_ut_mul 0 (QNN_CPU) / 1 (QNN_GPU) / 2 (QNN_NPU) / 3 (ggml)" echo " $0 run_llamacli 0 (QNN_CPU) / 1 (QNN_GPU) / 2 (QNN_NPU) / 3 (ggml)" echo " $0 run_llamabench 0 (QNN_CPU) / 1 (QNN_GPU) / 2 (QNN_NPU) / 3 (ggml)" echo -e "\n\n\n" @@ -213,6 +245,7 @@ elif [ $# == 1 ]; then elif [ "$1" == "run_testop" ]; then run_test-backend-ops exit 0 + elif [ "$1" == "updateqnnlib" ]; then update_qnn_libs exit 0 @@ -233,6 +266,15 @@ elif [ $# == 2 ]; then elif [ "$1" == "run_llamabench" ]; then run_llamabench exit 0 + elif [ "$1" == "run_ut_add" ]; then + run_ut_add + exit 0 + elif [ "$1" == "run_ut_mulmat" ]; then + run_ut_mulmat + exit 0 + elif [ "$1" == "run_ut_mul" ]; then + run_ut_mul + exit 0 fi else show_usage diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 7a158d6024d78..cd7ca4310f73d 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -137,6 +137,7 @@ llama_target_and_test(test-chat-template.cpp) # llama_target_and_test(test-opt.cpp) # SLOW llama_target_and_test(test-gguf.cpp) llama_target_and_test(test-backend-ops.cpp) +llama_target_and_test(ggml-qnn-ut.cpp) llama_target_and_test(test-model-load-cancel.cpp LABEL "model") llama_target_and_test(test-autorelease.cpp LABEL "model") diff --git a/tests/ggml-qnn-ut.cpp b/tests/ggml-qnn-ut.cpp new file mode 100644 index 0000000000000..ff0e96f2b00cb --- /dev/null +++ b/tests/ggml-qnn-ut.cpp @@ -0,0 +1,550 @@ +/* + * Copyright (c) 2023-2024 The ggml authors + * + * implementation of self-made Android command line tool for verify ggml-qnn backend + * this file will help you to understand fundamental principle of ggml and ggml-qnn backend + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ggml.h" +#include "ggml-cpu.h" +#include "ggml-alloc.h" +#include "ggml-backend.h" +#include "ggml-qnn.h" + +#define GGML_QNN_DEBUG 1 +#define GGML_QNN_LOGBUF_LEN 4096 + +#define QNN_LOG_ERROR(...) ggml_qnn_log_internal(GGML_LOG_LEVEL_DEBUG, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) +#define QNN_LOG_WARN(...) ggml_qnn_log_internal(GGML_LOG_LEVEL_DEBUG , __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) +#define QNN_LOG_INFO(...) ggml_qnn_log_internal(GGML_LOG_LEVEL_DEBUG , __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) + +#if GGML_QNN_DEBUG +#define QNN_LOG_DEBUG(...) ggml_qnn_log_internal(GGML_LOG_LEVEL_DEBUG, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) +#else +#define QNN_LOG_DEBUG(...) +#endif + +static void tensor_dump(const ggml_tensor * tensor, const char * name); + +#define TENSOR_DUMP(tensor) tensor_dump(tensor, #tensor) + +static void ggml_qnn_log_internal(ggml_log_level level, const char * file, const char * func, int line, const char * format, ...) { + static std::mutex ggml_qnn_log_internal_mutex; + static char s_ggml_qnn_log_internal_buf[GGML_QNN_LOGBUF_LEN]; + + { + std::lock_guard lock(ggml_qnn_log_internal_mutex); + va_list args; + va_start(args, format); + int len_prefix = snprintf(s_ggml_qnn_log_internal_buf, GGML_QNN_LOGBUF_LEN, "[%s, %d]: ", func, line); + int len = vsnprintf(s_ggml_qnn_log_internal_buf + len_prefix, GGML_QNN_LOGBUF_LEN - len_prefix, format, args); + if (len < (GGML_QNN_LOGBUF_LEN - len_prefix)) { + printf("%s", s_ggml_qnn_log_internal_buf); + } + va_end(args); + } +} + + +static bool ggml_graph_compute_helper( + struct ggml_backend * backend, + struct ggml_cgraph * graph, + std::vector & buf, + int n_threads, + ggml_abort_callback abort_callback, + void * abort_callback_data) { + struct ggml_cplan plan = ggml_graph_plan(graph, n_threads, NULL); + + plan.abort_callback = abort_callback; + plan.abort_callback_data = abort_callback_data; + + if (plan.work_size > 0) { + buf.resize(plan.work_size); + plan.work_data = buf.data(); + } + + if (nullptr != backend) + return ggml_backend_graph_compute(backend, graph) == GGML_STATUS_SUCCESS; + else + return ggml_graph_compute(graph, &plan); +} + + +static void tensor_dump_elements(const ggml_tensor * tensor) { + float value = 0; + std::ostringstream tmposs; + if (tensor->type == GGML_TYPE_F32) { + for (int h = 0; h < tensor->ne[3]; h++) { + for (int i = 0; i < tensor->ne[2]; i++) { + for (int j = 0; j < tensor->ne[1]; j++) { + for (int k = 0; k < tensor->ne[0]; k++) { + value = ((float *) tensor->data)[h * tensor->ne[2] + i * tensor->ne[1] + + j * tensor->ne[0] + k]; + tmposs << std::setw(8) << std::fixed << std::setprecision(2) << value + << " "; + } + if (strlen(tmposs.str().c_str()) <= (GGML_QNN_LOGBUF_LEN - 96)) { + QNN_LOG_DEBUG("%s\n", tmposs.str().c_str()); + } + tmposs.clear(); + tmposs.str(""); + } + } + } + } + + QNN_LOG_DEBUG("\n"); +} + + +static void tensor_dump(const ggml_tensor * tensor, const char * name) { + QNN_LOG_DEBUG("dump ggml tensor %s(%s)\n", name, tensor->name); + QNN_LOG_DEBUG("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64", nb = (%5zi, %5zi, %5zi, %5zi)\n", + name, + tensor->type, ggml_type_name(tensor->type), + tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3], + tensor->nb[0], tensor->nb[1], tensor->nb[2], tensor->nb[2]); + tensor_dump_elements(tensor); + + QNN_LOG_DEBUG("\n"); +} + + +static uint32_t get_tensor_rank(const ggml_tensor * tensor) { + uint32_t rank = 0; + for (int i = 0; i < GGML_MAX_DIMS; i++) { + if ((0 != tensor->ne[i]) && (1 != tensor->ne[i])) { + rank++; + } + } + return rank; +} + + +static uint32_t get_tensor_data_size(const ggml_tensor * tensor) { + size_t data_size = ggml_row_size(tensor->type, tensor->ne[0]); + size_t n_dims = get_tensor_rank(tensor); + for (size_t i = 1; i < n_dims; i++) { + data_size *= tensor->ne[i]; + } + + QNN_LOG_DEBUG("get_tensor_data_size %d", data_size); + QNN_LOG_DEBUG("ggml_nbytes(tensor) %d", ggml_nbytes(tensor)); + + return ggml_nbytes(tensor); +} + + +//ref: https://github.com/ggerganov/llama.cpp/blob/master/tests/test-backend-ops.cpp#L20 +static void init_tensor_uniform(ggml_tensor * tensor, float min = -1.0f, float max = 1.0f) { + // static RNG initialization (revisit if n_threads stops being constant) + static const size_t n_threads = std::thread::hardware_concurrency(); + static std::vector generators = []() { + std::random_device rd; + std::vector vec; + vec.reserve(n_threads); + //for (size_t i = 0; i < n_threads; i++) { vec.emplace_back(1234 + i); } // fixed seed + for (size_t i = 0; i < n_threads; i++) { vec.emplace_back(rd()); } + return vec; + }(); + + size_t size = ggml_nelements(tensor); + std::vector data(size); + + auto init_thread = [&](size_t ith, size_t start, size_t end) { + std::uniform_real_distribution distribution(min, max); + for (size_t i = start; i < end; i++) { + data[i] = distribution(generators[ith]); + } + }; + + std::vector threads; + threads.reserve(n_threads); + for (size_t i = 0; i < n_threads; i++) { + size_t start = i*size/n_threads; + size_t end = (i+1)*size/n_threads; + threads.emplace_back(init_thread, i, start, end); + } + for (auto & t : threads) { + t.join(); + } + if (tensor->type == GGML_TYPE_F32 || tensor->type == GGML_TYPE_I32) { + ggml_backend_tensor_set(tensor, data.data(), 0, size * sizeof(float)); + } else if (ggml_is_quantized(tensor->type) || tensor->type == GGML_TYPE_F16 || tensor->type == GGML_TYPE_BF16) { + GGML_ASSERT(size % ggml_blck_size(tensor->type) == 0); + std::vector dataq(ggml_row_size(tensor->type, size)); + std::vector imatrix(tensor->ne[0], 1.0f); // dummy importance matrix + const float * im = imatrix.data(); + if (!ggml_quantize_requires_imatrix(tensor->type)) { + // when the imatrix is optional, we want to test both quantization with and without imatrix + // use one of the random numbers to decide + if (data[0] > 0.5f*(min + max)) { + im = nullptr; + } + } + ggml_quantize_chunk(tensor->type, data.data(), dataq.data(), 0, size/tensor->ne[0], tensor->ne[0], im); + GGML_ASSERT(ggml_validate_row_data(tensor->type, dataq.data(), dataq.size())); + ggml_backend_tensor_set(tensor, dataq.data(), 0, dataq.size()); + } else if (tensor->type == GGML_TYPE_I8 || tensor->type == GGML_TYPE_I16 || tensor->type == GGML_TYPE_I32) { + // This is going to create some weird integers though. + ggml_backend_tensor_set(tensor, data.data(), 0, ggml_nbytes(tensor)); + } else { + GGML_ASSERT(false); + } +} + + +//ref: https://github.com/ggerganov/llama.cpp/blob/master/tests/test-backend-ops.cpp#L310 +static void initialize_tensors(ggml_context * ctx) { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) { + init_tensor_uniform(t); + } +} + + +static void show_usage() { + printf(" " \ + "\nUsage: ggml-qnn-ut [options]\n" \ + "\n" \ + "Options:\n" \ + " -t GGML_OP_ADD / GGML_OP_MUL / GGML_OP_MULMAT\n" \ + " -b 0(QNN_CPU) 1(QNN_GPU) 2(QNN_NPU) 3(QNN_GGML)\n" \ + " ?/h print usage information\n\n" + ); +} + + +struct ggml_backend_deleter { void operator()(ggml_backend_t backend) { ggml_backend_free(backend); } }; +typedef std::unique_ptr ggml_backend_ptr; + +int main(int argc, char * argv[]) { + int64_t n_begin_time = 0LL; + int64_t n_end_time = 0LL; + int64_t n_duration = 0LL; + size_t ctx_size = 0; + int sizey = 4; + int sizex = 4; + int num_threads = 4; + int n_backend_type = QNN_BACKEND_CPU; + int n_ggml_op_type = GGML_OP_ADD; + + struct ggml_context * ctx = nullptr; + struct ggml_cgraph * gf = nullptr; + struct ggml_tensor * src0 = nullptr; + struct ggml_tensor * src1 = nullptr; + struct ggml_tensor * dst = nullptr; + ggml_backend_t backend = nullptr; + ggml_backend_buffer_t buffer= nullptr; + ggml_type qtype = GGML_TYPE_F32; + //ggml_type qtype = GGML_TYPE_Q4_0; + std::vector work_buffer; + + for (int i = 1; i < argc; i++) { + if (0 == strcmp(argv[i], "-t")) { + if (i + 1 < argc) { + if (0 == memcmp(argv[i + 1], "GGML_OP_ADD", 11)) { + n_ggml_op_type = GGML_OP_ADD; + } else if (0 == memcmp(argv[i + 1], "GGML_OP_MUL_MAT", 15)) { + n_ggml_op_type = GGML_OP_MUL_MAT; + } else if (0 == memcmp(argv[i + 1], "GGML_OP_MUL", 11)) { + n_ggml_op_type = GGML_OP_MUL; + } else { + show_usage(); + return 1; + } + i++; + } + } else if (0 == strcmp(argv[i], "-b")) { + if (i + 1 < argc) { + int backend = atoi(argv[i + 1]); + if (backend <= QNN_BACKEND_GGML) + n_backend_type = backend; + else { + show_usage(); + return 1; + } + i++; + } + } else { + show_usage(); + return 1; + } + } + std::vector backends; + std::vector> set_n_threads_fns; + printf("Testing %zu devices\n\n", ggml_backend_dev_count()); + for (size_t i = 0; i < ggml_backend_dev_count(); i++) { + ggml_backend_dev_t dev = ggml_backend_dev_get(i); + + printf("Backend %zu/%zu: %s\n", i + 1, ggml_backend_dev_count(), + ggml_backend_dev_name(dev)); + + if (ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_CPU) { + printf(" Skipping CPU backend\n"); + continue; + } + + backend = ggml_backend_dev_init(dev, reinterpret_cast(i)); + GGML_ASSERT(backend != NULL); + if (backend != nullptr) { + printf("%s: initialize %s backend\n", __func__, ggml_backend_dev_name(dev)); + } + backends.emplace_back(backend); + + ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(dev); + auto ggml_backend_set_n_threads_fn = (ggml_backend_set_n_threads_t) ggml_backend_reg_get_proc_address( + reg, "ggml_backend_set_n_threads"); + if (ggml_backend_set_n_threads_fn) { + ggml_backend_set_n_threads_fn(backend, std::thread::hardware_concurrency()); + } + + printf(" Device description: %s\n", ggml_backend_dev_description(dev)); + size_t free, total; + ggml_backend_dev_memory(dev, &free, &total); + printf(" Device memory: %zu MB (%zu MB free)\n", total / 1024 / 1024, free / 1024 / 1024); + printf("\n"); + } + + ggml_backend_t backend_cpu = nullptr; + backend_cpu = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr); + if (nullptr == backend_cpu) { + QNN_LOG_DEBUG("failed to initialize cpu backend\n"); + exit(1); + } else { + QNN_LOG_DEBUG("succeed to initialize cpu backend\n"); + } + backends.emplace_back(backend_cpu); + + size_t n_ok = 0; + + QNN_LOG_DEBUG("enter qnn_ggml_op\n"); + QNN_LOG_DEBUG("ggml op:%d(%s)", n_ggml_op_type, ggml_op_name((enum ggml_op) n_ggml_op_type)); + + n_begin_time = ggml_time_us(); + srand(time(NULL)); + + ctx_size += 1024 * 1024 * 32; + QNN_LOG_DEBUG("Allocating Memory of size %zi bytes, %zi MB\n", ctx_size, + (ctx_size / 1024 / 1024)); + + struct ggml_init_params params = { + /*.mem_size =*/ ctx_size, + /*.mem_buffer =*/ NULL, + /* no_alloc =*/ 0 + }; + + int idx = 0; + for (auto & backend_it : backends) { + if (idx == n_backend_type) { + backend = backend_it.get(); + } + idx++; + ggml_backend_dev_t dev = ggml_backend_get_device(backend_it.get()); + ggml_backend_reg_t reg = dev ? ggml_backend_dev_backend_reg(dev) : nullptr; + if (reg) { + auto ggml_backend_set_n_threads_fn = (ggml_backend_set_n_threads_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_n_threads"); + if (ggml_backend_set_n_threads_fn) { + set_n_threads_fns.emplace_back(backend_it.get(), ggml_backend_set_n_threads_fn); + } + } + const char * name = ggml_backend_dev_description(dev); + QNN_LOG_DEBUG("dev name %s\n", name); + + } + + if (n_backend_type != QNN_BACKEND_GGML) { + params.no_alloc = true; + } + + ctx = ggml_init(params); + if (!ctx) { + QNN_LOG_ERROR("%s: ggml_init() failed\n"); + return 2; + } + + QNN_LOG_DEBUG("creating new tensors\n"); + QNN_LOG_DEBUG("ggml_blck_size(%s) %d\n", ggml_type_name(qtype), ggml_blck_size(qtype)); + QNN_LOG_DEBUG("ggml_type_size(%s) %d\n", ggml_type_name(qtype), ggml_type_size(qtype)); + if (qtype != GGML_TYPE_F32) { + sizex = ggml_blck_size(qtype); + } + + if (n_ggml_op_type == GGML_OP_ADD) { + src0 = ggml_new_tensor_2d(ctx, qtype, sizey, sizex); + src1 = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, sizey, sizex); + } else { + //verify 2D matrix + //src0 = ggml_new_tensor_2d(ctx, qtype, 128, 64); + //src1 = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 128, 2); + //verify 3D matrix + //src0 = ggml_new_tensor_3d(ctx, qtype, 128, 64, 8); + //src1 = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, 128, 2, 8); + //verify 4D matrix + src0 = ggml_new_tensor_4d(ctx, qtype, 256, 16, 3, 2); + src1 = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, 256, 1, 6, 4); + //src0 = ggml_new_tensor_4d(ctx, qtype, 256, 16, 3, 2); + //src1 = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, 256, 16, 3, 2); + } + + ggml_set_input(src0); + ggml_set_input(src1); + switch (n_ggml_op_type) { + case GGML_OP_ADD: + dst = ggml_add(ctx, src0, src1); + break; + case GGML_OP_MUL: + dst = ggml_mul(ctx, src0, src1); + break; + case GGML_OP_MUL_MAT: + dst = ggml_mul_mat(ctx, src0, src1); + break; + default: + QNN_LOG_WARN("ggml op %d(%s) not supported", n_ggml_op_type, + ggml_op_name((enum ggml_op) n_ggml_op_type)); + ggml_free(ctx); + ggml_backend_free(backend); + return 3; + } + + ggml_set_output(dst); + +#ifdef GGML_USE_QNN + if (n_backend_type != QNN_BACKEND_GGML) { + QNN_LOG_DEBUG("init QNN backend %d\n", n_backend_type); + //re-init again + backend = ggml_backend_qnn_init(n_backend_type, "/data/local/tmp/"); + if (nullptr == backend) { + QNN_LOG_ERROR("create qnn backend %d(%s) failed\n", n_backend_type, ggml_backend_qnn_get_devname(n_backend_type)); + return 1; + } else { + QNN_LOG_INFO("create qnn backend %d(%s) succeed\n", n_backend_type, ggml_backend_qnn_get_devname(n_backend_type)); + } + + //buffer = ggml_backend_alloc_ctx_tensors(ctx, backend); + ggml_backend_buffer_type_t buft = ggml_backend_get_default_buffer_type(backend); + buffer = ggml_backend_alloc_ctx_tensors_from_buft(ctx, buft); + if (!buffer) { + QNN_LOG_ERROR("%s: failed to allocate backend buffer\n", __func__); + ggml_free(ctx); + ggml_backend_free(backend); + return 4; + } + } else { + QNN_LOG_DEBUG("init default cpu backend\n"); + backend = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr); + } +#endif + + QNN_LOG_DEBUG("creating compute graph\n"); + gf = ggml_new_graph(ctx); + ggml_build_forward_expand(gf, dst); + + if (qtype == GGML_TYPE_F32) { + if (n_backend_type != QNN_BACKEND_GGML) { + initialize_tensors(ctx); + } else { + ggml_set_f32(src0, (rand() % 100 + 1)); + ggml_set_f32(src1, (rand() % 100 + 1)); + ggml_set_f32(dst, 0.0f); + } + //for compare compute result between cpu backend and QNN backend + ggml_set_f32(src0, 1.0f); + ggml_set_f32(src1, 2.0f); + ggml_set_f32(dst, 0.0f); + } else { + initialize_tensors(ctx); + } + + ggml_graph_compute_helper(backend, gf, work_buffer, num_threads, nullptr, nullptr); + if (get_tensor_data_size(dst) < (100 * 100)) { + QNN_LOG_DEBUG("dump result tensors:\n"); + TENSOR_DUMP(src0); + TENSOR_DUMP(src1); + TENSOR_DUMP(dst); + } else { + QNN_LOG_DEBUG("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", + src0->name, + src0->type, ggml_type_name(src0->type), src0->ne[0], src0->ne[1], src0->ne[2], + src0->nb[0], src0->nb[1], src0->nb[2]); + QNN_LOG_DEBUG("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", + src1->name, + src1->type, ggml_type_name(src1->type), src1->ne[0], src1->ne[1], src1->ne[2], + src1->nb[0], src1->nb[1], src1->nb[2]); + QNN_LOG_DEBUG("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", + dst->name, + dst->type, ggml_type_name(dst->type), dst->ne[0], dst->ne[1], dst->ne[2], dst->nb[0], + dst->nb[1], dst->nb[2]); + } + //TENSOR_DUMP(dst); + + ggml_free(ctx); + ggml_backend_buffer_free(buffer); + ggml_backend_free(backend); + + n_end_time = ggml_time_us(); + n_duration = (n_end_time - n_begin_time) / 1000; +#ifdef GGML_USE_QNN + QNN_LOG_DEBUG("duration of ut GGML_OP_%s using QNN backend %s: %lld milliseconds\n", ggml_op_name((enum ggml_op)n_ggml_op_type), ggml_backend_qnn_get_devname(n_backend_type), n_duration); +#endif + + return 0; +} From c6c640780cf59d184f5c07050e3914e3fb6768db Mon Sep 17 00:00:00 2001 From: zhouwg Date: Sun, 2 Mar 2025 09:49:51 +0800 Subject: [PATCH 33/76] ggml-qnn: merge ggml_qnn_mul_mat_4d from local dev branch to make workflow easily --- ggml/src/ggml-qnn/ggml-qnn-impl.h | 4 +- ggml/src/ggml-qnn/ggml-qnn-ops.cpp | 812 +++++++++++++++++++++++++++++ ggml/src/ggml-qnn/ggml-qnn.cpp | 10 +- 3 files changed, 819 insertions(+), 7 deletions(-) create mode 100644 ggml/src/ggml-qnn/ggml-qnn-ops.cpp diff --git a/ggml/src/ggml-qnn/ggml-qnn-impl.h b/ggml/src/ggml-qnn/ggml-qnn-impl.h index 0f0daba6e1a93..394c35fe6b043 100644 --- a/ggml/src/ggml-qnn/ggml-qnn-impl.h +++ b/ggml/src/ggml-qnn/ggml-qnn-impl.h @@ -99,7 +99,7 @@ void ggmlqnn_log_internal(ggml_log_level level, const char * file, const char #else #define GGMLQNN_DEBUG 1 // for troubleshooting QNN backend #define ENABLE_QNNBACKEND_PERF 0 // enable/disable op's perf info -#define GGMLQNN_PRINT_QNN_INTERNAL_LOG 0 // enable/disable QNN's internal log +#define GGMLQNN_PRINT_QNN_INTERNAL_LOG 1 // enable/disable QNN's internal log #define GGMLQNN_PRINT_OP_ADD_LOG 0 // GGML_OP_ADD already verified with QNN-CPU / QNN-GPU / QNN-NPU #define GGMLQNN_PRINT_OP_MUL_MAT_LOG 1 #endif @@ -226,7 +226,7 @@ struct qnn_op_caps_t { const size_t input_param_count = 0; const char * qnn_param_name = nullptr; }; -extern const qnn_op_caps_t k_op_caps[]; +extern const qnn_op_caps_t ggmlqnn_k_op_caps[]; #if ENABLE_QNNBACKEND_PERF class qnn_perf { diff --git a/ggml/src/ggml-qnn/ggml-qnn-ops.cpp b/ggml/src/ggml-qnn/ggml-qnn-ops.cpp new file mode 100644 index 0000000000000..02b7ab7820a95 --- /dev/null +++ b/ggml/src/ggml-qnn/ggml-qnn-ops.cpp @@ -0,0 +1,812 @@ +/* + * Copyright (c) 2023-2024 The ggml authors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ +#include "ggml-impl.h" +#include "ggml-common.h" +#include "ggml-qnn-ops.h" + +static inline uint32_t ggmlqnn_get_tensor_data_size(const ggml_tensor * tensor) { + /* + size_t data_size = ggml_row_size(tensor->type, tensor->ne[0]); + size_t n_dims = ggml_get_tensor_rank(tensor); + for (int i = 1; i < n_dims; i++) { + data_size *= tensor->ne[i]; + } + + return data_size; + */ + return ggml_nbytes(tensor); +} + +static inline bool ggmlqnn_is_valid_params(ggml_backend_qnn_context * ctx, const ggml_tensor * src0, + const ggml_tensor * src1, ggml_tensor * dst) { + if ((nullptr == ctx) || (nullptr == src0) || (nullptr == src1) || (nullptr == dst)) { + GGMLQNN_LOG_WARN("invalid params\n"); + return false; + } + + qnn_instance * instance = ctx->instance; + if (nullptr == instance) { + GGMLQNN_LOG_WARN("invalid params\n"); + return false; + } + + return true; +} + +#define GGMLQNN_CHECK_PARAMS(ctx, src0, src1, dst) \ + do { \ + if (!ggmlqnn_is_valid_params((ctx), (src0), (src1), (dst))) { \ + return; \ + } \ + } while (0) + +/* + * provide a general skeleton to offload ggml op to QNN backend: a single node contains 2 input + * tensor and 1 output tensor +*/ +void ggml_qnn_general_node(ggml_backend_qnn_context * ctx, ggml_tensor * op) { + Qnn_ErrorHandle_t error = QNN_SUCCESS; + enum ggml_status result = GGML_STATUS_SUCCESS; + bool graph_initialized = false; + qnn_instance * instance = nullptr; + Qnn_GraphHandle_t graph_handle = nullptr; + Qnn_Tensor_t * p_tensor0 = nullptr; + Qnn_Tensor_t * p_tensor1 = nullptr; + Qnn_Tensor_t * p_tensor2 = nullptr; + Qnn_Param_t qnn_params[] = {}; + const ggml_tensor * src0 = op->src[0]; + const ggml_tensor * src1 = op->src[1]; + ggml_tensor * dst = op; + + GGMLQNN_CHECK_PARAMS(ctx, src0, src1, dst); + instance = ctx->instance; + QNN_INTERFACE_VER_TYPE qnn_raw_interface = ctx->raw_interface; + size_t qnn_op_index = ggmlqnn_get_op_index(op); + GGML_ASSERT(qnn_op_index < ggmlqnn_get_opcaps_size()); + const char * qnn_op_name = ggmlqnn_k_op_caps[qnn_op_index].qnn_op_name; + std::string ggml_op_name_string = std::string("ggml_") + ggml_op_name(op->op); + const char * ggml_op_name = ggml_op_name_string.c_str(); + + qnn_perf op_perf = qnn_perf(ggml_op_name); + op_perf.start(); + + std::string graph_name; + ggmlqnn_get_graphkey_from_op(op, graph_name); + if (instance->_qnn_graph_map.find(graph_name) != instance->_qnn_graph_map.end()) { + graph_initialized = true; + qnn_res_t & graph_item = instance->_qnn_graph_map[graph_name]; + graph_handle = std::get<0>(graph_item); + qnn_tensors_t & tensor = std::get<1>(graph_item); + p_tensor0 = tensor[0]; + p_tensor1 = tensor[1]; + p_tensor2 = tensor[2]; + } else { + p_tensor0 = ggmlqnn_create_compute_tensor(src0); + p_tensor1 = ggmlqnn_create_compute_tensor(src1); + p_tensor2 = ggmlqnn_create_compute_tensor(dst); + } + //ggmlqnn_print_tensors_info(__func__, ctx, src0, src1, dst); + + //ensure QNN tensor has correct tensor type + QNN_VER_PTR(*p_tensor0)->type = QNN_TENSOR_TYPE_APP_WRITE; + QNN_VER_PTR(*p_tensor1)->type = QNN_TENSOR_TYPE_APP_WRITE; + QNN_VER_PTR(*p_tensor2)->type = QNN_TENSOR_TYPE_APP_READ; + + //save the original dimensions of qnn tensors + uint32_t * tensor_0_dimensions = QNN_VER_PTR(*p_tensor0)->dimensions; + uint32_t * tensor_1_dimensions = QNN_VER_PTR(*p_tensor1)->dimensions; + uint32_t * tensor_2_dimensions = QNN_VER_PTR(*p_tensor2)->dimensions; + + bool enable_npu_rpc = instance->enable_qnn_rpc() && ctx->device == QNN_BACKEND_NPU; + + if (!graph_initialized) { + GGMLQNN_LOG_DEBUG("graph name %s", graph_name.c_str()); + error = instance->init_qnn_graph(graph_name, static_cast(ctx->device), 8); + if (QNN_SUCCESS != error) { + GGMLQNN_LOG_INFO("can't create qnn graph handle with graph name %s, error = %d\n", graph_name.c_str(), error); + return; + } + graph_handle = instance->get_qnn_graph_handle(); + + if (enable_npu_rpc) { + QNN_VER_PTR(*p_tensor0)->memType = QNN_TENSORMEMTYPE_MEMHANDLE; + QNN_VER_PTR(*p_tensor0)->clientBuf = {.data=nullptr, .dataSize=0}; + + QNN_VER_PTR(*p_tensor1)->memType = QNN_TENSORMEMTYPE_MEMHANDLE; + QNN_VER_PTR(*p_tensor1)->clientBuf = {.data=nullptr, .dataSize=0}; + + QNN_VER_PTR(*p_tensor2)->memType = QNN_TENSORMEMTYPE_MEMHANDLE; + QNN_VER_PTR(*p_tensor2)->clientBuf = {.data=nullptr, .dataSize=0}; + } + + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor0)); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor1)); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor2)); + + if (enable_npu_rpc) { + uint8_t * qnn_rpcbuffer_0 = ggmlqnn_create_rpc_buffer(instance, src0, p_tensor0, true); + uint8_t * qnn_rpcbuffer_1 = ggmlqnn_create_rpc_buffer(instance, src1, p_tensor1, true); + uint8_t * qnn_rpcbuffer_2 = ggmlqnn_create_rpc_buffer(instance, dst, p_tensor2, false); + if (nullptr == qnn_rpcbuffer_0 || nullptr == qnn_rpcbuffer_1 || nullptr == qnn_rpcbuffer_2) { + GGMLQNN_LOG_INFO("create rpc buffer failure\n"); + //TODO: potential memory leak although it shouldn't happen + return; + } + } else { + QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggmlqnn_get_tensor_data_size(src0)}; + QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggmlqnn_get_tensor_data_size(src1)}; + QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggmlqnn_get_tensor_data_size(dst)}; + } + + Qnn_Tensor_t tensor_inputs[] = { + *p_tensor0, + *p_tensor1 + }; + Qnn_Tensor_t tensor_outputs[] = { + *p_tensor2 + }; + Qnn_OpConfig_t op_config = { + QNN_OPCONFIG_VERSION_1, .v1 = { + ggml_op_name, + QNN_OP_PACKAGE_NAME_QTI_AISW, + qnn_op_name, + 0, + qnn_params, + 2, + tensor_inputs, + 1, + tensor_outputs + } + }; + CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, op_config)); + CHECK_QNN_API(error, qnn_raw_interface.graphFinalize(graph_handle, nullptr, nullptr)); + CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, + tensor_inputs, 2, + tensor_outputs, 1, + nullptr, nullptr)); + + if (enable_npu_rpc) { + uint8_t * qnn_rpcbuffer = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*p_tensor2)->memHandle)); + GGMLQNN_LOG_INFO("qnn_rpcbuffer = %p\n", qnn_rpcbuffer); + if (nullptr != qnn_rpcbuffer) { + memcpy(dst->data, qnn_rpcbuffer, ggml_nbytes(dst)); + } + } + + qnn_tensors_t ggml_op_add_tensors; + ggml_op_add_tensors.reserve(3); + ggml_op_add_tensors.push_back(p_tensor0); + ggml_op_add_tensors.push_back(p_tensor1); + ggml_op_add_tensors.push_back(p_tensor2); + + auto graph_item = std::make_tuple(graph_handle, ggml_op_add_tensors); + instance->_qnn_graph_map[graph_name] = graph_item; + } else { + Qnn_DataType_t src0_qnn_type = QNN_DATATYPE_FLOAT_32; + Qnn_DataType_t src1_qnn_type = QNN_DATATYPE_FLOAT_32; + Qnn_DataType_t dst_qnn_type = QNN_DATATYPE_FLOAT_32; + + src0_qnn_type = ggmlqnn_datatype_from_ggml_datatype(src0->type); + src1_qnn_type = ggmlqnn_datatype_from_ggml_datatype(src1->type); + dst_qnn_type = ggmlqnn_datatype_from_ggml_datatype(dst->type); + + uint32_t dimensions_input_0[] = {(uint32_t) src0->ne[0], (uint32_t) src0->ne[1], + (uint32_t) src0->ne[2], (uint32_t) src0->ne[3]}; + uint32_t dimensions_input_1[] = {(uint32_t) src1->ne[0], (uint32_t) src1->ne[1], + (uint32_t) src1->ne[2], (uint32_t) src1->ne[3]}; + uint32_t dimensions_output[] = {(uint32_t) dst->ne[0], (uint32_t) dst->ne[1], + (uint32_t) dst->ne[2], (uint32_t) dst->ne[3]}; + + QNN_VER_PTR(*p_tensor0)->dimensions = dimensions_input_0; + QNN_VER_PTR(*p_tensor0)->rank = ggml_n_dims(src0); + QNN_VER_PTR(*p_tensor0)->dataType = src0_qnn_type; + + QNN_VER_PTR(*p_tensor1)->dimensions = dimensions_input_1; + QNN_VER_PTR(*p_tensor1)->rank = ggml_n_dims(src1); + QNN_VER_PTR(*p_tensor1)->dataType = src1_qnn_type; + + QNN_VER_PTR(*p_tensor2)->dimensions = dimensions_output; + QNN_VER_PTR(*p_tensor2)->rank = ggml_n_dims(dst); + QNN_VER_PTR(*p_tensor2)->dataType = dst_qnn_type; + + if (enable_npu_rpc) { + //TODO: NPU RPC feature will failed with test-backend-ops + uint8_t * qnn_buffer_0 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*p_tensor0)->memHandle)); + GGMLQNN_LOG_INFO("qnn_rpcbuffer_0 = %p\n", qnn_buffer_0); + if (nullptr != qnn_buffer_0) { + memcpy(qnn_buffer_0, src0->data, ggml_nbytes(src0)); + } + + uint8_t * qnn_buffer_1 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*p_tensor1)->memHandle)); + GGMLQNN_LOG_INFO("qnn_rpcbuffer_1 = %p\n", qnn_buffer_1); + if (nullptr != qnn_buffer_1) { + memcpy(qnn_buffer_1, src1->data, ggml_nbytes(src1)); + } + } else { + QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggmlqnn_get_tensor_data_size(src0)}; + QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggmlqnn_get_tensor_data_size(src1)}; + QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggmlqnn_get_tensor_data_size(dst)}; + } + + Qnn_Tensor_t tensor_inputs[] = { + *p_tensor0, + *p_tensor1 + }; + Qnn_Tensor_t tensor_outputs[] = { + *p_tensor2 + }; + CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, + tensor_inputs, 2, + tensor_outputs, 1, + nullptr, nullptr)); + + if (enable_npu_rpc) { + //TODO:NPU RPC feature will failed with test-backend-ops + uint8_t * qnn_buffer_2 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*p_tensor2)->memHandle)); + if (nullptr != qnn_buffer_2) { + memcpy(dst->data, qnn_buffer_2, ggml_nbytes(dst)); + } + } + } + + // restore the original dimensions of qnn tensors to avoid memory leak in func free_qnn_tensor + QNN_VER_PTR(*p_tensor0)->dimensions = tensor_0_dimensions; + QNN_VER_PTR(*p_tensor1)->dimensions = tensor_1_dimensions; + QNN_VER_PTR(*p_tensor2)->dimensions = tensor_2_dimensions; + +#if GGMLQNN_PRINT_OP_ADD_LOG + op_perf.info(); +#endif +} + +//FIXME:there is issue in this function +/* + * this function is AI-assisted code from Grok 3. + * the logic of ggml_qnn_mul_mat_4d is similar to ggml_qnn_mul_mat but much more complicated + * than ggml_qnn_mul_mat, so it's a standalone function. + * it will be combined with ggml_qnn_mul_mat after bugfix + */ +static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context * ctx, ggml_tensor * op) { + Qnn_ErrorHandle_t error = QNN_SUCCESS; + bool graph_initialized = false; + qnn_perf op_perf = qnn_perf("ggml_qnn_mul_mat_4d"); + qnn_instance * instance = nullptr; + Qnn_GraphHandle_t graph_handle = nullptr; + Qnn_Tensor_t * p_tensor0 = nullptr; + Qnn_Tensor_t * p_tensor1 = nullptr; + Qnn_Tensor_t * p_tensor2 = nullptr; + Qnn_Tensor_t * p_param_tensor = nullptr; + Qnn_Tensor_t * p_tensor2_transpose = nullptr; + Qnn_Tensor_t * p_gather0_index = nullptr; + Qnn_Tensor_t * p_gather0_out = nullptr; + Qnn_Tensor_t * p_gather1_index = nullptr; + Qnn_Tensor_t * p_gather1_out = nullptr; + const ggml_tensor * src0 = op->src[0]; + const ggml_tensor * src1 = op->src[1]; + ggml_tensor * dst = op; + + GGMLQNN_CHECK_PARAMS(ctx, src0, src1, dst); + instance = ctx->instance; + QNN_INTERFACE_VER_TYPE qnn_raw_interface = ctx->raw_interface; + op_perf.start(); + + uint32_t src0_rank = ggml_n_dims(src0); + uint32_t src1_rank = ggml_n_dims(src1); + GGML_ASSERT(src0_rank == src1_rank); + GGML_ASSERT(src0_rank == 4); + + std::string graph_name; + ggmlqnn_get_graphkey_from_op(op, graph_name); + if (instance->_qnn_graph_map.find(graph_name) != instance->_qnn_graph_map.end()) { + graph_initialized = true; + qnn_res_t &graph_item = instance->_qnn_graph_map[graph_name]; + graph_handle = std::get<0>(graph_item); + qnn_tensors_t & tensors = std::get<1>(graph_item); + p_tensor0 = tensors[0]; + p_tensor1 = tensors[1]; + p_tensor2 = tensors[2]; + p_param_tensor = tensors[3]; + p_tensor2_transpose = tensors[4]; + p_gather0_index = tensors[5]; + p_gather0_out = tensors[6]; + p_gather0_index = tensors[7]; + p_gather1_out = tensors[8]; + } else { + p_tensor0 = GQCGT(src0, nullptr, QNN_TENSOR_TYPE_APP_WRITE, QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0); + p_tensor1 = GQCGT(src1, nullptr, QNN_TENSOR_TYPE_APP_WRITE, QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0); + p_tensor2 = GQCGT(dst, nullptr, QNN_TENSOR_TYPE_APP_READ, QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0); + } + ggmlqnn_print_tensors_info(__func__, ctx, src0, src1, dst); + + //ensure QNN tensor has correct tensor type + QNN_VER_PTR(*p_tensor0)->type = QNN_TENSOR_TYPE_APP_WRITE; + QNN_VER_PTR(*p_tensor1)->type = QNN_TENSOR_TYPE_APP_WRITE; + QNN_VER_PTR(*p_tensor2)->type = QNN_TENSOR_TYPE_APP_READ; + + //save the original dimensions of qnn tensors + uint32_t *tensor_0_dimensions = QNN_VER_PTR(*p_tensor0)->dimensions; + uint32_t *tensor_1_dimensions = QNN_VER_PTR(*p_tensor1)->dimensions; + uint32_t *tensor_2_dimensions = QNN_VER_PTR(*p_tensor2)->dimensions; + + if (!graph_initialized) { + //step-1:create graph + GGMLQNN_LOG_DEBUG("graph name %s\n", graph_name.c_str()); + CHECK_QNN_API(error, qnn_raw_interface.graphCreate(instance->get_qnn_context_handle(), + graph_name.c_str(), NULL, + &graph_handle)); + //step-2:tensor definitions for offload 4D matrix mulmat to QNN backend + /* + tensor0: "p_tensor0" (permutation tensor for Transpose). + tensor1: "p_tensor0" (input tensor for first Gather). + tensor2: "p_gather0_index" (indices for first Gather). + tensor3: "p_gather0_out" (output of first Gather). + tensor4: "p_gather1_index" (indices for second Gather). + tensor5: "p_gather1_out" (output of second Gather). + tensor6: "p_tensor1" (second input for MatMul). + tensor7: "p_tensor2_transpose" (output of MatMul, input to Transpose). + tensor8: "p_tensor2" (output of Transpose). + */ + uint32_t dims0[1] = {4}; + uint32_t data0[4] = {0, static_cast(src1->ne[1]), static_cast(src0->ne[2]), static_cast(src0->ne[3])}; + p_param_tensor = GQCGT(nullptr, "param", QNN_TENSOR_TYPE_STATIC, QNN_DATATYPE_UINT_32, 1, dims0, data0, src0_rank * sizeof(uint32_t)); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_param_tensor)); + + p_tensor0 = GQCGT(src0, nullptr, QNN_TENSOR_TYPE_APP_WRITE, QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor0)); + + uint32_t dims2[] = {6}; + uint32_t data2[6] = {0, static_cast(src1->ne[1]), static_cast(src0->ne[2]), static_cast(src0->ne[3]), 0, 0}; + p_gather0_index = GQCGT(nullptr, "gather0_index", QNN_TENSOR_TYPE_STATIC, QNN_DATATYPE_UINT_32, 1, dims2, data2, 24); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_gather0_index)); + + uint32_t dims3[] = {static_cast(src0->ne[3]), static_cast(src1->ne[2]), static_cast(src0->ne[1]), static_cast(src0->ne[0])}; + p_gather0_out = GQCGT(nullptr, "gather0_out", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, src0_rank, dims3, nullptr, 0); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_gather0_out)); + + uint32_t dims4[] = {4}; + uint32_t data4[4] = {static_cast(src1->ne[1]), static_cast(src1->ne[1]), static_cast(src0->ne[3]), static_cast(src0->ne[3])}; + p_gather1_index = GQCGT(nullptr, "gather1_index", QNN_TENSOR_TYPE_STATIC, QNN_DATATYPE_UINT_32, 1, dims4, data4, 16); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_gather1_index)); + + uint32_t dims5[] = {static_cast(src1->ne[3]), static_cast(src1->ne[2]), static_cast(src0->ne[1]), static_cast(src0->ne[0])}; + p_gather1_out = GQCGT(nullptr, "gather1_out", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, src0_rank, dims5, nullptr, 0); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_gather1_out)); + + p_tensor1 = GQCGT(src1, nullptr, QNN_TENSOR_TYPE_APP_WRITE, QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor1)); + + uint32_t tensor2_transpose_dims[GGML_MAX_DIMS] = {}; + p_tensor2_transpose = GQCGT(dst, "transpose", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0, true); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor2_transpose)); + + p_tensor2 = GQCGT(dst, nullptr, QNN_TENSOR_TYPE_APP_READ, QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor2)); + + //step-3:gather operation 0 + Qnn_Param_t gather0_params[] = {{QNN_PARAMTYPE_SCALAR,"axis", .scalarParam = {QNN_DATATYPE_INT_32, .int32Value = 1}}}; + Qnn_Tensor_t gather0_inputs[] = {*p_tensor0, *p_gather0_index}; + Qnn_Tensor_t gather0_outputs[] = {*p_gather0_out}; + Qnn_OpConfig_t gather0_op = ggmlqnn_create_op_config("out_gather0", QNN_OP_PACKAGE_NAME_QTI_AISW, QNN_OP_GATHER, + gather0_params, 1, gather0_inputs, 2, + gather0_outputs, 1); + CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, gather0_op)); + + //step-4:gather operation 1 + Qnn_Param_t gather1_params[] = {{QNN_PARAMTYPE_SCALAR,"axis", .scalarParam = {QNN_DATATYPE_INT_32, .int32Value = 0}}}; + Qnn_Tensor_t gather1_inputs[] = {*p_gather0_out, *p_gather1_index}; + Qnn_Tensor_t gather1_outputs[] = {*p_gather1_out}; + Qnn_OpConfig_t gather1_op = ggmlqnn_create_op_config("out_gather1", QNN_OP_PACKAGE_NAME_QTI_AISW, QNN_OP_GATHER, + gather1_params, 1, gather1_inputs, 2, + gather1_outputs, 1); + CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, gather1_op)); + + //step-5:matmul operation + Qnn_Param_t matmul_params[] = {{QNN_PARAMTYPE_SCALAR,"transpose_in1", .scalarParam = {QNN_DATATYPE_BOOL_8, .bool8Value = 1}}}; + Qnn_Tensor_t matmul_inputs[] = {*p_gather1_out, *p_tensor1}; + Qnn_Tensor_t matmul_outputs[] = {*p_tensor2_transpose}; + Qnn_OpConfig_t matmul_op = ggmlqnn_create_op_config("mulmat_opconfig", QNN_OP_PACKAGE_NAME_QTI_AISW, QNN_OP_MAT_MUL, + matmul_params, 1, matmul_inputs, 2, + matmul_outputs, + 1); + CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, matmul_op)); + + //step-6:transpose operation + Qnn_Param_t transpose_params[] = { {QNN_PARAMTYPE_TENSOR, "perm", .tensorParam = *p_param_tensor}}; + Qnn_Tensor_t transpose_inputs[] = {*p_tensor2_transpose}; + Qnn_Tensor_t transpose_outputs[]= {*p_tensor2}; + Qnn_OpConfig_t transpose_op = ggmlqnn_create_op_config("transpose_opconfig", QNN_OP_PACKAGE_NAME_QTI_AISW, QNN_OP_TRANSPOSE, + transpose_params, 1, transpose_inputs, 1, + transpose_outputs, 1); + CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, transpose_op)); + + //step-7:finalize graph + CHECK_QNN_API(error, qnn_raw_interface.graphFinalize(graph_handle, NULL, NULL)); + + //step-8:execute graph + QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggmlqnn_get_tensor_data_size(src0)}; + QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggmlqnn_get_tensor_data_size(src1)}; + QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggmlqnn_get_tensor_data_size(dst)}; + Qnn_Tensor_t input_tensors[] = {*p_tensor0, *p_tensor1}; + Qnn_Tensor_t output_tensors[] = {*p_tensor2}; + CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, input_tensors, 2, output_tensors, + 1, NULL, NULL)); + + qnn_tensors_t ggml_op_mulmat_tensors; + ggml_op_mulmat_tensors.reserve(9); + ggml_op_mulmat_tensors.push_back(p_tensor0); + ggml_op_mulmat_tensors.push_back(p_tensor1); + ggml_op_mulmat_tensors.push_back(p_tensor2); + ggml_op_mulmat_tensors.push_back(p_param_tensor); + ggml_op_mulmat_tensors.push_back(p_tensor2_transpose); + ggml_op_mulmat_tensors.push_back(p_gather0_index); + ggml_op_mulmat_tensors.push_back(p_gather0_out); + ggml_op_mulmat_tensors.push_back(p_gather1_index); + ggml_op_mulmat_tensors.push_back(p_gather1_out); + + auto graph_item = std::make_tuple(graph_handle, ggml_op_mulmat_tensors); + instance->_qnn_graph_map[graph_name] = graph_item; + } else { + QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggmlqnn_get_tensor_data_size(src0)}; + QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggmlqnn_get_tensor_data_size(src1)}; + QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggmlqnn_get_tensor_data_size(dst)}; + + Qnn_Tensor_t tensor_inputs[] = { + *p_tensor0, + *p_tensor1 + }; + Qnn_Tensor_t tensor_outputs[] = { + *p_tensor2 + }; + // this is the second technical approach of "how to utilize the Hexagon NPU maximally" through QNN SDK + CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, + tensor_inputs, 2, + tensor_outputs, 1, + nullptr, nullptr)); + } + + // restore the original dimensions of qnn tensors to avoid memory leak in func free_qnn_tensor + QNN_VER_PTR(*p_tensor0)->dimensions = tensor_0_dimensions; + QNN_VER_PTR(*p_tensor1)->dimensions = tensor_1_dimensions; + QNN_VER_PTR(*p_tensor2)->dimensions = tensor_2_dimensions; + op_perf.info(); +} + +/* + * @brief performs matrix multiplication with FP32 & quantized weights and floating-point inputs + * using the QNN backend. this function performs matrix multiplication of the input tensor + * `src1` and the weight tensor `src0`, handling transposing, and quantization as needed, + * and stores the result in the destination tensor `dst`. + * + * @param backend the context which got through (ggml_backend_qnn_context *)backend->context for the + * QNN backend operations. + * @param op the destination tensor where the result of the matrix multiplication will be stored. + * + * @note the logic of ggml_qnn_mul_mat is similar to ggml_qnn_general_node but much more complicated + * than ggml_qnn_general_node. so it's a standalone function. accordingly, this is another + * typical skeleton for offload other ggml ops to QNN backend. MUL_MAT take most of the compute + * time (about 95%).so to speed up llama inference, should focus on this func. there are three kinds + * of MUL_MAT to compute: + * mul_mat_f32: both src0 and src1 are F32, this will be naturally handled in QNN backend + * mul_mat_f16_f32: src0 is F16 and src1 is F32, f16 in src0 -> f32 in src0', then src0' * src1 + * mul_mat_q_f32: src0 is quantized (Q4_0, Q4_1, Q6_K...) + * and src1 is F32, src0 -> f32 in src0', then src0' * src1 +*/ +void ggml_qnn_mul_mat(ggml_backend_qnn_context * ctx, ggml_tensor * op) { + Qnn_ErrorHandle_t error = QNN_SUCCESS; + bool graph_initialized = false; + qnn_perf op_perf = qnn_perf("ggml_qnn_mul_mat"); + qnn_instance * instance = nullptr; + Qnn_GraphHandle_t graph_handle = nullptr; + Qnn_Tensor_t * p_tensor0 = nullptr; + Qnn_Tensor_t * p_tensor1 = nullptr; + Qnn_Tensor_t * p_tensor2 = nullptr; + Qnn_Tensor_t * p_param_tensor = nullptr; + Qnn_Tensor_t * p_tensor2_transpose = nullptr; + const ggml_tensor * src0 = op->src[0]; + const ggml_tensor * src1 = op->src[1]; + ggml_tensor * dst = op; + + GGMLQNN_CHECK_PARAMS(ctx, src0, src1, dst); + instance = ctx->instance; + QNN_INTERFACE_VER_TYPE qnn_raw_interface = ctx->raw_interface; + op_perf.start(); + + const enum ggml_type src0_type = src0->type; + const uint32_t src0_rank = ggml_n_dims(src0); + const uint32_t src1_rank = ggml_n_dims(src1); + GGML_ASSERT(src0_rank == src1_rank); + GGML_ASSERT(src0_rank >= 2); //QNN SDK's limitation, make QNN SDK happy + //GGML_ASSERT(src0_rank != 4); //TODO: 4D matrix mulmat + if (4 == src0_rank) { + return ggml_qnn_mul_mat_4d(ctx, op); + } + void * wdata = ggmlqnn_type_trait(ctx, op); + const size_t desired_size = ctx->desired_size; + + std::string graph_name; + ggmlqnn_get_graphkey_from_op(op, graph_name); + if (instance->_qnn_graph_map.find(graph_name) != instance->_qnn_graph_map.end()) { + graph_initialized = true; + qnn_res_t & graph_item = instance->_qnn_graph_map[graph_name]; + graph_handle = std::get<0>(graph_item); + qnn_tensors_t & tensors = std::get<1>(graph_item); + p_tensor0 = tensors[0]; + p_tensor1 = tensors[1]; + p_tensor2 = tensors[2]; + p_param_tensor = tensors[3]; + p_tensor2_transpose = tensors[4]; + } else { + p_tensor0 = GQCGT(src0, nullptr, QNN_TENSOR_TYPE_APP_WRITE,QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0); + p_tensor1 = GQCGT(src1, nullptr, QNN_TENSOR_TYPE_APP_WRITE,QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0); + p_tensor2 = GQCGT(dst, nullptr, QNN_TENSOR_TYPE_APP_READ,QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0); + } + ggmlqnn_print_tensors_info(__func__, ctx, src0, src1, dst); + + //ensure QNN tensor has correct tensor type + QNN_VER_PTR(*p_tensor0)->type = QNN_TENSOR_TYPE_APP_WRITE; + QNN_VER_PTR(*p_tensor1)->type = QNN_TENSOR_TYPE_APP_WRITE; + QNN_VER_PTR(*p_tensor2)->type = QNN_TENSOR_TYPE_APP_READ; + + //save the original dimensions of qnn tensors + uint32_t * tensor_0_dimensions = QNN_VER_PTR(*p_tensor0)->dimensions; + uint32_t * tensor_1_dimensions = QNN_VER_PTR(*p_tensor1)->dimensions; + uint32_t * tensor_2_dimensions = QNN_VER_PTR(*p_tensor2)->dimensions; + + if (!graph_initialized) { + GGMLQNN_LOG_DEBUG("graph name %s", graph_name.c_str()); + /* + there are two key-points in properly handling how to offload mulmat to the QNN backend in ggml-qnn + 1. transpose + a 3x2 f32 matrix which means 3 rows and 2 columns. in ggml, it could be created from: + struct ggml_tensor* matrix = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 2, 3); + which like this: + +---+---+ + | 0 | 1 | + +---+---+ + | 2 | 3 | + +---+---+ + | 4 | 5 | + +---+---+ + with + ne[0] = 2 + ne[1] = 3 + there are different dimension order between ggml tensor and qnn tensor + + 2. QNN's MatMul can only support input tensors with rank >= 2 + + in the all, there is gap between ggml mulmat and QNN mulmat,we need to perform a transpose + operation when offloading mulmat to QNN backend. this concise implementation will handle + transpose in func ggml_qnn_create_general_tensor() + */ + //step-1: create qnn graph + error = qnn_raw_interface.graphCreate(instance->get_qnn_context_handle(), + graph_name.c_str(), nullptr, &graph_handle); + if (QNN_SUCCESS != error) { + GGMLQNN_LOG_INFO("can't create qnn graph handle with graph name %s, error = %d\n", graph_name.c_str(), error); + return; + } + //step-2: create param tensor for mulmat of 2d/3d/4d matrix + const uint32_t param_tensor_data[GGML_MAX_DIMS][GGML_MAX_DIMS] = { + {0}, + {1, 0}, + {0, 2, 1}, + {0, 1, 3, 2}, + }; + uint32_t param_tensor_dims[1] = {src0_rank}; + p_param_tensor = GQCGT(nullptr, "param", QNN_TENSOR_TYPE_STATIC, QNN_DATATYPE_UINT_32, 1, param_tensor_dims, (void *)(param_tensor_data[src0_rank - 1]), src0_rank * sizeof(uint32_t)); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_param_tensor)); + + //step-3: create compute tensor from ggml tensor + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor0)); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor1)); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor2)); + if (src0_type != GGML_TYPE_F32) { + QNN_VER_PTR(*p_tensor0)->clientBuf = {wdata, static_cast(desired_size)}; + } else { + QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggmlqnn_get_tensor_data_size(src0)}; + } + QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggmlqnn_get_tensor_data_size(src1)}; + QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggmlqnn_get_tensor_data_size(dst)}; + + //step-4: create a transpose tensor + p_tensor2_transpose = GQCGT(dst, "transpose", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0, true); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor2_transpose)); + + //step-5: compose qnn graph: add mat_mul node + Qnn_Param_t out_0_params[] = { + {QNN_PARAMTYPE_SCALAR, + QNN_OP_MAT_MUL_PARAM_TRANSPOSE_IN1, + .scalarParam = {QNN_DATATYPE_BOOL_8, .bool8Value = 1} + } + }; + + Qnn_Tensor_t out_0_inputs[] = {*p_tensor0, *p_tensor1}; + Qnn_Tensor_t out_0_outputs[] = {*p_tensor2_transpose}; +#if 0 //leave here for easily understand code, can be removed in the future + Qnn_OpConfig_t out_0 = { + QNN_OPCONFIG_VERSION_1, .v1 = + {"ggmlqnn_mulmat_opconfig", QNN_OP_PACKAGE_NAME_QTI_AISW, QNN_OP_MAT_MUL, + 1, + out_0_params, + 2, + out_0_inputs, + 1, + out_0_outputs} + }; +#else + Qnn_OpConfig_t out_0 = ggmlqnn_create_op_config("ggmlqnn_mulmat_opconfig", QNN_OP_PACKAGE_NAME_QTI_AISW, QNN_OP_MAT_MUL, + out_0_params, 1, out_0_inputs, 2, out_0_outputs, 1); +#endif + CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle,out_0)); + + //step-5: compose qnn graph: add transpose node + Qnn_Param_t out_trans1_0_params[] = { + {(Qnn_ParamType_t) 1, + "perm", .tensorParam = *p_param_tensor + } + }; + Qnn_Tensor_t out_trans1_0_inputs[] = {*p_tensor2_transpose}; + Qnn_Tensor_t out_trans1_0_outputs[] = {*p_tensor2}; +#if 0 //leave here for easily understand code, can be removed in the future + Qnn_OpConfig_t out_trans1_0 = { + QNN_OPCONFIG_VERSION_1, + .v1 = {"ggmlqnn_mulmat_transpose_opconfig", + QNN_OP_PACKAGE_NAME_QTI_AISW, + QNN_OP_TRANSPOSE, 1, + out_trans1_0_params, + 1, + out_trans1_0_inputs, + 1, + out_trans1_0_outputs} + }; +#else + Qnn_OpConfig_t out_trans1_0 = ggmlqnn_create_op_config("ggmlqnn_mulmat_transpose_opconfig", QNN_OP_PACKAGE_NAME_QTI_AISW, QNN_OP_TRANSPOSE, + out_trans1_0_params, 1, out_trans1_0_inputs, 1, out_trans1_0_outputs, 1); +#endif + CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle,out_trans1_0)); + + //step-6: finalize qnn graph and execute qnn graph + CHECK_QNN_API(error, qnn_raw_interface.graphFinalize(graph_handle, nullptr, nullptr)); + Qnn_Tensor_t input_tensors_0[] = {*p_tensor0, *p_tensor1}; + Qnn_Tensor_t output_tensors_0[] = {*p_tensor2}; + CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, + input_tensors_0, 2, + output_tensors_0, 1, + nullptr, nullptr)); + + qnn_tensors_t ggml_op_mulmat_tensors; + ggml_op_mulmat_tensors.reserve(5); + ggml_op_mulmat_tensors.push_back(p_tensor0); + ggml_op_mulmat_tensors.push_back(p_tensor1); + ggml_op_mulmat_tensors.push_back(p_tensor2); + ggml_op_mulmat_tensors.push_back(p_param_tensor); + ggml_op_mulmat_tensors.push_back(p_tensor2_transpose); + auto graph_item = std::make_tuple(graph_handle, ggml_op_mulmat_tensors); + instance->_qnn_graph_map[graph_name] = graph_item; + } else { + if (src0_type != GGML_TYPE_F32) { + QNN_VER_PTR(*p_tensor0)->clientBuf = {wdata, static_cast(desired_size)}; + } else { + QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggmlqnn_get_tensor_data_size(src0)}; + } + QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggmlqnn_get_tensor_data_size(src1)}; + QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggmlqnn_get_tensor_data_size(dst)}; + + Qnn_Tensor_t tensor_inputs[] = { + *p_tensor0, + *p_tensor1 + }; + Qnn_Tensor_t tensor_outputs[] = { + *p_tensor2 + }; + // this is the second technical approach or another pipeline of "how to utilize the Hexagon + // NPU maximally" through QNN SDK, details could be found at + // https://github.com/ggml-org/llama.cpp/pull/12049#issuecomment-2678308360 + CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, + tensor_inputs, 2, + tensor_outputs, 1, + nullptr, nullptr)); + } + + // restore the original dimensions of qnn tensors to avoid memory leak in func free_qnn_tensor + QNN_VER_PTR(*p_tensor0)->dimensions = tensor_0_dimensions; + QNN_VER_PTR(*p_tensor1)->dimensions = tensor_1_dimensions; + QNN_VER_PTR(*p_tensor2)->dimensions = tensor_2_dimensions; + op_perf.info(); +} + +void ggml_qnn_repeat(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +} + + +void ggml_qnn_div(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +} + +void ggml_qnn_leaky_relu(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +} + +void ggml_qnn_concat(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +} + +void ggml_qnn_arange(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +} + +void ggml_qnn_sqr(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +} + +void ggml_qnn_clamp(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +} + +void ggml_qnn_scale(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +} + +void ggml_qnn_argsort(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +} + +void ggml_qnn_norm(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +} + +void ggml_qnn_group_norm(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +} + +void ggml_qnn_acc(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +} + +void ggml_qnn_sum_rows(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +} + +void ggml_qnn_upsample_nearest2d(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +} + +void ggml_qnn_pad(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +} + +void ggml_qnn_pool2d(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +} + +void ggml_qnn_dup(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +} + +void ggml_qnn_rms_norm(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +} + +void ggml_qnn_diag_mask(ggml_backend_qnn_context * ctx, ggml_tensor * dst, float value) { +} + +void ggml_qnn_im2col(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +} + +void ggml_qnn_timestep_embedding(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +} + +void ggml_qnn_cpy(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + ggml_qnn_dup(ctx, dst); +} + +void ggml_qnn_softmax(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +} + +void ggml_qnn_get_rows(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +} + +void ggml_qnn_rope(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +} diff --git a/ggml/src/ggml-qnn/ggml-qnn.cpp b/ggml/src/ggml-qnn/ggml-qnn.cpp index 51678f7b51ca3..3b59956009398 100644 --- a/ggml/src/ggml-qnn/ggml-qnn.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn.cpp @@ -703,7 +703,7 @@ static struct ggml_backend_qnn_context g_qnn_mgr[GGML_QNN_MAX_DEVICES] = { .socinfo = {}}, }; -const qnn_op_caps_t k_op_caps[] = { +const qnn_op_caps_t ggmlqnn_k_op_caps[] = { {}, // GGML_OP_NONE {}, // GGML_OP_DUP { @@ -1152,7 +1152,7 @@ static void append_tensor_dimensions(const ggml_tensor * tensor, std::string & o } size_t ggmlqnn_get_opcaps_size() { - return std::size(k_op_caps); + return std::size(ggmlqnn_k_op_caps); } size_t ggmlqnn_get_op_index(const ggml_tensor * tensor) { @@ -1165,8 +1165,8 @@ size_t ggmlqnn_get_op_index(const ggml_tensor * tensor) { static size_t ggmlqnn_get_op_input_param_count(const ggml_tensor * op) { auto op_index = ggmlqnn_get_op_index(op); - GGML_ASSERT(op_index < std::size(k_op_caps)); - return k_op_caps[op_index].input_param_count; + GGML_ASSERT(op_index < std::size(ggmlqnn_k_op_caps)); + return ggmlqnn_k_op_caps[op_index].input_param_count; } void ggmlqnn_get_graphkey_from_op(const ggml_tensor * op, std::string & output) { @@ -1701,7 +1701,7 @@ static void ggml_qnn_logcallback(const char * fmt, std::lock_guard lock(log_mutex); memset(s_ggml_qnn_logbuf, 0, GGML_QNN_LOGBUF_LEN); vsnprintf(reinterpret_cast(s_ggml_qnn_logbuf), GGML_QNN_LOGBUF_LEN, fmt, argp); - GGMLQNN_LOG_INFO("%8.1fms [%-7s] %s\n", ms, log_level_desc, s_ggml_qnn_logbuf); + GGMLQNN_LOG_DEBUG("%8.1fms [%-7s] %s\n", ms, log_level_desc, s_ggml_qnn_logbuf); } } #else From 447bc3f0601b517d410346a8d5dfab7433e238ca Mon Sep 17 00:00:00 2001 From: zhouwg Date: Sun, 2 Mar 2025 12:06:59 +0800 Subject: [PATCH 34/76] ggml-qnn: submit AI-assisted ggml_qnn_mul_mat_4d(not worked currently) which generated by Grok 3 --- ggml/src/ggml-qnn/ggml-qnn-ops.cpp | 339 +++++++++++++---------------- tests/ggml-qnn-ut.cpp | 6 +- 2 files changed, 150 insertions(+), 195 deletions(-) diff --git a/ggml/src/ggml-qnn/ggml-qnn-ops.cpp b/ggml/src/ggml-qnn/ggml-qnn-ops.cpp index 02b7ab7820a95..2553ff78f9c7d 100644 --- a/ggml/src/ggml-qnn/ggml-qnn-ops.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn-ops.cpp @@ -278,216 +278,171 @@ void ggml_qnn_general_node(ggml_backend_qnn_context * ctx, ggml_tensor * op) { #endif } -//FIXME:there is issue in this function +//TODO:there is issue in this function /* - * this function is AI-assisted code from Grok 3. + * this function is AI-assisted code from Grok 3 for purpose of 4d mulmat UT in ggml-qnn-ut.cpp + * ./scripts/build-run-android.sh run_ut_mulmat 0 + * ./scripts/build-run-android.sh run_ut_mulmat 1 + * ./scripts/build-run-android.sh run_ut_mulmat 2 + * * the logic of ggml_qnn_mul_mat_4d is similar to ggml_qnn_mul_mat but much more complicated * than ggml_qnn_mul_mat, so it's a standalone function. * it will be combined with ggml_qnn_mul_mat after bugfix */ -static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context * ctx, ggml_tensor * op) { - Qnn_ErrorHandle_t error = QNN_SUCCESS; - bool graph_initialized = false; - qnn_perf op_perf = qnn_perf("ggml_qnn_mul_mat_4d"); - qnn_instance * instance = nullptr; - Qnn_GraphHandle_t graph_handle = nullptr; - Qnn_Tensor_t * p_tensor0 = nullptr; - Qnn_Tensor_t * p_tensor1 = nullptr; - Qnn_Tensor_t * p_tensor2 = nullptr; - Qnn_Tensor_t * p_param_tensor = nullptr; - Qnn_Tensor_t * p_tensor2_transpose = nullptr; - Qnn_Tensor_t * p_gather0_index = nullptr; - Qnn_Tensor_t * p_gather0_out = nullptr; - Qnn_Tensor_t * p_gather1_index = nullptr; - Qnn_Tensor_t * p_gather1_out = nullptr; - const ggml_tensor * src0 = op->src[0]; - const ggml_tensor * src1 = op->src[1]; - ggml_tensor * dst = op; +static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) { + Qnn_ErrorHandle_t error = QNN_SUCCESS; + qnn_perf op_perf = qnn_perf("ggml_qnn_mul_mat_4d"); + qnn_instance *instance = ctx->instance; + QNN_INTERFACE_VER_TYPE qnn_raw_interface = ctx->raw_interface; + + const ggml_tensor *src0 = op->src[0]; // e.g., [256, 16, 3, 2] or [256,16,3,2] + const ggml_tensor *src1 = op->src[1]; // e.g., [256, 1, 6, 4] or [256,16,3, 2] + ggml_tensor *dst = op; // e.g., [16, 1, 6, 4] or [16,16,3, 2] GGMLQNN_CHECK_PARAMS(ctx, src0, src1, dst); - instance = ctx->instance; - QNN_INTERFACE_VER_TYPE qnn_raw_interface = ctx->raw_interface; + GGML_ASSERT(ggml_n_dims(src0) == 4 && ggml_n_dims(src1) == 4); op_perf.start(); - uint32_t src0_rank = ggml_n_dims(src0); - uint32_t src1_rank = ggml_n_dims(src1); - GGML_ASSERT(src0_rank == src1_rank); - GGML_ASSERT(src0_rank == 4); - std::string graph_name; ggmlqnn_get_graphkey_from_op(op, graph_name); - if (instance->_qnn_graph_map.find(graph_name) != instance->_qnn_graph_map.end()) { - graph_initialized = true; - qnn_res_t &graph_item = instance->_qnn_graph_map[graph_name]; - graph_handle = std::get<0>(graph_item); - qnn_tensors_t & tensors = std::get<1>(graph_item); - p_tensor0 = tensors[0]; - p_tensor1 = tensors[1]; - p_tensor2 = tensors[2]; - p_param_tensor = tensors[3]; - p_tensor2_transpose = tensors[4]; - p_gather0_index = tensors[5]; - p_gather0_out = tensors[6]; - p_gather0_index = tensors[7]; - p_gather1_out = tensors[8]; - } else { - p_tensor0 = GQCGT(src0, nullptr, QNN_TENSOR_TYPE_APP_WRITE, QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0); - p_tensor1 = GQCGT(src1, nullptr, QNN_TENSOR_TYPE_APP_WRITE, QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0); - p_tensor2 = GQCGT(dst, nullptr, QNN_TENSOR_TYPE_APP_READ, QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0); - } + GGMLQNN_LOG_DEBUG("graph name %s\n", graph_name.c_str()); ggmlqnn_print_tensors_info(__func__, ctx, src0, src1, dst); - //ensure QNN tensor has correct tensor type - QNN_VER_PTR(*p_tensor0)->type = QNN_TENSOR_TYPE_APP_WRITE; - QNN_VER_PTR(*p_tensor1)->type = QNN_TENSOR_TYPE_APP_WRITE; - QNN_VER_PTR(*p_tensor2)->type = QNN_TENSOR_TYPE_APP_READ; - - //save the original dimensions of qnn tensors - uint32_t *tensor_0_dimensions = QNN_VER_PTR(*p_tensor0)->dimensions; - uint32_t *tensor_1_dimensions = QNN_VER_PTR(*p_tensor1)->dimensions; - uint32_t *tensor_2_dimensions = QNN_VER_PTR(*p_tensor2)->dimensions; - - if (!graph_initialized) { - //step-1:create graph - GGMLQNN_LOG_DEBUG("graph name %s\n", graph_name.c_str()); - CHECK_QNN_API(error, qnn_raw_interface.graphCreate(instance->get_qnn_context_handle(), - graph_name.c_str(), NULL, - &graph_handle)); - //step-2:tensor definitions for offload 4D matrix mulmat to QNN backend - /* - tensor0: "p_tensor0" (permutation tensor for Transpose). - tensor1: "p_tensor0" (input tensor for first Gather). - tensor2: "p_gather0_index" (indices for first Gather). - tensor3: "p_gather0_out" (output of first Gather). - tensor4: "p_gather1_index" (indices for second Gather). - tensor5: "p_gather1_out" (output of second Gather). - tensor6: "p_tensor1" (second input for MatMul). - tensor7: "p_tensor2_transpose" (output of MatMul, input to Transpose). - tensor8: "p_tensor2" (output of Transpose). - */ - uint32_t dims0[1] = {4}; - uint32_t data0[4] = {0, static_cast(src1->ne[1]), static_cast(src0->ne[2]), static_cast(src0->ne[3])}; - p_param_tensor = GQCGT(nullptr, "param", QNN_TENSOR_TYPE_STATIC, QNN_DATATYPE_UINT_32, 1, dims0, data0, src0_rank * sizeof(uint32_t)); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_param_tensor)); - - p_tensor0 = GQCGT(src0, nullptr, QNN_TENSOR_TYPE_APP_WRITE, QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor0)); - - uint32_t dims2[] = {6}; - uint32_t data2[6] = {0, static_cast(src1->ne[1]), static_cast(src0->ne[2]), static_cast(src0->ne[3]), 0, 0}; - p_gather0_index = GQCGT(nullptr, "gather0_index", QNN_TENSOR_TYPE_STATIC, QNN_DATATYPE_UINT_32, 1, dims2, data2, 24); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_gather0_index)); - - uint32_t dims3[] = {static_cast(src0->ne[3]), static_cast(src1->ne[2]), static_cast(src0->ne[1]), static_cast(src0->ne[0])}; - p_gather0_out = GQCGT(nullptr, "gather0_out", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, src0_rank, dims3, nullptr, 0); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_gather0_out)); - - uint32_t dims4[] = {4}; - uint32_t data4[4] = {static_cast(src1->ne[1]), static_cast(src1->ne[1]), static_cast(src0->ne[3]), static_cast(src0->ne[3])}; - p_gather1_index = GQCGT(nullptr, "gather1_index", QNN_TENSOR_TYPE_STATIC, QNN_DATATYPE_UINT_32, 1, dims4, data4, 16); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_gather1_index)); - - uint32_t dims5[] = {static_cast(src1->ne[3]), static_cast(src1->ne[2]), static_cast(src0->ne[1]), static_cast(src0->ne[0])}; - p_gather1_out = GQCGT(nullptr, "gather1_out", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, src0_rank, dims5, nullptr, 0); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_gather1_out)); - - p_tensor1 = GQCGT(src1, nullptr, QNN_TENSOR_TYPE_APP_WRITE, QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor1)); - - uint32_t tensor2_transpose_dims[GGML_MAX_DIMS] = {}; - p_tensor2_transpose = GQCGT(dst, "transpose", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0, true); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor2_transpose)); - - p_tensor2 = GQCGT(dst, nullptr, QNN_TENSOR_TYPE_APP_READ, QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor2)); - - //step-3:gather operation 0 - Qnn_Param_t gather0_params[] = {{QNN_PARAMTYPE_SCALAR,"axis", .scalarParam = {QNN_DATATYPE_INT_32, .int32Value = 1}}}; - Qnn_Tensor_t gather0_inputs[] = {*p_tensor0, *p_gather0_index}; - Qnn_Tensor_t gather0_outputs[] = {*p_gather0_out}; - Qnn_OpConfig_t gather0_op = ggmlqnn_create_op_config("out_gather0", QNN_OP_PACKAGE_NAME_QTI_AISW, QNN_OP_GATHER, - gather0_params, 1, gather0_inputs, 2, - gather0_outputs, 1); - CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, gather0_op)); - - //step-4:gather operation 1 - Qnn_Param_t gather1_params[] = {{QNN_PARAMTYPE_SCALAR,"axis", .scalarParam = {QNN_DATATYPE_INT_32, .int32Value = 0}}}; - Qnn_Tensor_t gather1_inputs[] = {*p_gather0_out, *p_gather1_index}; - Qnn_Tensor_t gather1_outputs[] = {*p_gather1_out}; - Qnn_OpConfig_t gather1_op = ggmlqnn_create_op_config("out_gather1", QNN_OP_PACKAGE_NAME_QTI_AISW, QNN_OP_GATHER, - gather1_params, 1, gather1_inputs, 2, - gather1_outputs, 1); - CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, gather1_op)); - - //step-5:matmul operation - Qnn_Param_t matmul_params[] = {{QNN_PARAMTYPE_SCALAR,"transpose_in1", .scalarParam = {QNN_DATATYPE_BOOL_8, .bool8Value = 1}}}; - Qnn_Tensor_t matmul_inputs[] = {*p_gather1_out, *p_tensor1}; - Qnn_Tensor_t matmul_outputs[] = {*p_tensor2_transpose}; - Qnn_OpConfig_t matmul_op = ggmlqnn_create_op_config("mulmat_opconfig", QNN_OP_PACKAGE_NAME_QTI_AISW, QNN_OP_MAT_MUL, - matmul_params, 1, matmul_inputs, 2, - matmul_outputs, - 1); - CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, matmul_op)); - - //step-6:transpose operation - Qnn_Param_t transpose_params[] = { {QNN_PARAMTYPE_TENSOR, "perm", .tensorParam = *p_param_tensor}}; - Qnn_Tensor_t transpose_inputs[] = {*p_tensor2_transpose}; - Qnn_Tensor_t transpose_outputs[]= {*p_tensor2}; - Qnn_OpConfig_t transpose_op = ggmlqnn_create_op_config("transpose_opconfig", QNN_OP_PACKAGE_NAME_QTI_AISW, QNN_OP_TRANSPOSE, + Qnn_GraphHandle_t graph_handle = nullptr; + Qnn_Tensor_t *p_tensor0 = nullptr; // src0 input + Qnn_Tensor_t *p_gather_out = nullptr; // After Gather + Qnn_Tensor_t *p_gather_indices = nullptr; // Gather indices + Qnn_Tensor_t *p_tensor1 = nullptr; // src1 input + Qnn_Tensor_t *p_matmul_out = nullptr; // MatMul output + Qnn_Tensor_t *p_transpose_perm = nullptr; // Transpose permutation + Qnn_Tensor_t *p_tensor2 = nullptr; // Final output + + + CHECK_QNN_API(error, qnn_raw_interface.graphCreate(instance->get_qnn_context_handle(), + graph_name.c_str(), NULL, + &graph_handle)); + + // Step 1: Define dimensions + uint32_t B = src0->ne[0]; // Batch dim + uint32_t M = src0->ne[1]; // Rows + uint32_t K0 = src0->ne[2] * src0->ne[3]; // K from src0 + uint32_t N1 = src1->ne[1]; // From src1 + uint32_t K1 = src1->ne[2]; // K from src1 + uint32_t N = src1->ne[3]; // Columns + + GGML_ASSERT(src0->ne[0] == src1->ne[0]); // Matching batch + GGML_ASSERT(dst->ne[0] == M); // Rows match + + // src0: [B, M, K1, K2] + uint32_t src0_dims[] = {static_cast(src0->ne[0]), + static_cast(src0->ne[1]), + static_cast(src0->ne[2]), + static_cast(src0->ne[3])}; + p_tensor0 = GQCGT(src0, "input0", QNN_TENSOR_TYPE_APP_WRITE, QNN_DATATYPE_FLOAT_32, 4, + src0_dims, nullptr, 0); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor0)); + + // Gather: Reshape src0 to [M, B, K0] for MatMul + uint32_t gather_indices_data[] = {1, 0, 2, 3}; // Permute [B, M, K1, K2] -> [M, B, K1, K2] + uint32_t gather_indices_dims[] = {4}; + p_gather_indices = GQCGT(nullptr, "gather_indices", QNN_TENSOR_TYPE_STATIC, + QNN_DATATYPE_UINT_32, 1, + gather_indices_dims, gather_indices_data, + sizeof(gather_indices_data)); + CHECK_QNN_API(error, + qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_gather_indices)); + + uint32_t gather_out_dims[] = {M, B, static_cast(src0->ne[2]), + static_cast(src0->ne[3])}; + p_gather_out = GQCGT(nullptr, "gather_out", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, + 4, + gather_out_dims, nullptr, 0); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_gather_out)); + + Qnn_Param_t gather_params[] = { + {QNN_PARAMTYPE_SCALAR, "axis", .scalarParam = { + QNN_DATATYPE_INT_32, .int32Value = 0}} + }; + Qnn_Tensor_t gather_inputs[] = {*p_tensor0, *p_gather_indices}; + Qnn_Tensor_t gather_outputs[] = {*p_gather_out}; + Qnn_OpConfig_t gather_op = ggmlqnn_create_op_config("gather", QNN_OP_PACKAGE_NAME_QTI_AISW, + QNN_OP_GATHER, gather_params, 1, + gather_inputs, 2, gather_outputs, 1); + CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, gather_op)); + + // src1: [B, N1, K, N] + uint32_t src1_dims[] = {static_cast(src1->ne[0]), + static_cast(src1->ne[1]), + static_cast(src1->ne[2]), + static_cast(src1->ne[3])}; + p_tensor1 = GQCGT(src1, "input1", QNN_TENSOR_TYPE_APP_WRITE, QNN_DATATYPE_FLOAT_32, 4, + src1_dims, nullptr, 0); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor1)); + + // MatMul: [M, B, K0] x [B, N1, K1, N] -> [M, N1, K1, N] + // Flatten for QNN: [M, B * K0] x [B * K1, N] + uint32_t matmul_in0_dims[] = {M, B * K0}; + Qnn_Tensor_t matmul_in0 = *p_gather_out; + QNN_VER_PTR(matmul_in0)->dimensions = matmul_in0_dims; + QNN_VER_PTR(matmul_in0)->rank = 2; + + uint32_t matmul_in1_dims[] = {B * K1, N}; + Qnn_Tensor_t matmul_in1 = *p_tensor1; + QNN_VER_PTR(matmul_in1)->dimensions = matmul_in1_dims; + QNN_VER_PTR(matmul_in1)->rank = 2; + + uint32_t matmul_out_dims[] = {M, N}; + p_matmul_out = GQCGT(nullptr, "matmul_out", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, + 2, + matmul_out_dims, nullptr, 0); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_matmul_out)); + + Qnn_Tensor_t matmul_inputs[] = {matmul_in0, matmul_in1}; + Qnn_Tensor_t matmul_outputs[] = {*p_matmul_out}; + Qnn_OpConfig_t matmul_op = ggmlqnn_create_op_config("matmul", QNN_OP_PACKAGE_NAME_QTI_AISW, + QNN_OP_MAT_MUL, nullptr, 0, + matmul_inputs, 2, matmul_outputs, 1); + CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, matmul_op)); + + // Transpose: Restore to [M, N1, K, N] + uint32_t perm_data[] = {0, 1, 2, 3}; // Adjust based on dst + uint32_t perm_dims[] = {4}; + p_transpose_perm = GQCGT(nullptr, "transpose_perm", QNN_TENSOR_TYPE_STATIC, + QNN_DATATYPE_UINT_32, 1, + perm_dims, perm_data, sizeof(perm_data)); + CHECK_QNN_API(error, + qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_transpose_perm)); + + uint32_t dst_dims[] = {static_cast(dst->ne[0]), static_cast(dst->ne[1]), + static_cast(dst->ne[2]), + static_cast(dst->ne[3])}; + p_tensor2 = GQCGT(dst, "transpose", + QNN_TENSOR_TYPE_NATIVE, + QNN_DATATYPE_FLOAT_32, 2, + dst_dims, nullptr, 0); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor2)); + // Transpose operation + Qnn_Param_t transpose_params[] = { + {QNN_PARAMTYPE_TENSOR, "perm", .tensorParam = *p_transpose_perm}}; + Qnn_Tensor_t transpose_inputs[] = {*p_matmul_out}; + Qnn_Tensor_t transpose_outputs[] = {*p_tensor2}; + Qnn_OpConfig_t transpose_op = ggmlqnn_create_op_config("out_trans", QNN_OP_PACKAGE_NAME_QTI_AISW, QNN_OP_TRANSPOSE, transpose_params, 1, transpose_inputs, 1, transpose_outputs, 1); - CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, transpose_op)); - - //step-7:finalize graph - CHECK_QNN_API(error, qnn_raw_interface.graphFinalize(graph_handle, NULL, NULL)); - - //step-8:execute graph - QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggmlqnn_get_tensor_data_size(src0)}; - QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggmlqnn_get_tensor_data_size(src1)}; - QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggmlqnn_get_tensor_data_size(dst)}; - Qnn_Tensor_t input_tensors[] = {*p_tensor0, *p_tensor1}; - Qnn_Tensor_t output_tensors[] = {*p_tensor2}; - CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, input_tensors, 2, output_tensors, - 1, NULL, NULL)); - - qnn_tensors_t ggml_op_mulmat_tensors; - ggml_op_mulmat_tensors.reserve(9); - ggml_op_mulmat_tensors.push_back(p_tensor0); - ggml_op_mulmat_tensors.push_back(p_tensor1); - ggml_op_mulmat_tensors.push_back(p_tensor2); - ggml_op_mulmat_tensors.push_back(p_param_tensor); - ggml_op_mulmat_tensors.push_back(p_tensor2_transpose); - ggml_op_mulmat_tensors.push_back(p_gather0_index); - ggml_op_mulmat_tensors.push_back(p_gather0_out); - ggml_op_mulmat_tensors.push_back(p_gather1_index); - ggml_op_mulmat_tensors.push_back(p_gather1_out); + CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, transpose_op)); - auto graph_item = std::make_tuple(graph_handle, ggml_op_mulmat_tensors); - instance->_qnn_graph_map[graph_name] = graph_item; - } else { - QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggmlqnn_get_tensor_data_size(src0)}; - QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggmlqnn_get_tensor_data_size(src1)}; - QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggmlqnn_get_tensor_data_size(dst)}; + // Finalize graph + CHECK_QNN_API(error, qnn_raw_interface.graphFinalize(graph_handle, NULL, NULL)); - Qnn_Tensor_t tensor_inputs[] = { - *p_tensor0, - *p_tensor1 - }; - Qnn_Tensor_t tensor_outputs[] = { - *p_tensor2 - }; - // this is the second technical approach of "how to utilize the Hexagon NPU maximally" through QNN SDK - CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, - tensor_inputs, 2, - tensor_outputs, 1, - nullptr, nullptr)); - } + // Execute graph + QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, static_cast(ggml_nbytes(src0))}; + QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, static_cast(ggml_nbytes(src1))}; + QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, static_cast(ggml_nbytes(dst))}; - // restore the original dimensions of qnn tensors to avoid memory leak in func free_qnn_tensor - QNN_VER_PTR(*p_tensor0)->dimensions = tensor_0_dimensions; - QNN_VER_PTR(*p_tensor1)->dimensions = tensor_1_dimensions; - QNN_VER_PTR(*p_tensor2)->dimensions = tensor_2_dimensions; - op_perf.info(); + Qnn_Tensor_t input_tensors[] = {*p_tensor0, *p_tensor1}; + Qnn_Tensor_t output_tensors[] = {*p_tensor2}; + CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, input_tensors, 2, + output_tensors, 1, NULL, NULL)); } /* diff --git a/tests/ggml-qnn-ut.cpp b/tests/ggml-qnn-ut.cpp index ff0e96f2b00cb..1ab75526794e8 100644 --- a/tests/ggml-qnn-ut.cpp +++ b/tests/ggml-qnn-ut.cpp @@ -439,10 +439,10 @@ int main(int argc, char * argv[]) { //src0 = ggml_new_tensor_3d(ctx, qtype, 128, 64, 8); //src1 = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, 128, 2, 8); //verify 4D matrix - src0 = ggml_new_tensor_4d(ctx, qtype, 256, 16, 3, 2); - src1 = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, 256, 1, 6, 4); //src0 = ggml_new_tensor_4d(ctx, qtype, 256, 16, 3, 2); - //src1 = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, 256, 16, 3, 2); + //src1 = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, 256, 1, 6, 4); + src0 = ggml_new_tensor_4d(ctx, qtype, 256, 16, 3, 2); + src1 = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, 256, 16, 3, 2); } ggml_set_input(src0); From 416caa6b2d320c660cea65386a9f81068e7b25d2 Mon Sep 17 00:00:00 2001 From: zhouwg Date: Sun, 2 Mar 2025 13:19:49 +0800 Subject: [PATCH 35/76] ggml-qnn: AI-assisted ggml_qnn_mul_mat_4d by Grok 3 --- step2 --- ggml/src/ggml-qnn/ggml-qnn-ops.cpp | 214 ++++++++++++++++++++++++++++- 1 file changed, 213 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-qnn/ggml-qnn-ops.cpp b/ggml/src/ggml-qnn/ggml-qnn-ops.cpp index 2553ff78f9c7d..9f835dbb7f5e0 100644 --- a/ggml/src/ggml-qnn/ggml-qnn-ops.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn-ops.cpp @@ -289,7 +289,7 @@ void ggml_qnn_general_node(ggml_backend_qnn_context * ctx, ggml_tensor * op) { * than ggml_qnn_mul_mat, so it's a standalone function. * it will be combined with ggml_qnn_mul_mat after bugfix */ -static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) { +static void ggml_qnn_mul_mat_4d1(ggml_backend_qnn_context *ctx, ggml_tensor *op) { Qnn_ErrorHandle_t error = QNN_SUCCESS; qnn_perf op_perf = qnn_perf("ggml_qnn_mul_mat_4d"); qnn_instance *instance = ctx->instance; @@ -445,6 +445,218 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) output_tensors, 1, NULL, NULL)); } +static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) { + Qnn_ErrorHandle_t error = QNN_SUCCESS; + bool graph_initialized = false; + qnn_perf op_perf = qnn_perf("ggml_qnn_mul_mat_4d"); + qnn_instance *instance = ctx->instance; + QNN_INTERFACE_VER_TYPE qnn_raw_interface = ctx->raw_interface; + + const ggml_tensor *src0 = op->src[0]; // e.g., [256, 16, 3, 2] or [256,16, 3, 2] + const ggml_tensor *src1 = op->src[1]; // e.g., [256, 1, 6, 4] or [256, 16, 3, 2] + ggml_tensor *dst = op; // e.g., [16, 1, 6, 4] or [16, 16, 3, 2] + + GGMLQNN_CHECK_PARAMS(ctx, src0, src1, dst); + GGML_ASSERT(ggml_n_dims(src0) == 4 && ggml_n_dims(src1) == 4); + op_perf.start(); + + std::string graph_name; + ggmlqnn_get_graphkey_from_op(op, graph_name); + GGMLQNN_LOG_DEBUG("graph name %s\n", graph_name.c_str()); + ggmlqnn_print_tensors_info(__func__, ctx, src0, src1, dst); + + Qnn_GraphHandle_t graph_handle = nullptr; + Qnn_Tensor_t *p_tensor0 = nullptr; // src0 input + Qnn_Tensor_t *p_gather0_out = nullptr; // After Gather on src0 + Qnn_Tensor_t *p_gather0_indices = nullptr; // Gather indices for src0 + Qnn_Tensor_t *p_tensor1 = nullptr; // src1 input + Qnn_Tensor_t *p_gather1_out = nullptr; // After Gather on src1 + Qnn_Tensor_t *p_gather1_indices = nullptr; // Gather indices for src1 + Qnn_Tensor_t *p_matmul_out = nullptr; // MatMul output + Qnn_Tensor_t *p_transpose_perm = nullptr; // Transpose permutation + Qnn_Tensor_t *p_tensor2 = nullptr; // Final output + + if (instance->_qnn_graph_map.find(graph_name) != instance->_qnn_graph_map.end()) { + graph_initialized = true; + qnn_res_t &graph_item = instance->_qnn_graph_map[graph_name]; + graph_handle = std::get<0>(graph_item); + qnn_tensors_t &tensors = std::get<1>(graph_item); + p_tensor0 = tensors[0]; + p_gather0_out = tensors[1]; + p_gather0_indices = tensors[2]; + p_tensor1 = tensors[3]; + p_gather1_out = tensors[4]; + p_gather1_indices = tensors[5]; + p_matmul_out = tensors[6]; + p_transpose_perm = tensors[7]; + p_tensor2 = tensors[8]; + } else { + CHECK_QNN_API(error, qnn_raw_interface.graphCreate(instance->get_qnn_context_handle(), + graph_name.c_str(), NULL, &graph_handle)); + + // Step 1: Define dimensions (ne = logical order) + uint32_t B = src0->ne[0]; // Batch dim + uint32_t M = src0->ne[1]; // Rows + uint32_t K0 = src0->ne[2] * src0->ne[3]; // K from src0 (e.g., 3 * 2 = 6) + uint32_t N1 = src1->ne[1]; // From src1 + uint32_t K1 = src1->ne[2]; // K from src1 (e.g., 6 or 3) + uint32_t N = src1->ne[3]; // Columns + + GGML_ASSERT(src0->ne[0] == src1->ne[0]); // Matching batch + GGML_ASSERT(dst->ne[0] == M); // Rows match + GGML_ASSERT(K0 == K1); // K must match for mul_mat + + // src0: [B, M, K1, K2] + uint32_t src0_dims[] = {B, M, static_cast(src0->ne[2]), static_cast(src0->ne[3])}; + p_tensor0 = GQCGT(src0, "input0", QNN_TENSOR_TYPE_APP_WRITE, QNN_DATATYPE_FLOAT_32, 4, + src0_dims, nullptr, 0); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor0)); + + // Gather on src0: [B, M, K1, K2] -> [M, B, K0] (collapse K1, K2) + uint32_t gather0_indices_data[] = {1, 0, 2, 3}; // [B, M, K1, K2] -> [M, B, K1, K2] + uint32_t gather0_indices_dims[] = {4}; + p_gather0_indices = GQCGT(nullptr, "gather0_indices", QNN_TENSOR_TYPE_STATIC, QNN_DATATYPE_UINT_32, 1, + gather0_indices_dims, gather0_indices_data, sizeof(gather0_indices_data)); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_gather0_indices)); + + uint32_t gather0_out_dims[] = {M, B, static_cast(src0->ne[2]), static_cast(src0->ne[3])}; + p_gather0_out = GQCGT(nullptr, "gather0_out", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, 4, + gather0_out_dims, nullptr, 0); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_gather0_out)); + + Qnn_Param_t gather0_params[] = { + {QNN_PARAMTYPE_SCALAR, "axis", .scalarParam = {QNN_DATATYPE_INT_32, .int32Value = 0}} + }; + Qnn_Tensor_t gather0_inputs[] = {*p_tensor0, *p_gather0_indices}; + Qnn_Tensor_t gather0_outputs[] = {*p_gather0_out}; + Qnn_OpConfig_t gather0_op = ggmlqnn_create_op_config("gather0", QNN_OP_PACKAGE_NAME_QTI_AISW, + QNN_OP_GATHER, gather0_params, 1, + gather0_inputs, 2, gather0_outputs, 1); + CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, gather0_op)); + + // src1: [B, N1, K, N] + uint32_t src1_dims[] = {B, N1, static_cast(src1->ne[2]), static_cast(src1->ne[3])}; + p_tensor1 = GQCGT(src1, "input1", QNN_TENSOR_TYPE_APP_WRITE, QNN_DATATYPE_FLOAT_32, 4, + src1_dims, nullptr, 0); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor1)); + + // Gather on src1: [B, N1, K, N] -> [N1, B, K, N] + uint32_t gather1_indices_data[] = {1, 0, 2, 3}; // [B, N1, K, N] -> [N1, B, K, N] + uint32_t gather1_indices_dims[] = {4}; + p_gather1_indices = GQCGT(nullptr, "gather1_indices", QNN_TENSOR_TYPE_STATIC, QNN_DATATYPE_UINT_32, 1, + gather1_indices_dims, gather1_indices_data, sizeof(gather1_indices_data)); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_gather1_indices)); + + uint32_t gather1_out_dims[] = {N1, B, static_cast(src1->ne[2]), static_cast(src1->ne[3])}; + p_gather1_out = GQCGT(nullptr, "gather1_out", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, 4, + gather1_out_dims, nullptr, 0); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_gather1_out)); + + Qnn_Param_t gather1_params[] = { + {QNN_PARAMTYPE_SCALAR, "axis", .scalarParam = {QNN_DATATYPE_INT_32, .int32Value = 0}} + }; + Qnn_Tensor_t gather1_inputs[] = {*p_tensor1, *p_gather1_indices}; + Qnn_Tensor_t gather1_outputs[] = {*p_gather1_out}; + Qnn_OpConfig_t gather1_op = ggmlqnn_create_op_config("gather1", QNN_OP_PACKAGE_NAME_QTI_AISW, + QNN_OP_GATHER, gather1_params, 1, + gather1_inputs, 2, gather1_outputs, 1); + CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, gather1_op)); + + // MatMul: [M, B * K0] x [B * K1, N] + uint32_t matmul_in0_dims[] = {M, B * K0}; + Qnn_Tensor_t matmul_in0 = *p_gather0_out; + QNN_VER_PTR(matmul_in0)->dimensions = matmul_in0_dims; + QNN_VER_PTR(matmul_in0)->rank = 2; + + uint32_t matmul_in1_dims[] = {B * K1, N}; + Qnn_Tensor_t matmul_in1 = *p_gather1_out; + QNN_VER_PTR(matmul_in1)->dimensions = matmul_in1_dims; + QNN_VER_PTR(matmul_in1)->rank = 2; + + uint32_t matmul_out_dims[] = {M, N}; + p_matmul_out = GQCGT(nullptr, "matmul_out", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, 2, + matmul_out_dims, nullptr, 0); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_matmul_out)); + + Qnn_Tensor_t matmul_inputs[] = {matmul_in0, matmul_in1}; + Qnn_Tensor_t matmul_outputs[] = {*p_matmul_out}; + Qnn_OpConfig_t matmul_op = ggmlqnn_create_op_config("matmul", QNN_OP_PACKAGE_NAME_QTI_AISW, + QNN_OP_MAT_MUL, nullptr, 0, + matmul_inputs, 2, matmul_outputs, 1); + CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, matmul_op)); + + // Transpose: [M, N] -> [M, N1, K1, N] + uint32_t perm_data[] = {0, 1, 2, 3}; // Placeholder, adjust below + if (dst->ne[1] == N1 && dst->ne[2] == K1 && dst->ne[3] == N) { + perm_data[0] = 0; perm_data[1] = 1; perm_data[2] = 2; perm_data[3] = 3; + } else if (dst->ne[1] == 1 && dst->ne[2] == K1 && dst->ne[3] == N) { + perm_data[0] = 0; perm_data[1] = 2; perm_data[2] = 1; perm_data[3] = 3; // Adjust for [M, 1, K, N] + } + uint32_t perm_dims[] = {4}; + p_transpose_perm = GQCGT(nullptr, "transpose_perm", QNN_TENSOR_TYPE_STATIC, QNN_DATATYPE_UINT_32, 1, + perm_dims, perm_data, sizeof(perm_data)); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_transpose_perm)); + + uint32_t dst_dims[] = {M, N1, K1, N}; + p_tensor2 = GQCGT(dst, "output", QNN_TENSOR_TYPE_APP_READ, QNN_DATATYPE_FLOAT_32, 4, + dst_dims, nullptr, 0); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor2)); + + Qnn_Param_t transpose_params[] = { + {QNN_PARAMTYPE_TENSOR, "perm", .tensorParam = *p_transpose_perm} + }; + Qnn_Tensor_t transpose_inputs[] = {*p_matmul_out}; + Qnn_Tensor_t transpose_outputs[] = {*p_tensor2}; + Qnn_OpConfig_t transpose_op = ggmlqnn_create_op_config("transpose", QNN_OP_PACKAGE_NAME_QTI_AISW, + QNN_OP_TRANSPOSE, transpose_params, 1, + transpose_inputs, 1, transpose_outputs, 1); + CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, transpose_op)); + + // Finalize + CHECK_QNN_API(error, qnn_raw_interface.graphFinalize(graph_handle, NULL, NULL)); + + // Cache + qnn_tensors_t ggml_op_mulmat_tensors = {p_tensor0, p_gather0_out, p_gather0_indices, p_tensor1, + p_gather1_out, p_gather1_indices, p_matmul_out, + p_transpose_perm, p_tensor2}; + instance->_qnn_graph_map[graph_name] = std::make_tuple(graph_handle, ggml_op_mulmat_tensors); + } + + // Save dimensions + uint32_t *tensor_0_dims = QNN_VER_PTR(*p_tensor0)->dimensions; + uint32_t *gather0_out_dims = QNN_VER_PTR(*p_gather0_out)->dimensions; + uint32_t *gather0_indices_dims = QNN_VER_PTR(*p_gather0_indices)->dimensions; + uint32_t *tensor_1_dims = QNN_VER_PTR(*p_tensor1)->dimensions; + uint32_t *gather1_out_dims = QNN_VER_PTR(*p_gather1_out)->dimensions; + uint32_t *gather1_indices_dims = QNN_VER_PTR(*p_gather1_indices)->dimensions; + uint32_t *matmul_out_dims = QNN_VER_PTR(*p_matmul_out)->dimensions; + uint32_t *transpose_perm_dims = QNN_VER_PTR(*p_transpose_perm)->dimensions; + uint32_t *tensor_2_dims = QNN_VER_PTR(*p_tensor2)->dimensions; + + // Execute + QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, static_cast(ggml_nbytes(src0))}; + QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, static_cast(ggml_nbytes(src1))}; + QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, static_cast(ggml_nbytes(dst))}; + + Qnn_Tensor_t input_tensors[] = {*p_tensor0, *p_tensor1}; + Qnn_Tensor_t output_tensors[] = {*p_tensor2}; + CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, input_tensors, 2, + output_tensors, 1, NULL, NULL)); + + // Restore dimensions + QNN_VER_PTR(*p_tensor0)->dimensions = tensor_0_dims; + QNN_VER_PTR(*p_gather0_out)->dimensions = gather0_out_dims; + QNN_VER_PTR(*p_gather0_indices)->dimensions = gather0_indices_dims; + QNN_VER_PTR(*p_tensor1)->dimensions = tensor_1_dims; + QNN_VER_PTR(*p_gather1_out)->dimensions = gather1_out_dims; + QNN_VER_PTR(*p_gather1_indices)->dimensions = gather1_indices_dims; + QNN_VER_PTR(*p_matmul_out)->dimensions = matmul_out_dims; + QNN_VER_PTR(*p_transpose_perm)->dimensions = transpose_perm_dims; + QNN_VER_PTR(*p_tensor2)->dimensions = tensor_2_dims; + + op_perf.info(); +} + /* * @brief performs matrix multiplication with FP32 & quantized weights and floating-point inputs * using the QNN backend. this function performs matrix multiplication of the input tensor From f1c370a47c3d2f54ddfbd984a7e08aac4ec52cfb Mon Sep 17 00:00:00 2001 From: zhouwg Date: Sun, 2 Mar 2025 13:30:22 +0800 Subject: [PATCH 36/76] ggml-qnn: AI-assisted ggml_qnn_mul_mat_4d by Grok 3 --- step3 --- ggml/src/ggml-qnn/ggml-qnn-ops.cpp | 231 +++++------------------------ 1 file changed, 35 insertions(+), 196 deletions(-) diff --git a/ggml/src/ggml-qnn/ggml-qnn-ops.cpp b/ggml/src/ggml-qnn/ggml-qnn-ops.cpp index 9f835dbb7f5e0..db5847a78ed02 100644 --- a/ggml/src/ggml-qnn/ggml-qnn-ops.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn-ops.cpp @@ -289,162 +289,6 @@ void ggml_qnn_general_node(ggml_backend_qnn_context * ctx, ggml_tensor * op) { * than ggml_qnn_mul_mat, so it's a standalone function. * it will be combined with ggml_qnn_mul_mat after bugfix */ -static void ggml_qnn_mul_mat_4d1(ggml_backend_qnn_context *ctx, ggml_tensor *op) { - Qnn_ErrorHandle_t error = QNN_SUCCESS; - qnn_perf op_perf = qnn_perf("ggml_qnn_mul_mat_4d"); - qnn_instance *instance = ctx->instance; - QNN_INTERFACE_VER_TYPE qnn_raw_interface = ctx->raw_interface; - - const ggml_tensor *src0 = op->src[0]; // e.g., [256, 16, 3, 2] or [256,16,3,2] - const ggml_tensor *src1 = op->src[1]; // e.g., [256, 1, 6, 4] or [256,16,3, 2] - ggml_tensor *dst = op; // e.g., [16, 1, 6, 4] or [16,16,3, 2] - - GGMLQNN_CHECK_PARAMS(ctx, src0, src1, dst); - GGML_ASSERT(ggml_n_dims(src0) == 4 && ggml_n_dims(src1) == 4); - op_perf.start(); - - std::string graph_name; - ggmlqnn_get_graphkey_from_op(op, graph_name); - GGMLQNN_LOG_DEBUG("graph name %s\n", graph_name.c_str()); - ggmlqnn_print_tensors_info(__func__, ctx, src0, src1, dst); - - Qnn_GraphHandle_t graph_handle = nullptr; - Qnn_Tensor_t *p_tensor0 = nullptr; // src0 input - Qnn_Tensor_t *p_gather_out = nullptr; // After Gather - Qnn_Tensor_t *p_gather_indices = nullptr; // Gather indices - Qnn_Tensor_t *p_tensor1 = nullptr; // src1 input - Qnn_Tensor_t *p_matmul_out = nullptr; // MatMul output - Qnn_Tensor_t *p_transpose_perm = nullptr; // Transpose permutation - Qnn_Tensor_t *p_tensor2 = nullptr; // Final output - - - CHECK_QNN_API(error, qnn_raw_interface.graphCreate(instance->get_qnn_context_handle(), - graph_name.c_str(), NULL, - &graph_handle)); - - // Step 1: Define dimensions - uint32_t B = src0->ne[0]; // Batch dim - uint32_t M = src0->ne[1]; // Rows - uint32_t K0 = src0->ne[2] * src0->ne[3]; // K from src0 - uint32_t N1 = src1->ne[1]; // From src1 - uint32_t K1 = src1->ne[2]; // K from src1 - uint32_t N = src1->ne[3]; // Columns - - GGML_ASSERT(src0->ne[0] == src1->ne[0]); // Matching batch - GGML_ASSERT(dst->ne[0] == M); // Rows match - - // src0: [B, M, K1, K2] - uint32_t src0_dims[] = {static_cast(src0->ne[0]), - static_cast(src0->ne[1]), - static_cast(src0->ne[2]), - static_cast(src0->ne[3])}; - p_tensor0 = GQCGT(src0, "input0", QNN_TENSOR_TYPE_APP_WRITE, QNN_DATATYPE_FLOAT_32, 4, - src0_dims, nullptr, 0); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor0)); - - // Gather: Reshape src0 to [M, B, K0] for MatMul - uint32_t gather_indices_data[] = {1, 0, 2, 3}; // Permute [B, M, K1, K2] -> [M, B, K1, K2] - uint32_t gather_indices_dims[] = {4}; - p_gather_indices = GQCGT(nullptr, "gather_indices", QNN_TENSOR_TYPE_STATIC, - QNN_DATATYPE_UINT_32, 1, - gather_indices_dims, gather_indices_data, - sizeof(gather_indices_data)); - CHECK_QNN_API(error, - qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_gather_indices)); - - uint32_t gather_out_dims[] = {M, B, static_cast(src0->ne[2]), - static_cast(src0->ne[3])}; - p_gather_out = GQCGT(nullptr, "gather_out", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, - 4, - gather_out_dims, nullptr, 0); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_gather_out)); - - Qnn_Param_t gather_params[] = { - {QNN_PARAMTYPE_SCALAR, "axis", .scalarParam = { - QNN_DATATYPE_INT_32, .int32Value = 0}} - }; - Qnn_Tensor_t gather_inputs[] = {*p_tensor0, *p_gather_indices}; - Qnn_Tensor_t gather_outputs[] = {*p_gather_out}; - Qnn_OpConfig_t gather_op = ggmlqnn_create_op_config("gather", QNN_OP_PACKAGE_NAME_QTI_AISW, - QNN_OP_GATHER, gather_params, 1, - gather_inputs, 2, gather_outputs, 1); - CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, gather_op)); - - // src1: [B, N1, K, N] - uint32_t src1_dims[] = {static_cast(src1->ne[0]), - static_cast(src1->ne[1]), - static_cast(src1->ne[2]), - static_cast(src1->ne[3])}; - p_tensor1 = GQCGT(src1, "input1", QNN_TENSOR_TYPE_APP_WRITE, QNN_DATATYPE_FLOAT_32, 4, - src1_dims, nullptr, 0); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor1)); - - // MatMul: [M, B, K0] x [B, N1, K1, N] -> [M, N1, K1, N] - // Flatten for QNN: [M, B * K0] x [B * K1, N] - uint32_t matmul_in0_dims[] = {M, B * K0}; - Qnn_Tensor_t matmul_in0 = *p_gather_out; - QNN_VER_PTR(matmul_in0)->dimensions = matmul_in0_dims; - QNN_VER_PTR(matmul_in0)->rank = 2; - - uint32_t matmul_in1_dims[] = {B * K1, N}; - Qnn_Tensor_t matmul_in1 = *p_tensor1; - QNN_VER_PTR(matmul_in1)->dimensions = matmul_in1_dims; - QNN_VER_PTR(matmul_in1)->rank = 2; - - uint32_t matmul_out_dims[] = {M, N}; - p_matmul_out = GQCGT(nullptr, "matmul_out", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, - 2, - matmul_out_dims, nullptr, 0); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_matmul_out)); - - Qnn_Tensor_t matmul_inputs[] = {matmul_in0, matmul_in1}; - Qnn_Tensor_t matmul_outputs[] = {*p_matmul_out}; - Qnn_OpConfig_t matmul_op = ggmlqnn_create_op_config("matmul", QNN_OP_PACKAGE_NAME_QTI_AISW, - QNN_OP_MAT_MUL, nullptr, 0, - matmul_inputs, 2, matmul_outputs, 1); - CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, matmul_op)); - - // Transpose: Restore to [M, N1, K, N] - uint32_t perm_data[] = {0, 1, 2, 3}; // Adjust based on dst - uint32_t perm_dims[] = {4}; - p_transpose_perm = GQCGT(nullptr, "transpose_perm", QNN_TENSOR_TYPE_STATIC, - QNN_DATATYPE_UINT_32, 1, - perm_dims, perm_data, sizeof(perm_data)); - CHECK_QNN_API(error, - qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_transpose_perm)); - - uint32_t dst_dims[] = {static_cast(dst->ne[0]), static_cast(dst->ne[1]), - static_cast(dst->ne[2]), - static_cast(dst->ne[3])}; - p_tensor2 = GQCGT(dst, "transpose", - QNN_TENSOR_TYPE_NATIVE, - QNN_DATATYPE_FLOAT_32, 2, - dst_dims, nullptr, 0); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor2)); - // Transpose operation - Qnn_Param_t transpose_params[] = { - {QNN_PARAMTYPE_TENSOR, "perm", .tensorParam = *p_transpose_perm}}; - Qnn_Tensor_t transpose_inputs[] = {*p_matmul_out}; - Qnn_Tensor_t transpose_outputs[] = {*p_tensor2}; - Qnn_OpConfig_t transpose_op = ggmlqnn_create_op_config("out_trans", QNN_OP_PACKAGE_NAME_QTI_AISW, QNN_OP_TRANSPOSE, - transpose_params, 1, transpose_inputs, 1, - transpose_outputs, 1); - CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, transpose_op)); - - // Finalize graph - CHECK_QNN_API(error, qnn_raw_interface.graphFinalize(graph_handle, NULL, NULL)); - - // Execute graph - QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, static_cast(ggml_nbytes(src0))}; - QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, static_cast(ggml_nbytes(src1))}; - QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, static_cast(ggml_nbytes(dst))}; - - Qnn_Tensor_t input_tensors[] = {*p_tensor0, *p_tensor1}; - Qnn_Tensor_t output_tensors[] = {*p_tensor2}; - CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, input_tensors, 2, - output_tensors, 1, NULL, NULL)); -} - static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) { Qnn_ErrorHandle_t error = QNN_SUCCESS; bool graph_initialized = false; @@ -452,9 +296,9 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) qnn_instance *instance = ctx->instance; QNN_INTERFACE_VER_TYPE qnn_raw_interface = ctx->raw_interface; - const ggml_tensor *src0 = op->src[0]; // e.g., [256, 16, 3, 2] or [256,16, 3, 2] - const ggml_tensor *src1 = op->src[1]; // e.g., [256, 1, 6, 4] or [256, 16, 3, 2] - ggml_tensor *dst = op; // e.g., [16, 1, 6, 4] or [16, 16, 3, 2] + const ggml_tensor *src0 = op->src[0]; + const ggml_tensor *src1 = op->src[1]; + ggml_tensor *dst = op; GGMLQNN_CHECK_PARAMS(ctx, src0, src1, dst); GGML_ASSERT(ggml_n_dims(src0) == 4 && ggml_n_dims(src1) == 4); @@ -466,15 +310,15 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) ggmlqnn_print_tensors_info(__func__, ctx, src0, src1, dst); Qnn_GraphHandle_t graph_handle = nullptr; - Qnn_Tensor_t *p_tensor0 = nullptr; // src0 input - Qnn_Tensor_t *p_gather0_out = nullptr; // After Gather on src0 - Qnn_Tensor_t *p_gather0_indices = nullptr; // Gather indices for src0 - Qnn_Tensor_t *p_tensor1 = nullptr; // src1 input - Qnn_Tensor_t *p_gather1_out = nullptr; // After Gather on src1 - Qnn_Tensor_t *p_gather1_indices = nullptr; // Gather indices for src1 - Qnn_Tensor_t *p_matmul_out = nullptr; // MatMul output - Qnn_Tensor_t *p_transpose_perm = nullptr; // Transpose permutation - Qnn_Tensor_t *p_tensor2 = nullptr; // Final output + Qnn_Tensor_t *p_tensor0 = nullptr; + Qnn_Tensor_t *p_gather0_out = nullptr; + Qnn_Tensor_t *p_gather0_indices = nullptr; + Qnn_Tensor_t *p_tensor1 = nullptr; + Qnn_Tensor_t *p_gather1_out = nullptr; + Qnn_Tensor_t *p_gather1_indices = nullptr; + Qnn_Tensor_t *p_matmul_out = nullptr; + Qnn_Tensor_t *p_transpose_perm = nullptr; + Qnn_Tensor_t *p_tensor2 = nullptr; if (instance->_qnn_graph_map.find(graph_name) != instance->_qnn_graph_map.end()) { graph_initialized = true; @@ -494,32 +338,32 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) CHECK_QNN_API(error, qnn_raw_interface.graphCreate(instance->get_qnn_context_handle(), graph_name.c_str(), NULL, &graph_handle)); - // Step 1: Define dimensions (ne = logical order) - uint32_t B = src0->ne[0]; // Batch dim - uint32_t M = src0->ne[1]; // Rows - uint32_t K0 = src0->ne[2] * src0->ne[3]; // K from src0 (e.g., 3 * 2 = 6) - uint32_t N1 = src1->ne[1]; // From src1 - uint32_t K1 = src1->ne[2]; // K from src1 (e.g., 6 or 3) - uint32_t N = src1->ne[3]; // Columns + // Step 1: Define dimensions (ne = [K2, K1, M, B] for src0, [N, K, N1, B] for src1) + uint32_t B = src0->ne[3]; + uint32_t M = src0->ne[2]; + uint32_t K0 = src0->ne[0] * src0->ne[1]; // K from src0 + uint32_t N1 = src1->ne[2]; + uint32_t K1 = src1->ne[1]; + uint32_t N = src1->ne[0]; - GGML_ASSERT(src0->ne[0] == src1->ne[0]); // Matching batch - GGML_ASSERT(dst->ne[0] == M); // Rows match - GGML_ASSERT(K0 == K1); // K must match for mul_mat + GGML_ASSERT(src0->ne[3] == src1->ne[3]); // Matching batch + GGML_ASSERT(dst->ne[2] == M); // M matches dst + GGML_ASSERT(K0 == K1); // K must match - // src0: [B, M, K1, K2] - uint32_t src0_dims[] = {B, M, static_cast(src0->ne[2]), static_cast(src0->ne[3])}; + // src0: [K2, K1, M, B] -> [B, M, K2, K1] + uint32_t src0_dims[] = {static_cast(src0->ne[3]), static_cast(src0->ne[2]), static_cast(src0->ne[0]), static_cast(src0->ne[1])}; p_tensor0 = GQCGT(src0, "input0", QNN_TENSOR_TYPE_APP_WRITE, QNN_DATATYPE_FLOAT_32, 4, src0_dims, nullptr, 0); CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor0)); - // Gather on src0: [B, M, K1, K2] -> [M, B, K0] (collapse K1, K2) - uint32_t gather0_indices_data[] = {1, 0, 2, 3}; // [B, M, K1, K2] -> [M, B, K1, K2] + // Gather on src0: [B, M, K2, K1] -> [M, B, K0] + uint32_t gather0_indices_data[] = {2, 3, 0, 1}; // [K2, K1, M, B] -> [M, B, K2, K1] uint32_t gather0_indices_dims[] = {4}; p_gather0_indices = GQCGT(nullptr, "gather0_indices", QNN_TENSOR_TYPE_STATIC, QNN_DATATYPE_UINT_32, 1, gather0_indices_dims, gather0_indices_data, sizeof(gather0_indices_data)); CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_gather0_indices)); - uint32_t gather0_out_dims[] = {M, B, static_cast(src0->ne[2]), static_cast(src0->ne[3])}; + uint32_t gather0_out_dims[] = {M, B, static_cast(src0->ne[0]), static_cast(src0->ne[1])}; p_gather0_out = GQCGT(nullptr, "gather0_out", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, 4, gather0_out_dims, nullptr, 0); CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_gather0_out)); @@ -534,20 +378,20 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) gather0_inputs, 2, gather0_outputs, 1); CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, gather0_op)); - // src1: [B, N1, K, N] - uint32_t src1_dims[] = {B, N1, static_cast(src1->ne[2]), static_cast(src1->ne[3])}; + // src1: [N, K, N1, B] + uint32_t src1_dims[] = {static_cast(src1->ne[3]), static_cast(src1->ne[2]), static_cast(src1->ne[1]), static_cast(src1->ne[0])}; p_tensor1 = GQCGT(src1, "input1", QNN_TENSOR_TYPE_APP_WRITE, QNN_DATATYPE_FLOAT_32, 4, src1_dims, nullptr, 0); CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor1)); - // Gather on src1: [B, N1, K, N] -> [N1, B, K, N] - uint32_t gather1_indices_data[] = {1, 0, 2, 3}; // [B, N1, K, N] -> [N1, B, K, N] + // Gather on src1: [N, K, N1, B] -> [N1, B, K, N] + uint32_t gather1_indices_data[] = {2, 3, 1, 0}; // [N, K, N1, B] -> [N1, B, K, N] uint32_t gather1_indices_dims[] = {4}; p_gather1_indices = GQCGT(nullptr, "gather1_indices", QNN_TENSOR_TYPE_STATIC, QNN_DATATYPE_UINT_32, 1, gather1_indices_dims, gather1_indices_data, sizeof(gather1_indices_data)); CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_gather1_indices)); - uint32_t gather1_out_dims[] = {N1, B, static_cast(src1->ne[2]), static_cast(src1->ne[3])}; + uint32_t gather1_out_dims[] = {N1, B, static_cast(src1->ne[1]), static_cast(src1->ne[0])}; p_gather1_out = GQCGT(nullptr, "gather1_out", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, 4, gather1_out_dims, nullptr, 0); CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_gather1_out)); @@ -585,19 +429,14 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) matmul_inputs, 2, matmul_outputs, 1); CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, matmul_op)); - // Transpose: [M, N] -> [M, N1, K1, N] - uint32_t perm_data[] = {0, 1, 2, 3}; // Placeholder, adjust below - if (dst->ne[1] == N1 && dst->ne[2] == K1 && dst->ne[3] == N) { - perm_data[0] = 0; perm_data[1] = 1; perm_data[2] = 2; perm_data[3] = 3; - } else if (dst->ne[1] == 1 && dst->ne[2] == K1 && dst->ne[3] == N) { - perm_data[0] = 0; perm_data[1] = 2; perm_data[2] = 1; perm_data[3] = 3; // Adjust for [M, 1, K, N] - } + // Transpose: [M, N] -> [M, N1, K, N] + uint32_t perm_data[] = {0, 2, 1, 3}; // [M, N] -> [M, N1, K, N] based on dst uint32_t perm_dims[] = {4}; p_transpose_perm = GQCGT(nullptr, "transpose_perm", QNN_TENSOR_TYPE_STATIC, QNN_DATATYPE_UINT_32, 1, perm_dims, perm_data, sizeof(perm_data)); CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_transpose_perm)); - uint32_t dst_dims[] = {M, N1, K1, N}; + uint32_t dst_dims[] = {static_cast(dst->ne[2]), static_cast(dst->ne[3]), static_cast(dst->ne[1]), static_cast(dst->ne[0])}; // Match dst->ne order p_tensor2 = GQCGT(dst, "output", QNN_TENSOR_TYPE_APP_READ, QNN_DATATYPE_FLOAT_32, 4, dst_dims, nullptr, 0); CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor2)); From 60be94e0939047ce909e65ecb074771766342084 Mon Sep 17 00:00:00 2001 From: zhouwg Date: Sun, 2 Mar 2025 13:41:44 +0800 Subject: [PATCH 37/76] ggml-qnn: AI-assisted ggml_qnn_mul_mat_4d by Grok 3 --- step4 --- ggml/src/ggml-qnn/ggml-qnn-ops.cpp | 32 +++++++++++++++--------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/ggml/src/ggml-qnn/ggml-qnn-ops.cpp b/ggml/src/ggml-qnn/ggml-qnn-ops.cpp index db5847a78ed02..1092e2ffac811 100644 --- a/ggml/src/ggml-qnn/ggml-qnn-ops.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn-ops.cpp @@ -296,9 +296,9 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) qnn_instance *instance = ctx->instance; QNN_INTERFACE_VER_TYPE qnn_raw_interface = ctx->raw_interface; - const ggml_tensor *src0 = op->src[0]; - const ggml_tensor *src1 = op->src[1]; - ggml_tensor *dst = op; + const ggml_tensor *src0 = op->src[0]; // e.g., ne = [3, 2, 16, 256] + const ggml_tensor *src1 = op->src[1]; // e.g., ne = [4, 6, 1, 256] or [2, 3, 16, 256] + ggml_tensor *dst = op; // e.g., ne = [4, 6, 1, 16] or [2, 3, 16, 16] GGMLQNN_CHECK_PARAMS(ctx, src0, src1, dst); GGML_ASSERT(ggml_n_dims(src0) == 4 && ggml_n_dims(src1) == 4); @@ -341,22 +341,22 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) // Step 1: Define dimensions (ne = [K2, K1, M, B] for src0, [N, K, N1, B] for src1) uint32_t B = src0->ne[3]; uint32_t M = src0->ne[2]; - uint32_t K0 = src0->ne[0] * src0->ne[1]; // K from src0 + uint32_t K0 = src0->ne[0] * src0->ne[1]; // K from src0 (e.g., 3 * 2 = 6) uint32_t N1 = src1->ne[2]; - uint32_t K1 = src1->ne[1]; - uint32_t N = src1->ne[0]; + uint32_t K1 = src1->ne[1] * src1->ne[0]; // K from src1 (e.g., 6 or 3 * 2 = 6) + uint32_t N = src1->ne[3]; GGML_ASSERT(src0->ne[3] == src1->ne[3]); // Matching batch - GGML_ASSERT(dst->ne[2] == M); // M matches dst - GGML_ASSERT(K0 == K1); // K must match + GGML_ASSERT(dst->ne[2] == M); // M matches dst + GGML_ASSERT(K0 == K1); // K must match - // src0: [K2, K1, M, B] -> [B, M, K2, K1] - uint32_t src0_dims[] = {static_cast(src0->ne[3]), static_cast(src0->ne[2]), static_cast(src0->ne[0]), static_cast(src0->ne[1])}; + // src0: [K2, K1, M, B] -> [B, M, K0] (logical order for QNN) + uint32_t src0_dims[] = {B, M, static_cast(src0->ne[0]), static_cast(src0->ne[1])}; p_tensor0 = GQCGT(src0, "input0", QNN_TENSOR_TYPE_APP_WRITE, QNN_DATATYPE_FLOAT_32, 4, src0_dims, nullptr, 0); CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor0)); - // Gather on src0: [B, M, K2, K1] -> [M, B, K0] + // Gather on src0: [K2, K1, M, B] -> [M, B, K2, K1] uint32_t gather0_indices_data[] = {2, 3, 0, 1}; // [K2, K1, M, B] -> [M, B, K2, K1] uint32_t gather0_indices_dims[] = {4}; p_gather0_indices = GQCGT(nullptr, "gather0_indices", QNN_TENSOR_TYPE_STATIC, QNN_DATATYPE_UINT_32, 1, @@ -378,8 +378,8 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) gather0_inputs, 2, gather0_outputs, 1); CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, gather0_op)); - // src1: [N, K, N1, B] - uint32_t src1_dims[] = {static_cast(src1->ne[3]), static_cast(src1->ne[2]), static_cast(src1->ne[1]), static_cast(src1->ne[0])}; + // src1: [N, K, N1, B] -> [B, N1, K, N] (logical order for QNN) + uint32_t src1_dims[] = {B, N1, static_cast(src1->ne[1]), static_cast(src1->ne[0])}; p_tensor1 = GQCGT(src1, "input1", QNN_TENSOR_TYPE_APP_WRITE, QNN_DATATYPE_FLOAT_32, 4, src1_dims, nullptr, 0); CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor1)); @@ -429,14 +429,14 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) matmul_inputs, 2, matmul_outputs, 1); CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, matmul_op)); - // Transpose: [M, N] -> [M, N1, K, N] - uint32_t perm_data[] = {0, 2, 1, 3}; // [M, N] -> [M, N1, K, N] based on dst + // Transpose: [M, N] -> [N, K, N1, M] to match dst->ne + uint32_t perm_data[] = {3, 2, 1, 0}; // Adjust to dst->ne order uint32_t perm_dims[] = {4}; p_transpose_perm = GQCGT(nullptr, "transpose_perm", QNN_TENSOR_TYPE_STATIC, QNN_DATATYPE_UINT_32, 1, perm_dims, perm_data, sizeof(perm_data)); CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_transpose_perm)); - uint32_t dst_dims[] = {static_cast(dst->ne[2]), static_cast(dst->ne[3]), static_cast(dst->ne[1]), static_cast(dst->ne[0])}; // Match dst->ne order + uint32_t dst_dims[] = {static_cast(dst->ne[0]), static_cast(dst->ne[1]), static_cast(dst->ne[2]), static_cast(dst->ne[3])}; // Match dst->ne directly p_tensor2 = GQCGT(dst, "output", QNN_TENSOR_TYPE_APP_READ, QNN_DATATYPE_FLOAT_32, 4, dst_dims, nullptr, 0); CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor2)); From b3ca9c95ac5b0f35b470db186c7b7297022d06c2 Mon Sep 17 00:00:00 2001 From: zhouwg Date: Sun, 2 Mar 2025 13:50:24 +0800 Subject: [PATCH 38/76] ggml-qnn: AI-assisted ggml_qnn_mul_mat_4d by Grok 3 --- step5 --- ggml/src/ggml-qnn/ggml-qnn-ops.cpp | 33 +++++++++++++++--------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/ggml/src/ggml-qnn/ggml-qnn-ops.cpp b/ggml/src/ggml-qnn/ggml-qnn-ops.cpp index 1092e2ffac811..72c241bb60d50 100644 --- a/ggml/src/ggml-qnn/ggml-qnn-ops.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn-ops.cpp @@ -296,9 +296,9 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) qnn_instance *instance = ctx->instance; QNN_INTERFACE_VER_TYPE qnn_raw_interface = ctx->raw_interface; - const ggml_tensor *src0 = op->src[0]; // e.g., ne = [3, 2, 16, 256] - const ggml_tensor *src1 = op->src[1]; // e.g., ne = [4, 6, 1, 256] or [2, 3, 16, 256] - ggml_tensor *dst = op; // e.g., ne = [4, 6, 1, 16] or [2, 3, 16, 16] + const ggml_tensor *src0 = op->src[0]; + const ggml_tensor *src1 = op->src[1]; + ggml_tensor *dst = op; GGMLQNN_CHECK_PARAMS(ctx, src0, src1, dst); GGML_ASSERT(ggml_n_dims(src0) == 4 && ggml_n_dims(src1) == 4); @@ -307,7 +307,6 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) std::string graph_name; ggmlqnn_get_graphkey_from_op(op, graph_name); GGMLQNN_LOG_DEBUG("graph name %s\n", graph_name.c_str()); - ggmlqnn_print_tensors_info(__func__, ctx, src0, src1, dst); Qnn_GraphHandle_t graph_handle = nullptr; Qnn_Tensor_t *p_tensor0 = nullptr; @@ -320,6 +319,8 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) Qnn_Tensor_t *p_transpose_perm = nullptr; Qnn_Tensor_t *p_tensor2 = nullptr; + ggmlqnn_print_tensors_info(__func__, ctx, src0, src1, dst); // Keep debug line + if (instance->_qnn_graph_map.find(graph_name) != instance->_qnn_graph_map.end()) { graph_initialized = true; qnn_res_t &graph_item = instance->_qnn_graph_map[graph_name]; @@ -341,23 +342,23 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) // Step 1: Define dimensions (ne = [K2, K1, M, B] for src0, [N, K, N1, B] for src1) uint32_t B = src0->ne[3]; uint32_t M = src0->ne[2]; - uint32_t K0 = src0->ne[0] * src0->ne[1]; // K from src0 (e.g., 3 * 2 = 6) + uint32_t K0 = src0->ne[0] * src0->ne[1]; // K from src0 uint32_t N1 = src1->ne[2]; - uint32_t K1 = src1->ne[1] * src1->ne[0]; // K from src1 (e.g., 6 or 3 * 2 = 6) - uint32_t N = src1->ne[3]; + uint32_t K1 = src1->ne[1] * src1->ne[0]; // K from src1 + uint32_t N = src1->ne[0]; GGML_ASSERT(src0->ne[3] == src1->ne[3]); // Matching batch GGML_ASSERT(dst->ne[2] == M); // M matches dst GGML_ASSERT(K0 == K1); // K must match - // src0: [K2, K1, M, B] -> [B, M, K0] (logical order for QNN) - uint32_t src0_dims[] = {B, M, static_cast(src0->ne[0]), static_cast(src0->ne[1])}; + // src0: Use GGML's ne directly, let GQCGT reverse to QNN order + uint32_t src0_dims[] = {static_cast(src0->ne[0]), static_cast(src0->ne[1]), static_cast(src0->ne[2]), static_cast(src0->ne[3])}; p_tensor0 = GQCGT(src0, "input0", QNN_TENSOR_TYPE_APP_WRITE, QNN_DATATYPE_FLOAT_32, 4, src0_dims, nullptr, 0); CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor0)); // Gather on src0: [K2, K1, M, B] -> [M, B, K2, K1] - uint32_t gather0_indices_data[] = {2, 3, 0, 1}; // [K2, K1, M, B] -> [M, B, K2, K1] + uint32_t gather0_indices_data[] = {2, 3, 0, 1}; // Correct for QNN's reversed order uint32_t gather0_indices_dims[] = {4}; p_gather0_indices = GQCGT(nullptr, "gather0_indices", QNN_TENSOR_TYPE_STATIC, QNN_DATATYPE_UINT_32, 1, gather0_indices_dims, gather0_indices_data, sizeof(gather0_indices_data)); @@ -378,14 +379,14 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) gather0_inputs, 2, gather0_outputs, 1); CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, gather0_op)); - // src1: [N, K, N1, B] -> [B, N1, K, N] (logical order for QNN) - uint32_t src1_dims[] = {B, N1, static_cast(src1->ne[1]), static_cast(src1->ne[0])}; + // src1: Use GGML's ne directly + uint32_t src1_dims[] = {static_cast(src1->ne[0]), static_cast(src1->ne[1]), static_cast(src1->ne[2]), static_cast(src1->ne[3])}; p_tensor1 = GQCGT(src1, "input1", QNN_TENSOR_TYPE_APP_WRITE, QNN_DATATYPE_FLOAT_32, 4, src1_dims, nullptr, 0); CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor1)); // Gather on src1: [N, K, N1, B] -> [N1, B, K, N] - uint32_t gather1_indices_data[] = {2, 3, 1, 0}; // [N, K, N1, B] -> [N1, B, K, N] + uint32_t gather1_indices_data[] = {2, 3, 1, 0}; // Correct for QNN's reversed order uint32_t gather1_indices_dims[] = {4}; p_gather1_indices = GQCGT(nullptr, "gather1_indices", QNN_TENSOR_TYPE_STATIC, QNN_DATATYPE_UINT_32, 1, gather1_indices_dims, gather1_indices_data, sizeof(gather1_indices_data)); @@ -429,14 +430,14 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) matmul_inputs, 2, matmul_outputs, 1); CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, matmul_op)); - // Transpose: [M, N] -> [N, K, N1, M] to match dst->ne - uint32_t perm_data[] = {3, 2, 1, 0}; // Adjust to dst->ne order + // Transpose: [M, N] -> Match dst->ne + uint32_t perm_data[] = {3, 2, 1, 0}; // [M, N] -> [N, K, N1, M] for dst->ne uint32_t perm_dims[] = {4}; p_transpose_perm = GQCGT(nullptr, "transpose_perm", QNN_TENSOR_TYPE_STATIC, QNN_DATATYPE_UINT_32, 1, perm_dims, perm_data, sizeof(perm_data)); CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_transpose_perm)); - uint32_t dst_dims[] = {static_cast(dst->ne[0]), static_cast(dst->ne[1]), static_cast(dst->ne[2]), static_cast(dst->ne[3])}; // Match dst->ne directly + uint32_t dst_dims[] = {static_cast(dst->ne[0]), static_cast(dst->ne[1]), static_cast(dst->ne[2]), static_cast(dst->ne[3])}; p_tensor2 = GQCGT(dst, "output", QNN_TENSOR_TYPE_APP_READ, QNN_DATATYPE_FLOAT_32, 4, dst_dims, nullptr, 0); CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor2)); From 4e01e326060f349312a3ea448b9d91523028ca29 Mon Sep 17 00:00:00 2001 From: zhouwg Date: Sun, 2 Mar 2025 14:00:46 +0800 Subject: [PATCH 39/76] ggml-qnn: AI-assisted ggml_qnn_mul_mat_4d by Grok 3 --- step6 --- ggml/src/ggml-qnn/ggml-qnn-ops.cpp | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/ggml/src/ggml-qnn/ggml-qnn-ops.cpp b/ggml/src/ggml-qnn/ggml-qnn-ops.cpp index 72c241bb60d50..55b941c4bcec9 100644 --- a/ggml/src/ggml-qnn/ggml-qnn-ops.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn-ops.cpp @@ -342,29 +342,29 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) // Step 1: Define dimensions (ne = [K2, K1, M, B] for src0, [N, K, N1, B] for src1) uint32_t B = src0->ne[3]; uint32_t M = src0->ne[2]; - uint32_t K0 = src0->ne[0] * src0->ne[1]; // K from src0 + uint32_t K0 = src0->ne[0] * src0->ne[1]; uint32_t N1 = src1->ne[2]; - uint32_t K1 = src1->ne[1] * src1->ne[0]; // K from src1 + uint32_t K1 = src1->ne[1] * src1->ne[0]; uint32_t N = src1->ne[0]; GGML_ASSERT(src0->ne[3] == src1->ne[3]); // Matching batch GGML_ASSERT(dst->ne[2] == M); // M matches dst GGML_ASSERT(K0 == K1); // K must match - // src0: Use GGML's ne directly, let GQCGT reverse to QNN order - uint32_t src0_dims[] = {static_cast(src0->ne[0]), static_cast(src0->ne[1]), static_cast(src0->ne[2]), static_cast(src0->ne[3])}; + // src0: [K2, K1, M, B] -> QNN sees [B, M, K1, K2] after GQCGT reversal + uint32_t src0_dims[] = {static_cast(src0->ne[3]), static_cast(src0->ne[2]), static_cast(src0->ne[1]), static_cast(src0->ne[0])}; p_tensor0 = GQCGT(src0, "input0", QNN_TENSOR_TYPE_APP_WRITE, QNN_DATATYPE_FLOAT_32, 4, src0_dims, nullptr, 0); CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor0)); - // Gather on src0: [K2, K1, M, B] -> [M, B, K2, K1] - uint32_t gather0_indices_data[] = {2, 3, 0, 1}; // Correct for QNN's reversed order + // Gather on src0: [B, M, K1, K2] -> [M, B, K1, K2] + uint32_t gather0_indices_data[] = {1, 0, 2, 3}; // [B, M, K1, K2] -> [M, B, K1, K2] uint32_t gather0_indices_dims[] = {4}; p_gather0_indices = GQCGT(nullptr, "gather0_indices", QNN_TENSOR_TYPE_STATIC, QNN_DATATYPE_UINT_32, 1, gather0_indices_dims, gather0_indices_data, sizeof(gather0_indices_data)); CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_gather0_indices)); - uint32_t gather0_out_dims[] = {M, B, static_cast(src0->ne[0]), static_cast(src0->ne[1])}; + uint32_t gather0_out_dims[] = {M, B, static_cast(src0->ne[1]), static_cast(src0->ne[0])}; p_gather0_out = GQCGT(nullptr, "gather0_out", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, 4, gather0_out_dims, nullptr, 0); CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_gather0_out)); @@ -379,14 +379,14 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) gather0_inputs, 2, gather0_outputs, 1); CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, gather0_op)); - // src1: Use GGML's ne directly - uint32_t src1_dims[] = {static_cast(src1->ne[0]), static_cast(src1->ne[1]), static_cast(src1->ne[2]), static_cast(src1->ne[3])}; + // src1: [N, K, N1, B] -> QNN sees [B, N1, K, N] + uint32_t src1_dims[] = {static_cast(src1->ne[3]), static_cast(src1->ne[2]), static_cast(src1->ne[1]), static_cast(src1->ne[0])}; p_tensor1 = GQCGT(src1, "input1", QNN_TENSOR_TYPE_APP_WRITE, QNN_DATATYPE_FLOAT_32, 4, src1_dims, nullptr, 0); CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor1)); - // Gather on src1: [N, K, N1, B] -> [N1, B, K, N] - uint32_t gather1_indices_data[] = {2, 3, 1, 0}; // Correct for QNN's reversed order + // Gather on src1: [B, N1, K, N] -> [N1, B, K, N] + uint32_t gather1_indices_data[] = {1, 0, 2, 3}; // [B, N1, K, N] -> [N1, B, K, N] uint32_t gather1_indices_dims[] = {4}; p_gather1_indices = GQCGT(nullptr, "gather1_indices", QNN_TENSOR_TYPE_STATIC, QNN_DATATYPE_UINT_32, 1, gather1_indices_dims, gather1_indices_data, sizeof(gather1_indices_data)); @@ -430,9 +430,9 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) matmul_inputs, 2, matmul_outputs, 1); CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, matmul_op)); - // Transpose: [M, N] -> Match dst->ne - uint32_t perm_data[] = {3, 2, 1, 0}; // [M, N] -> [N, K, N1, M] for dst->ne - uint32_t perm_dims[] = {4}; + // Transpose: [M, N] -> Match dst->ne ([N, K, N1, M] reversed) + uint32_t perm_data[] = {1, 0}; // [M, N] -> [N, M] for 2D + uint32_t perm_dims[] = {2}; p_transpose_perm = GQCGT(nullptr, "transpose_perm", QNN_TENSOR_TYPE_STATIC, QNN_DATATYPE_UINT_32, 1, perm_dims, perm_data, sizeof(perm_data)); CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_transpose_perm)); From 8869350652308d27fc6e3c17d095c4141a8b0325 Mon Sep 17 00:00:00 2001 From: zhouwg Date: Sun, 2 Mar 2025 14:12:24 +0800 Subject: [PATCH 40/76] ggml-qnn: AI-assisted ggml_qnn_mul_mat_4d by Grok 3 --- step7 --- ggml/src/ggml-qnn/ggml-qnn-ops.cpp | 46 +++++++++--------------------- 1 file changed, 13 insertions(+), 33 deletions(-) diff --git a/ggml/src/ggml-qnn/ggml-qnn-ops.cpp b/ggml/src/ggml-qnn/ggml-qnn-ops.cpp index 55b941c4bcec9..c5f4cb9008270 100644 --- a/ggml/src/ggml-qnn/ggml-qnn-ops.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn-ops.cpp @@ -316,7 +316,6 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) Qnn_Tensor_t *p_gather1_out = nullptr; Qnn_Tensor_t *p_gather1_indices = nullptr; Qnn_Tensor_t *p_matmul_out = nullptr; - Qnn_Tensor_t *p_transpose_perm = nullptr; Qnn_Tensor_t *p_tensor2 = nullptr; ggmlqnn_print_tensors_info(__func__, ctx, src0, src1, dst); // Keep debug line @@ -333,13 +332,12 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) p_gather1_out = tensors[4]; p_gather1_indices = tensors[5]; p_matmul_out = tensors[6]; - p_transpose_perm = tensors[7]; - p_tensor2 = tensors[8]; + p_tensor2 = tensors[7]; } else { CHECK_QNN_API(error, qnn_raw_interface.graphCreate(instance->get_qnn_context_handle(), graph_name.c_str(), NULL, &graph_handle)); - // Step 1: Define dimensions (ne = [K2, K1, M, B] for src0, [N, K, N1, B] for src1) + // Step 1: Define dimensions uint32_t B = src0->ne[3]; uint32_t M = src0->ne[2]; uint32_t K0 = src0->ne[0] * src0->ne[1]; @@ -351,7 +349,7 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) GGML_ASSERT(dst->ne[2] == M); // M matches dst GGML_ASSERT(K0 == K1); // K must match - // src0: [K2, K1, M, B] -> QNN sees [B, M, K1, K2] after GQCGT reversal + // src0: [K2, K1, M, B] -> QNN: [B, M, K1, K2] uint32_t src0_dims[] = {static_cast(src0->ne[3]), static_cast(src0->ne[2]), static_cast(src0->ne[1]), static_cast(src0->ne[0])}; p_tensor0 = GQCGT(src0, "input0", QNN_TENSOR_TYPE_APP_WRITE, QNN_DATATYPE_FLOAT_32, 4, src0_dims, nullptr, 0); @@ -379,7 +377,7 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) gather0_inputs, 2, gather0_outputs, 1); CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, gather0_op)); - // src1: [N, K, N1, B] -> QNN sees [B, N1, K, N] + // src1: [N, K, N1, B] -> QNN: [B, N1, K, N] uint32_t src1_dims[] = {static_cast(src1->ne[3]), static_cast(src1->ne[2]), static_cast(src1->ne[1]), static_cast(src1->ne[0])}; p_tensor1 = GQCGT(src1, "input1", QNN_TENSOR_TYPE_APP_WRITE, QNN_DATATYPE_FLOAT_32, 4, src1_dims, nullptr, 0); @@ -407,19 +405,19 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) gather1_inputs, 2, gather1_outputs, 1); CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, gather1_op)); - // MatMul: [M, B * K0] x [B * K1, N] - uint32_t matmul_in0_dims[] = {M, B * K0}; + // MatMul: [M, B, K0] x [N1, B, K1] -> [M, N1, N] + uint32_t matmul_in0_dims[] = {M, B, K0}; Qnn_Tensor_t matmul_in0 = *p_gather0_out; QNN_VER_PTR(matmul_in0)->dimensions = matmul_in0_dims; - QNN_VER_PTR(matmul_in0)->rank = 2; + QNN_VER_PTR(matmul_in0)->rank = 3; - uint32_t matmul_in1_dims[] = {B * K1, N}; + uint32_t matmul_in1_dims[] = {N1, B, K1}; Qnn_Tensor_t matmul_in1 = *p_gather1_out; QNN_VER_PTR(matmul_in1)->dimensions = matmul_in1_dims; - QNN_VER_PTR(matmul_in1)->rank = 2; + QNN_VER_PTR(matmul_in1)->rank = 3; - uint32_t matmul_out_dims[] = {M, N}; - p_matmul_out = GQCGT(nullptr, "matmul_out", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, 2, + uint32_t matmul_out_dims[] = {M, N1, N}; + p_matmul_out = GQCGT(nullptr, "matmul_out", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, 3, matmul_out_dims, nullptr, 0); CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_matmul_out)); @@ -430,35 +428,19 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) matmul_inputs, 2, matmul_outputs, 1); CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, matmul_op)); - // Transpose: [M, N] -> Match dst->ne ([N, K, N1, M] reversed) - uint32_t perm_data[] = {1, 0}; // [M, N] -> [N, M] for 2D - uint32_t perm_dims[] = {2}; - p_transpose_perm = GQCGT(nullptr, "transpose_perm", QNN_TENSOR_TYPE_STATIC, QNN_DATATYPE_UINT_32, 1, - perm_dims, perm_data, sizeof(perm_data)); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_transpose_perm)); - + // Output: Match dst->ne directly uint32_t dst_dims[] = {static_cast(dst->ne[0]), static_cast(dst->ne[1]), static_cast(dst->ne[2]), static_cast(dst->ne[3])}; p_tensor2 = GQCGT(dst, "output", QNN_TENSOR_TYPE_APP_READ, QNN_DATATYPE_FLOAT_32, 4, dst_dims, nullptr, 0); CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor2)); - Qnn_Param_t transpose_params[] = { - {QNN_PARAMTYPE_TENSOR, "perm", .tensorParam = *p_transpose_perm} - }; - Qnn_Tensor_t transpose_inputs[] = {*p_matmul_out}; - Qnn_Tensor_t transpose_outputs[] = {*p_tensor2}; - Qnn_OpConfig_t transpose_op = ggmlqnn_create_op_config("transpose", QNN_OP_PACKAGE_NAME_QTI_AISW, - QNN_OP_TRANSPOSE, transpose_params, 1, - transpose_inputs, 1, transpose_outputs, 1); - CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, transpose_op)); - // Finalize CHECK_QNN_API(error, qnn_raw_interface.graphFinalize(graph_handle, NULL, NULL)); // Cache qnn_tensors_t ggml_op_mulmat_tensors = {p_tensor0, p_gather0_out, p_gather0_indices, p_tensor1, p_gather1_out, p_gather1_indices, p_matmul_out, - p_transpose_perm, p_tensor2}; + p_tensor2}; instance->_qnn_graph_map[graph_name] = std::make_tuple(graph_handle, ggml_op_mulmat_tensors); } @@ -470,7 +452,6 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) uint32_t *gather1_out_dims = QNN_VER_PTR(*p_gather1_out)->dimensions; uint32_t *gather1_indices_dims = QNN_VER_PTR(*p_gather1_indices)->dimensions; uint32_t *matmul_out_dims = QNN_VER_PTR(*p_matmul_out)->dimensions; - uint32_t *transpose_perm_dims = QNN_VER_PTR(*p_transpose_perm)->dimensions; uint32_t *tensor_2_dims = QNN_VER_PTR(*p_tensor2)->dimensions; // Execute @@ -491,7 +472,6 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) QNN_VER_PTR(*p_gather1_out)->dimensions = gather1_out_dims; QNN_VER_PTR(*p_gather1_indices)->dimensions = gather1_indices_dims; QNN_VER_PTR(*p_matmul_out)->dimensions = matmul_out_dims; - QNN_VER_PTR(*p_transpose_perm)->dimensions = transpose_perm_dims; QNN_VER_PTR(*p_tensor2)->dimensions = tensor_2_dims; op_perf.info(); From 330e90664ceb2c8c828493995221725849a77f9c Mon Sep 17 00:00:00 2001 From: zhouwg Date: Sun, 2 Mar 2025 14:17:40 +0800 Subject: [PATCH 41/76] ggml-qnn: AI-assisted ggml_qnn_mul_mat_4d by Grok 3 --- step8 --- ggml/src/ggml-qnn/ggml-qnn-ops.cpp | 96 +++++++----------------------- 1 file changed, 20 insertions(+), 76 deletions(-) diff --git a/ggml/src/ggml-qnn/ggml-qnn-ops.cpp b/ggml/src/ggml-qnn/ggml-qnn-ops.cpp index c5f4cb9008270..5b2552e45868b 100644 --- a/ggml/src/ggml-qnn/ggml-qnn-ops.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn-ops.cpp @@ -310,11 +310,7 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) Qnn_GraphHandle_t graph_handle = nullptr; Qnn_Tensor_t *p_tensor0 = nullptr; - Qnn_Tensor_t *p_gather0_out = nullptr; - Qnn_Tensor_t *p_gather0_indices = nullptr; Qnn_Tensor_t *p_tensor1 = nullptr; - Qnn_Tensor_t *p_gather1_out = nullptr; - Qnn_Tensor_t *p_gather1_indices = nullptr; Qnn_Tensor_t *p_matmul_out = nullptr; Qnn_Tensor_t *p_tensor2 = nullptr; @@ -326,18 +322,14 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) graph_handle = std::get<0>(graph_item); qnn_tensors_t &tensors = std::get<1>(graph_item); p_tensor0 = tensors[0]; - p_gather0_out = tensors[1]; - p_gather0_indices = tensors[2]; - p_tensor1 = tensors[3]; - p_gather1_out = tensors[4]; - p_gather1_indices = tensors[5]; - p_matmul_out = tensors[6]; - p_tensor2 = tensors[7]; + p_tensor1 = tensors[1]; + p_matmul_out = tensors[2]; + p_tensor2 = tensors[3]; } else { CHECK_QNN_API(error, qnn_raw_interface.graphCreate(instance->get_qnn_context_handle(), graph_name.c_str(), NULL, &graph_handle)); - // Step 1: Define dimensions + // Define dimensions uint32_t B = src0->ne[3]; uint32_t M = src0->ne[2]; uint32_t K0 = src0->ne[0] * src0->ne[1]; @@ -350,73 +342,29 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) GGML_ASSERT(K0 == K1); // K must match // src0: [K2, K1, M, B] -> QNN: [B, M, K1, K2] - uint32_t src0_dims[] = {static_cast(src0->ne[3]), static_cast(src0->ne[2]), static_cast(src0->ne[1]), static_cast(src0->ne[0])}; + uint32_t src0_dims[] = {B, M, static_cast(src0->ne[1]), static_cast(src0->ne[0])}; p_tensor0 = GQCGT(src0, "input0", QNN_TENSOR_TYPE_APP_WRITE, QNN_DATATYPE_FLOAT_32, 4, src0_dims, nullptr, 0); CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor0)); - // Gather on src0: [B, M, K1, K2] -> [M, B, K1, K2] - uint32_t gather0_indices_data[] = {1, 0, 2, 3}; // [B, M, K1, K2] -> [M, B, K1, K2] - uint32_t gather0_indices_dims[] = {4}; - p_gather0_indices = GQCGT(nullptr, "gather0_indices", QNN_TENSOR_TYPE_STATIC, QNN_DATATYPE_UINT_32, 1, - gather0_indices_dims, gather0_indices_data, sizeof(gather0_indices_data)); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_gather0_indices)); - - uint32_t gather0_out_dims[] = {M, B, static_cast(src0->ne[1]), static_cast(src0->ne[0])}; - p_gather0_out = GQCGT(nullptr, "gather0_out", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, 4, - gather0_out_dims, nullptr, 0); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_gather0_out)); - - Qnn_Param_t gather0_params[] = { - {QNN_PARAMTYPE_SCALAR, "axis", .scalarParam = {QNN_DATATYPE_INT_32, .int32Value = 0}} - }; - Qnn_Tensor_t gather0_inputs[] = {*p_tensor0, *p_gather0_indices}; - Qnn_Tensor_t gather0_outputs[] = {*p_gather0_out}; - Qnn_OpConfig_t gather0_op = ggmlqnn_create_op_config("gather0", QNN_OP_PACKAGE_NAME_QTI_AISW, - QNN_OP_GATHER, gather0_params, 1, - gather0_inputs, 2, gather0_outputs, 1); - CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, gather0_op)); - // src1: [N, K, N1, B] -> QNN: [B, N1, K, N] - uint32_t src1_dims[] = {static_cast(src1->ne[3]), static_cast(src1->ne[2]), static_cast(src1->ne[1]), static_cast(src1->ne[0])}; + uint32_t src1_dims[] = {B, N1, static_cast(src1->ne[1]), static_cast(src1->ne[0])}; p_tensor1 = GQCGT(src1, "input1", QNN_TENSOR_TYPE_APP_WRITE, QNN_DATATYPE_FLOAT_32, 4, src1_dims, nullptr, 0); CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor1)); - // Gather on src1: [B, N1, K, N] -> [N1, B, K, N] - uint32_t gather1_indices_data[] = {1, 0, 2, 3}; // [B, N1, K, N] -> [N1, B, K, N] - uint32_t gather1_indices_dims[] = {4}; - p_gather1_indices = GQCGT(nullptr, "gather1_indices", QNN_TENSOR_TYPE_STATIC, QNN_DATATYPE_UINT_32, 1, - gather1_indices_dims, gather1_indices_data, sizeof(gather1_indices_data)); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_gather1_indices)); - - uint32_t gather1_out_dims[] = {N1, B, static_cast(src1->ne[1]), static_cast(src1->ne[0])}; - p_gather1_out = GQCGT(nullptr, "gather1_out", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, 4, - gather1_out_dims, nullptr, 0); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_gather1_out)); - - Qnn_Param_t gather1_params[] = { - {QNN_PARAMTYPE_SCALAR, "axis", .scalarParam = {QNN_DATATYPE_INT_32, .int32Value = 0}} - }; - Qnn_Tensor_t gather1_inputs[] = {*p_tensor1, *p_gather1_indices}; - Qnn_Tensor_t gather1_outputs[] = {*p_gather1_out}; - Qnn_OpConfig_t gather1_op = ggmlqnn_create_op_config("gather1", QNN_OP_PACKAGE_NAME_QTI_AISW, - QNN_OP_GATHER, gather1_params, 1, - gather1_inputs, 2, gather1_outputs, 1); - CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, gather1_op)); - - // MatMul: [M, B, K0] x [N1, B, K1] -> [M, N1, N] - uint32_t matmul_in0_dims[] = {M, B, K0}; - Qnn_Tensor_t matmul_in0 = *p_gather0_out; + // MatMul: [B, M, K0] x [B, N1, K1] -> [B, M, N1] + uint32_t matmul_in0_dims[] = {B, M, K0}; + Qnn_Tensor_t matmul_in0 = *p_tensor0; QNN_VER_PTR(matmul_in0)->dimensions = matmul_in0_dims; QNN_VER_PTR(matmul_in0)->rank = 3; - uint32_t matmul_in1_dims[] = {N1, B, K1}; - Qnn_Tensor_t matmul_in1 = *p_gather1_out; + uint32_t matmul_in1_dims[] = {B, N1, K1}; + Qnn_Tensor_t matmul_in1 = *p_tensor1; QNN_VER_PTR(matmul_in1)->dimensions = matmul_in1_dims; QNN_VER_PTR(matmul_in1)->rank = 3; - uint32_t matmul_out_dims[] = {M, N1, N}; + uint32_t matmul_out_dims[] = {B, M, N1}; p_matmul_out = GQCGT(nullptr, "matmul_out", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, 3, matmul_out_dims, nullptr, 0); CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_matmul_out)); @@ -428,7 +376,7 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) matmul_inputs, 2, matmul_outputs, 1); CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, matmul_op)); - // Output: Match dst->ne directly + // Output: [M, N1, K2', K1'] matches dst->ne uint32_t dst_dims[] = {static_cast(dst->ne[0]), static_cast(dst->ne[1]), static_cast(dst->ne[2]), static_cast(dst->ne[3])}; p_tensor2 = GQCGT(dst, "output", QNN_TENSOR_TYPE_APP_READ, QNN_DATATYPE_FLOAT_32, 4, dst_dims, nullptr, 0); @@ -438,19 +386,13 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) CHECK_QNN_API(error, qnn_raw_interface.graphFinalize(graph_handle, NULL, NULL)); // Cache - qnn_tensors_t ggml_op_mulmat_tensors = {p_tensor0, p_gather0_out, p_gather0_indices, p_tensor1, - p_gather1_out, p_gather1_indices, p_matmul_out, - p_tensor2}; + qnn_tensors_t ggml_op_mulmat_tensors = {p_tensor0, p_tensor1, p_matmul_out, p_tensor2}; instance->_qnn_graph_map[graph_name] = std::make_tuple(graph_handle, ggml_op_mulmat_tensors); } // Save dimensions uint32_t *tensor_0_dims = QNN_VER_PTR(*p_tensor0)->dimensions; - uint32_t *gather0_out_dims = QNN_VER_PTR(*p_gather0_out)->dimensions; - uint32_t *gather0_indices_dims = QNN_VER_PTR(*p_gather0_indices)->dimensions; uint32_t *tensor_1_dims = QNN_VER_PTR(*p_tensor1)->dimensions; - uint32_t *gather1_out_dims = QNN_VER_PTR(*p_gather1_out)->dimensions; - uint32_t *gather1_indices_dims = QNN_VER_PTR(*p_gather1_indices)->dimensions; uint32_t *matmul_out_dims = QNN_VER_PTR(*p_matmul_out)->dimensions; uint32_t *tensor_2_dims = QNN_VER_PTR(*p_tensor2)->dimensions; @@ -466,14 +408,16 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) // Restore dimensions QNN_VER_PTR(*p_tensor0)->dimensions = tensor_0_dims; - QNN_VER_PTR(*p_gather0_out)->dimensions = gather0_out_dims; - QNN_VER_PTR(*p_gather0_indices)->dimensions = gather0_indices_dims; QNN_VER_PTR(*p_tensor1)->dimensions = tensor_1_dims; - QNN_VER_PTR(*p_gather1_out)->dimensions = gather1_out_dims; - QNN_VER_PTR(*p_gather1_indices)->dimensions = gather1_indices_dims; QNN_VER_PTR(*p_matmul_out)->dimensions = matmul_out_dims; QNN_VER_PTR(*p_tensor2)->dimensions = tensor_2_dims; + // Log dst data for debugging + float *dst_data = (float *)dst->data; + for (int i = 0; i < dst->ne[0] * dst->ne[1] * dst->ne[2] * dst->ne[3]; i++) { + GGMLQNN_LOG_DEBUG("dst[%d] = %f\n", i, dst_data[i]); + } + op_perf.info(); } From c2be898703ff1da16b069416d8b8dfc7668368c5 Mon Sep 17 00:00:00 2001 From: zhouwg Date: Sun, 2 Mar 2025 14:54:38 +0800 Subject: [PATCH 42/76] ggml-qnn: AI-assisted ggml_qnn_mul_mat_4d by Grok 3 --- good in step9 --- ggml/src/ggml-qnn/ggml-qnn-ops.cpp | 137 +++++++++++++++++------------ 1 file changed, 82 insertions(+), 55 deletions(-) diff --git a/ggml/src/ggml-qnn/ggml-qnn-ops.cpp b/ggml/src/ggml-qnn/ggml-qnn-ops.cpp index 5b2552e45868b..f54f19ec7263e 100644 --- a/ggml/src/ggml-qnn/ggml-qnn-ops.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn-ops.cpp @@ -307,14 +307,16 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) std::string graph_name; ggmlqnn_get_graphkey_from_op(op, graph_name); GGMLQNN_LOG_DEBUG("graph name %s\n", graph_name.c_str()); + ggmlqnn_print_tensors_info(__func__, ctx, src0, src1, dst); Qnn_GraphHandle_t graph_handle = nullptr; Qnn_Tensor_t *p_tensor0 = nullptr; + Qnn_Tensor_t *p_reshape0_out = nullptr; Qnn_Tensor_t *p_tensor1 = nullptr; + Qnn_Tensor_t *p_permute1_out = nullptr; + Qnn_Tensor_t *p_reshape1_out = nullptr; Qnn_Tensor_t *p_matmul_out = nullptr; - Qnn_Tensor_t *p_tensor2 = nullptr; - - ggmlqnn_print_tensors_info(__func__, ctx, src0, src1, dst); // Keep debug line + Qnn_Tensor_t *p_reshape2_out = nullptr; if (instance->_qnn_graph_map.find(graph_name) != instance->_qnn_graph_map.end()) { graph_initialized = true; @@ -322,97 +324,122 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) graph_handle = std::get<0>(graph_item); qnn_tensors_t &tensors = std::get<1>(graph_item); p_tensor0 = tensors[0]; - p_tensor1 = tensors[1]; - p_matmul_out = tensors[2]; - p_tensor2 = tensors[3]; + p_reshape0_out = tensors[1]; + p_tensor1 = tensors[2]; + p_permute1_out = tensors[3]; + p_reshape1_out = tensors[4]; + p_matmul_out = tensors[5]; + p_reshape2_out = tensors[6]; } else { CHECK_QNN_API(error, qnn_raw_interface.graphCreate(instance->get_qnn_context_handle(), graph_name.c_str(), NULL, &graph_handle)); - // Define dimensions - uint32_t B = src0->ne[3]; - uint32_t M = src0->ne[2]; - uint32_t K0 = src0->ne[0] * src0->ne[1]; - uint32_t N1 = src1->ne[2]; - uint32_t K1 = src1->ne[1] * src1->ne[0]; - uint32_t N = src1->ne[0]; + // Define dimensions (GGML order: [K, M, H, B]) + uint32_t B = src0->ne[2] * src0->ne[3]; // 3 * 2 = 6 + uint32_t M = src0->ne[1]; // 16 + uint32_t K = src0->ne[0]; // 256 + uint32_t N = src1->ne[1]; // 16 - GGML_ASSERT(src0->ne[3] == src1->ne[3]); // Matching batch - GGML_ASSERT(dst->ne[2] == M); // M matches dst - GGML_ASSERT(K0 == K1); // K must match + GGML_ASSERT(src0->ne[2] == src1->ne[2] && src0->ne[3] == src1->ne[3]); // Matching batch dimensions + GGML_ASSERT(dst->ne[0] == N && dst->ne[1] == M && dst->ne[2] == src0->ne[2] && dst->ne[3] == src0->ne[3]); - // src0: [K2, K1, M, B] -> QNN: [B, M, K1, K2] - uint32_t src0_dims[] = {B, M, static_cast(src0->ne[1]), static_cast(src0->ne[0])}; + // src0: [256, 16, 3, 2] -> QNN: [B, H, M, K] = [2, 3, 16, 256] + uint32_t src0_dims[] = {static_cast(src0->ne[3]), static_cast(src0->ne[2]), static_cast(src0->ne[1]), static_cast(src0->ne[0])}; p_tensor0 = GQCGT(src0, "input0", QNN_TENSOR_TYPE_APP_WRITE, QNN_DATATYPE_FLOAT_32, 4, src0_dims, nullptr, 0); CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor0)); - // src1: [N, K, N1, B] -> QNN: [B, N1, K, N] - uint32_t src1_dims[] = {B, N1, static_cast(src1->ne[1]), static_cast(src1->ne[0])}; + // Reshape src0 to [6, 16, 256] for [B, M, K] + uint32_t reshape0_out_dims[] = {B, M, K}; + p_reshape0_out = GQCGT(nullptr, "reshape0_out", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, 3, + reshape0_out_dims, nullptr, 0); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_reshape0_out)); + Qnn_Tensor_t reshape0_inputs[] = {*p_tensor0}; + Qnn_Tensor_t reshape0_outputs[] = {*p_reshape0_out}; + Qnn_OpConfig_t reshape0_op = ggmlqnn_create_op_config("reshape0", QNN_OP_PACKAGE_NAME_QTI_AISW, + QNN_OP_RESHAPE, nullptr, 0, + reshape0_inputs, 1, reshape0_outputs, 1); + CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, reshape0_op)); + + // src1: [256, 16, 3, 2] -> QNN: [B, H, N, K] = [2, 3, 16, 256] + uint32_t src1_dims[] = {static_cast(src1->ne[3]), static_cast(src1->ne[2]), static_cast(src1->ne[1]), static_cast(src1->ne[0])}; p_tensor1 = GQCGT(src1, "input1", QNN_TENSOR_TYPE_APP_WRITE, QNN_DATATYPE_FLOAT_32, 4, src1_dims, nullptr, 0); CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor1)); - // MatMul: [B, M, K0] x [B, N1, K1] -> [B, M, N1] - uint32_t matmul_in0_dims[] = {B, M, K0}; - Qnn_Tensor_t matmul_in0 = *p_tensor0; - QNN_VER_PTR(matmul_in0)->dimensions = matmul_in0_dims; - QNN_VER_PTR(matmul_in0)->rank = 3; - - uint32_t matmul_in1_dims[] = {B, N1, K1}; - Qnn_Tensor_t matmul_in1 = *p_tensor1; - QNN_VER_PTR(matmul_in1)->dimensions = matmul_in1_dims; - QNN_VER_PTR(matmul_in1)->rank = 3; - - uint32_t matmul_out_dims[] = {B, M, N1}; + // Permute src1 to [2, 3, 256, 16] to align K and N + uint32_t perm_data[] = {0, 1, 3, 2}; // [B, H, N, K] -> [B, H, K, N] + uint32_t perm_dims[] = {4}; + Qnn_Tensor_t * p_perm = GQCGT(nullptr, "perm", QNN_TENSOR_TYPE_STATIC, QNN_DATATYPE_UINT_32, 1, + perm_dims, perm_data, sizeof(perm_data)); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_perm)); + uint32_t permute1_out_dims[] = {static_cast(src1->ne[3]), static_cast(src1->ne[2]), static_cast(src1->ne[0]), static_cast(src1->ne[1])}; // [2, 3, 256, 16] + p_permute1_out = GQCGT(nullptr, "permute1_out", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, 4, + permute1_out_dims, nullptr, 0); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_permute1_out)); + Qnn_Param_t permute1_params[] = {{QNN_PARAMTYPE_TENSOR, "perm", .tensorParam = *p_perm}}; + Qnn_Tensor_t permute1_inputs[] = {*p_tensor1}; + Qnn_Tensor_t permute1_outputs[] = {*p_permute1_out}; + Qnn_OpConfig_t permute1_op = ggmlqnn_create_op_config("permute1", QNN_OP_PACKAGE_NAME_QTI_AISW, + QNN_OP_TRANSPOSE, permute1_params, 1, + permute1_inputs, 1, permute1_outputs, 1); + CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, permute1_op)); + + // Reshape src1 to [6, 256, 16] for [B, K, N] + uint32_t reshape1_out_dims[] = {B, K, N}; + p_reshape1_out = GQCGT(nullptr, "reshape1_out", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, 3, + reshape1_out_dims, nullptr, 0); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_reshape1_out)); + Qnn_Tensor_t reshape1_inputs[] = {*p_permute1_out}; + Qnn_Tensor_t reshape1_outputs[] = {*p_reshape1_out}; + Qnn_OpConfig_t reshape1_op = ggmlqnn_create_op_config("reshape1", QNN_OP_PACKAGE_NAME_QTI_AISW, + QNN_OP_RESHAPE, nullptr, 0, + reshape1_inputs, 1, reshape1_outputs, 1); + CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, reshape1_op)); + + // MatMul: [6, 16, 256] x [6, 256, 16] -> [6, 16, 16] + uint32_t matmul_out_dims[] = {B, M, N}; p_matmul_out = GQCGT(nullptr, "matmul_out", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, 3, matmul_out_dims, nullptr, 0); CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_matmul_out)); - - Qnn_Tensor_t matmul_inputs[] = {matmul_in0, matmul_in1}; + Qnn_Tensor_t matmul_inputs[] = {*p_reshape0_out, *p_reshape1_out}; Qnn_Tensor_t matmul_outputs[] = {*p_matmul_out}; Qnn_OpConfig_t matmul_op = ggmlqnn_create_op_config("matmul", QNN_OP_PACKAGE_NAME_QTI_AISW, QNN_OP_MAT_MUL, nullptr, 0, matmul_inputs, 2, matmul_outputs, 1); CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, matmul_op)); - // Output: [M, N1, K2', K1'] matches dst->ne - uint32_t dst_dims[] = {static_cast(dst->ne[0]), static_cast(dst->ne[1]), static_cast(dst->ne[2]), static_cast(dst->ne[3])}; - p_tensor2 = GQCGT(dst, "output", QNN_TENSOR_TYPE_APP_READ, QNN_DATATYPE_FLOAT_32, 4, - dst_dims, nullptr, 0); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor2)); + // Output: [16, 16, 3, 2] -> QNN: [2, 3, 16, 16] + uint32_t reshape2_out_dims[] = {static_cast(dst->ne[3]), static_cast(dst->ne[2]), static_cast(dst->ne[1]), static_cast(dst->ne[0])}; + p_reshape2_out = GQCGT(dst, "output", QNN_TENSOR_TYPE_APP_READ, QNN_DATATYPE_FLOAT_32, 4, + reshape2_out_dims, nullptr, 0); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_reshape2_out)); + Qnn_Tensor_t reshape2_inputs[] = {*p_matmul_out}; + Qnn_Tensor_t reshape2_outputs[] = {*p_reshape2_out}; + Qnn_OpConfig_t reshape2_op = ggmlqnn_create_op_config("reshape2", QNN_OP_PACKAGE_NAME_QTI_AISW, + QNN_OP_RESHAPE, nullptr, 0, + reshape2_inputs, 1, reshape2_outputs, 1); + CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, reshape2_op)); // Finalize CHECK_QNN_API(error, qnn_raw_interface.graphFinalize(graph_handle, NULL, NULL)); // Cache - qnn_tensors_t ggml_op_mulmat_tensors = {p_tensor0, p_tensor1, p_matmul_out, p_tensor2}; + qnn_tensors_t ggml_op_mulmat_tensors = {p_tensor0, p_reshape0_out, p_tensor1, p_permute1_out, p_reshape1_out, p_matmul_out, p_reshape2_out}; instance->_qnn_graph_map[graph_name] = std::make_tuple(graph_handle, ggml_op_mulmat_tensors); } - // Save dimensions - uint32_t *tensor_0_dims = QNN_VER_PTR(*p_tensor0)->dimensions; - uint32_t *tensor_1_dims = QNN_VER_PTR(*p_tensor1)->dimensions; - uint32_t *matmul_out_dims = QNN_VER_PTR(*p_matmul_out)->dimensions; - uint32_t *tensor_2_dims = QNN_VER_PTR(*p_tensor2)->dimensions; - // Execute QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, static_cast(ggml_nbytes(src0))}; QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, static_cast(ggml_nbytes(src1))}; - QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, static_cast(ggml_nbytes(dst))}; + QNN_VER_PTR(*p_reshape2_out)->clientBuf = {dst->data, static_cast(ggml_nbytes(dst))}; Qnn_Tensor_t input_tensors[] = {*p_tensor0, *p_tensor1}; - Qnn_Tensor_t output_tensors[] = {*p_tensor2}; + Qnn_Tensor_t output_tensors[] = {*p_reshape2_out}; CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, input_tensors, 2, output_tensors, 1, NULL, NULL)); - // Restore dimensions - QNN_VER_PTR(*p_tensor0)->dimensions = tensor_0_dims; - QNN_VER_PTR(*p_tensor1)->dimensions = tensor_1_dims; - QNN_VER_PTR(*p_matmul_out)->dimensions = matmul_out_dims; - QNN_VER_PTR(*p_tensor2)->dimensions = tensor_2_dims; - - // Log dst data for debugging + // Log dst for debugging float *dst_data = (float *)dst->data; for (int i = 0; i < dst->ne[0] * dst->ne[1] * dst->ne[2] * dst->ne[3]; i++) { GGMLQNN_LOG_DEBUG("dst[%d] = %f\n", i, dst_data[i]); From 7b795494490da24c8ae5c064b85529d19ea1316b Mon Sep 17 00:00:00 2001 From: zhouwg Date: Sun, 2 Mar 2025 15:05:45 +0800 Subject: [PATCH 43/76] ggml-qnn: AI-assisted ggml_qnn_mul_mat_4d by Grok 3 --- narrow down to make AI happy --- tests/ggml-qnn-ut.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/ggml-qnn-ut.cpp b/tests/ggml-qnn-ut.cpp index 1ab75526794e8..26e7a8847ae09 100644 --- a/tests/ggml-qnn-ut.cpp +++ b/tests/ggml-qnn-ut.cpp @@ -439,10 +439,10 @@ int main(int argc, char * argv[]) { //src0 = ggml_new_tensor_3d(ctx, qtype, 128, 64, 8); //src1 = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, 128, 2, 8); //verify 4D matrix - //src0 = ggml_new_tensor_4d(ctx, qtype, 256, 16, 3, 2); - //src1 = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, 256, 1, 6, 4); - src0 = ggml_new_tensor_4d(ctx, qtype, 256, 16, 3, 2); - src1 = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, 256, 16, 3, 2); + src0 = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, 256, 16, 3, 2); + src1 = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, 256, 1, 6, 4); + //src0 = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, 256, 16, 3, 2); + //src1 = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, 256, 16, 3, 2); } ggml_set_input(src0); From 4bd6dd400ed1805db8ad993425fc63a251626b45 Mon Sep 17 00:00:00 2001 From: zhouwg Date: Sun, 2 Mar 2025 15:10:45 +0800 Subject: [PATCH 44/76] ggml-qnn: AI-assisted ggml_qnn_mul_mat_4d by Grok 3 --- step10 --- ggml/src/ggml-qnn/ggml-qnn-ops.cpp | 44 ++++++++++++++++++------------ 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/ggml/src/ggml-qnn/ggml-qnn-ops.cpp b/ggml/src/ggml-qnn/ggml-qnn-ops.cpp index f54f19ec7263e..2f2491f619a5f 100644 --- a/ggml/src/ggml-qnn/ggml-qnn-ops.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn-ops.cpp @@ -307,6 +307,7 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) std::string graph_name; ggmlqnn_get_graphkey_from_op(op, graph_name); GGMLQNN_LOG_DEBUG("graph name %s\n", graph_name.c_str()); + ggmlqnn_print_tensors_info(__func__, ctx, src0, src1, dst); Qnn_GraphHandle_t graph_handle = nullptr; @@ -334,23 +335,26 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) CHECK_QNN_API(error, qnn_raw_interface.graphCreate(instance->get_qnn_context_handle(), graph_name.c_str(), NULL, &graph_handle)); - // Define dimensions (GGML order: [K, M, H, B]) - uint32_t B = src0->ne[2] * src0->ne[3]; // 3 * 2 = 6 - uint32_t M = src0->ne[1]; // 16 - uint32_t K = src0->ne[0]; // 256 - uint32_t N = src1->ne[1]; // 16 + // Define dimensions + uint32_t B0 = src0->ne[2] * src0->ne[3]; // src0 batch: 3 * 2 = 6 + uint32_t B1 = src1->ne[2] * src1->ne[3]; // src1 batch: 6 * 4 = 24 + uint32_t M = src0->ne[1]; // 16 + uint32_t K = src0->ne[0]; // 256 + uint32_t N = src1->ne[1]; // 1 (second case), 16 (first case) - GGML_ASSERT(src0->ne[2] == src1->ne[2] && src0->ne[3] == src1->ne[3]); // Matching batch dimensions - GGML_ASSERT(dst->ne[0] == N && dst->ne[1] == M && dst->ne[2] == src0->ne[2] && dst->ne[3] == src0->ne[3]); + // Validate K matches + GGML_ASSERT(src0->ne[0] == src1->ne[0]); // K must match: 256 == 256 + // Output shape should match src1's batch dims + GGML_ASSERT(dst->ne[0] == N && dst->ne[1] == M && dst->ne[2] == src1->ne[2] && dst->ne[3] == src1->ne[3]); - // src0: [256, 16, 3, 2] -> QNN: [B, H, M, K] = [2, 3, 16, 256] + // src0: [256, 16, 3, 2] -> QNN: [2, 3, 16, 256] (B, H, M, K) uint32_t src0_dims[] = {static_cast(src0->ne[3]), static_cast(src0->ne[2]), static_cast(src0->ne[1]), static_cast(src0->ne[0])}; p_tensor0 = GQCGT(src0, "input0", QNN_TENSOR_TYPE_APP_WRITE, QNN_DATATYPE_FLOAT_32, 4, src0_dims, nullptr, 0); CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor0)); - // Reshape src0 to [6, 16, 256] for [B, M, K] - uint32_t reshape0_out_dims[] = {B, M, K}; + // Reshape src0 to [6, 16, 256] for [B0, M, K] + uint32_t reshape0_out_dims[] = {B0, M, K}; p_reshape0_out = GQCGT(nullptr, "reshape0_out", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, 3, reshape0_out_dims, nullptr, 0); CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_reshape0_out)); @@ -361,19 +365,19 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) reshape0_inputs, 1, reshape0_outputs, 1); CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, reshape0_op)); - // src1: [256, 16, 3, 2] -> QNN: [B, H, N, K] = [2, 3, 16, 256] + // src1: [256, 1, 6, 4] -> QNN: [4, 6, 1, 256] (B, H, N, K) uint32_t src1_dims[] = {static_cast(src1->ne[3]), static_cast(src1->ne[2]), static_cast(src1->ne[1]), static_cast(src1->ne[0])}; p_tensor1 = GQCGT(src1, "input1", QNN_TENSOR_TYPE_APP_WRITE, QNN_DATATYPE_FLOAT_32, 4, src1_dims, nullptr, 0); CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor1)); - // Permute src1 to [2, 3, 256, 16] to align K and N + // Permute src1 to [4, 6, 256, 1] to align K and N uint32_t perm_data[] = {0, 1, 3, 2}; // [B, H, N, K] -> [B, H, K, N] uint32_t perm_dims[] = {4}; Qnn_Tensor_t * p_perm = GQCGT(nullptr, "perm", QNN_TENSOR_TYPE_STATIC, QNN_DATATYPE_UINT_32, 1, perm_dims, perm_data, sizeof(perm_data)); CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_perm)); - uint32_t permute1_out_dims[] = {static_cast(src1->ne[3]), static_cast(src1->ne[2]), static_cast(src1->ne[0]), static_cast(src1->ne[1])}; // [2, 3, 256, 16] + uint32_t permute1_out_dims[] = {static_cast(src1->ne[3]), static_cast(src1->ne[2]), static_cast(src1->ne[0]), static_cast(src1->ne[1])}; // [4, 6, 256, 1] p_permute1_out = GQCGT(nullptr, "permute1_out", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, 4, permute1_out_dims, nullptr, 0); CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_permute1_out)); @@ -385,8 +389,8 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) permute1_inputs, 1, permute1_outputs, 1); CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, permute1_op)); - // Reshape src1 to [6, 256, 16] for [B, K, N] - uint32_t reshape1_out_dims[] = {B, K, N}; + // Reshape src1 to [24, 256, 1] for [B1, K, N] + uint32_t reshape1_out_dims[] = {B1, K, N}; p_reshape1_out = GQCGT(nullptr, "reshape1_out", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, 3, reshape1_out_dims, nullptr, 0); CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_reshape1_out)); @@ -397,11 +401,15 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) reshape1_inputs, 1, reshape1_outputs, 1); CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, reshape1_op)); - // MatMul: [6, 16, 256] x [6, 256, 16] -> [6, 16, 16] - uint32_t matmul_out_dims[] = {B, M, N}; + // MatMul: [6, 16, 256] x [24, 256, 1] -> Needs adjustment for broadcasting + // Adjust src0 to match B1 by repeating or reshaping + uint32_t matmul_out_dims[] = {B1, M, N}; // [24, 16, 1] p_matmul_out = GQCGT(nullptr, "matmul_out", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, 3, matmul_out_dims, nullptr, 0); CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_matmul_out)); + + // Note: QNN MatMul doesn't broadcast; we need to tile src0 + // For simplicity, assume dst shape drives execution; adjust src0 later if needed Qnn_Tensor_t matmul_inputs[] = {*p_reshape0_out, *p_reshape1_out}; Qnn_Tensor_t matmul_outputs[] = {*p_matmul_out}; Qnn_OpConfig_t matmul_op = ggmlqnn_create_op_config("matmul", QNN_OP_PACKAGE_NAME_QTI_AISW, @@ -409,7 +417,7 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) matmul_inputs, 2, matmul_outputs, 1); CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, matmul_op)); - // Output: [16, 16, 3, 2] -> QNN: [2, 3, 16, 16] + // Output: [1, 16, 6, 4] -> QNN: [4, 6, 16, 1] uint32_t reshape2_out_dims[] = {static_cast(dst->ne[3]), static_cast(dst->ne[2]), static_cast(dst->ne[1]), static_cast(dst->ne[0])}; p_reshape2_out = GQCGT(dst, "output", QNN_TENSOR_TYPE_APP_READ, QNN_DATATYPE_FLOAT_32, 4, reshape2_out_dims, nullptr, 0); From 97296e55dc83cf8ae725a9d69de2023232b0ea94 Mon Sep 17 00:00:00 2001 From: zhouwg Date: Sun, 2 Mar 2025 15:19:38 +0800 Subject: [PATCH 45/76] ggml-qnn: AI-assisted ggml_qnn_mul_mat_4d by Grok 3 --- narrow down to make AI happy --- tests/ggml-qnn-ut.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/ggml-qnn-ut.cpp b/tests/ggml-qnn-ut.cpp index 26e7a8847ae09..a7d8cb5619732 100644 --- a/tests/ggml-qnn-ut.cpp +++ b/tests/ggml-qnn-ut.cpp @@ -439,10 +439,13 @@ int main(int argc, char * argv[]) { //src0 = ggml_new_tensor_3d(ctx, qtype, 128, 64, 8); //src1 = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, 128, 2, 8); //verify 4D matrix +#if 1 //failure src0 = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, 256, 16, 3, 2); src1 = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, 256, 1, 6, 4); - //src0 = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, 256, 16, 3, 2); - //src1 = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, 256, 16, 3, 2); +#else //ok + src0 = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, 256, 16, 3, 2); + src1 = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, 256, 16, 3, 2); +#endif } ggml_set_input(src0); From a5e8cbecefee02d325da94cac4f024c183dc39a3 Mon Sep 17 00:00:00 2001 From: zhouwg Date: Sun, 2 Mar 2025 15:25:50 +0800 Subject: [PATCH 46/76] ggml-qnn: AI-assisted ggml_qnn_mul_mat_4d by Grok 3 --- step11 --- ggml/src/ggml-qnn/ggml-qnn-ops.cpp | 81 ++++++++++++++++++------------ 1 file changed, 48 insertions(+), 33 deletions(-) diff --git a/ggml/src/ggml-qnn/ggml-qnn-ops.cpp b/ggml/src/ggml-qnn/ggml-qnn-ops.cpp index 2f2491f619a5f..43ec5ee16a8c0 100644 --- a/ggml/src/ggml-qnn/ggml-qnn-ops.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn-ops.cpp @@ -289,6 +289,7 @@ void ggml_qnn_general_node(ggml_backend_qnn_context * ctx, ggml_tensor * op) { * than ggml_qnn_mul_mat, so it's a standalone function. * it will be combined with ggml_qnn_mul_mat after bugfix */ + static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) { Qnn_ErrorHandle_t error = QNN_SUCCESS; bool graph_initialized = false; @@ -313,6 +314,7 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) Qnn_GraphHandle_t graph_handle = nullptr; Qnn_Tensor_t *p_tensor0 = nullptr; Qnn_Tensor_t *p_reshape0_out = nullptr; + Qnn_Tensor_t *p_tile0_out = nullptr; Qnn_Tensor_t *p_tensor1 = nullptr; Qnn_Tensor_t *p_permute1_out = nullptr; Qnn_Tensor_t *p_reshape1_out = nullptr; @@ -326,34 +328,34 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) qnn_tensors_t &tensors = std::get<1>(graph_item); p_tensor0 = tensors[0]; p_reshape0_out = tensors[1]; - p_tensor1 = tensors[2]; - p_permute1_out = tensors[3]; - p_reshape1_out = tensors[4]; - p_matmul_out = tensors[5]; - p_reshape2_out = tensors[6]; + p_tile0_out = tensors[2]; + p_tensor1 = tensors[3]; + p_permute1_out = tensors[4]; + p_reshape1_out = tensors[5]; + p_matmul_out = tensors[6]; + p_reshape2_out = tensors[7]; } else { CHECK_QNN_API(error, qnn_raw_interface.graphCreate(instance->get_qnn_context_handle(), graph_name.c_str(), NULL, &graph_handle)); // Define dimensions - uint32_t B0 = src0->ne[2] * src0->ne[3]; // src0 batch: 3 * 2 = 6 - uint32_t B1 = src1->ne[2] * src1->ne[3]; // src1 batch: 6 * 4 = 24 - uint32_t M = src0->ne[1]; // 16 - uint32_t K = src0->ne[0]; // 256 - uint32_t N = src1->ne[1]; // 1 (second case), 16 (first case) - - // Validate K matches - GGML_ASSERT(src0->ne[0] == src1->ne[0]); // K must match: 256 == 256 - // Output shape should match src1's batch dims + uint32_t K = src0->ne[0]; // Inner dimension + uint32_t M = src0->ne[1]; // Rows of src0 + uint32_t N = src1->ne[1]; // Columns of src1 + uint32_t B0 = src0->ne[2] * src0->ne[3]; // src0 batch + uint32_t B1 = src1->ne[2] * src1->ne[3]; // src1 batch (drives output) + + // Validate + GGML_ASSERT(src0->ne[0] == src1->ne[0]); // K must match GGML_ASSERT(dst->ne[0] == N && dst->ne[1] == M && dst->ne[2] == src1->ne[2] && dst->ne[3] == src1->ne[3]); - // src0: [256, 16, 3, 2] -> QNN: [2, 3, 16, 256] (B, H, M, K) + // src0: [K, M, H0, B0] -> QNN: [B0, H0, M, K] uint32_t src0_dims[] = {static_cast(src0->ne[3]), static_cast(src0->ne[2]), static_cast(src0->ne[1]), static_cast(src0->ne[0])}; p_tensor0 = GQCGT(src0, "input0", QNN_TENSOR_TYPE_APP_WRITE, QNN_DATATYPE_FLOAT_32, 4, src0_dims, nullptr, 0); CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor0)); - // Reshape src0 to [6, 16, 256] for [B0, M, K] + // Reshape src0 to [B0, M, K] uint32_t reshape0_out_dims[] = {B0, M, K}; p_reshape0_out = GQCGT(nullptr, "reshape0_out", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, 3, reshape0_out_dims, nullptr, 0); @@ -365,19 +367,37 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) reshape0_inputs, 1, reshape0_outputs, 1); CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, reshape0_op)); - // src1: [256, 1, 6, 4] -> QNN: [4, 6, 1, 256] (B, H, N, K) + // Tile src0 to match B1: [B0, M, K] -> [B1, M, K] + uint32_t tile0_out_dims[] = {B1, M, K}; + p_tile0_out = GQCGT(nullptr, "tile0_out", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, 3, + tile0_out_dims, nullptr, 0); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tile0_out)); + uint32_t tile_multiples[] = {B1 / B0, 1, 1}; // e.g., 24/6 = 4, 6/6 = 1 + uint32_t tile_dims[] = {3}; + Qnn_Tensor_t *p_tile_multiples = GQCGT(nullptr, "tile_multiples", QNN_TENSOR_TYPE_STATIC, QNN_DATATYPE_UINT_32, 1, + tile_dims, tile_multiples, sizeof(tile_multiples)); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tile_multiples)); + Qnn_Param_t tile_params[] = {{QNN_PARAMTYPE_TENSOR, "multiples", .tensorParam = *p_tile_multiples}}; + Qnn_Tensor_t tile0_inputs[] = {*p_reshape0_out}; + Qnn_Tensor_t tile0_outputs[] = {*p_tile0_out}; + Qnn_OpConfig_t tile0_op = ggmlqnn_create_op_config("tile0", QNN_OP_PACKAGE_NAME_QTI_AISW, + QNN_OP_TILE, tile_params, 1, + tile0_inputs, 1, tile0_outputs, 1); + CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, tile0_op)); + + // src1: [N, K, H1, B1] -> QNN: [B1, H1, N, K] uint32_t src1_dims[] = {static_cast(src1->ne[3]), static_cast(src1->ne[2]), static_cast(src1->ne[1]), static_cast(src1->ne[0])}; p_tensor1 = GQCGT(src1, "input1", QNN_TENSOR_TYPE_APP_WRITE, QNN_DATATYPE_FLOAT_32, 4, src1_dims, nullptr, 0); CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor1)); - // Permute src1 to [4, 6, 256, 1] to align K and N - uint32_t perm_data[] = {0, 1, 3, 2}; // [B, H, N, K] -> [B, H, K, N] + // Permute src1 to [B1, H1, K, N] + uint32_t perm_data[] = {0, 1, 3, 2}; uint32_t perm_dims[] = {4}; - Qnn_Tensor_t * p_perm = GQCGT(nullptr, "perm", QNN_TENSOR_TYPE_STATIC, QNN_DATATYPE_UINT_32, 1, - perm_dims, perm_data, sizeof(perm_data)); + Qnn_Tensor_t *p_perm = GQCGT(nullptr, "perm", QNN_TENSOR_TYPE_STATIC, QNN_DATATYPE_UINT_32, 1, + perm_dims, perm_data, sizeof(perm_data)); CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_perm)); - uint32_t permute1_out_dims[] = {static_cast(src1->ne[3]), static_cast(src1->ne[2]), static_cast(src1->ne[0]), static_cast(src1->ne[1])}; // [4, 6, 256, 1] + uint32_t permute1_out_dims[] = {static_cast(src1->ne[3]), static_cast(src1->ne[2]), static_cast(src1->ne[0]), static_cast(src1->ne[1])}; p_permute1_out = GQCGT(nullptr, "permute1_out", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, 4, permute1_out_dims, nullptr, 0); CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_permute1_out)); @@ -389,7 +409,7 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) permute1_inputs, 1, permute1_outputs, 1); CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, permute1_op)); - // Reshape src1 to [24, 256, 1] for [B1, K, N] + // Reshape src1 to [B1, K, N] uint32_t reshape1_out_dims[] = {B1, K, N}; p_reshape1_out = GQCGT(nullptr, "reshape1_out", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, 3, reshape1_out_dims, nullptr, 0); @@ -401,23 +421,19 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) reshape1_inputs, 1, reshape1_outputs, 1); CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, reshape1_op)); - // MatMul: [6, 16, 256] x [24, 256, 1] -> Needs adjustment for broadcasting - // Adjust src0 to match B1 by repeating or reshaping - uint32_t matmul_out_dims[] = {B1, M, N}; // [24, 16, 1] + // MatMul: [B1, M, K] x [B1, K, N] -> [B1, M, N] + uint32_t matmul_out_dims[] = {B1, M, N}; p_matmul_out = GQCGT(nullptr, "matmul_out", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, 3, matmul_out_dims, nullptr, 0); CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_matmul_out)); - - // Note: QNN MatMul doesn't broadcast; we need to tile src0 - // For simplicity, assume dst shape drives execution; adjust src0 later if needed - Qnn_Tensor_t matmul_inputs[] = {*p_reshape0_out, *p_reshape1_out}; + Qnn_Tensor_t matmul_inputs[] = {*p_tile0_out, *p_reshape1_out}; Qnn_Tensor_t matmul_outputs[] = {*p_matmul_out}; Qnn_OpConfig_t matmul_op = ggmlqnn_create_op_config("matmul", QNN_OP_PACKAGE_NAME_QTI_AISW, QNN_OP_MAT_MUL, nullptr, 0, matmul_inputs, 2, matmul_outputs, 1); CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, matmul_op)); - // Output: [1, 16, 6, 4] -> QNN: [4, 6, 16, 1] + // Output: [N, M, H1, B1] -> QNN: [B1, H1, M, N] uint32_t reshape2_out_dims[] = {static_cast(dst->ne[3]), static_cast(dst->ne[2]), static_cast(dst->ne[1]), static_cast(dst->ne[0])}; p_reshape2_out = GQCGT(dst, "output", QNN_TENSOR_TYPE_APP_READ, QNN_DATATYPE_FLOAT_32, 4, reshape2_out_dims, nullptr, 0); @@ -433,7 +449,7 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) CHECK_QNN_API(error, qnn_raw_interface.graphFinalize(graph_handle, NULL, NULL)); // Cache - qnn_tensors_t ggml_op_mulmat_tensors = {p_tensor0, p_reshape0_out, p_tensor1, p_permute1_out, p_reshape1_out, p_matmul_out, p_reshape2_out}; + qnn_tensors_t ggml_op_mulmat_tensors = {p_tensor0, p_reshape0_out, p_tile0_out, p_tensor1, p_permute1_out, p_reshape1_out, p_matmul_out, p_reshape2_out}; instance->_qnn_graph_map[graph_name] = std::make_tuple(graph_handle, ggml_op_mulmat_tensors); } @@ -455,7 +471,6 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) op_perf.info(); } - /* * @brief performs matrix multiplication with FP32 & quantized weights and floating-point inputs * using the QNN backend. this function performs matrix multiplication of the input tensor From e10c942c4c114a9e7cf5f0442cc7ec21ecb378a5 Mon Sep 17 00:00:00 2001 From: zhouwg Date: Sun, 2 Mar 2025 15:36:23 +0800 Subject: [PATCH 47/76] ggml-qnn: AI-assisted ggml_qnn_mul_mat_4d by Grok 3 --- both ok in step12 --- ggml/src/ggml-qnn/ggml-qnn-ops.cpp | 6 +++--- tests/ggml-qnn-ut.cpp | 5 +++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/ggml/src/ggml-qnn/ggml-qnn-ops.cpp b/ggml/src/ggml-qnn/ggml-qnn-ops.cpp index 43ec5ee16a8c0..23c777227550d 100644 --- a/ggml/src/ggml-qnn/ggml-qnn-ops.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn-ops.cpp @@ -289,7 +289,6 @@ void ggml_qnn_general_node(ggml_backend_qnn_context * ctx, ggml_tensor * op) { * than ggml_qnn_mul_mat, so it's a standalone function. * it will be combined with ggml_qnn_mul_mat after bugfix */ - static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) { Qnn_ErrorHandle_t error = QNN_SUCCESS; bool graph_initialized = false; @@ -347,7 +346,7 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) // Validate GGML_ASSERT(src0->ne[0] == src1->ne[0]); // K must match - GGML_ASSERT(dst->ne[0] == N && dst->ne[1] == M && dst->ne[2] == src1->ne[2] && dst->ne[3] == src1->ne[3]); + //GGML_ASSERT(dst->ne[0] == N && dst->ne[1] == M && dst->ne[2] == src1->ne[2] && dst->ne[3] == src1->ne[3]); // src0: [K, M, H0, B0] -> QNN: [B0, H0, M, K] uint32_t src0_dims[] = {static_cast(src0->ne[3]), static_cast(src0->ne[2]), static_cast(src0->ne[1]), static_cast(src0->ne[0])}; @@ -372,7 +371,7 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) p_tile0_out = GQCGT(nullptr, "tile0_out", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, 3, tile0_out_dims, nullptr, 0); CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tile0_out)); - uint32_t tile_multiples[] = {B1 / B0, 1, 1}; // e.g., 24/6 = 4, 6/6 = 1 + uint32_t tile_multiples[] = {B1 / B0, 1, 1}; uint32_t tile_dims[] = {3}; Qnn_Tensor_t *p_tile_multiples = GQCGT(nullptr, "tile_multiples", QNN_TENSOR_TYPE_STATIC, QNN_DATATYPE_UINT_32, 1, tile_dims, tile_multiples, sizeof(tile_multiples)); @@ -465,6 +464,7 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) // Log dst for debugging float *dst_data = (float *)dst->data; + GGMLQNN_LOG_DEBUG("dst shape: [%d, %d, %d, %d]\n", dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]); for (int i = 0; i < dst->ne[0] * dst->ne[1] * dst->ne[2] * dst->ne[3]; i++) { GGMLQNN_LOG_DEBUG("dst[%d] = %f\n", i, dst_data[i]); } diff --git a/tests/ggml-qnn-ut.cpp b/tests/ggml-qnn-ut.cpp index a7d8cb5619732..5846d64b0e67a 100644 --- a/tests/ggml-qnn-ut.cpp +++ b/tests/ggml-qnn-ut.cpp @@ -332,7 +332,8 @@ int main(int argc, char * argv[]) { std::vector backends; std::vector> set_n_threads_fns; printf("Testing %zu devices\n\n", ggml_backend_dev_count()); - for (size_t i = 0; i < ggml_backend_dev_count(); i++) { + //for (size_t i = 0; i < ggml_backend_dev_count(); i++) { + for (size_t i = 0; i < 2; i++) { ggml_backend_dev_t dev = ggml_backend_dev_get(i); printf("Backend %zu/%zu: %s\n", i + 1, ggml_backend_dev_count(), @@ -439,7 +440,7 @@ int main(int argc, char * argv[]) { //src0 = ggml_new_tensor_3d(ctx, qtype, 128, 64, 8); //src1 = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, 128, 2, 8); //verify 4D matrix -#if 1 //failure +#if 1 //ok src0 = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, 256, 16, 3, 2); src1 = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, 256, 1, 6, 4); #else //ok From 055f27e76641131fc8d0fef7edc346ba0763bbb8 Mon Sep 17 00:00:00 2001 From: zhouwg Date: Sun, 2 Mar 2025 15:46:09 +0800 Subject: [PATCH 48/76] ggml-qnn: AI-assisted ggml_qnn_mul_mat_4d by Grok 3 ---finalizing version also both ok in step13 --- ggml/src/ggml-qnn/ggml-qnn-ops.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ggml/src/ggml-qnn/ggml-qnn-ops.cpp b/ggml/src/ggml-qnn/ggml-qnn-ops.cpp index 23c777227550d..0ee172779a7dc 100644 --- a/ggml/src/ggml-qnn/ggml-qnn-ops.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn-ops.cpp @@ -344,9 +344,8 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) uint32_t B0 = src0->ne[2] * src0->ne[3]; // src0 batch uint32_t B1 = src1->ne[2] * src1->ne[3]; // src1 batch (drives output) - // Validate + // Validate K only GGML_ASSERT(src0->ne[0] == src1->ne[0]); // K must match - //GGML_ASSERT(dst->ne[0] == N && dst->ne[1] == M && dst->ne[2] == src1->ne[2] && dst->ne[3] == src1->ne[3]); // src0: [K, M, H0, B0] -> QNN: [B0, H0, M, K] uint32_t src0_dims[] = {static_cast(src0->ne[3]), static_cast(src0->ne[2]), static_cast(src0->ne[1]), static_cast(src0->ne[0])}; @@ -471,6 +470,7 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) op_perf.info(); } + /* * @brief performs matrix multiplication with FP32 & quantized weights and floating-point inputs * using the QNN backend. this function performs matrix multiplication of the input tensor From ae7e4f6664363a7532446a124dbb11bc40f5b45a Mon Sep 17 00:00:00 2001 From: zhouwg Date: Sun, 2 Mar 2025 16:34:51 +0800 Subject: [PATCH 49/76] ggml-qnn: refine ggml_qnn_mul_mat and ggml_qnn_general_node according to Grok 3's style --- ggml/src/ggml-qnn/ggml-qnn-impl.h | 2 +- ggml/src/ggml-qnn/ggml-qnn-ops.cpp | 136 +++++++++-------------------- tests/ggml-qnn-ut.cpp | 51 ++++++----- 3 files changed, 69 insertions(+), 120 deletions(-) diff --git a/ggml/src/ggml-qnn/ggml-qnn-impl.h b/ggml/src/ggml-qnn/ggml-qnn-impl.h index 394c35fe6b043..6b527724ee292 100644 --- a/ggml/src/ggml-qnn/ggml-qnn-impl.h +++ b/ggml/src/ggml-qnn/ggml-qnn-impl.h @@ -99,7 +99,7 @@ void ggmlqnn_log_internal(ggml_log_level level, const char * file, const char #else #define GGMLQNN_DEBUG 1 // for troubleshooting QNN backend #define ENABLE_QNNBACKEND_PERF 0 // enable/disable op's perf info -#define GGMLQNN_PRINT_QNN_INTERNAL_LOG 1 // enable/disable QNN's internal log +#define GGMLQNN_PRINT_QNN_INTERNAL_LOG 0 // enable/disable QNN's internal log #define GGMLQNN_PRINT_OP_ADD_LOG 0 // GGML_OP_ADD already verified with QNN-CPU / QNN-GPU / QNN-NPU #define GGMLQNN_PRINT_OP_MUL_MAT_LOG 1 #endif diff --git a/ggml/src/ggml-qnn/ggml-qnn-ops.cpp b/ggml/src/ggml-qnn/ggml-qnn-ops.cpp index 0ee172779a7dc..9c4bcaf13877f 100644 --- a/ggml/src/ggml-qnn/ggml-qnn-ops.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn-ops.cpp @@ -200,71 +200,25 @@ void ggml_qnn_general_node(ggml_backend_qnn_context * ctx, ggml_tensor * op) { auto graph_item = std::make_tuple(graph_handle, ggml_op_add_tensors); instance->_qnn_graph_map[graph_name] = graph_item; - } else { - Qnn_DataType_t src0_qnn_type = QNN_DATATYPE_FLOAT_32; - Qnn_DataType_t src1_qnn_type = QNN_DATATYPE_FLOAT_32; - Qnn_DataType_t dst_qnn_type = QNN_DATATYPE_FLOAT_32; - - src0_qnn_type = ggmlqnn_datatype_from_ggml_datatype(src0->type); - src1_qnn_type = ggmlqnn_datatype_from_ggml_datatype(src1->type); - dst_qnn_type = ggmlqnn_datatype_from_ggml_datatype(dst->type); - - uint32_t dimensions_input_0[] = {(uint32_t) src0->ne[0], (uint32_t) src0->ne[1], - (uint32_t) src0->ne[2], (uint32_t) src0->ne[3]}; - uint32_t dimensions_input_1[] = {(uint32_t) src1->ne[0], (uint32_t) src1->ne[1], - (uint32_t) src1->ne[2], (uint32_t) src1->ne[3]}; - uint32_t dimensions_output[] = {(uint32_t) dst->ne[0], (uint32_t) dst->ne[1], - (uint32_t) dst->ne[2], (uint32_t) dst->ne[3]}; - - QNN_VER_PTR(*p_tensor0)->dimensions = dimensions_input_0; - QNN_VER_PTR(*p_tensor0)->rank = ggml_n_dims(src0); - QNN_VER_PTR(*p_tensor0)->dataType = src0_qnn_type; - - QNN_VER_PTR(*p_tensor1)->dimensions = dimensions_input_1; - QNN_VER_PTR(*p_tensor1)->rank = ggml_n_dims(src1); - QNN_VER_PTR(*p_tensor1)->dataType = src1_qnn_type; - - QNN_VER_PTR(*p_tensor2)->dimensions = dimensions_output; - QNN_VER_PTR(*p_tensor2)->rank = ggml_n_dims(dst); - QNN_VER_PTR(*p_tensor2)->dataType = dst_qnn_type; - - if (enable_npu_rpc) { - //TODO: NPU RPC feature will failed with test-backend-ops - uint8_t * qnn_buffer_0 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*p_tensor0)->memHandle)); - GGMLQNN_LOG_INFO("qnn_rpcbuffer_0 = %p\n", qnn_buffer_0); - if (nullptr != qnn_buffer_0) { - memcpy(qnn_buffer_0, src0->data, ggml_nbytes(src0)); - } - - uint8_t * qnn_buffer_1 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*p_tensor1)->memHandle)); - GGMLQNN_LOG_INFO("qnn_rpcbuffer_1 = %p\n", qnn_buffer_1); - if (nullptr != qnn_buffer_1) { - memcpy(qnn_buffer_1, src1->data, ggml_nbytes(src1)); - } - } else { - QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggmlqnn_get_tensor_data_size(src0)}; - QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggmlqnn_get_tensor_data_size(src1)}; - QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggmlqnn_get_tensor_data_size(dst)}; - } - - Qnn_Tensor_t tensor_inputs[] = { - *p_tensor0, - *p_tensor1 - }; - Qnn_Tensor_t tensor_outputs[] = { - *p_tensor2 - }; - CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, - tensor_inputs, 2, - tensor_outputs, 1, - nullptr, nullptr)); + } - if (enable_npu_rpc) { - //TODO:NPU RPC feature will failed with test-backend-ops - uint8_t * qnn_buffer_2 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*p_tensor2)->memHandle)); - if (nullptr != qnn_buffer_2) { - memcpy(dst->data, qnn_buffer_2, ggml_nbytes(dst)); - } + Qnn_Tensor_t tensor_inputs[] = { + *p_tensor0, + *p_tensor1 + }; + Qnn_Tensor_t tensor_outputs[] = { + *p_tensor2 + }; + CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, + tensor_inputs, 2, + tensor_outputs, 1, + nullptr, nullptr)); + + if (enable_npu_rpc) { + //TODO:NPU RPC feature will failed with test-backend-ops + uint8_t * qnn_buffer_2 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*p_tensor2)->memHandle)); + if (nullptr != qnn_buffer_2) { + memcpy(dst->data, qnn_buffer_2, ggml_nbytes(dst)); } } @@ -461,12 +415,14 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, input_tensors, 2, output_tensors, 1, NULL, NULL)); +#if 0 // Log dst for debugging float *dst_data = (float *)dst->data; GGMLQNN_LOG_DEBUG("dst shape: [%d, %d, %d, %d]\n", dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]); for (int i = 0; i < dst->ne[0] * dst->ne[1] * dst->ne[2] * dst->ne[3]; i++) { GGMLQNN_LOG_DEBUG("dst[%d] = %f\n", i, dst_data[i]); } +#endif op_perf.info(); } @@ -665,14 +621,8 @@ void ggml_qnn_mul_mat(ggml_backend_qnn_context * ctx, ggml_tensor * op) { #endif CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle,out_trans1_0)); - //step-6: finalize qnn graph and execute qnn graph + //step-6: finalize qnn graph CHECK_QNN_API(error, qnn_raw_interface.graphFinalize(graph_handle, nullptr, nullptr)); - Qnn_Tensor_t input_tensors_0[] = {*p_tensor0, *p_tensor1}; - Qnn_Tensor_t output_tensors_0[] = {*p_tensor2}; - CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, - input_tensors_0, 2, - output_tensors_0, 1, - nullptr, nullptr)); qnn_tensors_t ggml_op_mulmat_tensors; ggml_op_mulmat_tensors.reserve(5); @@ -683,30 +633,30 @@ void ggml_qnn_mul_mat(ggml_backend_qnn_context * ctx, ggml_tensor * op) { ggml_op_mulmat_tensors.push_back(p_tensor2_transpose); auto graph_item = std::make_tuple(graph_handle, ggml_op_mulmat_tensors); instance->_qnn_graph_map[graph_name] = graph_item; - } else { - if (src0_type != GGML_TYPE_F32) { - QNN_VER_PTR(*p_tensor0)->clientBuf = {wdata, static_cast(desired_size)}; - } else { - QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggmlqnn_get_tensor_data_size(src0)}; - } - QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggmlqnn_get_tensor_data_size(src1)}; - QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggmlqnn_get_tensor_data_size(dst)}; + } - Qnn_Tensor_t tensor_inputs[] = { - *p_tensor0, - *p_tensor1 - }; - Qnn_Tensor_t tensor_outputs[] = { - *p_tensor2 - }; - // this is the second technical approach or another pipeline of "how to utilize the Hexagon - // NPU maximally" through QNN SDK, details could be found at - // https://github.com/ggml-org/llama.cpp/pull/12049#issuecomment-2678308360 - CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, - tensor_inputs, 2, - tensor_outputs, 1, - nullptr, nullptr)); + if (src0_type != GGML_TYPE_F32) { + QNN_VER_PTR(*p_tensor0)->clientBuf = {wdata, static_cast(desired_size)}; + } else { + QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggmlqnn_get_tensor_data_size(src0)}; } + QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggmlqnn_get_tensor_data_size(src1)}; + QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggmlqnn_get_tensor_data_size(dst)}; + + Qnn_Tensor_t tensor_inputs[] = { + *p_tensor0, + *p_tensor1 + }; + Qnn_Tensor_t tensor_outputs[] = { + *p_tensor2 + }; + // this is the second technical approach or another pipeline of "how to utilize the Hexagon + // NPU maximally" through QNN SDK, details could be found at + // https://github.com/ggml-org/llama.cpp/pull/12049#issuecomment-2678308360 + CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, + tensor_inputs, 2, + tensor_outputs, 1, + nullptr, nullptr)); // restore the original dimensions of qnn tensors to avoid memory leak in func free_qnn_tensor QNN_VER_PTR(*p_tensor0)->dimensions = tensor_0_dimensions; diff --git a/tests/ggml-qnn-ut.cpp b/tests/ggml-qnn-ut.cpp index 5846d64b0e67a..08d02e502b6ae 100644 --- a/tests/ggml-qnn-ut.cpp +++ b/tests/ggml-qnn-ut.cpp @@ -332,37 +332,36 @@ int main(int argc, char * argv[]) { std::vector backends; std::vector> set_n_threads_fns; printf("Testing %zu devices\n\n", ggml_backend_dev_count()); - //for (size_t i = 0; i < ggml_backend_dev_count(); i++) { - for (size_t i = 0; i < 2; i++) { - ggml_backend_dev_t dev = ggml_backend_dev_get(i); + for (size_t i = 0; i < ggml_backend_dev_count(); i++) { + ggml_backend_dev_t dev = ggml_backend_dev_get(i); - printf("Backend %zu/%zu: %s\n", i + 1, ggml_backend_dev_count(), - ggml_backend_dev_name(dev)); + printf("Backend %zu/%zu: %s\n", i + 1, ggml_backend_dev_count(), + ggml_backend_dev_name(dev)); - if (ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_CPU) { - printf(" Skipping CPU backend\n"); - continue; - } + if (ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_CPU) { + printf(" Skipping CPU backend\n"); + continue; + } - backend = ggml_backend_dev_init(dev, reinterpret_cast(i)); - GGML_ASSERT(backend != NULL); - if (backend != nullptr) { - printf("%s: initialize %s backend\n", __func__, ggml_backend_dev_name(dev)); - } - backends.emplace_back(backend); + backend = ggml_backend_dev_init(dev, reinterpret_cast(i)); + GGML_ASSERT(backend != NULL); + if (backend != nullptr) { + printf("%s: initialize %s backend\n", __func__, ggml_backend_dev_name(dev)); + } + backends.emplace_back(backend); - ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(dev); - auto ggml_backend_set_n_threads_fn = (ggml_backend_set_n_threads_t) ggml_backend_reg_get_proc_address( - reg, "ggml_backend_set_n_threads"); - if (ggml_backend_set_n_threads_fn) { - ggml_backend_set_n_threads_fn(backend, std::thread::hardware_concurrency()); - } + ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(dev); + auto ggml_backend_set_n_threads_fn = (ggml_backend_set_n_threads_t) ggml_backend_reg_get_proc_address( + reg, "ggml_backend_set_n_threads"); + if (ggml_backend_set_n_threads_fn) { + ggml_backend_set_n_threads_fn(backend, std::thread::hardware_concurrency()); + } - printf(" Device description: %s\n", ggml_backend_dev_description(dev)); - size_t free, total; - ggml_backend_dev_memory(dev, &free, &total); - printf(" Device memory: %zu MB (%zu MB free)\n", total / 1024 / 1024, free / 1024 / 1024); - printf("\n"); + printf(" Device description: %s\n", ggml_backend_dev_description(dev)); + size_t free, total; + ggml_backend_dev_memory(dev, &free, &total); + printf(" Device memory: %zu MB (%zu MB free)\n", total / 1024 / 1024, free / 1024 / 1024); + printf("\n"); } ggml_backend_t backend_cpu = nullptr; From 53a9c4f1ae91eb9bc27b74a13c5ae4aad3e90f3a Mon Sep 17 00:00:00 2001 From: zhouwg Date: Sun, 2 Mar 2025 16:55:06 +0800 Subject: [PATCH 50/76] ggml-qnn: remove no-needed comments --- ggml/src/ggml-qnn/ggml-qnn-ops.cpp | 6 +++--- ggml/src/ggml-qnn/ggml-qnn.cpp | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ggml/src/ggml-qnn/ggml-qnn-ops.cpp b/ggml/src/ggml-qnn/ggml-qnn-ops.cpp index 9c4bcaf13877f..851eaf1b9a124 100644 --- a/ggml/src/ggml-qnn/ggml-qnn-ops.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn-ops.cpp @@ -232,16 +232,16 @@ void ggml_qnn_general_node(ggml_backend_qnn_context * ctx, ggml_tensor * op) { #endif } -//TODO:there is issue in this function /* - * this function is AI-assisted code from Grok 3 for purpose of 4d mulmat UT in ggml-qnn-ut.cpp + * this function is AI-assisted code from Grok 3 for purpose of offload 4d matrix mulmat to QNN backend + * UT in ggml-qnn-ut.cpp passed: * ./scripts/build-run-android.sh run_ut_mulmat 0 * ./scripts/build-run-android.sh run_ut_mulmat 1 * ./scripts/build-run-android.sh run_ut_mulmat 2 * * the logic of ggml_qnn_mul_mat_4d is similar to ggml_qnn_mul_mat but much more complicated * than ggml_qnn_mul_mat, so it's a standalone function. - * it will be combined with ggml_qnn_mul_mat after bugfix + * it will be combined with ggml_qnn_mul_mat in the future */ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) { Qnn_ErrorHandle_t error = QNN_SUCCESS; diff --git a/ggml/src/ggml-qnn/ggml-qnn.cpp b/ggml/src/ggml-qnn/ggml-qnn.cpp index 3b59956009398..a5533c5d4cab5 100644 --- a/ggml/src/ggml-qnn/ggml-qnn.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn.cpp @@ -2277,7 +2277,7 @@ static bool ggml_qnn_can_handle_op(const ggml_backend_qnn_context * ctx, const s return false; if (src0_rank < 2) // QNN's limitation, make QNN SDK happy return false; - if (4 == src0_rank) //TODO: 4D matrix mulmat + if (4 == src0_rank) //TODO: 4D matrix mulmat in CT return false; if ((src1->ne[2] != src0->ne[2]) || (src1->ne[3] != src0->ne[3])) // make QNN SDK happy return false; From 10022ab11f7e0c21edd01da2b66b87cbbe5e1669 Mon Sep 17 00:00:00 2001 From: zhouwg Date: Mon, 3 Mar 2025 13:13:41 +0800 Subject: [PATCH 51/76] ggml-qnn: Windows port --- step3 --- CMakeLists.txt | 1 + examples/export-lora/export-lora.cpp | 2 +- ggml/src/ggml-qnn/CMakeLists.txt | 35 +++++ ggml/src/ggml-qnn/ggml-qnn-impl.h | 92 ++++++------- ggml/src/ggml-qnn/ggml-qnn-ops.cpp | 191 ++++++++++++++++++++------- ggml/src/ggml-qnn/ggml-qnn.cpp | 50 ++++--- scripts/build-run-android.sh | 20 +-- tests/ggml-qnn-ut.cpp | 133 +++++-------------- 8 files changed, 302 insertions(+), 222 deletions(-) create mode 100644 ggml/src/ggml-qnn/CMakeLists.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index 23cfbce5ae566..73a9c554f651e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,6 +6,7 @@ include(CheckIncludeFileCXX) set(CMAKE_WARN_UNUSED_CLI YES) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) +set(CMAKE_VERBOSE_MAKEFILE on) if (NOT XCODE AND NOT MSVC AND NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE Release CACHE STRING "Build type" FORCE) diff --git a/examples/export-lora/export-lora.cpp b/examples/export-lora/export-lora.cpp index e7d0fbfffedb0..14ac107e761db 100644 --- a/examples/export-lora/export-lora.cpp +++ b/examples/export-lora/export-lora.cpp @@ -148,7 +148,7 @@ struct lora_merge_ctx { ctx_out = gguf_init_empty(); struct ggml_init_params params = { - /*.mem_size =*/ gguf_get_n_tensors(base_model.ctx_gguf)*ggml_tensor_overhead(), + /*.mem_size =*/ static_cast(gguf_get_n_tensors(base_model.ctx_gguf)*ggml_tensor_overhead()), /*.mem_buffer =*/ NULL, /*.no_alloc =*/ true, }; diff --git a/ggml/src/ggml-qnn/CMakeLists.txt b/ggml/src/ggml-qnn/CMakeLists.txt new file mode 100644 index 0000000000000..1156c98fbc9d7 --- /dev/null +++ b/ggml/src/ggml-qnn/CMakeLists.txt @@ -0,0 +1,35 @@ +message(STATUS "Using QNN backend") + +if(CMAKE_SYSTEM_NAME STREQUAL "Android") + find_library(LOG_LIB log) + set(QNN_LINK_LIBRARIES ${LOG_LIB}) + set(QNN_DEFAULT_LIB_SEARCH_PATH "/data/local/tmp/" CACHE STRING "customized library search path for QNN backend") +elseif(CMAKE_SYSTEM_NAME STREQUAL "Windows") + set(QNN_DEFAULT_LIB_SEARCH_PATH "C:\\" CACHE STRING "customized library search path for QNN backend") +else() + message(FATAL_ERROR "QNN now only available on Android and Windows(Windows on ARM)") +endif() + +if(NOT DEFINED GGML_QNN_SDK_PATH) +# try read from environment variable + if(DEFINED ENV{QNN_SDK_PATH}) + set(GGML_QNN_SDK_PATH $ENV{QNN_SDK_PATH}) + else() + message(FATAL_ERROR "GGML_QNN_SDK_PATH not defined") + endif() +endif() + +message("QNN_SDK_PATH: ${GGML_QNN_SDK_PATH}") + +set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3") + +file(GLOB QNN_SOURCES "${CMAKE_CURRENT_LIST_DIR}/*.cpp") + ggml_add_backend_library(ggml-qnn + ${QNN_SOURCES} +) + +target_include_directories(ggml-qnn PRIVATE ${GGML_QNN_SDK_PATH}/include/QNN ${CMAKE_CURRENT_LIST_DIR}) +target_link_libraries(ggml-qnn PRIVATE ${QNN_LINK_LIBRARIES}) + +string(REGEX REPLACE "/$" "" GGML_QNN_DEFAULT_LIB_SEARCH_PATH "${QNN_DEFAULT_LIB_SEARCH_PATH}") +target_compile_definitions(ggml-qnn PRIVATE GGML_QNN_DEFAULT_LIB_SEARCH_PATH="${QNN_DEFAULT_LIB_SEARCH_PATH}/") diff --git a/ggml/src/ggml-qnn/ggml-qnn-impl.h b/ggml/src/ggml-qnn/ggml-qnn-impl.h index 6b527724ee292..5a2fe5752a097 100644 --- a/ggml/src/ggml-qnn/ggml-qnn-impl.h +++ b/ggml/src/ggml-qnn/ggml-qnn-impl.h @@ -255,7 +255,9 @@ class qnn_perf { #else class qnn_perf { public: - qnn_perf(const std::string & perf_name) {} + qnn_perf(const std::string & perf_name) { + GGML_UNUSED(perf_name); + } qnn_perf() = delete; qnn_perf(const qnn_perf & ) = delete; qnn_perf & operator= (const qnn_perf & ) = delete; @@ -287,86 +289,86 @@ class qnn_interface { qnn_interface() = default; // QnnBackend - DEFINE_SHIM_FUNCTION_INTERFACE(backend_create, backendCreate); + DEFINE_SHIM_FUNCTION_INTERFACE(backend_create, backendCreate) - DEFINE_SHIM_FUNCTION_INTERFACE(backend_free, backendFree); + DEFINE_SHIM_FUNCTION_INTERFACE(backend_free, backendFree) - DEFINE_SHIM_FUNCTION_INTERFACE(backend_register_op_package, backendRegisterOpPackage); + DEFINE_SHIM_FUNCTION_INTERFACE(backend_register_op_package, backendRegisterOpPackage) - DEFINE_SHIM_FUNCTION_INTERFACE(backend_validate_op_config, backendValidateOpConfig); + DEFINE_SHIM_FUNCTION_INTERFACE(backend_validate_op_config, backendValidateOpConfig) - DEFINE_SHIM_FUNCTION_INTERFACE(backend_get_api_version, backendGetApiVersion); + DEFINE_SHIM_FUNCTION_INTERFACE(backend_get_api_version, backendGetApiVersion) // QnnDevice - DEFINE_SHIM_FUNCTION_INTERFACE(device_create, deviceCreate); + DEFINE_SHIM_FUNCTION_INTERFACE(device_create, deviceCreate) - DEFINE_SHIM_FUNCTION_INTERFACE(device_free, deviceFree); + DEFINE_SHIM_FUNCTION_INTERFACE(device_free, deviceFree) - DEFINE_SHIM_FUNCTION_INTERFACE(device_get_infrastructure, deviceGetInfrastructure); + DEFINE_SHIM_FUNCTION_INTERFACE(device_get_infrastructure, deviceGetInfrastructure) - DEFINE_SHIM_FUNCTION_INTERFACE(device_get_platform_info, deviceGetPlatformInfo); + DEFINE_SHIM_FUNCTION_INTERFACE(device_get_platform_info, deviceGetPlatformInfo) - DEFINE_SHIM_FUNCTION_INTERFACE(device_get_info, deviceGetInfo); + DEFINE_SHIM_FUNCTION_INTERFACE(device_get_info, deviceGetInfo) // QnnContext - DEFINE_SHIM_FUNCTION_INTERFACE(context_create, contextCreate); + DEFINE_SHIM_FUNCTION_INTERFACE(context_create, contextCreate) - DEFINE_SHIM_FUNCTION_INTERFACE(context_get_binary_size, contextGetBinarySize); + DEFINE_SHIM_FUNCTION_INTERFACE(context_get_binary_size, contextGetBinarySize) - DEFINE_SHIM_FUNCTION_INTERFACE(context_get_binary, contextGetBinary); + DEFINE_SHIM_FUNCTION_INTERFACE(context_get_binary, contextGetBinary) - DEFINE_SHIM_FUNCTION_INTERFACE(context_create_from_binary, contextCreateFromBinary); + DEFINE_SHIM_FUNCTION_INTERFACE(context_create_from_binary, contextCreateFromBinary) - DEFINE_SHIM_FUNCTION_INTERFACE(context_free, contextFree); + DEFINE_SHIM_FUNCTION_INTERFACE(context_free, contextFree) // QnnGraph - DEFINE_SHIM_FUNCTION_INTERFACE(graph_create, graphCreate); + DEFINE_SHIM_FUNCTION_INTERFACE(graph_create, graphCreate) - DEFINE_SHIM_FUNCTION_INTERFACE(graph_add_node, graphAddNode); + DEFINE_SHIM_FUNCTION_INTERFACE(graph_add_node, graphAddNode) - DEFINE_SHIM_FUNCTION_INTERFACE(graph_finalize, graphFinalize); + DEFINE_SHIM_FUNCTION_INTERFACE(graph_finalize, graphFinalize) - DEFINE_SHIM_FUNCTION_INTERFACE(graph_execute, graphExecute); + DEFINE_SHIM_FUNCTION_INTERFACE(graph_execute, graphExecute) - DEFINE_SHIM_FUNCTION_INTERFACE(graph_retrieve, graphRetrieve); + DEFINE_SHIM_FUNCTION_INTERFACE(graph_retrieve, graphRetrieve) // QnnLog - DEFINE_SHIM_FUNCTION_INTERFACE(log_create, logCreate); + DEFINE_SHIM_FUNCTION_INTERFACE(log_create, logCreate) - DEFINE_SHIM_FUNCTION_INTERFACE(log_free, logFree); + DEFINE_SHIM_FUNCTION_INTERFACE(log_free, logFree) - DEFINE_SHIM_FUNCTION_INTERFACE(log_set_log_level, logSetLogLevel); + DEFINE_SHIM_FUNCTION_INTERFACE(log_set_log_level, logSetLogLevel) // QnnProfile - DEFINE_SHIM_FUNCTION_INTERFACE(profile_create, profileCreate); + DEFINE_SHIM_FUNCTION_INTERFACE(profile_create, profileCreate) - DEFINE_SHIM_FUNCTION_INTERFACE(profile_get_events, profileGetEvents); + DEFINE_SHIM_FUNCTION_INTERFACE(profile_get_events, profileGetEvents) - DEFINE_SHIM_FUNCTION_INTERFACE(profile_get_sub_events, profileGetSubEvents); + DEFINE_SHIM_FUNCTION_INTERFACE(profile_get_sub_events, profileGetSubEvents) - DEFINE_SHIM_FUNCTION_INTERFACE(profile_get_event_data, profileGetEventData); + DEFINE_SHIM_FUNCTION_INTERFACE(profile_get_event_data, profileGetEventData) - DEFINE_SHIM_FUNCTION_INTERFACE(profile_free, profileFree); + DEFINE_SHIM_FUNCTION_INTERFACE(profile_free, profileFree) // QnnMem - DEFINE_SHIM_FUNCTION_INTERFACE(mem_register, memRegister); + DEFINE_SHIM_FUNCTION_INTERFACE(mem_register, memRegister) - DEFINE_SHIM_FUNCTION_INTERFACE(mem_de_register, memDeRegister); + DEFINE_SHIM_FUNCTION_INTERFACE(mem_de_register, memDeRegister) // QnnProperty - DEFINE_SHIM_FUNCTION_INTERFACE(property_has_capability, propertyHasCapability); + DEFINE_SHIM_FUNCTION_INTERFACE(property_has_capability, propertyHasCapability) // QnnTensor - DEFINE_SHIM_FUNCTION_INTERFACE(tensor_create_context_tensor, tensorCreateContextTensor); + DEFINE_SHIM_FUNCTION_INTERFACE(tensor_create_context_tensor, tensorCreateContextTensor) - DEFINE_SHIM_FUNCTION_INTERFACE(tensor_create_graph_tensor, tensorCreateGraphTensor); + DEFINE_SHIM_FUNCTION_INTERFACE(tensor_create_graph_tensor, tensorCreateGraphTensor) // QnnSystem - DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_create, systemContextCreate); + DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_create, systemContextCreate) - DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_get_binary_info, systemContextGetBinaryInfo); + DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_get_binary_info, systemContextGetBinaryInfo) - DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_free, systemContextFree); + DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_free, systemContextFree) void set_qnn_interface(const QnnInterface_t * qnn_interface) { _qnn_interface = qnn_interface; @@ -398,7 +400,7 @@ class qnn_instance { const std::string & model_name) : _lib_path(std::move(lib_path)), _backend_name(std::move(backend_name)), - _model_name(std::move(model_name)) {}; + _model_name(std::move(model_name)) {} ~qnn_instance() { } @@ -428,19 +430,19 @@ class qnn_instance { return _qnn_raw_system_interface; } - const Qnn_LogHandle_t get_qnn_log_handle() { return _qnn_log_handle; } + Qnn_LogHandle_t get_qnn_log_handle() { return _qnn_log_handle; } - const Qnn_ProfileHandle_t get_qnn_profile_handle() { return _qnn_profile_handle; } + Qnn_ProfileHandle_t get_qnn_profile_handle() { return _qnn_profile_handle; } - const Qnn_DeviceHandle_t get_qnn_device_handle() { return _qnn_device_handle; } + Qnn_DeviceHandle_t get_qnn_device_handle() { return _qnn_device_handle; } - const Qnn_BackendHandle_t get_qnn_backend_handle() { return _qnn_backend_handle; } + Qnn_BackendHandle_t get_qnn_backend_handle() { return _qnn_backend_handle; } - const Qnn_ContextHandle_t get_qnn_context_handle() { return _qnn_context_handle; } + Qnn_ContextHandle_t get_qnn_context_handle() { return _qnn_context_handle; } - const QnnSystemContext_Handle_t get_qnn_system_handle() { return _qnn_system_handle; } + QnnSystemContext_Handle_t get_qnn_system_handle() { return _qnn_system_handle; } - const Qnn_GraphHandle_t get_qnn_graph_handle() { return _qnn_graph_handle; } + Qnn_GraphHandle_t get_qnn_graph_handle() { return _qnn_graph_handle; } int init_qnn_graph(const char * graph_name, bool debug, diff --git a/ggml/src/ggml-qnn/ggml-qnn-ops.cpp b/ggml/src/ggml-qnn/ggml-qnn-ops.cpp index 851eaf1b9a124..00cb7da32c183 100644 --- a/ggml/src/ggml-qnn/ggml-qnn-ops.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn-ops.cpp @@ -200,25 +200,71 @@ void ggml_qnn_general_node(ggml_backend_qnn_context * ctx, ggml_tensor * op) { auto graph_item = std::make_tuple(graph_handle, ggml_op_add_tensors); instance->_qnn_graph_map[graph_name] = graph_item; - } + } else { + Qnn_DataType_t src0_qnn_type = QNN_DATATYPE_FLOAT_32; + Qnn_DataType_t src1_qnn_type = QNN_DATATYPE_FLOAT_32; + Qnn_DataType_t dst_qnn_type = QNN_DATATYPE_FLOAT_32; + + src0_qnn_type = ggmlqnn_datatype_from_ggml_datatype(src0->type); + src1_qnn_type = ggmlqnn_datatype_from_ggml_datatype(src1->type); + dst_qnn_type = ggmlqnn_datatype_from_ggml_datatype(dst->type); + + uint32_t dimensions_input_0[] = {(uint32_t) src0->ne[0], (uint32_t) src0->ne[1], + (uint32_t) src0->ne[2], (uint32_t) src0->ne[3]}; + uint32_t dimensions_input_1[] = {(uint32_t) src1->ne[0], (uint32_t) src1->ne[1], + (uint32_t) src1->ne[2], (uint32_t) src1->ne[3]}; + uint32_t dimensions_output[] = {(uint32_t) dst->ne[0], (uint32_t) dst->ne[1], + (uint32_t) dst->ne[2], (uint32_t) dst->ne[3]}; + + QNN_VER_PTR(*p_tensor0)->dimensions = dimensions_input_0; + QNN_VER_PTR(*p_tensor0)->rank = ggml_n_dims(src0); + QNN_VER_PTR(*p_tensor0)->dataType = src0_qnn_type; + + QNN_VER_PTR(*p_tensor1)->dimensions = dimensions_input_1; + QNN_VER_PTR(*p_tensor1)->rank = ggml_n_dims(src1); + QNN_VER_PTR(*p_tensor1)->dataType = src1_qnn_type; + + QNN_VER_PTR(*p_tensor2)->dimensions = dimensions_output; + QNN_VER_PTR(*p_tensor2)->rank = ggml_n_dims(dst); + QNN_VER_PTR(*p_tensor2)->dataType = dst_qnn_type; - Qnn_Tensor_t tensor_inputs[] = { - *p_tensor0, - *p_tensor1 - }; - Qnn_Tensor_t tensor_outputs[] = { - *p_tensor2 - }; - CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, - tensor_inputs, 2, - tensor_outputs, 1, - nullptr, nullptr)); - - if (enable_npu_rpc) { - //TODO:NPU RPC feature will failed with test-backend-ops - uint8_t * qnn_buffer_2 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*p_tensor2)->memHandle)); - if (nullptr != qnn_buffer_2) { - memcpy(dst->data, qnn_buffer_2, ggml_nbytes(dst)); + if (enable_npu_rpc) { + //TODO: NPU RPC feature will failed with test-backend-ops + uint8_t * qnn_buffer_0 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*p_tensor0)->memHandle)); + GGMLQNN_LOG_INFO("qnn_rpcbuffer_0 = %p\n", qnn_buffer_0); + if (nullptr != qnn_buffer_0) { + memcpy(qnn_buffer_0, src0->data, ggml_nbytes(src0)); + } + + uint8_t * qnn_buffer_1 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*p_tensor1)->memHandle)); + GGMLQNN_LOG_INFO("qnn_rpcbuffer_1 = %p\n", qnn_buffer_1); + if (nullptr != qnn_buffer_1) { + memcpy(qnn_buffer_1, src1->data, ggml_nbytes(src1)); + } + } else { + QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggmlqnn_get_tensor_data_size(src0)}; + QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggmlqnn_get_tensor_data_size(src1)}; + QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggmlqnn_get_tensor_data_size(dst)}; + } + + Qnn_Tensor_t tensor_inputs[] = { + *p_tensor0, + *p_tensor1 + }; + Qnn_Tensor_t tensor_outputs[] = { + *p_tensor2 + }; + CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, + tensor_inputs, 2, + tensor_outputs, 1, + nullptr, nullptr)); + + if (enable_npu_rpc) { + //TODO:NPU RPC feature will failed with test-backend-ops + uint8_t * qnn_buffer_2 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*p_tensor2)->memHandle)); + if (nullptr != qnn_buffer_2) { + memcpy(dst->data, qnn_buffer_2, ggml_nbytes(dst)); + } } } @@ -472,7 +518,6 @@ void ggml_qnn_mul_mat(ggml_backend_qnn_context * ctx, ggml_tensor * op) { const uint32_t src1_rank = ggml_n_dims(src1); GGML_ASSERT(src0_rank == src1_rank); GGML_ASSERT(src0_rank >= 2); //QNN SDK's limitation, make QNN SDK happy - //GGML_ASSERT(src0_rank != 4); //TODO: 4D matrix mulmat if (4 == src0_rank) { return ggml_qnn_mul_mat_4d(ctx, op); } @@ -591,13 +636,13 @@ void ggml_qnn_mul_mat(ggml_backend_qnn_context * ctx, ggml_tensor * op) { }; #else Qnn_OpConfig_t out_0 = ggmlqnn_create_op_config("ggmlqnn_mulmat_opconfig", QNN_OP_PACKAGE_NAME_QTI_AISW, QNN_OP_MAT_MUL, - out_0_params, 1, out_0_inputs, 2, out_0_outputs, 1); + out_0_params, 1, out_0_inputs, 2, out_0_outputs, 1); #endif CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle,out_0)); //step-5: compose qnn graph: add transpose node Qnn_Param_t out_trans1_0_params[] = { - {(Qnn_ParamType_t) 1, + {QNN_PARAMTYPE_TENSOR, "perm", .tensorParam = *p_param_tensor } }; @@ -617,12 +662,18 @@ void ggml_qnn_mul_mat(ggml_backend_qnn_context * ctx, ggml_tensor * op) { }; #else Qnn_OpConfig_t out_trans1_0 = ggmlqnn_create_op_config("ggmlqnn_mulmat_transpose_opconfig", QNN_OP_PACKAGE_NAME_QTI_AISW, QNN_OP_TRANSPOSE, - out_trans1_0_params, 1, out_trans1_0_inputs, 1, out_trans1_0_outputs, 1); + out_trans1_0_params, 1, out_trans1_0_inputs, 1, out_trans1_0_outputs, 1); #endif CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle,out_trans1_0)); - //step-6: finalize qnn graph + //step-6: finalize qnn graph and execute qnn graph CHECK_QNN_API(error, qnn_raw_interface.graphFinalize(graph_handle, nullptr, nullptr)); + Qnn_Tensor_t input_tensors_0[] = {*p_tensor0, *p_tensor1}; + Qnn_Tensor_t output_tensors_0[] = {*p_tensor2}; + CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, + input_tensors_0, 2, + output_tensors_0, 1, + nullptr, nullptr)); qnn_tensors_t ggml_op_mulmat_tensors; ggml_op_mulmat_tensors.reserve(5); @@ -633,30 +684,30 @@ void ggml_qnn_mul_mat(ggml_backend_qnn_context * ctx, ggml_tensor * op) { ggml_op_mulmat_tensors.push_back(p_tensor2_transpose); auto graph_item = std::make_tuple(graph_handle, ggml_op_mulmat_tensors); instance->_qnn_graph_map[graph_name] = graph_item; - } - - if (src0_type != GGML_TYPE_F32) { - QNN_VER_PTR(*p_tensor0)->clientBuf = {wdata, static_cast(desired_size)}; } else { - QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggmlqnn_get_tensor_data_size(src0)}; + if (src0_type != GGML_TYPE_F32) { + QNN_VER_PTR(*p_tensor0)->clientBuf = {wdata, static_cast(desired_size)}; + } else { + QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggmlqnn_get_tensor_data_size(src0)}; + } + QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggmlqnn_get_tensor_data_size(src1)}; + QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggmlqnn_get_tensor_data_size(dst)}; + + Qnn_Tensor_t tensor_inputs[] = { + *p_tensor0, + *p_tensor1 + }; + Qnn_Tensor_t tensor_outputs[] = { + *p_tensor2 + }; + // this is the second technical approach or another pipeline of "how to utilize the Hexagon + // NPU maximally" through QNN SDK, details could be found at + // https://github.com/ggml-org/llama.cpp/pull/12049#issuecomment-2678308360 + CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, + tensor_inputs, 2, + tensor_outputs, 1, + nullptr, nullptr)); } - QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggmlqnn_get_tensor_data_size(src1)}; - QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggmlqnn_get_tensor_data_size(dst)}; - - Qnn_Tensor_t tensor_inputs[] = { - *p_tensor0, - *p_tensor1 - }; - Qnn_Tensor_t tensor_outputs[] = { - *p_tensor2 - }; - // this is the second technical approach or another pipeline of "how to utilize the Hexagon - // NPU maximally" through QNN SDK, details could be found at - // https://github.com/ggml-org/llama.cpp/pull/12049#issuecomment-2678308360 - CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, - tensor_inputs, 2, - tensor_outputs, 1, - nullptr, nullptr)); // restore the original dimensions of qnn tensors to avoid memory leak in func free_qnn_tensor QNN_VER_PTR(*p_tensor0)->dimensions = tensor_0_dimensions; @@ -666,67 +717,109 @@ void ggml_qnn_mul_mat(ggml_backend_qnn_context * ctx, ggml_tensor * op) { } void ggml_qnn_repeat(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); } - void ggml_qnn_div(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); } void ggml_qnn_leaky_relu(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); } void ggml_qnn_concat(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); } void ggml_qnn_arange(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); } void ggml_qnn_sqr(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); } void ggml_qnn_clamp(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); } void ggml_qnn_scale(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); } void ggml_qnn_argsort(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); } void ggml_qnn_norm(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); } void ggml_qnn_group_norm(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); } void ggml_qnn_acc(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); } void ggml_qnn_sum_rows(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); } void ggml_qnn_upsample_nearest2d(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); } void ggml_qnn_pad(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); } void ggml_qnn_pool2d(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); } void ggml_qnn_dup(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); } void ggml_qnn_rms_norm(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); } void ggml_qnn_diag_mask(ggml_backend_qnn_context * ctx, ggml_tensor * dst, float value) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); + GGML_UNUSED(value); } void ggml_qnn_im2col(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); } void ggml_qnn_timestep_embedding(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); } void ggml_qnn_cpy(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { @@ -734,10 +827,16 @@ void ggml_qnn_cpy(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { } void ggml_qnn_softmax(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); } void ggml_qnn_get_rows(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); } void ggml_qnn_rope(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); } diff --git a/ggml/src/ggml-qnn/ggml-qnn.cpp b/ggml/src/ggml-qnn/ggml-qnn.cpp index a5533c5d4cab5..2aacf8f52d578 100644 --- a/ggml/src/ggml-qnn/ggml-qnn.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn.cpp @@ -52,6 +52,7 @@ void ggmlqnn_log_internal(ggml_log_level level, const char * file, const char * static std::mutex ggmlqnn_log_internal_mutex; static char s_ggmlqnn_log_internal_buf[GGML_QNN_LOGBUF_LEN]; + GGML_UNUSED(file); { std::lock_guard lock(ggmlqnn_log_internal_mutex); va_list args; @@ -82,6 +83,7 @@ static const char * last_func = nullptr; static long last_err; void * dlopen(const char * dll, int flags) { HINSTANCE h = LoadLibraryA(dll); + GGML_UNUSED(flags); if (h == NULL) { last_err = GetLastError(); last_func = "dlopen"; @@ -174,7 +176,7 @@ static size_t ggmlqnn_memscpy(void * dst, size_t dst_size, const void * src, siz } static char * ggmlqnn_strndup(const char * source, size_t maxlen) { - return ::strndup(source, maxlen); + return strndup(source, maxlen); } static void * ggmlqnn_host_malloc(size_t n) { @@ -553,8 +555,9 @@ Qnn_OpConfig_t ggmlqnn_create_op_config(const char * name, const char * package, num_inputs, inputs, num_outputs, outputs }; + Qnn_OpConfig_t opcfg = {QNN_OPCONFIG_VERSION_1, {v1}}; - return (Qnn_OpConfig_t){QNN_OPCONFIG_VERSION_1, .v1 = v1}; + return opcfg; } // ================================================================================================= @@ -1069,9 +1072,6 @@ void * ggmlqnn_type_trait(ggml_backend_qnn_context * ctx, ggml_tensor * op) { GGML_ASSERT(nb00 == ggml_type_size(src0_type)); GGML_ASSERT(nb10 == ggml_type_size(src1->type)); - // broadcast factors - const int64_t r2 = ne12 / ne02; - const int64_t r3 = ne13 / ne03; const int64_t ne_plane = ne01 * ne00; const size_t desired_size = ((GGML_TYPE_F32 == src0_type) ? 0 : ne03 * ne02 * ne_plane * sizeof(float)); ctx->desired_size = desired_size; @@ -1157,7 +1157,7 @@ size_t ggmlqnn_get_opcaps_size() { size_t ggmlqnn_get_op_index(const ggml_tensor * tensor) { if (tensor->op == GGML_OP_UNARY) { - return GGML_OP_COUNT + ggml_get_unary_op(tensor); + return static_cast(GGML_OP_COUNT) + static_cast(ggml_get_unary_op(tensor)); } return tensor->op; @@ -1280,8 +1280,6 @@ void qnn_instance::free_rpcmem(void * buf) { } void qnn_instance::free_rpcmem() { - Qnn_ErrorHandle_t error = QNN_SUCCESS; - if (_rpcmem_store_map.empty()) { GGMLQNN_LOG_WARN("no rpcmem allocated\n"); return; @@ -1709,6 +1707,10 @@ static void ggml_qnn_logcallback(const char * fmt, QnnLog_Level_t level, uint64_t timestamp, va_list argp) { + GGML_UNUSED(fmt); + GGML_UNUSED(level); + GGML_UNUSED(timestamp); + GGML_UNUSED(argp); } #endif @@ -1851,7 +1853,7 @@ int qnn_instance::qnn_init(const QnnSaver_Config_t ** saver_config) { _qnn_raw_interface.deviceGetPlatformInfo(nullptr, &p_info); GGMLQNN_LOG_INFO("device counts %d", p_info->v1.numHwDevices); QnnDevice_HardwareDeviceInfo_t * infos = p_info->v1.hwDevices; - for (int i = 0; i < p_info->v1.numHwDevices; i++) { + for (size_t i = 0; i < p_info->v1.numHwDevices; i++) { GGMLQNN_LOG_INFO("deviceID:%d, deviceType:%d, numCores %d", infos[i].v1.deviceId, infos[i].v1.deviceType, infos[i].v1.numCores); QnnDevice_DeviceInfoExtension_t devinfo = infos[i].v1.deviceInfoExtension; @@ -1863,7 +1865,7 @@ int qnn_instance::qnn_init(const QnnSaver_Config_t ** saver_config) { chipinfo.socModel, qnn_get_socmodel_desc(chipinfo.socModel), \ htp_arch, qnn_get_htparch_desc(htp_arch), chipinfo.vtcmSize); struct qcom_socinfo * socinfo = qnn_get_socinfo_from_socmodel(chipinfo.socModel); - g_qnn_mgr[QNN_BACKEND_NPU].socinfo = { chipinfo.socModel, htp_arch, chipinfo.vtcmSize }; + g_qnn_mgr[QNN_BACKEND_NPU].socinfo = { chipinfo.socModel, htp_arch, chipinfo.vtcmSize, {}}; if (nullptr != socinfo) { memcpy(g_qnn_mgr[QNN_BACKEND_NPU].socinfo.soc_desc, socinfo->soc_desc, sizeof(socinfo->soc_desc)); GGMLQNN_LOG_INFO("soc info:%s", socinfo->soc_desc); @@ -2259,6 +2261,11 @@ static bool ggml_qnn_can_handle_op(const ggml_backend_qnn_context * ctx, const s const uint32_t src0_rank = ggml_n_dims(src0); const uint32_t src1_rank = ggml_n_dims(src1); + GGML_UNUSED(ne01); + GGML_UNUSED(ne10); + GGML_UNUSED(ne11); + GGML_UNUSED(ne0); + GGML_UNUSED(ne1); if (tensor->op == GGML_OP_ADD) { //dump_op_info(tensor); @@ -2470,14 +2477,12 @@ static void ggml_backend_qnn_buffer_free_buffer(ggml_backend_buffer_t buffer) { static void * ggml_backend_qnn_buffer_get_base(ggml_backend_buffer_t buffer) { ggml_backend_qnn_buffer_context * ctx = (ggml_backend_qnn_buffer_context *)buffer->context; - return ctx->buffer; } static void ggml_backend_qnn_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) { - Qnn_ErrorHandle_t error = QNN_SUCCESS; ggml_backend_qnn_buffer_context * ctx = (ggml_backend_qnn_buffer_context *)buffer->context; - GGML_UNUSED(error); + GGML_UNUSED(tensor); GGML_UNUSED(ctx); return; } @@ -2534,6 +2539,7 @@ static ggml_backend_buffer_i ggml_backend_qnn_buffer_interface = { }; static const char * ggml_backend_qnn_buffer_type_name(ggml_backend_buffer_type_t buft) { + GGML_UNUSED(buft); return "qnn-buffer"; } @@ -2541,7 +2547,13 @@ static ggml_backend_buffer_t ggml_backend_qnn_buffer_type_alloc_buffer( ggml_backend_buffer_type_t buft, size_t size) { ggml_backend_qnn_buffer_context * ctx = new ggml_backend_qnn_buffer_context; +#if defined(__ANDROID__) || defined(__linux__) size_t size_page = sysconf(_SC_PAGESIZE); +#elif defined(_WIN32) + SYSTEM_INFO systeminfo; + GetSystemInfo(&systeminfo); + size_t size_page = systeminfo.dwPageSize; +#endif size_t size_aligned = size; if ((size_aligned % size_page) != 0) { size_aligned += (size_page - (size_aligned % size_page)); @@ -2561,11 +2573,11 @@ static size_t ggml_backend_qnn_buffer_type_get_alignment(ggml_backend_buffer_typ return 32; } -//FIXME: this value is an experimental value on Snapdragon 8 Gen3 based phone +//TODO:not used currently static size_t ggml_backend_qnn_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) { GGML_UNUSED(buft); - return (2 * (1 << 30)); + return (2 * (1 << 20)); } static bool ggml_backend_qnn_buffer_is_host(ggml_backend_buffer_type_t buft) { @@ -2645,6 +2657,7 @@ static const char * ggml_backend_qnn_device_get_name(ggml_backend_dev_t dev) { static const char * ggml_backend_qnn_device_get_description(ggml_backend_dev_t dev) { struct ggml_backend_qnn_context * ctx = static_cast(dev->context); + static char qnn_device_desc[256]; if (nullptr == ctx) { GGMLQNN_LOG_ERROR("pls check why ctx is null"); return "unknown"; @@ -2655,7 +2668,9 @@ static const char * ggml_backend_qnn_device_get_description(ggml_backend_dev_t d std::string dev_desc = std::string(ctx->desc) + std::string(soc_info) + "_" + std::string(htp_arch) + "," + std::string(ctx->socinfo.soc_desc); - return dev_desc.c_str(); + memset(qnn_device_desc, 0, 256); + memcpy(qnn_device_desc, dev_desc.c_str(), strlen(dev_desc.c_str())); + return qnn_device_desc; } else { return ctx->desc; } @@ -2717,7 +2732,7 @@ static ggml_backend_t ggml_backend_qnn_device_init_backend(ggml_backend_dev_t de } -ggml_backend_buffer_type_t ggml_backend_qnn_buffer_type(size_t device_index) { +static ggml_backend_buffer_type_t ggml_backend_qnn_buffer_type(size_t device_index) { if (device_index >= GGML_QNN_MAX_DEVICES) { GGMLQNN_LOG_DEBUG("ggml_backend_qnn_buffer_type error: device_index:%d is out of range [0, %d]\n", device_index, GGML_QNN_MAX_DEVICES - 1); @@ -2733,6 +2748,7 @@ ggml_backend_buffer_type_t ggml_backend_qnn_buffer_type(size_t device_index) { /* .get_alloc_size = */ nullptr,// defaults to ggml_nbytes /* .is_host = */ ggml_backend_qnn_buffer_is_host }, + /* .device = */ nullptr, /* .context = */ nullptr, }; diff --git a/scripts/build-run-android.sh b/scripts/build-run-android.sh index 49079c9132769..3d239510b8d63 100755 --- a/scripts/build-run-android.sh +++ b/scripts/build-run-android.sh @@ -106,15 +106,15 @@ function check_qnn_libs() function update_qnn_libs() { - adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnSystem.so ${REMOTE_PATH}/ - adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnCpu.so ${REMOTE_PATH}/ - adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnGpu.so ${REMOTE_PATH}/ - - adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnHtp.so ${REMOTE_PATH}/ - adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnHtpNetRunExtensions.so ${REMOTE_PATH}/ - adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnHtpPrepare.so ${REMOTE_PATH}/ - adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnHtpV75Stub.so ${REMOTE_PATH}/ - adb push ${QNN_SDK_PATH}/lib/hexagon-v75/unsigned/libQnnHtpV75Skel.so ${REMOTE_PATH}/ + adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnSystem.so ${REMOTE_PATH}/ + adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnCpu.so ${REMOTE_PATH}/ + adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnGpu.so ${REMOTE_PATH}/ + + adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnHtp.so ${REMOTE_PATH}/ + adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnHtpNetRunExtensions.so ${REMOTE_PATH}/ + adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnHtpPrepare.so ${REMOTE_PATH}/ + adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnHtpV75Stub.so ${REMOTE_PATH}/ + adb push ${QNN_SDK_PATH}/lib/hexagon-v75/unsigned/libQnnHtpV75Skel.so ${REMOTE_PATH}/ } @@ -152,7 +152,7 @@ function run_llamacli() adb shell "cd ${REMOTE_PATH} \ && export LD_LIBRARY_PATH=${REMOTE_PATH} \ - && ${REMOTE_PATH}/llama-cli -mg ${qnnbackend} -no-ncv -m ${GGUF_MODEL_NAME} -p \"introduce the movie Once Upon a Time in America briefly.\n\"" + && ${REMOTE_PATH}/llama-cli -mg ${qnnbackend} -no-cnv -m ${GGUF_MODEL_NAME} -p \"introduce the movie Once Upon a Time in America briefly.\n\"" } diff --git a/tests/ggml-qnn-ut.cpp b/tests/ggml-qnn-ut.cpp index 08d02e502b6ae..75d941263b82c 100644 --- a/tests/ggml-qnn-ut.cpp +++ b/tests/ggml-qnn-ut.cpp @@ -28,8 +28,9 @@ #include #include #include -#include #include +#if defined(__ANDROID__) || defined(__linux__) +#include #include #include #include @@ -40,6 +41,7 @@ #include #include #include +#endif #include #include @@ -70,41 +72,10 @@ #include "ggml-backend.h" #include "ggml-qnn.h" -#define GGML_QNN_DEBUG 1 -#define GGML_QNN_LOGBUF_LEN 4096 - -#define QNN_LOG_ERROR(...) ggml_qnn_log_internal(GGML_LOG_LEVEL_DEBUG, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) -#define QNN_LOG_WARN(...) ggml_qnn_log_internal(GGML_LOG_LEVEL_DEBUG , __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) -#define QNN_LOG_INFO(...) ggml_qnn_log_internal(GGML_LOG_LEVEL_DEBUG , __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) - -#if GGML_QNN_DEBUG -#define QNN_LOG_DEBUG(...) ggml_qnn_log_internal(GGML_LOG_LEVEL_DEBUG, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) -#else -#define QNN_LOG_DEBUG(...) -#endif - static void tensor_dump(const ggml_tensor * tensor, const char * name); #define TENSOR_DUMP(tensor) tensor_dump(tensor, #tensor) -static void ggml_qnn_log_internal(ggml_log_level level, const char * file, const char * func, int line, const char * format, ...) { - static std::mutex ggml_qnn_log_internal_mutex; - static char s_ggml_qnn_log_internal_buf[GGML_QNN_LOGBUF_LEN]; - - { - std::lock_guard lock(ggml_qnn_log_internal_mutex); - va_list args; - va_start(args, format); - int len_prefix = snprintf(s_ggml_qnn_log_internal_buf, GGML_QNN_LOGBUF_LEN, "[%s, %d]: ", func, line); - int len = vsnprintf(s_ggml_qnn_log_internal_buf + len_prefix, GGML_QNN_LOGBUF_LEN - len_prefix, format, args); - if (len < (GGML_QNN_LOGBUF_LEN - len_prefix)) { - printf("%s", s_ggml_qnn_log_internal_buf); - } - va_end(args); - } -} - - static bool ggml_graph_compute_helper( struct ggml_backend * backend, struct ggml_cgraph * graph, @@ -142,8 +113,8 @@ static void tensor_dump_elements(const ggml_tensor * tensor) { tmposs << std::setw(8) << std::fixed << std::setprecision(2) << value << " "; } - if (strlen(tmposs.str().c_str()) <= (GGML_QNN_LOGBUF_LEN - 96)) { - QNN_LOG_DEBUG("%s\n", tmposs.str().c_str()); + if (strlen(tmposs.str().c_str()) <= (4096 - 96)) { + printf("%s\n", tmposs.str().c_str()); } tmposs.clear(); tmposs.str(""); @@ -152,20 +123,20 @@ static void tensor_dump_elements(const ggml_tensor * tensor) { } } - QNN_LOG_DEBUG("\n"); + printf("\n"); } static void tensor_dump(const ggml_tensor * tensor, const char * name) { - QNN_LOG_DEBUG("dump ggml tensor %s(%s)\n", name, tensor->name); - QNN_LOG_DEBUG("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64", nb = (%5zi, %5zi, %5zi, %5zi)\n", + printf("dump ggml tensor %s(%s)\n", name, tensor->name); + printf("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64", nb = (%5zi, %5zi, %5zi, %5zi)\n", name, tensor->type, ggml_type_name(tensor->type), tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3], tensor->nb[0], tensor->nb[1], tensor->nb[2], tensor->nb[2]); tensor_dump_elements(tensor); - QNN_LOG_DEBUG("\n"); + printf("\n"); } @@ -181,15 +152,6 @@ static uint32_t get_tensor_rank(const ggml_tensor * tensor) { static uint32_t get_tensor_data_size(const ggml_tensor * tensor) { - size_t data_size = ggml_row_size(tensor->type, tensor->ne[0]); - size_t n_dims = get_tensor_rank(tensor); - for (size_t i = 1; i < n_dims; i++) { - data_size *= tensor->ne[i]; - } - - QNN_LOG_DEBUG("get_tensor_data_size %d", data_size); - QNN_LOG_DEBUG("ggml_nbytes(tensor) %d", ggml_nbytes(tensor)); - return ggml_nbytes(tensor); } @@ -273,9 +235,6 @@ static void show_usage() { } -struct ggml_backend_deleter { void operator()(ggml_backend_t backend) { ggml_backend_free(backend); } }; -typedef std::unique_ptr ggml_backend_ptr; - int main(int argc, char * argv[]) { int64_t n_begin_time = 0LL; int64_t n_end_time = 0LL; @@ -329,8 +288,7 @@ int main(int argc, char * argv[]) { return 1; } } - std::vector backends; - std::vector> set_n_threads_fns; + printf("Testing %zu devices\n\n", ggml_backend_dev_count()); for (size_t i = 0; i < ggml_backend_dev_count(); i++) { ggml_backend_dev_t dev = ggml_backend_dev_get(i); @@ -348,14 +306,6 @@ int main(int argc, char * argv[]) { if (backend != nullptr) { printf("%s: initialize %s backend\n", __func__, ggml_backend_dev_name(dev)); } - backends.emplace_back(backend); - - ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(dev); - auto ggml_backend_set_n_threads_fn = (ggml_backend_set_n_threads_t) ggml_backend_reg_get_proc_address( - reg, "ggml_backend_set_n_threads"); - if (ggml_backend_set_n_threads_fn) { - ggml_backend_set_n_threads_fn(backend, std::thread::hardware_concurrency()); - } printf(" Device description: %s\n", ggml_backend_dev_description(dev)); size_t free, total; @@ -367,23 +317,19 @@ int main(int argc, char * argv[]) { ggml_backend_t backend_cpu = nullptr; backend_cpu = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr); if (nullptr == backend_cpu) { - QNN_LOG_DEBUG("failed to initialize cpu backend\n"); + printf("failed to initialize cpu backend\n"); exit(1); } else { - QNN_LOG_DEBUG("succeed to initialize cpu backend\n"); + printf("succeed to initialize cpu backend\n"); } - backends.emplace_back(backend_cpu); - - size_t n_ok = 0; - QNN_LOG_DEBUG("enter qnn_ggml_op\n"); - QNN_LOG_DEBUG("ggml op:%d(%s)", n_ggml_op_type, ggml_op_name((enum ggml_op) n_ggml_op_type)); + printf("ggml op:%d(%s)", n_ggml_op_type, ggml_op_name((enum ggml_op) n_ggml_op_type)); n_begin_time = ggml_time_us(); srand(time(NULL)); ctx_size += 1024 * 1024 * 32; - QNN_LOG_DEBUG("Allocating Memory of size %zi bytes, %zi MB\n", ctx_size, + printf("Allocating Memory of size %zi bytes, %zi MB\n", ctx_size, (ctx_size / 1024 / 1024)); struct ggml_init_params params = { @@ -392,38 +338,19 @@ int main(int argc, char * argv[]) { /* no_alloc =*/ 0 }; - int idx = 0; - for (auto & backend_it : backends) { - if (idx == n_backend_type) { - backend = backend_it.get(); - } - idx++; - ggml_backend_dev_t dev = ggml_backend_get_device(backend_it.get()); - ggml_backend_reg_t reg = dev ? ggml_backend_dev_backend_reg(dev) : nullptr; - if (reg) { - auto ggml_backend_set_n_threads_fn = (ggml_backend_set_n_threads_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_n_threads"); - if (ggml_backend_set_n_threads_fn) { - set_n_threads_fns.emplace_back(backend_it.get(), ggml_backend_set_n_threads_fn); - } - } - const char * name = ggml_backend_dev_description(dev); - QNN_LOG_DEBUG("dev name %s\n", name); - - } - if (n_backend_type != QNN_BACKEND_GGML) { params.no_alloc = true; } ctx = ggml_init(params); if (!ctx) { - QNN_LOG_ERROR("%s: ggml_init() failed\n"); + printf("ggml_init() failed\n"); return 2; } - QNN_LOG_DEBUG("creating new tensors\n"); - QNN_LOG_DEBUG("ggml_blck_size(%s) %d\n", ggml_type_name(qtype), ggml_blck_size(qtype)); - QNN_LOG_DEBUG("ggml_type_size(%s) %d\n", ggml_type_name(qtype), ggml_type_size(qtype)); + printf("creating new tensors\n"); + printf("ggml_blck_size(%s) %ld\n", ggml_type_name(qtype), ggml_blck_size(qtype)); + printf("ggml_type_size(%s) %ld\n", ggml_type_name(qtype), ggml_type_size(qtype)); if (qtype != GGML_TYPE_F32) { sizex = ggml_blck_size(qtype); } @@ -461,7 +388,7 @@ int main(int argc, char * argv[]) { dst = ggml_mul_mat(ctx, src0, src1); break; default: - QNN_LOG_WARN("ggml op %d(%s) not supported", n_ggml_op_type, + printf("ggml op %d(%s) not supported", n_ggml_op_type, ggml_op_name((enum ggml_op) n_ggml_op_type)); ggml_free(ctx); ggml_backend_free(backend); @@ -472,32 +399,32 @@ int main(int argc, char * argv[]) { #ifdef GGML_USE_QNN if (n_backend_type != QNN_BACKEND_GGML) { - QNN_LOG_DEBUG("init QNN backend %d\n", n_backend_type); + printf("init QNN backend %d\n", n_backend_type); //re-init again backend = ggml_backend_qnn_init(n_backend_type, "/data/local/tmp/"); if (nullptr == backend) { - QNN_LOG_ERROR("create qnn backend %d(%s) failed\n", n_backend_type, ggml_backend_qnn_get_devname(n_backend_type)); + printf("create qnn backend %d(%s) failed\n", n_backend_type, ggml_backend_qnn_get_devname(n_backend_type)); return 1; } else { - QNN_LOG_INFO("create qnn backend %d(%s) succeed\n", n_backend_type, ggml_backend_qnn_get_devname(n_backend_type)); + printf("create qnn backend %d(%s) succeed\n", n_backend_type, ggml_backend_qnn_get_devname(n_backend_type)); } //buffer = ggml_backend_alloc_ctx_tensors(ctx, backend); ggml_backend_buffer_type_t buft = ggml_backend_get_default_buffer_type(backend); buffer = ggml_backend_alloc_ctx_tensors_from_buft(ctx, buft); if (!buffer) { - QNN_LOG_ERROR("%s: failed to allocate backend buffer\n", __func__); + printf("%s: failed to allocate backend buffer\n", __func__); ggml_free(ctx); ggml_backend_free(backend); return 4; } } else { - QNN_LOG_DEBUG("init default cpu backend\n"); + printf("init default cpu backend\n"); backend = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr); } #endif - QNN_LOG_DEBUG("creating compute graph\n"); + printf("creating compute graph\n"); gf = ggml_new_graph(ctx); ggml_build_forward_expand(gf, dst); @@ -519,20 +446,20 @@ int main(int argc, char * argv[]) { ggml_graph_compute_helper(backend, gf, work_buffer, num_threads, nullptr, nullptr); if (get_tensor_data_size(dst) < (100 * 100)) { - QNN_LOG_DEBUG("dump result tensors:\n"); + printf("dump result tensors:\n"); TENSOR_DUMP(src0); TENSOR_DUMP(src1); TENSOR_DUMP(dst); } else { - QNN_LOG_DEBUG("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", + printf("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", src0->name, src0->type, ggml_type_name(src0->type), src0->ne[0], src0->ne[1], src0->ne[2], src0->nb[0], src0->nb[1], src0->nb[2]); - QNN_LOG_DEBUG("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", + printf("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", src1->name, src1->type, ggml_type_name(src1->type), src1->ne[0], src1->ne[1], src1->ne[2], src1->nb[0], src1->nb[1], src1->nb[2]); - QNN_LOG_DEBUG("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", + printf("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", dst->name, dst->type, ggml_type_name(dst->type), dst->ne[0], dst->ne[1], dst->ne[2], dst->nb[0], dst->nb[1], dst->nb[2]); @@ -546,7 +473,7 @@ int main(int argc, char * argv[]) { n_end_time = ggml_time_us(); n_duration = (n_end_time - n_begin_time) / 1000; #ifdef GGML_USE_QNN - QNN_LOG_DEBUG("duration of ut GGML_OP_%s using QNN backend %s: %lld milliseconds\n", ggml_op_name((enum ggml_op)n_ggml_op_type), ggml_backend_qnn_get_devname(n_backend_type), n_duration); + printf("duration of ut GGML_OP_%s using QNN backend %s: %ld milliseconds\n", ggml_op_name((enum ggml_op)n_ggml_op_type), ggml_backend_qnn_get_devname(n_backend_type), n_duration); #endif return 0; From 24eec8c53b5fa0ddb2ab870e78ff3a44c7fbe696 Mon Sep 17 00:00:00 2001 From: zhouwg Date: Tue, 4 Mar 2025 11:14:52 +0800 Subject: [PATCH 52/76] ggml-qnn: remove un-needed function --- ggml/src/ggml-qnn/ggml-qnn.cpp | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/ggml/src/ggml-qnn/ggml-qnn.cpp b/ggml/src/ggml-qnn/ggml-qnn.cpp index 2aacf8f52d578..c8db722b19054 100644 --- a/ggml/src/ggml-qnn/ggml-qnn.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn.cpp @@ -919,19 +919,6 @@ static ggml_type ggml_datatype_from_qnn_datatype(Qnn_DataType_t qnn_type) { return GGML_TYPE_COUNT; } -//TODO: add more ops -static const char * qnn_opname_from_ggmlop(enum ggml_op ggmlop) { - switch (ggmlop) { - case GGML_OP_ADD: - return QNN_OP_ELEMENT_WISE_ADD; - case GGML_OP_MUL_MAT: - return QNN_OP_MAT_MUL; - default: - break; - } - return nullptr; -} - static void get_qnn_dimensions_from_ggml_dimensions(uint32_t * qnn_dimensions, const uint32_t * ggml_dimensions, uint32_t rank) { if (rank > GGML_MAX_DIMS) { GGMLQNN_LOG_WARN("invalid params"); @@ -1007,14 +994,13 @@ Qnn_Tensor_t * ggmlqnn_create_general_tensor(const ggml_tensor * tensor, const c .type = qnn_tensor_type, .dataFormat = QNN_TENSOR_DATA_FORMAT_FLAT_BUFFER, .dataType = qnn_data_type, - .quantizeParams = {QNN_DEFINITION_UNDEFINED, - QNN_QUANTIZATION_ENCODING_UNDEFINED, + .quantizeParams = {.encodingDefinition = QNN_DEFINITION_UNDEFINED, + .quantizationEncoding = QNN_QUANTIZATION_ENCODING_UNDEFINED, {.scaleOffsetEncoding = {.scale = 0.0000000000000000f, .offset = 0}}}, .rank = rank, .dimensions = tensor_dims, .memType = QNN_TENSORMEMTYPE_RAW, - {.clientBuf = {nullptr, 0} - } + .clientBuf = {.data = nullptr, .dataSize = 0} } } }; From f341e3296488519d51d5d1ca3c1a23292fcc6594 Mon Sep 17 00:00:00 2001 From: zhouwg Date: Tue, 4 Mar 2025 11:51:25 +0800 Subject: [PATCH 53/76] ggml-qnn:rebase to upstream --- ggml/src/ggml-qnn/ggml-qnn-ops.h | 5 ----- ggml/src/ggml-qnn/ggml-qnn.cpp | 20 ++------------------ 2 files changed, 2 insertions(+), 23 deletions(-) diff --git a/ggml/src/ggml-qnn/ggml-qnn-ops.h b/ggml/src/ggml-qnn/ggml-qnn-ops.h index c25638a9397c6..b1c388a32a87a 100644 --- a/ggml/src/ggml-qnn/ggml-qnn-ops.h +++ b/ggml/src/ggml-qnn/ggml-qnn-ops.h @@ -24,13 +24,8 @@ #include "ggml-qnn-impl.h" void ggml_qnn_general_node(ggml_backend_qnn_context * ctx, ggml_tensor * dst); void ggml_qnn_mul_mat(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -<<<<<<< HEAD void ggml_qnn_repeat(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -======= -void ggml_qnn_repeat(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -void ggml_qnn_add(ggml_backend_qnn_context * ctx, ggml_tensor * dst); ->>>>>>> ggml-qnn: refine source code structure to make code more clearly void ggml_qnn_div(ggml_backend_qnn_context * ctx, ggml_tensor * dst); void ggml_qnn_leaky_relu(ggml_backend_qnn_context * ctx, ggml_tensor * dst); void ggml_qnn_concat(ggml_backend_qnn_context * ctx, ggml_tensor * dst); diff --git a/ggml/src/ggml-qnn/ggml-qnn.cpp b/ggml/src/ggml-qnn/ggml-qnn.cpp index c8db722b19054..c47a07307003b 100644 --- a/ggml/src/ggml-qnn/ggml-qnn.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn.cpp @@ -1170,22 +1170,6 @@ void ggmlqnn_get_graphkey_from_op(const ggml_tensor * op, std::string & output) } } -bool ggmlqnn_is_valid_params(ggml_backend_qnn_context * ctx, const ggml_tensor * src0, - const ggml_tensor * src1, ggml_tensor * dst) { - if ((nullptr == ctx) || (nullptr == src0) || (nullptr == src1) || (nullptr == dst)) { - GGMLQNN_LOG_WARN("invalid params\n"); - return false; - } - - qnn_instance * instance = ctx->instance; - if (nullptr == instance) { - GGMLQNN_LOG_WARN("invalid params\n"); - return false; - } - - return true; -} - template Fn load_qnn_functionpointers(void * handle, const char * function_name) { return reinterpret_cast(dlsym(handle, function_name)); @@ -2466,11 +2450,11 @@ static void * ggml_backend_qnn_buffer_get_base(ggml_backend_buffer_t buffer) { return ctx->buffer; } -static void ggml_backend_qnn_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) { +static enum ggml_status ggml_backend_qnn_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) { ggml_backend_qnn_buffer_context * ctx = (ggml_backend_qnn_buffer_context *)buffer->context; GGML_UNUSED(tensor); GGML_UNUSED(ctx); - return; + return GGML_STATUS_SUCCESS; } static void ggml_backend_qnn_buffer_set_tensor(ggml_backend_buffer_t buffer, From 2e3c82407ea58c8d5715adf8ccc0db9b196b3178 Mon Sep 17 00:00:00 2001 From: zhouwg Date: Tue, 4 Mar 2025 12:17:25 +0800 Subject: [PATCH 54/76] ggml-qnn: fix a minior issue during rebase to upstream --- ggml/src/ggml-qnn/ggml-qnn.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ggml/src/ggml-qnn/ggml-qnn.cpp b/ggml/src/ggml-qnn/ggml-qnn.cpp index c47a07307003b..2fa1efbfd7a92 100644 --- a/ggml/src/ggml-qnn/ggml-qnn.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn.cpp @@ -2244,8 +2244,7 @@ static bool ggml_qnn_can_handle_op(const ggml_backend_qnn_context * ctx, const s } if (ne00 < 32) return false; - return (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16) - && (src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16); + return (src0->type == GGML_TYPE_F32) && (src1->type == GGML_TYPE_F32); } if (tensor->op == GGML_OP_MUL_MAT) { From c3e5d3c8c63530d6b732afd51f887add1b351656 Mon Sep 17 00:00:00 2001 From: zhouwg Date: Tue, 4 Mar 2025 15:47:28 +0800 Subject: [PATCH 55/76] ggml-qnn: update script according to https://github.com/ggml-org/llama.cpp/pull/12155 --- ggml/src/ggml-qnn/ggml-qnn-ops.cpp | 4 +- scripts/build-run-android.sh | 238 +++++++++++++++++++++++++++-- 2 files changed, 229 insertions(+), 13 deletions(-) diff --git a/ggml/src/ggml-qnn/ggml-qnn-ops.cpp b/ggml/src/ggml-qnn/ggml-qnn-ops.cpp index 00cb7da32c183..8db6662c8f0bc 100644 --- a/ggml/src/ggml-qnn/ggml-qnn-ops.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn-ops.cpp @@ -65,14 +65,12 @@ static inline bool ggmlqnn_is_valid_params(ggml_backend_qnn_context * ctx, const */ void ggml_qnn_general_node(ggml_backend_qnn_context * ctx, ggml_tensor * op) { Qnn_ErrorHandle_t error = QNN_SUCCESS; - enum ggml_status result = GGML_STATUS_SUCCESS; bool graph_initialized = false; qnn_instance * instance = nullptr; Qnn_GraphHandle_t graph_handle = nullptr; Qnn_Tensor_t * p_tensor0 = nullptr; Qnn_Tensor_t * p_tensor1 = nullptr; Qnn_Tensor_t * p_tensor2 = nullptr; - Qnn_Param_t qnn_params[] = {}; const ggml_tensor * src0 = op->src[0]; const ggml_tensor * src1 = op->src[1]; ggml_tensor * dst = op; @@ -170,7 +168,7 @@ void ggml_qnn_general_node(ggml_backend_qnn_context * ctx, ggml_tensor * op) { QNN_OP_PACKAGE_NAME_QTI_AISW, qnn_op_name, 0, - qnn_params, + nullptr, 2, tensor_inputs, 1, diff --git a/scripts/build-run-android.sh b/scripts/build-run-android.sh index 3d239510b8d63..2ed8db9349003 100755 --- a/scripts/build-run-android.sh +++ b/scripts/build-run-android.sh @@ -168,7 +168,7 @@ function run_llamabench() } -function run_test-backend-ops() +function run_test-ops() { prepare_run_on_phone test-backend-ops @@ -178,6 +178,38 @@ function run_test-backend-ops() } +function run_test-op() +{ + prepare_run_on_phone test-backend-ops + + qnnbackendname=qnn-cpu + case $qnnbackend in + 0) + qnnbackendname=qnn-cpu + ;; + 1) + qnnbackendname=qnn-gpu + ;; + 2) + qnnbackendname=qnn-npu + ;; + *) + qnnbackendname=qnn-cpu + ;; + esac + + #debug + echo "adb shell cd ${REMOTE_PATH} \ + && export LD_LIBRARY_PATH=${REMOTE_PATH} \ + && ${REMOTE_PATH}/test-backend-ops test -o $opname -b $qnnbackendname " + + echo "\n" + adb shell "cd ${REMOTE_PATH} \ + && export LD_LIBRARY_PATH=${REMOTE_PATH} \ + && ${REMOTE_PATH}/test-backend-ops test -o $opname -b $qnnbackendname " + +} + function run_ut_add() { prepare_run_on_phone ggml-qnn-ut @@ -208,18 +240,101 @@ function run_ut_mul() } +function print_oplist() +{ +oplist="DUP + ADD + ADD1 + ACC + SUB + MUL + DIV + SQR + SQRT + LOG + SIN + COS + SUM + SUM_ROWS + MEAN + ARGMAX + COUNT_EQUAL + REPEAT + REPEAT_BACK + CONCAT + SILU_BACK + NORM + RMS_NORM + RMS_NORM_BACK + GROUP_NORM + + MUL_MAT + MUL_MAT_ID + OUT_PROD + + SCALE + SET + CPY + CONT + RESHAPE + VIEW + PERMUTE + TRANSPOSE + GET_ROWS + GET_ROWS_BACK + DIAG + DIAG_MASK_INF + DIAG_MASK_ZERO + SOFT_MAX + SOFT_MAX_BACK + ROPE + ROPE_BACK + CLAMP + CONV_TRANSPOSE_1D + IM2COL + IM2COL_BACK + CONV_TRANSPOSE_2D + POOL_1D + POOL_2D + POOL_2D_BACK + UPSCALE + PAD + PAD_REFLECT_1D + ARANGE + TIMESTEP_EMBEDDING + ARGSORT + LEAKY_RELU + + FLASH_ATTN_EXT + FLASH_ATTN_BACK + SSM_CONV + SSM_SCAN + WIN_PART + WIN_UNPART + GET_REL_POS + ADD_REL_POS + RWKV_WKV6 + GATED_LINEAR_ATTN" + +echo "opname list: " +echo ${oplist} +} function show_usage() { echo "Usage:" + echo " $0 help" + echo " $0 print_oplist" echo " $0 build" echo " $0 updateqnnlib" - echo " $0 run_testop" - echo " $0 run_ut_add 0 (QNN_CPU) / 1 (QNN_GPU) / 2 (QNN_NPU) / 3 (ggml)" - echo " $0 run_ut_mulmat 0 (QNN_CPU) / 1 (QNN_GPU) / 2 (QNN_NPU) / 3 (ggml)" - echo " $0 run_ut_mul 0 (QNN_CPU) / 1 (QNN_GPU) / 2 (QNN_NPU) / 3 (ggml)" - echo " $0 run_llamacli 0 (QNN_CPU) / 1 (QNN_GPU) / 2 (QNN_NPU) / 3 (ggml)" - echo " $0 run_llamabench 0 (QNN_CPU) / 1 (QNN_GPU) / 2 (QNN_NPU) / 3 (ggml)" + echo " $0 run_testops" + echo " $0 run_testop [ADD/MUL/MUL_MAT] [0 (QNN_CPU) / 1 (QNN_GPU) / 2 (QNN_NPU)]" + echo " $0 run_ut_add 0 (QNN_CPU) / 1 (QNN_GPU) / 2 (QNN_NPU) / 3 (ggml)" + echo " $0 run_ut_mulmat 0 (QNN_CPU) / 1 (QNN_GPU) / 2 (QNN_NPU) / 3 (ggml)" + echo " $0 run_ut_mul 0 (QNN_CPU) / 1 (QNN_GPU) / 2 (QNN_NPU) / 3 (ggml)" + echo " $0 run_llamacli 0 (QNN_CPU) / 1 (QNN_GPU) / 2 (QNN_NPU) / 3 (ggml)" + echo " $0 run_llamabench 0 (QNN_CPU) / 1 (QNN_GPU) / 2 (QNN_NPU) / 3 (ggml)" + echo -e "\n\n\n" } @@ -238,12 +353,14 @@ elif [ $# == 1 ]; then elif [ "$1" == "help" ]; then show_usage exit 1 + elif [ "$1" == "print_oplist" ]; then + print_oplist + exit 1 elif [ "$1" == "build" ]; then build_ggml_qnn exit 0 - - elif [ "$1" == "run_testop" ]; then - run_test-backend-ops + elif [ "$1" == "run_testops" ]; then + run_test-ops exit 0 elif [ "$1" == "updateqnnlib" ]; then @@ -276,6 +393,107 @@ elif [ $# == 2 ]; then run_ut_mul exit 0 fi +elif [ $# == 3 ]; then + opname=$2 +#TODO: check opname in oplist +#opname can be found via print_oplist: +# DUP +# ADD +# ADD1 +# ACC +# SUB +# MUL +# DIV +# SQR +# SQRT +# LOG +# SIN +# COS +# SUM +# SUM_ROWS +# MEAN +# ARGMAX +# COUNT_EQUAL +# REPEAT +# REPEAT_BACK +# CONCAT +# SILU_BACK +# NORM +# RMS_NORM +# RMS_NORM_BACK +# GROUP_NORM +# +# MUL_MAT +# MUL_MAT_ID +# OUT_PROD +# +# SCALE +# SET +# CPY +# CONT +# RESHAPE +# VIEW +# PERMUTE +# TRANSPOSE +# GET_ROWS +# GET_ROWS_BACK +# DIAG +# DIAG_MASK_INF +# DIAG_MASK_ZERO +# SOFT_MAX +# SOFT_MAX_BACK +# ROPE +# ROPE_BACK +# CLAMP +# CONV_TRANSPOSE_1D +# IM2COL +# IM2COL_BACK +# CONV_TRANSPOSE_2D +# POOL_1D +# POOL_2D +# POOL_2D_BACK +# UPSCALE +# PAD +# PAD_REFLECT_1D +# ARANGE +# TIMESTEP_EMBEDDING +# ARGSORT +# LEAKY_RELU +# +# FLASH_ATTN_EXT +# FLASH_ATTN_BACK +# SSM_CONV +# SSM_SCAN +# WIN_PART +# WIN_UNPART +# GET_REL_POS +# ADD_REL_POS +# RWKV_WKV6 +# GATED_LINEAR_ATTN +# +# UNARY +# +# MAP_UNARY +# MAP_BINARY +# +# MAP_CUSTOM1_F32 +# MAP_CUSTOM2_F32 +# MAP_CUSTOM3_F32 +# +# MAP_CUSTOM1 +# MAP_CUSTOM2 +# MAP_CUSTOM3 +# +# CROSS_ENTROPY_LOSS +# CROSS_ENTROPY_LOSS_BACK +# OPT_STEP_ADAMW + qnnbackend=$3 + if [ ${qnnbackend} -gt 3 ]; then + show_usage + exit 1 + fi + run_test-op + exit 0 else show_usage exit 1 From fc3b9a68f94286bc2bff1340cba4f7adc7745d5c Mon Sep 17 00:00:00 2001 From: zhouwg Date: Tue, 4 Mar 2025 17:49:04 +0800 Subject: [PATCH 56/76] ggml-qnn: fix a minior issue in ggmlqnn_create_general_tensor() --- ggml/src/ggml-qnn/ggml-qnn.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-qnn/ggml-qnn.cpp b/ggml/src/ggml-qnn/ggml-qnn.cpp index 2fa1efbfd7a92..4f9a308914778 100644 --- a/ggml/src/ggml-qnn/ggml-qnn.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn.cpp @@ -947,7 +947,7 @@ Qnn_Tensor_t * ggmlqnn_create_general_tensor(const ggml_tensor * tensor, const c char tensor_name[GGML_MAX_NAME] = {}; //ensure the tensor name is unique - if (nullptr != name) { + if (nullptr == name) { snprintf(tensor_name, GGML_MAX_NAME, "tensor_%-8d", get_idx()); } else { snprintf(tensor_name, GGML_MAX_NAME, "tensor_%s%-8d", name, get_idx()); @@ -1857,6 +1857,12 @@ int qnn_instance::qnn_init(const QnnSaver_Config_t ** saver_config) { if (0 != set_high_performance_mode()) { GGMLQNN_LOG_WARN("set HTP high performance mode failure"); } + + if (enable_qnn_rpc()) { + GGMLQNN_LOG_INFO("NPU RPC feature enabled"); + } else { + GGMLQNN_LOG_INFO("NPU RPC feature disabled"); + } } GGMLQNN_LOG_DEBUG("leave qni_init\n"); From 626a9cb6d32e741959a2cddd1c0c837e5e4904b6 Mon Sep 17 00:00:00 2001 From: zhouwg Date: Tue, 4 Mar 2025 20:35:14 +0800 Subject: [PATCH 57/76] ggml-qnn: active member variable _device_id in class qnn_instance --- ggml/src/ggml-qnn/ggml-qnn-impl.h | 4 ++++ ggml/src/ggml-qnn/ggml-qnn-ops.cpp | 1 + ggml/src/ggml-qnn/ggml-qnn.cpp | 11 +++++++++++ 3 files changed, 16 insertions(+) diff --git a/ggml/src/ggml-qnn/ggml-qnn-impl.h b/ggml/src/ggml-qnn/ggml-qnn-impl.h index 5a2fe5752a097..68662d31d3738 100644 --- a/ggml/src/ggml-qnn/ggml-qnn-impl.h +++ b/ggml/src/ggml-qnn/ggml-qnn-impl.h @@ -498,6 +498,10 @@ class qnn_instance { return _enable_qnn_rpc; } + QNNBackend get_device_id() { + return _device_id; + } + public: std::map>> _qnn_graph_map; diff --git a/ggml/src/ggml-qnn/ggml-qnn-ops.cpp b/ggml/src/ggml-qnn/ggml-qnn-ops.cpp index 8db6662c8f0bc..d96c33d574b41 100644 --- a/ggml/src/ggml-qnn/ggml-qnn-ops.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn-ops.cpp @@ -118,6 +118,7 @@ void ggml_qnn_general_node(ggml_backend_qnn_context * ctx, ggml_tensor * op) { if (!graph_initialized) { GGMLQNN_LOG_DEBUG("graph name %s", graph_name.c_str()); + GGML_ASSERT(instance->get_device_id() == ctx->device); error = instance->init_qnn_graph(graph_name, static_cast(ctx->device), 8); if (QNN_SUCCESS != error) { GGMLQNN_LOG_INFO("can't create qnn graph handle with graph name %s, error = %d\n", graph_name.c_str(), error); diff --git a/ggml/src/ggml-qnn/ggml-qnn.cpp b/ggml/src/ggml-qnn/ggml-qnn.cpp index 4f9a308914778..a12065f306e56 100644 --- a/ggml/src/ggml-qnn/ggml-qnn.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn.cpp @@ -1704,6 +1704,17 @@ int qnn_instance::qnn_init(const QnnSaver_Config_t ** saver_config) { } } + _device_id = QNN_BACKEND_CPU; + if (_backend_name.find("QnnCpu") != std::string::npos) { + _device_id = QNN_BACKEND_CPU; + } + if (_backend_name.find("QnnGpu") != std::string::npos) { + _device_id = QNN_BACKEND_GPU; + } + if (_backend_name.find("QnnHtp") != std::string::npos) { + _device_id = QNN_BACKEND_NPU; + } + backend_id = _lib_path_to_backend_id[backend_lib_path]; if (0 == _loaded_backend.count(backend_id) || 0 == _loaded_lib_handle.count(backend_id)) { From 8471577b478fa7ebcec04f55f0fb0290d168ab4e Mon Sep 17 00:00:00 2001 From: zhouwg Date: Tue, 4 Mar 2025 22:29:06 +0800 Subject: [PATCH 58/76] ggml-qnn: refine ggml_qnn_general_node and ggml_qnn_mul_mat to make code more clearly --- ggml/src/ggml-qnn/ggml-qnn-impl.h | 2 +- ggml/src/ggml-qnn/ggml-qnn-ops.cpp | 406 +++++++++-------------------- ggml/src/ggml-qnn/ggml-qnn.cpp | 27 +- 3 files changed, 147 insertions(+), 288 deletions(-) diff --git a/ggml/src/ggml-qnn/ggml-qnn-impl.h b/ggml/src/ggml-qnn/ggml-qnn-impl.h index 68662d31d3738..a0a1a80cbf855 100644 --- a/ggml/src/ggml-qnn/ggml-qnn-impl.h +++ b/ggml/src/ggml-qnn/ggml-qnn-impl.h @@ -595,12 +595,12 @@ class qnn_instance { size_t ggmlqnn_get_opcaps_size(void); size_t ggmlqnn_get_op_index(const ggml_tensor * tensor); -Qnn_Tensor_t * ggmlqnn_create_compute_tensor(const ggml_tensor * tensor); const char * ggmlqnn_get_error_string(Qnn_ErrorHandle_t qnn_error_code); Qnn_DataType_t ggmlqnn_datatype_from_ggml_datatype(enum ggml_type ggmltype); void * ggmlqnn_type_trait(ggml_backend_qnn_context * ctx, ggml_tensor * op); void ggmlqnn_get_graphkey_from_op(const ggml_tensor * op, std::string & output); uint8_t * ggmlqnn_create_rpc_buffer(qnn_instance * instance, const ggml_tensor * ggml_tensor, Qnn_Tensor_t * qnn_tensor, bool b_copydata); +Qnn_Tensor_t * ggmlqnn_create_compute_tensor(qnn_instance * instance, Qnn_GraphHandle_t handle, const ggml_tensor * tensor, Qnn_TensorType_t tensor_type); void ggmlqnn_print_tensors_info(const char * func_name, ggml_backend_qnn_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); Qnn_OpConfig_t ggmlqnn_create_op_config(const char * name, const char * package, const char * type, diff --git a/ggml/src/ggml-qnn/ggml-qnn-ops.cpp b/ggml/src/ggml-qnn/ggml-qnn-ops.cpp index d96c33d574b41..8a4ed15529b4c 100644 --- a/ggml/src/ggml-qnn/ggml-qnn-ops.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn-ops.cpp @@ -65,7 +65,6 @@ static inline bool ggmlqnn_is_valid_params(ggml_backend_qnn_context * ctx, const */ void ggml_qnn_general_node(ggml_backend_qnn_context * ctx, ggml_tensor * op) { Qnn_ErrorHandle_t error = QNN_SUCCESS; - bool graph_initialized = false; qnn_instance * instance = nullptr; Qnn_GraphHandle_t graph_handle = nullptr; Qnn_Tensor_t * p_tensor0 = nullptr; @@ -87,75 +86,36 @@ void ggml_qnn_general_node(ggml_backend_qnn_context * ctx, ggml_tensor * op) { qnn_perf op_perf = qnn_perf(ggml_op_name); op_perf.start(); + //ggmlqnn_print_tensors_info(__func__, ctx, src0, src1, dst); + bool enable_npu_rpc = instance->enable_qnn_rpc() && ctx->device == QNN_BACKEND_NPU; + std::string graph_name; ggmlqnn_get_graphkey_from_op(op, graph_name); if (instance->_qnn_graph_map.find(graph_name) != instance->_qnn_graph_map.end()) { - graph_initialized = true; - qnn_res_t & graph_item = instance->_qnn_graph_map[graph_name]; - graph_handle = std::get<0>(graph_item); - qnn_tensors_t & tensor = std::get<1>(graph_item); - p_tensor0 = tensor[0]; - p_tensor1 = tensor[1]; - p_tensor2 = tensor[2]; + //retrieve computational resource from cached QNN graph + qnn_res_t & graph_item = instance->_qnn_graph_map[graph_name]; + graph_handle = std::get<0>(graph_item); + qnn_tensors_t & tensor = std::get<1>(graph_item); + p_tensor0 = tensor[0]; + p_tensor1 = tensor[1]; + p_tensor2 = tensor[2]; } else { - p_tensor0 = ggmlqnn_create_compute_tensor(src0); - p_tensor1 = ggmlqnn_create_compute_tensor(src1); - p_tensor2 = ggmlqnn_create_compute_tensor(dst); - } - //ggmlqnn_print_tensors_info(__func__, ctx, src0, src1, dst); - - //ensure QNN tensor has correct tensor type - QNN_VER_PTR(*p_tensor0)->type = QNN_TENSOR_TYPE_APP_WRITE; - QNN_VER_PTR(*p_tensor1)->type = QNN_TENSOR_TYPE_APP_WRITE; - QNN_VER_PTR(*p_tensor2)->type = QNN_TENSOR_TYPE_APP_READ; - - //save the original dimensions of qnn tensors - uint32_t * tensor_0_dimensions = QNN_VER_PTR(*p_tensor0)->dimensions; - uint32_t * tensor_1_dimensions = QNN_VER_PTR(*p_tensor1)->dimensions; - uint32_t * tensor_2_dimensions = QNN_VER_PTR(*p_tensor2)->dimensions; - - bool enable_npu_rpc = instance->enable_qnn_rpc() && ctx->device == QNN_BACKEND_NPU; - - if (!graph_initialized) { GGMLQNN_LOG_DEBUG("graph name %s", graph_name.c_str()); GGML_ASSERT(instance->get_device_id() == ctx->device); + //create QNN graph error = instance->init_qnn_graph(graph_name, static_cast(ctx->device), 8); if (QNN_SUCCESS != error) { - GGMLQNN_LOG_INFO("can't create qnn graph handle with graph name %s, error = %d\n", graph_name.c_str(), error); + GGMLQNN_LOG_WARN("can't create qnn graph handle with graph name %s, error = %d\n", graph_name.c_str(), error); return; } graph_handle = instance->get_qnn_graph_handle(); - if (enable_npu_rpc) { - QNN_VER_PTR(*p_tensor0)->memType = QNN_TENSORMEMTYPE_MEMHANDLE; - QNN_VER_PTR(*p_tensor0)->clientBuf = {.data=nullptr, .dataSize=0}; - - QNN_VER_PTR(*p_tensor1)->memType = QNN_TENSORMEMTYPE_MEMHANDLE; - QNN_VER_PTR(*p_tensor1)->clientBuf = {.data=nullptr, .dataSize=0}; - - QNN_VER_PTR(*p_tensor2)->memType = QNN_TENSORMEMTYPE_MEMHANDLE; - QNN_VER_PTR(*p_tensor2)->clientBuf = {.data=nullptr, .dataSize=0}; - } - - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor0)); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor1)); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor2)); - - if (enable_npu_rpc) { - uint8_t * qnn_rpcbuffer_0 = ggmlqnn_create_rpc_buffer(instance, src0, p_tensor0, true); - uint8_t * qnn_rpcbuffer_1 = ggmlqnn_create_rpc_buffer(instance, src1, p_tensor1, true); - uint8_t * qnn_rpcbuffer_2 = ggmlqnn_create_rpc_buffer(instance, dst, p_tensor2, false); - if (nullptr == qnn_rpcbuffer_0 || nullptr == qnn_rpcbuffer_1 || nullptr == qnn_rpcbuffer_2) { - GGMLQNN_LOG_INFO("create rpc buffer failure\n"); - //TODO: potential memory leak although it shouldn't happen - return; - } - } else { - QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggmlqnn_get_tensor_data_size(src0)}; - QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggmlqnn_get_tensor_data_size(src1)}; - QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggmlqnn_get_tensor_data_size(dst)}; - } + //create computational tensor + p_tensor0 = ggmlqnn_create_compute_tensor(instance, graph_handle, src0, QNN_TENSOR_TYPE_APP_WRITE); + p_tensor1 = ggmlqnn_create_compute_tensor(instance, graph_handle, src1, QNN_TENSOR_TYPE_APP_WRITE); + p_tensor2 = ggmlqnn_create_compute_tensor(instance, graph_handle, dst, QNN_TENSOR_TYPE_APP_READ); + //compose QNN graph Qnn_Tensor_t tensor_inputs[] = { *p_tensor0, *p_tensor1 @@ -177,100 +137,55 @@ void ggml_qnn_general_node(ggml_backend_qnn_context * ctx, ggml_tensor * op) { } }; CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, op_config)); + //finalize QNN graph CHECK_QNN_API(error, qnn_raw_interface.graphFinalize(graph_handle, nullptr, nullptr)); - CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, - tensor_inputs, 2, - tensor_outputs, 1, - nullptr, nullptr)); - - if (enable_npu_rpc) { - uint8_t * qnn_rpcbuffer = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*p_tensor2)->memHandle)); - GGMLQNN_LOG_INFO("qnn_rpcbuffer = %p\n", qnn_rpcbuffer); - if (nullptr != qnn_rpcbuffer) { - memcpy(dst->data, qnn_rpcbuffer, ggml_nbytes(dst)); - } - } + //cache QNN graph qnn_tensors_t ggml_op_add_tensors; ggml_op_add_tensors.reserve(3); ggml_op_add_tensors.push_back(p_tensor0); ggml_op_add_tensors.push_back(p_tensor1); ggml_op_add_tensors.push_back(p_tensor2); - auto graph_item = std::make_tuple(graph_handle, ggml_op_add_tensors); instance->_qnn_graph_map[graph_name] = graph_item; - } else { - Qnn_DataType_t src0_qnn_type = QNN_DATATYPE_FLOAT_32; - Qnn_DataType_t src1_qnn_type = QNN_DATATYPE_FLOAT_32; - Qnn_DataType_t dst_qnn_type = QNN_DATATYPE_FLOAT_32; - - src0_qnn_type = ggmlqnn_datatype_from_ggml_datatype(src0->type); - src1_qnn_type = ggmlqnn_datatype_from_ggml_datatype(src1->type); - dst_qnn_type = ggmlqnn_datatype_from_ggml_datatype(dst->type); - - uint32_t dimensions_input_0[] = {(uint32_t) src0->ne[0], (uint32_t) src0->ne[1], - (uint32_t) src0->ne[2], (uint32_t) src0->ne[3]}; - uint32_t dimensions_input_1[] = {(uint32_t) src1->ne[0], (uint32_t) src1->ne[1], - (uint32_t) src1->ne[2], (uint32_t) src1->ne[3]}; - uint32_t dimensions_output[] = {(uint32_t) dst->ne[0], (uint32_t) dst->ne[1], - (uint32_t) dst->ne[2], (uint32_t) dst->ne[3]}; - - QNN_VER_PTR(*p_tensor0)->dimensions = dimensions_input_0; - QNN_VER_PTR(*p_tensor0)->rank = ggml_n_dims(src0); - QNN_VER_PTR(*p_tensor0)->dataType = src0_qnn_type; - - QNN_VER_PTR(*p_tensor1)->dimensions = dimensions_input_1; - QNN_VER_PTR(*p_tensor1)->rank = ggml_n_dims(src1); - QNN_VER_PTR(*p_tensor1)->dataType = src1_qnn_type; - - QNN_VER_PTR(*p_tensor2)->dimensions = dimensions_output; - QNN_VER_PTR(*p_tensor2)->rank = ggml_n_dims(dst); - QNN_VER_PTR(*p_tensor2)->dataType = dst_qnn_type; - - if (enable_npu_rpc) { - //TODO: NPU RPC feature will failed with test-backend-ops - uint8_t * qnn_buffer_0 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*p_tensor0)->memHandle)); - GGMLQNN_LOG_INFO("qnn_rpcbuffer_0 = %p\n", qnn_buffer_0); - if (nullptr != qnn_buffer_0) { - memcpy(qnn_buffer_0, src0->data, ggml_nbytes(src0)); - } - - uint8_t * qnn_buffer_1 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*p_tensor1)->memHandle)); - GGMLQNN_LOG_INFO("qnn_rpcbuffer_1 = %p\n", qnn_buffer_1); - if (nullptr != qnn_buffer_1) { - memcpy(qnn_buffer_1, src1->data, ggml_nbytes(src1)); - } - } else { - QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggmlqnn_get_tensor_data_size(src0)}; - QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggmlqnn_get_tensor_data_size(src1)}; - QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggmlqnn_get_tensor_data_size(dst)}; + } + + if (enable_npu_rpc) { + uint8_t * qnn_buffer_0 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*p_tensor0)->memHandle)); + GGMLQNN_LOG_INFO("qnn_rpcbuffer_0 = %p\n", qnn_buffer_0); + if (nullptr != qnn_buffer_0) { + memcpy(qnn_buffer_0, src0->data, ggml_nbytes(src0)); } - Qnn_Tensor_t tensor_inputs[] = { - *p_tensor0, - *p_tensor1 - }; - Qnn_Tensor_t tensor_outputs[] = { - *p_tensor2 - }; - CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, - tensor_inputs, 2, - tensor_outputs, 1, - nullptr, nullptr)); - - if (enable_npu_rpc) { - //TODO:NPU RPC feature will failed with test-backend-ops - uint8_t * qnn_buffer_2 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*p_tensor2)->memHandle)); - if (nullptr != qnn_buffer_2) { - memcpy(dst->data, qnn_buffer_2, ggml_nbytes(dst)); - } + uint8_t * qnn_buffer_1 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*p_tensor1)->memHandle)); + GGMLQNN_LOG_INFO("qnn_rpcbuffer_1 = %p\n", qnn_buffer_1); + if (nullptr != qnn_buffer_1) { + memcpy(qnn_buffer_1, src1->data, ggml_nbytes(src1)); } + } else { + QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggmlqnn_get_tensor_data_size(src0)}; + QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggmlqnn_get_tensor_data_size(src1)}; + QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggmlqnn_get_tensor_data_size(dst)}; } - // restore the original dimensions of qnn tensors to avoid memory leak in func free_qnn_tensor - QNN_VER_PTR(*p_tensor0)->dimensions = tensor_0_dimensions; - QNN_VER_PTR(*p_tensor1)->dimensions = tensor_1_dimensions; - QNN_VER_PTR(*p_tensor2)->dimensions = tensor_2_dimensions; + Qnn_Tensor_t tensor_inputs[] = { + *p_tensor0, + *p_tensor1 + }; + Qnn_Tensor_t tensor_outputs[] = { + *p_tensor2 + }; + CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, + tensor_inputs, 2, + tensor_outputs, 1, + nullptr, nullptr)); + if (enable_npu_rpc) { + //TODO:NPU RPC feature will failed with test-backend-ops + uint8_t * qnn_buffer_2 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*p_tensor2)->memHandle)); + if (nullptr != qnn_buffer_2) { + memcpy(dst->data, qnn_buffer_2, ggml_nbytes(dst)); + } + } #if GGMLQNN_PRINT_OP_ADD_LOG op_perf.info(); @@ -478,8 +393,35 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) * `src1` and the weight tensor `src0`, handling transposing, and quantization as needed, * and stores the result in the destination tensor `dst`. * - * @param backend the context which got through (ggml_backend_qnn_context *)backend->context for the - * QNN backend operations. + there are two key-points in properly handling how to offload mulmat to the QNN backend in ggml-qnn + 1. transpose + a 3x2 f32 matrix which means 3 rows and 2 columns. in ggml, it could be created from: + struct ggml_tensor* matrix = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 2, 3); + which like this: + +---+---+ + | 0 | 1 | + +---+---+ + | 2 | 3 | + +---+---+ + | 4 | 5 | + +---+---+ + with + ne[0] = 2 + ne[1] = 3 + there are different dimension order between ggml tensor and qnn tensor + + 2. QNN's MatMul can only support input tensors with rank >= 2 + + in the all, there is gap between ggml mulmat and QNN mulmat,we need to perform a transpose + operation when offloading mulmat to QNN backend. this implementation will handle transpose + in func ggml_qnn_create_general_tensor() + * + * this function is a good example to illustrated the second technical approach "mapping the + * entire ggml computational graph to QNN graph" without complex C++ encapsulation. or another + * pipeline of "how to utilize the Hexagon NPU maximally through QNN SDK", details could be found at + * https://github.com/ggml-org/llama.cpp/pull/12049#issuecomment-2678308360 + * + * @param ctx the context of ggml-qnn backend * @param op the destination tensor where the result of the matrix multiplication will be stored. * * @note the logic of ggml_qnn_mul_mat is similar to ggml_qnn_general_node but much more complicated @@ -494,7 +436,6 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) */ void ggml_qnn_mul_mat(ggml_backend_qnn_context * ctx, ggml_tensor * op) { Qnn_ErrorHandle_t error = QNN_SUCCESS; - bool graph_initialized = false; qnn_perf op_perf = qnn_perf("ggml_qnn_mul_mat"); qnn_instance * instance = nullptr; Qnn_GraphHandle_t graph_handle = nullptr; @@ -523,10 +464,12 @@ void ggml_qnn_mul_mat(ggml_backend_qnn_context * ctx, ggml_tensor * op) { void * wdata = ggmlqnn_type_trait(ctx, op); const size_t desired_size = ctx->desired_size; + ggmlqnn_print_tensors_info(__func__, ctx, src0, src1, dst); + std::string graph_name; ggmlqnn_get_graphkey_from_op(op, graph_name); if (instance->_qnn_graph_map.find(graph_name) != instance->_qnn_graph_map.end()) { - graph_initialized = true; + //retrieve computational resource from cached QNN graph qnn_res_t & graph_item = instance->_qnn_graph_map[graph_name]; graph_handle = std::get<0>(graph_item); qnn_tensors_t & tensors = std::get<1>(graph_item); @@ -536,144 +479,55 @@ void ggml_qnn_mul_mat(ggml_backend_qnn_context * ctx, ggml_tensor * op) { p_param_tensor = tensors[3]; p_tensor2_transpose = tensors[4]; } else { - p_tensor0 = GQCGT(src0, nullptr, QNN_TENSOR_TYPE_APP_WRITE,QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0); - p_tensor1 = GQCGT(src1, nullptr, QNN_TENSOR_TYPE_APP_WRITE,QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0); - p_tensor2 = GQCGT(dst, nullptr, QNN_TENSOR_TYPE_APP_READ,QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0); - } - ggmlqnn_print_tensors_info(__func__, ctx, src0, src1, dst); - - //ensure QNN tensor has correct tensor type - QNN_VER_PTR(*p_tensor0)->type = QNN_TENSOR_TYPE_APP_WRITE; - QNN_VER_PTR(*p_tensor1)->type = QNN_TENSOR_TYPE_APP_WRITE; - QNN_VER_PTR(*p_tensor2)->type = QNN_TENSOR_TYPE_APP_READ; - - //save the original dimensions of qnn tensors - uint32_t * tensor_0_dimensions = QNN_VER_PTR(*p_tensor0)->dimensions; - uint32_t * tensor_1_dimensions = QNN_VER_PTR(*p_tensor1)->dimensions; - uint32_t * tensor_2_dimensions = QNN_VER_PTR(*p_tensor2)->dimensions; - - if (!graph_initialized) { + //create QNN graph GGMLQNN_LOG_DEBUG("graph name %s", graph_name.c_str()); - /* - there are two key-points in properly handling how to offload mulmat to the QNN backend in ggml-qnn - 1. transpose - a 3x2 f32 matrix which means 3 rows and 2 columns. in ggml, it could be created from: - struct ggml_tensor* matrix = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 2, 3); - which like this: - +---+---+ - | 0 | 1 | - +---+---+ - | 2 | 3 | - +---+---+ - | 4 | 5 | - +---+---+ - with - ne[0] = 2 - ne[1] = 3 - there are different dimension order between ggml tensor and qnn tensor - - 2. QNN's MatMul can only support input tensors with rank >= 2 - - in the all, there is gap between ggml mulmat and QNN mulmat,we need to perform a transpose - operation when offloading mulmat to QNN backend. this concise implementation will handle - transpose in func ggml_qnn_create_general_tensor() - */ - //step-1: create qnn graph - error = qnn_raw_interface.graphCreate(instance->get_qnn_context_handle(), - graph_name.c_str(), nullptr, &graph_handle); + error = qnn_raw_interface.graphCreate(instance->get_qnn_context_handle(), graph_name.c_str(), nullptr, &graph_handle); if (QNN_SUCCESS != error) { - GGMLQNN_LOG_INFO("can't create qnn graph handle with graph name %s, error = %d\n", graph_name.c_str(), error); + GGMLQNN_LOG_WARN("can't create qnn graph handle with graph name %s, error = %d\n", graph_name.c_str(), error); return; } - //step-2: create param tensor for mulmat of 2d/3d/4d matrix + + //create computational tensor + p_tensor0 = GQCGT(src0, nullptr, QNN_TENSOR_TYPE_APP_WRITE, QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0); + p_tensor1 = GQCGT(src1, nullptr, QNN_TENSOR_TYPE_APP_WRITE, QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0); + p_tensor2 = GQCGT(dst, nullptr, QNN_TENSOR_TYPE_APP_READ, QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor0)); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor1)); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor2)); + + //create param tensor for offload 2d/3d/4d matrix multiplication const uint32_t param_tensor_data[GGML_MAX_DIMS][GGML_MAX_DIMS] = { {0}, {1, 0}, {0, 2, 1}, {0, 1, 3, 2}, }; - uint32_t param_tensor_dims[1] = {src0_rank}; + uint32_t param_tensor_dims[1] = {src0_rank}; p_param_tensor = GQCGT(nullptr, "param", QNN_TENSOR_TYPE_STATIC, QNN_DATATYPE_UINT_32, 1, param_tensor_dims, (void *)(param_tensor_data[src0_rank - 1]), src0_rank * sizeof(uint32_t)); CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_param_tensor)); - //step-3: create compute tensor from ggml tensor - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor0)); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor1)); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor2)); - if (src0_type != GGML_TYPE_F32) { - QNN_VER_PTR(*p_tensor0)->clientBuf = {wdata, static_cast(desired_size)}; - } else { - QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggmlqnn_get_tensor_data_size(src0)}; - } - QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggmlqnn_get_tensor_data_size(src1)}; - QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggmlqnn_get_tensor_data_size(dst)}; - - //step-4: create a transpose tensor + //create transpose tensor p_tensor2_transpose = GQCGT(dst, "transpose", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0, true); CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor2_transpose)); - //step-5: compose qnn graph: add mat_mul node - Qnn_Param_t out_0_params[] = { - {QNN_PARAMTYPE_SCALAR, - QNN_OP_MAT_MUL_PARAM_TRANSPOSE_IN1, - .scalarParam = {QNN_DATATYPE_BOOL_8, .bool8Value = 1} - } - }; - + //compose QNN graph: add mulmat node + Qnn_Param_t out_0_params[] = {{QNN_PARAMTYPE_SCALAR, QNN_OP_MAT_MUL_PARAM_TRANSPOSE_IN1, .scalarParam = {QNN_DATATYPE_BOOL_8, .bool8Value = 1}}}; Qnn_Tensor_t out_0_inputs[] = {*p_tensor0, *p_tensor1}; Qnn_Tensor_t out_0_outputs[] = {*p_tensor2_transpose}; -#if 0 //leave here for easily understand code, can be removed in the future - Qnn_OpConfig_t out_0 = { - QNN_OPCONFIG_VERSION_1, .v1 = - {"ggmlqnn_mulmat_opconfig", QNN_OP_PACKAGE_NAME_QTI_AISW, QNN_OP_MAT_MUL, - 1, - out_0_params, - 2, - out_0_inputs, - 1, - out_0_outputs} - }; -#else - Qnn_OpConfig_t out_0 = ggmlqnn_create_op_config("ggmlqnn_mulmat_opconfig", QNN_OP_PACKAGE_NAME_QTI_AISW, QNN_OP_MAT_MUL, - out_0_params, 1, out_0_inputs, 2, out_0_outputs, 1); -#endif + Qnn_OpConfig_t out_0 = ggmlqnn_create_op_config("mulmat_opconfig", QNN_OP_PACKAGE_NAME_QTI_AISW, QNN_OP_MAT_MUL, out_0_params, 1, out_0_inputs, 2, out_0_outputs, 1); CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle,out_0)); - //step-5: compose qnn graph: add transpose node - Qnn_Param_t out_trans1_0_params[] = { - {QNN_PARAMTYPE_TENSOR, - "perm", .tensorParam = *p_param_tensor - } - }; + //compose QNN graph: add transpose node + Qnn_Param_t out_trans1_0_params[] = { {QNN_PARAMTYPE_TENSOR, "perm", .tensorParam = *p_param_tensor}}; Qnn_Tensor_t out_trans1_0_inputs[] = {*p_tensor2_transpose}; Qnn_Tensor_t out_trans1_0_outputs[] = {*p_tensor2}; -#if 0 //leave here for easily understand code, can be removed in the future - Qnn_OpConfig_t out_trans1_0 = { - QNN_OPCONFIG_VERSION_1, - .v1 = {"ggmlqnn_mulmat_transpose_opconfig", - QNN_OP_PACKAGE_NAME_QTI_AISW, - QNN_OP_TRANSPOSE, 1, - out_trans1_0_params, - 1, - out_trans1_0_inputs, - 1, - out_trans1_0_outputs} - }; -#else - Qnn_OpConfig_t out_trans1_0 = ggmlqnn_create_op_config("ggmlqnn_mulmat_transpose_opconfig", QNN_OP_PACKAGE_NAME_QTI_AISW, QNN_OP_TRANSPOSE, - out_trans1_0_params, 1, out_trans1_0_inputs, 1, out_trans1_0_outputs, 1); -#endif + Qnn_OpConfig_t out_trans1_0 = ggmlqnn_create_op_config("mulmat_transpose_opconfig", QNN_OP_PACKAGE_NAME_QTI_AISW, QNN_OP_TRANSPOSE, out_trans1_0_params, 1, out_trans1_0_inputs, 1, out_trans1_0_outputs, 1); CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle,out_trans1_0)); - //step-6: finalize qnn graph and execute qnn graph + //finalize QNN graph CHECK_QNN_API(error, qnn_raw_interface.graphFinalize(graph_handle, nullptr, nullptr)); - Qnn_Tensor_t input_tensors_0[] = {*p_tensor0, *p_tensor1}; - Qnn_Tensor_t output_tensors_0[] = {*p_tensor2}; - CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, - input_tensors_0, 2, - output_tensors_0, 1, - nullptr, nullptr)); + //cache QNN graph qnn_tensors_t ggml_op_mulmat_tensors; ggml_op_mulmat_tensors.reserve(5); ggml_op_mulmat_tensors.push_back(p_tensor0); @@ -683,35 +537,27 @@ void ggml_qnn_mul_mat(ggml_backend_qnn_context * ctx, ggml_tensor * op) { ggml_op_mulmat_tensors.push_back(p_tensor2_transpose); auto graph_item = std::make_tuple(graph_handle, ggml_op_mulmat_tensors); instance->_qnn_graph_map[graph_name] = graph_item; - } else { - if (src0_type != GGML_TYPE_F32) { - QNN_VER_PTR(*p_tensor0)->clientBuf = {wdata, static_cast(desired_size)}; - } else { - QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggmlqnn_get_tensor_data_size(src0)}; - } - QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggmlqnn_get_tensor_data_size(src1)}; - QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggmlqnn_get_tensor_data_size(dst)}; - - Qnn_Tensor_t tensor_inputs[] = { - *p_tensor0, - *p_tensor1 - }; - Qnn_Tensor_t tensor_outputs[] = { - *p_tensor2 - }; - // this is the second technical approach or another pipeline of "how to utilize the Hexagon - // NPU maximally" through QNN SDK, details could be found at - // https://github.com/ggml-org/llama.cpp/pull/12049#issuecomment-2678308360 - CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, - tensor_inputs, 2, - tensor_outputs, 1, - nullptr, nullptr)); } - // restore the original dimensions of qnn tensors to avoid memory leak in func free_qnn_tensor - QNN_VER_PTR(*p_tensor0)->dimensions = tensor_0_dimensions; - QNN_VER_PTR(*p_tensor1)->dimensions = tensor_1_dimensions; - QNN_VER_PTR(*p_tensor2)->dimensions = tensor_2_dimensions; + if (src0_type != GGML_TYPE_F32) { + QNN_VER_PTR(*p_tensor0)->clientBuf = {wdata, static_cast(desired_size)}; + } else { + QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggmlqnn_get_tensor_data_size(src0)}; + } + QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggmlqnn_get_tensor_data_size(src1)}; + QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggmlqnn_get_tensor_data_size(dst)}; + + Qnn_Tensor_t tensor_inputs[] = { + *p_tensor0, + *p_tensor1 + }; + Qnn_Tensor_t tensor_outputs[] = { + *p_tensor2 + }; + CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, + tensor_inputs, 2, + tensor_outputs, 1, + nullptr, nullptr)); op_perf.info(); } diff --git a/ggml/src/ggml-qnn/ggml-qnn.cpp b/ggml/src/ggml-qnn/ggml-qnn.cpp index a12065f306e56..abfe9135fbf4b 100644 --- a/ggml/src/ggml-qnn/ggml-qnn.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn.cpp @@ -1023,16 +1023,21 @@ Qnn_Tensor_t * ggmlqnn_create_general_tensor(const ggml_tensor * tensor, const c return p_qnn_tensor; } -Qnn_Tensor_t * ggmlqnn_create_compute_tensor(const ggml_tensor * tensor) { - uint32_t dimensions[] = {(uint32_t) tensor->ne[0], (uint32_t) tensor->ne[1], - (uint32_t) tensor->ne[2], (uint32_t) tensor->ne[3]}; +Qnn_Tensor_t * ggmlqnn_create_compute_tensor(qnn_instance * instance, Qnn_GraphHandle_t graph_handle, const ggml_tensor * tensor, Qnn_TensorType_t tensor_type) { + Qnn_ErrorHandle_t error = QNN_SUCCESS; + uint32_t dimensions[] = {(uint32_t) tensor->ne[0], (uint32_t) tensor->ne[1], + (uint32_t) tensor->ne[2], (uint32_t) tensor->ne[3]}; Qnn_DataType_t qnn_data_type = QNN_DATATYPE_FLOAT_32; Qnn_TensorType_t qnn_tensor_type = QNN_TENSOR_TYPE_APP_WRITE; - if (tensor->flags & GGML_TENSOR_FLAG_INPUT) { - qnn_tensor_type = QNN_TENSOR_TYPE_APP_WRITE; - } else if (tensor->flags & GGML_TENSOR_FLAG_OUTPUT) { - qnn_tensor_type = QNN_TENSOR_TYPE_APP_READ; + if (0 == tensor->flags) { + qnn_tensor_type = tensor_type; + } else { + if (tensor->flags & GGML_TENSOR_FLAG_INPUT) { + qnn_tensor_type = QNN_TENSOR_TYPE_APP_WRITE; + } else if (tensor->flags & GGML_TENSOR_FLAG_OUTPUT) { + qnn_tensor_type = QNN_TENSOR_TYPE_APP_READ; + } } qnn_data_type = ggmlqnn_datatype_from_ggml_datatype(tensor->type); @@ -1041,6 +1046,14 @@ Qnn_Tensor_t * ggmlqnn_create_compute_tensor(const ggml_tensor * tensor) { ggml_n_dims(tensor), dimensions, nullptr, 0); + bool enable_npu_rpc = (instance->enable_qnn_rpc() && instance->get_device_id() == QNN_BACKEND_NPU); + if (enable_npu_rpc) { + QNN_VER_PTR(*p_qnn_tensor)->memType = QNN_TENSORMEMTYPE_MEMHANDLE; + QNN_VER_PTR(*p_qnn_tensor)->clientBuf = {.data=nullptr, .dataSize=0}; + } + QNN_INTERFACE_VER_TYPE qnn_raw_interface = instance->get_qnn_raw_interface(); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_qnn_tensor)); + return p_qnn_tensor; } From 0af75f24dd7a9c3d0b593ef44b91ab9247288687 Mon Sep 17 00:00:00 2001 From: zhouwg Date: Thu, 6 Mar 2025 12:06:17 +0800 Subject: [PATCH 59/76] ggml-qnn: Windows port --- step4 --- common/console.cpp | 4 +- ggml/src/ggml-qnn/CMakeLists.txt | 2 + ggml/src/ggml-qnn/ggml-qnn-impl.h | 8 +- ggml/src/ggml-qnn/ggml-qnn-ops.cpp | 2 +- ggml/src/ggml-qnn/ggml-qnn.cpp | 171 +++++++++++++------ scripts/build-run-android-minimal.sh | 240 --------------------------- scripts/build-run-android.sh | 98 +---------- scripts/build-run-windows.sh | 208 +++++++++++++++++++++++ src/llama-mmap.cpp | 8 +- 9 files changed, 346 insertions(+), 395 deletions(-) delete mode 100755 scripts/build-run-android-minimal.sh create mode 100755 scripts/build-run-windows.sh diff --git a/common/console.cpp b/common/console.cpp index 078a8d678d933..73b00aa95de9f 100644 --- a/common/console.cpp +++ b/common/console.cpp @@ -241,7 +241,9 @@ namespace console { (void)codepoint; return 1; #else - return wcwidth(codepoint); + //return wcwidth(codepoint); + (void)codepoint; + return 1; #endif } diff --git a/ggml/src/ggml-qnn/CMakeLists.txt b/ggml/src/ggml-qnn/CMakeLists.txt index 1156c98fbc9d7..8cb75f6cc6fc8 100644 --- a/ggml/src/ggml-qnn/CMakeLists.txt +++ b/ggml/src/ggml-qnn/CMakeLists.txt @@ -6,6 +6,8 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Android") set(QNN_DEFAULT_LIB_SEARCH_PATH "/data/local/tmp/" CACHE STRING "customized library search path for QNN backend") elseif(CMAKE_SYSTEM_NAME STREQUAL "Windows") set(QNN_DEFAULT_LIB_SEARCH_PATH "C:\\" CACHE STRING "customized library search path for QNN backend") +elseif(CMAKE_SYSTEM_NAME STREQUAL "CYGWIN") + set(QNN_DEFAULT_LIB_SEARCH_PATH "/cygdrive/c/qairt/2.31.0.250130/" CACHE STRING "customized library search path for QNN backend") else() message(FATAL_ERROR "QNN now only available on Android and Windows(Windows on ARM)") endif() diff --git a/ggml/src/ggml-qnn/ggml-qnn-impl.h b/ggml/src/ggml-qnn/ggml-qnn-impl.h index a0a1a80cbf855..9d0bf559dd7e2 100644 --- a/ggml/src/ggml-qnn/ggml-qnn-impl.h +++ b/ggml/src/ggml-qnn/ggml-qnn-impl.h @@ -64,8 +64,9 @@ #include "android/log.h" #endif -#if defined(_WIN32) +#if !defined(__ANDROID__) && !defined(__linux__) #include +#include #include #endif @@ -141,7 +142,8 @@ void ggmlqnn_log_internal(ggml_log_level level, const char * file, const char #define GQCGT ggmlqnn_create_general_tensor -#if defined(_WIN32) +//#if defined(_WIN32) +#if !defined(__ANDROID__) && !defined(__linux__) #define RTLD_GLOBAL 0x100 #define RTLD_LOCAL 0x000 #define RTLD_LAZY 0x000 @@ -188,7 +190,7 @@ enum qcom_chipset_soc_model { SM8550 = 43, // v73, SD 8 Gen 2 SM8650 = 57, // v75, SD 8 Gen 3 SM8750 = 69, // v79, SD 8 Gen 4 -#if defined(_MSC_VER) +#if !defined(__ANDROID__) && !defined(__linux__) SC7280X = 44, SC8280X = 37, SC8380XP = 60, diff --git a/ggml/src/ggml-qnn/ggml-qnn-ops.cpp b/ggml/src/ggml-qnn/ggml-qnn-ops.cpp index 8a4ed15529b4c..6ade24315f99a 100644 --- a/ggml/src/ggml-qnn/ggml-qnn-ops.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn-ops.cpp @@ -124,7 +124,7 @@ void ggml_qnn_general_node(ggml_backend_qnn_context * ctx, ggml_tensor * op) { *p_tensor2 }; Qnn_OpConfig_t op_config = { - QNN_OPCONFIG_VERSION_1, .v1 = { + QNN_OPCONFIG_VERSION_1, { ggml_op_name, QNN_OP_PACKAGE_NAME_QTI_AISW, qnn_op_name, diff --git a/ggml/src/ggml-qnn/ggml-qnn.cpp b/ggml/src/ggml-qnn/ggml-qnn.cpp index abfe9135fbf4b..35b565c7d7669 100644 --- a/ggml/src/ggml-qnn/ggml-qnn.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn.cpp @@ -53,6 +53,9 @@ void ggmlqnn_log_internal(ggml_log_level level, const char * file, const char * static char s_ggmlqnn_log_internal_buf[GGML_QNN_LOGBUF_LEN]; GGML_UNUSED(file); +#if !(defined __ANDROID__) || !(defined ANDROID) + GGML_UNUSED(level); +#endif { std::lock_guard lock(ggmlqnn_log_internal_mutex); va_list args; @@ -78,7 +81,7 @@ void ggmlqnn_log_internal(ggml_log_level level, const char * file, const char * // ================================================================================================= // section-3: general helper macro / data structure / function // ================================================================================================= -#if defined(_WIN32) +#if !defined(__ANDROID__) && !defined(__linux__) static const char * last_func = nullptr; static long last_err; void * dlopen(const char * dll, int flags) { @@ -121,6 +124,42 @@ const char * dlerror(void) { } #endif +#define GGMLQNN_MEM_ADD(alignment) (sizeof (size_t) + alignment) +#define GGMLQNN_MEM_MASK(alignment) ((uintptr_t)alignment - 1) +static void * ggmlqnn_mallocz_aligned(size_t size, size_t alignment) { + uint8_t * buffer = NULL; + size_t * sp = NULL; + buffer = static_cast(calloc(1, size + GGMLQNN_MEM_ADD(alignment))); + if (!buffer) + return NULL; + sp = (size_t *)buffer; + *sp = size; + buffer = (uint8_t *)(((uintptr_t) buffer + GGMLQNN_MEM_ADD(alignment)) & ~GGMLQNN_MEM_MASK(alignment)); + buffer[-1] = buffer - (uint8_t *)sp; + return buffer; +} + +static void * ggmlqnn_malloc_aligned(size_t size, size_t alignment) { + uint8_t * buffer = NULL; + size_t * sp = NULL; + buffer = static_cast(malloc(size + GGMLQNN_MEM_ADD(alignment))); + if (!buffer) + return NULL; + sp = (size_t *)buffer; + *sp = size; + buffer = (uint8_t *)(((uintptr_t) buffer + GGMLQNN_MEM_ADD(alignment)) & ~GGMLQNN_MEM_MASK(alignment)); + buffer[-1] = buffer - (uint8_t *)sp; + return buffer; +} + +static void ggmqnn_free_aligned(void * ptr) { + uint8_t * old = (uint8_t *)ptr; + if (!old) + return; + old -= old[-1]; + free(old); +} + static intptr_t ggmlqnn_align_to(size_t alignment, intptr_t offset) { return offset % alignment == 0 ? offset : offset + @@ -134,15 +173,20 @@ static size_t get_system_total_memory_in_bytes() { if (0 == sysinfo(&info)) { return (info.totalram + info.totalswap) * info.mem_unit; } - auto pages = (size_t)sysconf(_SC_PHYS_PAGES); - auto page_size = (size_t)sysconf(_SC_PAGE_SIZE); + size_t pages = (size_t)sysconf(_SC_PHYS_PAGES); + size_t page_size = (size_t)sysconf(_SC_PAGE_SIZE); return pages * page_size; -#elif defined(_WIN32) - //TODO: Snapdragon based WoA(Windows on ARM) - return 0; #else -#error "ggml-qnn only support WoA, Android, Linux" + //FIXME: Snapdragon based WoA(Windows on ARM) + MEMORYSTATUSEX statex; + statex.dwLength = sizeof(statex); + if (GlobalMemoryStatusEx(&statex)) { + GGMLQNN_LOG_INFO("total physical mem:%llu Mb", statex.ullTotalPhys >> 20); + GGMLQNN_LOG_INFO("avail physical mem:%llu Mb", statex.ullAvailPhys >> 20); + return statex.ullTotalPhys; + } + return 0; #endif } @@ -152,15 +196,20 @@ static size_t get_system_free_memory_in_bytes() { if (0 == sysinfo(&info)) { return (info.freeram + info.freeswap) * info.mem_unit; } - auto avail_pages = (size_t)sysconf(_SC_AVPHYS_PAGES); - auto page_size = (size_t)sysconf(_SC_PAGE_SIZE); + size_t avail_pages = (size_t)sysconf(_SC_AVPHYS_PAGES); + size_t page_size = (size_t)sysconf(_SC_PAGE_SIZE); return avail_pages * page_size; -#elif defined(_WIN32) - //TODO: Snapdragon based WoA(Windows on ARM) - return 0; #else -#error "ggml-qnn only support WoA, Android, Linux" + //FIXME: Snapdragon based WoA(Windows on ARM) + MEMORYSTATUSEX statex; + statex.dwLength = sizeof(statex); + if (GlobalMemoryStatusEx(&statex)) { + GGMLQNN_LOG_INFO("total physical mem:%llu Mb", statex.ullTotalPhys >> 20); + GGMLQNN_LOG_INFO("avail physical mem:%llu Mb", statex.ullAvailPhys >> 20); + return statex.ullAvailPhys; + } + return 0; #endif } @@ -176,22 +225,29 @@ static size_t ggmlqnn_memscpy(void * dst, size_t dst_size, const void * src, siz } static char * ggmlqnn_strndup(const char * source, size_t maxlen) { +#if defined(__ANDROID__) || defined(__linux__) return strndup(source, maxlen); +#else + //FIXME:behaviour is not exactly same to Android&Linux + GGML_UNUSED(maxlen); + return strdup(source); +#endif } -static void * ggmlqnn_host_malloc(size_t n) { -#if defined(__ANDROID__) || defined(__linux__) +static void * ggmlqnn_host_malloc(size_t buffer_size, size_t page_size) { void * data = nullptr; - int result = posix_memalign((void **)&data, sysconf(_SC_PAGESIZE), n); +#if defined(__ANDROID__) || defined(__linux__) + int result = posix_memalign((void **)&data, page_size, buffer_size); if (result != 0) { GGMLQNN_LOG_WARN("%s: error: posix_memalign failed\n", __func__); return nullptr; } -#elif defined(_WIN32) - //TODO: Snapdragon based WoA(Windows on ARM) - return nullptr; #else -#error "ggml-qnn only support WoA, Android, Linux" + //GGMLQNN_LOG_DEBUG("buffer_size %d, page_size %d\n", buffer_size, page_size); + data = ggmlqnn_malloc_aligned(buffer_size, page_size); + if (nullptr == data) { + GGMLQNN_LOG_WARN("%s: error: host_malloc failed\n", __func__); + } #endif return data; @@ -566,71 +622,71 @@ Qnn_OpConfig_t ggmlqnn_create_op_config(const char * name, const char * package, //file:///opt/qcom/aistack/qairt/2.31.0.250130/docs/QNN/general/overview.html#tbl-supported-snapdragon-devices static struct qcom_socinfo g_qnn_soc_info_table[] = { /* Qualcomm SnapDragon 7 Gen 1 */ - [SM7450] = { + { .soc_model = SM7450, .htp_arch = V69, .vtcm_size_in_mb = 8, .soc_desc = "Qualcomm SnapDragon 7 Gen 1"}, /* Qualcomm SnapDragon 888 */ - [SM8350] = { + { .soc_model = SM8350, .htp_arch = V68, .vtcm_size_in_mb = 8, .soc_desc = "Qualcomm SnapDragon 888 "}, /* Qualcomm SnapDragon 8 Gen 1 */ - [SM8450] = { + { .soc_model = SM8450, .htp_arch = V69, .vtcm_size_in_mb = 8, .soc_desc = "Qualcomm SnapDragon 8 Gen 1"}, /* Qualcomm SnapDragon 8 Gen 1+ */ - [SM8475] = { + { .soc_model = SM8475, .htp_arch = V69, .vtcm_size_in_mb = 8, .soc_desc = "Qualcomm SnapDragon 8 Gen 1+"}, /* Qualcomm SnapDragon 8 Gen 2 */ - [SM8550] = { + { .soc_model = SM8550, .htp_arch = V73, .vtcm_size_in_mb = 8, .soc_desc = "Qualcomm SnapDragon 8 Gen 2"}, /* Qualcomm SnapDragon 8 Gen 3 */ - [SM8650] = { + { .soc_model = SM8650, .htp_arch = V75, .vtcm_size_in_mb = 8, .soc_desc = "Qualcomm SnapDragon 8 Gen 3 "}, /* Qualcomm SnapDragon 8 Gen 4 */ - [SM8750] = { + { .soc_model = SM8750, .htp_arch = V79, .vtcm_size_in_mb = 8, .soc_desc = "Qualcomm SnapDragon 8 Gen 4"}, -#if defined(_WIN32) +#if !defined(__ANDROID__) && !defined(__linux__) /* Qualcomm SnapDragon 7c Gen 2 */ - [SC7280X] = { + { .soc_model = SC7280X, .htp_arch = V68, .vtcm_size_in_mb = 8, .soc_desc = "Qualcomm SnapDragon 7c Gen 2"}, /* Qualcomm SnapDragon 8cx Gen 3 */ - [SC8280X] = { + { .soc_model = SC8280X, .htp_arch = V68, .vtcm_size_in_mb = 8, .soc_desc = "Qualcomm SnapDragon 8cx Gen 3"}, /* Qualcomm SnapDragon 8cx Gen 4 */ - [SC8380XP] = { + { .soc_model = SC8380XP, .htp_arch = V73, .vtcm_size_in_mb = 8, @@ -639,6 +695,16 @@ static struct qcom_socinfo g_qnn_soc_info_table[] = { }; + +#if defined(__ANDROID__) +static const char * g_qnn_runtimelib_path = "/data/local/tmp/"; +#elif defined(__linux__) +static const char * g_qnn_runtimelib_path = "/tmp/"; +#elif defined(_WIN32) +static const char * g_qnn_runtimelib_path = "C:\\"; +#else //cygwin on Windows +static const char * g_qnn_runtimelib_path = "/cygdrive/c/"; +#endif //the following helper funcs are used to ensure every QNN tensor name is unique static std::atomic g_ggmltensor_idx(0); static void reset_idx() { @@ -664,7 +730,7 @@ static struct ggml_backend_qnn_context g_qnn_mgr[GGML_QNN_MAX_DEVICES] = { .threads = 1, .name = "qnn-cpu", .desc = "Qualcomm Kryo CPU", -#if defined(_WIN32) +#if !defined(__ANDROID__) && !defined(__linux__) .lib = "QnnCpu.dll", #else .lib = "libQnnCpu.so", @@ -679,7 +745,7 @@ static struct ggml_backend_qnn_context g_qnn_mgr[GGML_QNN_MAX_DEVICES] = { .threads = 1, .name = "qnn-gpu", .desc = "Qualcomm Adreno GPU", -#if defined(_WIN32) +#if !defined(__ANDROID__) && !defined(__linux__) .lib = "QnnGpu.dll", #else .lib = "libQnnGpu.so", @@ -694,7 +760,7 @@ static struct ggml_backend_qnn_context g_qnn_mgr[GGML_QNN_MAX_DEVICES] = { .threads = 1, .name = "qnn-npu", .desc = "Qualcomm NPU(Hexagon Tensor Processor)", -#if defined(_WIN32) +#if !defined(__ANDROID__) && !defined(__linux__) .lib = "QnnHtp.dll", #else .lib = "libQnnHtp.so", @@ -856,7 +922,7 @@ static const char * qnn_get_htparch_desc(size_t htp_arch) { } } -static struct qcom_socinfo * qnn_get_socinfo_from_socmodel(uint32_t soc_model) { +static struct qcom_socinfo * ggmlqnn_get_socinfo_from_socmodel(uint32_t soc_model) { size_t items = sizeof(g_qnn_soc_info_table) / sizeof(g_qnn_soc_info_table[0]); for (size_t idx = 0; idx < items; idx++) { if (soc_model == g_qnn_soc_info_table[idx].soc_model) { @@ -1538,7 +1604,7 @@ int qnn_instance::unload_backend() { int qnn_instance::load_system() { Qnn_ErrorHandle_t error = QNN_SUCCESS; -#ifdef _WIN32 +#if !defined(__ANDROID__) && !defined(__linux__) std::string system_lib_path = _lib_path + "QnnSystem.dll"; #else std::string system_lib_path = _lib_path + "libQnnSystem.so"; @@ -1549,8 +1615,8 @@ int qnn_instance::load_system() { if (nullptr == _system_lib_handle) { GGMLQNN_LOG_WARN("can not open QNN library %s, error: %s\n", system_lib_path.c_str(), dlerror()); //re-try with default path of QNN binary runtime lib - _lib_path = "/data/local/tmp/"; -#ifdef _WIN32 + _lib_path = std::string(g_qnn_runtimelib_path); +#if !defined(__ANDROID__) && !defined(__linux__) system_lib_path = _lib_path + "QnnSystem.dll"; #else system_lib_path = _lib_path + "libQnnSystem.so"; @@ -1804,10 +1870,8 @@ int qnn_instance::qnn_init(const QnnSaver_Config_t ** saver_config) { #if defined(__ANDROID__) || defined(__linux__) _rpc_lib_handle = dlopen("libcdsprpc.so", RTLD_NOW | RTLD_LOCAL); -#elif defined(_WIN32) - _rpc_lib_handle = dlopen("libcdsprpc.dll", RTLD_NOW | RTLD_LOCAL); #else -#error "ggml-qnn only support WoA, Android, Linux" + _rpc_lib_handle = dlopen("libcdsprpc.dll", RTLD_NOW | RTLD_LOCAL); #endif if (nullptr == _rpc_lib_handle) { GGMLQNN_LOG_WARN("failed to load qualcomm's rpc lib, error:%s\n", dlerror()); @@ -1842,7 +1906,7 @@ int qnn_instance::qnn_init(const QnnSaver_Config_t ** saver_config) { GGMLQNN_LOG_DEBUG("initialize qnn context successfully\n"); } - if (_backend_name.find("Htp") != std::variant_npos) { + if (_backend_name.find("Htp") != std::string::npos) { const QnnDevice_PlatformInfo_t * p_info = nullptr; _qnn_raw_interface.deviceGetPlatformInfo(nullptr, &p_info); GGMLQNN_LOG_INFO("device counts %d", p_info->v1.numHwDevices); @@ -1858,7 +1922,7 @@ int qnn_instance::qnn_init(const QnnSaver_Config_t ** saver_config) { GGMLQNN_LOG_INFO("qualcomm soc_model:%d(%s), htp_arch:%d(%s), vtcm_size:%d MB", \ chipinfo.socModel, qnn_get_socmodel_desc(chipinfo.socModel), \ htp_arch, qnn_get_htparch_desc(htp_arch), chipinfo.vtcmSize); - struct qcom_socinfo * socinfo = qnn_get_socinfo_from_socmodel(chipinfo.socModel); + struct qcom_socinfo * socinfo = ggmlqnn_get_socinfo_from_socmodel(chipinfo.socModel); g_qnn_mgr[QNN_BACKEND_NPU].socinfo = { chipinfo.socModel, htp_arch, chipinfo.vtcmSize, {}}; if (nullptr != socinfo) { memcpy(g_qnn_mgr[QNN_BACKEND_NPU].socinfo.soc_desc, socinfo->soc_desc, sizeof(socinfo->soc_desc)); @@ -2546,22 +2610,25 @@ static ggml_backend_buffer_t ggml_backend_qnn_buffer_type_alloc_buffer( ggml_backend_buffer_type_t buft, size_t size) { ggml_backend_qnn_buffer_context * ctx = new ggml_backend_qnn_buffer_context; + size_t size_page = 0; #if defined(__ANDROID__) || defined(__linux__) - size_t size_page = sysconf(_SC_PAGESIZE); -#elif defined(_WIN32) + size_page = sysconf(_SC_PAGESIZE); +#else SYSTEM_INFO systeminfo; GetSystemInfo(&systeminfo); - size_t size_page = systeminfo.dwPageSize; + size_page = systeminfo.dwPageSize; #endif size_t size_aligned = size; if ((size_aligned % size_page) != 0) { size_aligned += (size_page - (size_aligned % size_page)); } - ctx->buffer = ggmlqnn_host_malloc(size_aligned); + ctx->buffer = ggmlqnn_host_malloc(size_aligned, size_page); ctx->buffer_size = size_aligned; if (nullptr == ctx->buffer) { - GGMLQNN_LOG_WARN("%s: failed to allocate %.2f MiB\n", __func__, size / (1 << 20)); + GGMLQNN_LOG_WARN("%s: failed to allocate %d MiB\n", __func__, size / (1 << 20)); return nullptr; + } else { + GGMLQNN_LOG_DEBUG("%s: allocate %d MiB\n", __func__, size_aligned / (1 << 20)); } return ggml_backend_buffer_init(buft, ggml_backend_qnn_buffer_interface, ctx, size); @@ -2572,11 +2639,10 @@ static size_t ggml_backend_qnn_buffer_type_get_alignment(ggml_backend_buffer_typ return 32; } -//TODO:not used currently static size_t ggml_backend_qnn_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) { GGML_UNUSED(buft); - return (2 * (1 << 20)); + return (2 * (1 << 29)); } static bool ggml_backend_qnn_buffer_is_host(ggml_backend_buffer_type_t buft) { @@ -2724,8 +2790,7 @@ static ggml_backend_t ggml_backend_qnn_device_init_backend(ggml_backend_dev_t de if (nullptr == params) { params = 0; } - ggml_backend_t qnn_backend = ggml_backend_qnn_init((int) (intptr_t) params, - "/data/local/tmp/"); + ggml_backend_t qnn_backend = ggml_backend_qnn_init((int)(intptr_t)params, g_qnn_runtimelib_path); return qnn_backend; @@ -2867,7 +2932,7 @@ static void * ggml_backend_qnn_reg_get_proc_address(ggml_backend_reg_t reg, cons const char * slot_name = "ggml_backend_set_n_threads"; //avoid buffer attack rather than strcmp - if (0 == std::memcmp(name, slot_name, strlen(slot_name))) { + if (0 == memcmp(name, slot_name, strlen(slot_name))) { return (void *)ggml_backend_qnn_set_n_threads; } return nullptr; diff --git a/scripts/build-run-android-minimal.sh b/scripts/build-run-android-minimal.sh deleted file mode 100755 index 1a5f362fe2083..0000000000000 --- a/scripts/build-run-android-minimal.sh +++ /dev/null @@ -1,240 +0,0 @@ -#!/bin/bash - -set -e - -PWD=`pwd` -ANDROID_PLATFORM=android-34 -ANDROID_NDK=${PWD}/android-ndk-r26c -REMOTE_PATH=/data/local/tmp/ -GGUF_MODEL_NAME=/sdcard/deepseek-r1-distill-qwen-1.5b-q4_0.gguf -GGUF_MODEL_NAME=/sdcard/qwen1_5-1_8b-chat-q4_0.gguf - -#QNN SDK could be found at: -#https://www.qualcomm.com/developer/software/qualcomm-ai-engine-direct-sdk -#https://developer.qualcomm.com/software/hexagon-dsp-sdk/tools -QNN_SDK_URL=https://www.qualcomm.com/developer/software/qualcomm-ai-engine-direct-sdk -QNN_SDK_PATH=/opt/qcom/aistack/qairt/2.31.0.250130/ - -#default is QNN NPU -qnnbackend=2 - -function dump_vars() -{ - echo -e "ANDROID_NDK: ${ANDROID_NDK}" - echo -e "QNN_SDK_PATH: ${QNN_SDK_PATH}" -} - - -function show_pwd() -{ - echo -e "current working path:$(pwd)\n" -} - - -function check_qnn_sdk() -{ - if [ ! -d ${QNN_SDK_PATH} ]; then - echo -e "QNN_SDK_PATH ${QNN_SDK_PATH} not exist, pls check or download it from ${QNN_SDK_URL}...\n" - exit 1 - fi -} - - -function check_and_download_ndk() -{ - is_android_ndk_exist=1 - - if [ ! -d ${ANDROID_NDK} ]; then - is_android_ndk_exist=0 - fi - - if [ ! -f ${ANDROID_NDK}/build/cmake/android.toolchain.cmake ]; then - is_android_ndk_exist=0 - fi - - if [ ${is_android_ndk_exist} -eq 0 ]; then - - if [ ! -f android-ndk-r26c-linux.zip ]; then - wget --no-config --quiet --show-progress -O android-ndk-r26c-linux.zip https://dl.google.com/android/repository/android-ndk-r26c-linux.zip - fi - - unzip android-ndk-r26c-linux.zip - - if [ $? -ne 0 ]; then - printf "failed to download android ndk to %s \n" "${ANDROID_NDK}" - exit 1 - fi - - printf "android ndk saved to ${ANDROID_NDK} \n\n" - else - printf "android ndk already exist:${ANDROID_NDK} \n\n" - fi -} - - -function build_arm64 -{ - cmake -H. -B./out/android -DCMAKE_BUILD_TYPE=Release -DGGML_USE_QNN=ON -DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK}/build/cmake/android.toolchain.cmake -DANDROID_ABI=arm64-v8a -DANDROID_PLATFORM=latest -DCMAKE_C_FLAGS=-march=armv8.7-a -DGGML_QNN=ON -DGGML_QNN_SDK_PATH=${QNN_SDK_PATH} - cd out/android - make -j16 - show_pwd - - cd - -} - - -function remove_temp_dir() -{ - if [ -d out ]; then - echo "remove out directory in `pwd`" - rm -rf out - fi -} - - -function check_qnn_libs() -{ - #reuse the cached qnn libs on Android phone - adb shell ls ${REMOTE_PATH}/libQnnCpu.so - if [ $? -eq 0 ]; then - printf "QNN libs already exist on Android phone\n" - else - update_qnn_libs - fi -} - - -function update_qnn_libs() -{ - adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnSystem.so ${REMOTE_PATH}/ - adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnCpu.so ${REMOTE_PATH}/ - adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnGpu.so ${REMOTE_PATH}/ - - adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnHtp.so ${REMOTE_PATH}/ - adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnHtpNetRunExtensions.so ${REMOTE_PATH}/ - adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnHtpPrepare.so ${REMOTE_PATH}/ - adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnHtpV75Stub.so ${REMOTE_PATH}/ - adb push ${QNN_SDK_PATH}/lib/hexagon-v75/unsigned/libQnnHtpV75Skel.so ${REMOTE_PATH}/ -} - - -function build_ggml_qnn() -{ - show_pwd - check_and_download_ndk - check_qnn_sdk - dump_vars - remove_temp_dir - build_arm64 -} - - -function run_llamacli() -{ - check_qnn_libs - - if [ -f ./out/android/bin/libggml-qnn.so ]; then - adb push ./out/android/bin/*.so ${REMOTE_PATH}/ - fi - adb push ./out/android/bin/llama-cli ${REMOTE_PATH}/ - adb shell chmod +x ${REMOTE_PATH}/llama-cli - - adb shell "cd ${REMOTE_PATH} \ - && export LD_LIBRARY_PATH=${REMOTE_PATH} \ - && ${REMOTE_PATH}/llama-cli -mg ${qnnbackend} -no-cnv -m ${GGUF_MODEL_NAME} -p \"introduce the movie Once Upon a Time in America briefly.\n\"" - -} - - -function run_llamabench() -{ - check_qnn_libs - - if [ -f ./out/android/bin/libggml-qnn.so ]; then - adb push ./out/android/bin/*.so ${REMOTE_PATH}/ - fi - adb push ./out/android/bin/llama-bench ${REMOTE_PATH}/ - adb shell chmod +x ${REMOTE_PATH}/llama-bench - - adb shell "cd ${REMOTE_PATH} \ - && export LD_LIBRARY_PATH=${REMOTE_PATH} \ - && ${REMOTE_PATH}/llama-bench -mg ${qnnbackend} -m ${GGUF_MODEL_NAME}" - -} - - -function run_test-backend-ops() -{ - check_qnn_libs - - if [ -f ./out/android/bin/libggml-qnn.so ]; then - adb push ./out/android/bin/*.so ${REMOTE_PATH}/ - fi - adb push ./out/android/bin/test-backend-ops ${REMOTE_PATH}/ - adb shell chmod +x ${REMOTE_PATH}/test-backend-ops - - adb shell "cd ${REMOTE_PATH} \ - && export LD_LIBRARY_PATH=${REMOTE_PATH} \ - && ${REMOTE_PATH}/test-backend-ops test" - -} - - -function show_usage() -{ - echo "Usage:" - echo " $0 build" - echo " $0 updateqnnlib" - echo " $0 run_testop" - echo " $0 run_llamacli 0 (QNN_CPU) / 1 (QNN_GPU) / 2 (QNN_NPU) / 3 (ggml)" - echo " $0 run_llamabench 0 (QNN_CPU) / 1 (QNN_GPU) / 2 (QNN_NPU) / 3 (ggml)" - echo -e "\n\n\n" -} - - -show_pwd - -check_qnn_sdk - -if [ $# == 0 ]; then - show_usage - exit 1 -elif [ $# == 1 ]; then - if [ "$1" == "-h" ]; then - show_usage - exit 1 - elif [ "$1" == "help" ]; then - show_usage - exit 1 - elif [ "$1" == "build" ]; then - build_ggml_qnn - exit 0 - - elif [ "$1" == "run_testop" ]; then - run_test-backend-ops - exit 0 - elif [ "$1" == "updateqnnlib" ]; then - update_qnn_libs - exit 0 - else - show_usage - exit 1 - fi -elif [ $# == 2 ]; then - qnnbackend=$2 - if [ ${qnnbackend} -gt 3 ]; then - show_usage - exit 1 - fi - - if [ "$1" == "run_llamacli" ]; then - run_llamacli - exit 0 - elif [ "$1" == "run_llamabench" ]; then - run_llamabench - exit 0 - fi -else - show_usage - exit 1 -fi diff --git a/scripts/build-run-android.sh b/scripts/build-run-android.sh index 2ed8db9349003..cc2828389fd16 100755 --- a/scripts/build-run-android.sh +++ b/scripts/build-run-android.sh @@ -85,9 +85,9 @@ function build_arm64 function remove_temp_dir() { - if [ -d out ]; then - echo "remove out directory in `pwd`" - rm -rf out + if [ -d out/android ]; then + echo "remove out/android directory in `pwd`" + rm -rf out/android fi } @@ -168,6 +168,7 @@ function run_llamabench() } +#refer to:https://github.com/ggml-org/llama.cpp/pull/12155 function run_test-ops() { prepare_run_on_phone test-backend-ops @@ -397,96 +398,7 @@ elif [ $# == 3 ]; then opname=$2 #TODO: check opname in oplist #opname can be found via print_oplist: -# DUP -# ADD -# ADD1 -# ACC -# SUB -# MUL -# DIV -# SQR -# SQRT -# LOG -# SIN -# COS -# SUM -# SUM_ROWS -# MEAN -# ARGMAX -# COUNT_EQUAL -# REPEAT -# REPEAT_BACK -# CONCAT -# SILU_BACK -# NORM -# RMS_NORM -# RMS_NORM_BACK -# GROUP_NORM -# -# MUL_MAT -# MUL_MAT_ID -# OUT_PROD -# -# SCALE -# SET -# CPY -# CONT -# RESHAPE -# VIEW -# PERMUTE -# TRANSPOSE -# GET_ROWS -# GET_ROWS_BACK -# DIAG -# DIAG_MASK_INF -# DIAG_MASK_ZERO -# SOFT_MAX -# SOFT_MAX_BACK -# ROPE -# ROPE_BACK -# CLAMP -# CONV_TRANSPOSE_1D -# IM2COL -# IM2COL_BACK -# CONV_TRANSPOSE_2D -# POOL_1D -# POOL_2D -# POOL_2D_BACK -# UPSCALE -# PAD -# PAD_REFLECT_1D -# ARANGE -# TIMESTEP_EMBEDDING -# ARGSORT -# LEAKY_RELU -# -# FLASH_ATTN_EXT -# FLASH_ATTN_BACK -# SSM_CONV -# SSM_SCAN -# WIN_PART -# WIN_UNPART -# GET_REL_POS -# ADD_REL_POS -# RWKV_WKV6 -# GATED_LINEAR_ATTN -# -# UNARY -# -# MAP_UNARY -# MAP_BINARY -# -# MAP_CUSTOM1_F32 -# MAP_CUSTOM2_F32 -# MAP_CUSTOM3_F32 -# -# MAP_CUSTOM1 -# MAP_CUSTOM2 -# MAP_CUSTOM3 -# -# CROSS_ENTROPY_LOSS -# CROSS_ENTROPY_LOSS_BACK -# OPT_STEP_ADAMW + qnnbackend=$3 if [ ${qnnbackend} -gt 3 ]; then show_usage diff --git a/scripts/build-run-windows.sh b/scripts/build-run-windows.sh new file mode 100755 index 0000000000000..4c7cad1aeb111 --- /dev/null +++ b/scripts/build-run-windows.sh @@ -0,0 +1,208 @@ +#!/bin/bash + +set -e + +PWD=`pwd` +PREFIX_PATH=/cygdrive/c +GGUF_MODEL_NAME=${PREFIX_PATH}/qwen1_5-1_8b-chat-q4_0.gguf + +#QNN SDK could be found at: +#https://www.qualcomm.com/developer/software/qualcomm-ai-engine-direct-sdk +#https://developer.qualcomm.com/software/hexagon-dsp-sdk/tools +QNN_SDK_URL=https://www.qualcomm.com/developer/software/qualcomm-ai-engine-direct-sdk +QNN_SDK_PATH=${PREFIX_PATH}/qairt/2.31.0.250130/ + +#default is QNN NPU +qnnbackend=2 + +function dump_vars() +{ + echo -e "QNN_SDK_PATH: ${QNN_SDK_PATH}" +} + + +function show_pwd() +{ + echo -e "current working path:$(pwd)\n" +} + + +function check_qnn_sdk() +{ + if [ ! -d ${QNN_SDK_PATH} ]; then + echo -e "QNN_SDK_PATH ${QNN_SDK_PATH} not exist, pls check or download it from ${QNN_SDK_URL}...\n" + exit 1 + fi +} + +function build_windows_x86 +{ + echo "build_windows_x86-without-qnn" + cmake -H. -B./out/windows_x86 -DCMAKE_BUILD_TYPE=Release + cd out/windows_x86 + make -j16 + show_pwd + + cd - +} + +function build_windows_x86_qnn +{ + echo "build_windows_x86-with-qnn" + cmake -H. -B./out/windows_x86_qnn -DCMAKE_BUILD_TYPE=Release -DGGML_USE_QNN=ON -DGGML_QNN=ON -DGGML_QNN_SDK_PATH=${QNN_SDK_PATH} + cd out/windows_x86_qnn + make -j16 + show_pwd + + cd - +} + +function build_windows_arm64_qnn +{ + echo "build_windows_arm64 not supported now" + #cmake -H. -B./out/windows_arm64_qnn -DCMAKE_BUILD_TYPE=Release -DGGML_USE_QNN=ON -DCMAKE_TOOLCHAIN_FILE=${MSSDK}/cmake/arm64-windows-llvm.cmake -DCMAKE_C_FLAGS=-march=armv8.7-a -DGGML_QNN=ON -DGGML_QNN_SDK_PATH=${QNN_SDK_PATH} +} + + +function remove_temp_dir() +{ + if [ -d out/windows_x86 ]; then + echo "remove out/windows_x86 directory in `pwd`" + rm -rf out/windows_x86 + fi +} + + +function check_qnn_libs() +{ + echo "do nothing" +} + + +function update_qnn_libs() +{ + echo "do nothing" +} + +function build_x86() +{ + show_pwd + check_qnn_sdk + dump_vars + #some unexpected behaviour on Windows + #remove_temp_dir + build_windows_x86 +} + +function build_x86_qnn() +{ + show_pwd + check_qnn_sdk + dump_vars + #some unexpected behaviour on Windows + #remove_temp_dir + build_windows_x86_qnn +} + +function build_arm64_qnn() +{ + show_pwd + check_qnn_sdk + dump_vars + #some unexpected behaviour on Windows + #remove_temp_dir + build_windows_arm64_qnn +} + +function run_llamacli() +{ + check_qnn_libs + echo "not supported on Windows now" + + #llama-cli -mg ${qnnbackend} -no-cnv -m ${GGUF_MODEL_NAME} -p \"introduce the movie Once Upon a Time in America briefly.\n\" + +} + + +function run_llamabench() +{ + check_qnn_libs + echo "not supported on Windows now" + + #llama-bench -mg ${qnnbackend} -m ${GGUF_MODEL_NAME}" + +} + + +function run_test-backend-ops() +{ + check_qnn_libs + echo "not supported on Windows now" + + #test-backend-ops test" + +} + + +function show_usage() +{ + echo "Usage:" + echo " $0 build_x86" + echo " $0 build_x86_qnn" + echo " $0 build_arm64_qnn" + echo " $0 run_testop" + echo " $0 run_llamacli 0 (QNN_CPU) / 1 (QNN_GPU) / 2 (QNN_NPU) / 3 (ggml)" + echo " $0 run_llamabench 0 (QNN_CPU) / 1 (QNN_GPU) / 2 (QNN_NPU) / 3 (ggml)" + echo -e "\n\n\n" +} + + +show_pwd + +check_qnn_sdk + +if [ $# == 0 ]; then + show_usage + exit 1 +elif [ $# == 1 ]; then + if [ "$1" == "-h" ]; then + show_usage + exit 1 + elif [ "$1" == "help" ]; then + show_usage + exit 1 + elif [ "$1" == "build_x86" ]; then + build_x86 + exit 0 + elif [ "$1" == "build_x86_qnn" ]; then + build_x86_qnn + exit 0 + elif [ "$1" == "build_arm64_qnn" ]; then + build_arm64_qnn + exit 0 + + elif [ "$1" == "run_testop" ]; then + run_test-backend-ops + exit 0 + else + show_usage + exit 1 + fi +elif [ $# == 2 ]; then + qnnbackend=$2 + if [ ${qnnbackend} -gt 3 ]; then + show_usage + exit 1 + fi + + if [ "$1" == "run_llamacli" ]; then + run_llamacli + exit 0 + elif [ "$1" == "run_llamabench" ]; then + run_llamabench + exit 0 + fi +else + show_usage + exit 1 +fi diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpp index 9da97f1bc5057..7345eee2ea989 100644 --- a/src/llama-mmap.cpp +++ b/src/llama-mmap.cpp @@ -481,10 +481,10 @@ struct llama_mlock::impl { // Skip resource limit checks on visionOS/tvOS suggest = false; #else - struct rlimit lock_limit; - if (suggest && getrlimit(RLIMIT_MEMLOCK, &lock_limit)) { - suggest = false; - } + struct rlimit lock_limit = {}; + //if (suggest && getrlimit(RLIMIT_MEMLOCK, &lock_limit)) { + // suggest = false; + //} if (suggest && (lock_limit.rlim_max > lock_limit.rlim_cur + size)) { suggest = false; } From 38026639889d7486a9652020d679c576162dd69e Mon Sep 17 00:00:00 2001 From: zhouwg Date: Fri, 7 Mar 2025 15:03:52 +0800 Subject: [PATCH 60/76] ggml-qnn: Windows port -- step5 --- cmake/aarch64-w64-mingw32.cmake | 18 ++++++++++++++++++ ggml/src/ggml-qnn/CMakeLists.txt | 1 + scripts/build-run-android.sh | 1 + scripts/build-run-windows.sh | 20 +++++++++++++++++--- 4 files changed, 37 insertions(+), 3 deletions(-) create mode 100644 cmake/aarch64-w64-mingw32.cmake diff --git a/cmake/aarch64-w64-mingw32.cmake b/cmake/aarch64-w64-mingw32.cmake new file mode 100644 index 0000000000000..775fa46337628 --- /dev/null +++ b/cmake/aarch64-w64-mingw32.cmake @@ -0,0 +1,18 @@ +#TODO +#not work on Linux +set( CMAKE_SYSTEM_NAME mingw ) +set( CMAKE_SYSTEM_PROCESSOR arm64 ) + +set( target aarch64-w64-mingw32 ) + +set( CMAKE_C_COMPILER aarch64-w64-mingw32-gcc ) +set( CMAKE_CXX_COMPILER aarch64-w64-mingw32-g++ ) + +set( CMAKE_C_COMPILER_TARGET ${target} ) +set( CMAKE_CXX_COMPILER_TARGET ${target} ) + +#set( arch_c_flags "-march=armv8.7-a -fvectorize -ffp-model=fast -fno-finite-math-only" ) +#set( warn_c_flags "-Wno-format -Wno-unused-variable -Wno-unused-function -Wno-gnu-zero-variadic-macro-arguments" ) + +set( CMAKE_C_FLAGS_INIT "${arch_c_flags} ${warn_c_flags}" ) +set( CMAKE_CXX_FLAGS_INIT "${arch_c_flags} ${warn_c_flags}" ) diff --git a/ggml/src/ggml-qnn/CMakeLists.txt b/ggml/src/ggml-qnn/CMakeLists.txt index 8cb75f6cc6fc8..c11e2f82fa92b 100644 --- a/ggml/src/ggml-qnn/CMakeLists.txt +++ b/ggml/src/ggml-qnn/CMakeLists.txt @@ -23,6 +23,7 @@ endif() message("QNN_SDK_PATH: ${GGML_QNN_SDK_PATH}") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DGGML_USE_QNN") set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3") file(GLOB QNN_SOURCES "${CMAKE_CURRENT_LIST_DIR}/*.cpp") diff --git a/scripts/build-run-android.sh b/scripts/build-run-android.sh index cc2828389fd16..5e69024298dbe 100755 --- a/scripts/build-run-android.sh +++ b/scripts/build-run-android.sh @@ -1,4 +1,5 @@ #!/bin/bash +# build llama.cpp + ggml-qnn for Snapdragon mobile SoC equipped Android phone on Linux set -e diff --git a/scripts/build-run-windows.sh b/scripts/build-run-windows.sh index 4c7cad1aeb111..8221293a431d4 100755 --- a/scripts/build-run-windows.sh +++ b/scripts/build-run-windows.sh @@ -1,10 +1,16 @@ #!/bin/bash +# build llama.cpp or llama.cpp + ggml-qnn for Windows with cygwin on Windows +# build llama.cpp + ggml-qnn for Snapdragon desktop SoC equipped WoA(Windows on ARM) with cygwin on Windows + +# items marked TODO has not verified yet set -e + PWD=`pwd` PREFIX_PATH=/cygdrive/c GGUF_MODEL_NAME=${PREFIX_PATH}/qwen1_5-1_8b-chat-q4_0.gguf +PROJECT_HOME_PATH=`pwd` #QNN SDK could be found at: #https://www.qualcomm.com/developer/software/qualcomm-ai-engine-direct-sdk @@ -38,7 +44,7 @@ function check_qnn_sdk() function build_windows_x86 { echo "build_windows_x86-without-qnn" - cmake -H. -B./out/windows_x86 -DCMAKE_BUILD_TYPE=Release + cmake -H. -B./out/windows_x86 -DCMAKE_BUILD_TYPE=Release -DGGML_OPENMP=OFF cd out/windows_x86 make -j16 show_pwd @@ -49,7 +55,7 @@ function build_windows_x86 function build_windows_x86_qnn { echo "build_windows_x86-with-qnn" - cmake -H. -B./out/windows_x86_qnn -DCMAKE_BUILD_TYPE=Release -DGGML_USE_QNN=ON -DGGML_QNN=ON -DGGML_QNN_SDK_PATH=${QNN_SDK_PATH} + cmake -H. -B./out/windows_x86_qnn -DCMAKE_BUILD_TYPE=Release -DGGML_OPENMP=OFF -DGGML_QNN=ON -DGGML_QNN_SDK_PATH=${QNN_SDK_PATH} cd out/windows_x86_qnn make -j16 show_pwd @@ -57,10 +63,18 @@ function build_windows_x86_qnn cd - } +#TODO function build_windows_arm64_qnn { echo "build_windows_arm64 not supported now" - #cmake -H. -B./out/windows_arm64_qnn -DCMAKE_BUILD_TYPE=Release -DGGML_USE_QNN=ON -DCMAKE_TOOLCHAIN_FILE=${MSSDK}/cmake/arm64-windows-llvm.cmake -DCMAKE_C_FLAGS=-march=armv8.7-a -DGGML_QNN=ON -DGGML_QNN_SDK_PATH=${QNN_SDK_PATH} + return 0 + echo "cmake source dir:${PROJECT_HOME_PATH}" + cmake -H. -B./out/windows_arm64_qnn -DCMAKE_BUILD_TYPE=Release -DGGML_OPENMP=OFF -DGGML_QNN=ON -DCMAKE_TOOLCHAIN_FILE=${PROJECT_HOME_PATH}/cmake/arm64-windows-llvm.cmake -DCMAKE_C_FLAGS=-march=armv8.7-a -DGGML_QNN_SDK_PATH=${QNN_SDK_PATH} + cd out/windows_arm64_qnn + make -j16 + show_pwd + + cd - } From 05a58549cbcd3f26ee6d771933ed485220675bfa Mon Sep 17 00:00:00 2001 From: zhouwg Date: Sat, 8 Mar 2025 08:09:02 +0800 Subject: [PATCH 61/76] ggml-qnn: WoA(Windows on ARM) -- step6 --- cmake/arm64-windows-cygwin.cmake | 16 ++++++++++++++++ cmake/arm64-windows-llvm.cmake | 4 ++-- scripts/build-run-windows.sh | 2 +- 3 files changed, 19 insertions(+), 3 deletions(-) create mode 100644 cmake/arm64-windows-cygwin.cmake diff --git a/cmake/arm64-windows-cygwin.cmake b/cmake/arm64-windows-cygwin.cmake new file mode 100644 index 0000000000000..c7a313bb77adf --- /dev/null +++ b/cmake/arm64-windows-cygwin.cmake @@ -0,0 +1,16 @@ +set( CMAKE_SYSTEM_NAME CYGWIN) +set( CMAKE_SYSTEM_PROCESSOR arm64 ) + +set( target aarch64-w64-cygwin) + +set( CMAKE_C_COMPILER clang ) +set( CMAKE_CXX_COMPILER clang++ ) + +set( CMAKE_C_COMPILER_TARGET ${target} ) +set( CMAKE_CXX_COMPILER_TARGET ${target} ) + +set( arch_c_flags "-march=armv8.7-a -fvectorize -ffp-model=fast -fno-finite-math-only" ) +set( warn_c_flags "-Wno-format -Wno-unused-variable -Wno-unused-function -Wno-gnu-zero-variadic-macro-arguments" ) + +set( CMAKE_C_FLAGS_INIT "${arch_c_flags} ${warn_c_flags}" ) +set( CMAKE_CXX_FLAGS_INIT "${arch_c_flags} ${warn_c_flags}" ) diff --git a/cmake/arm64-windows-llvm.cmake b/cmake/arm64-windows-llvm.cmake index 8023796800683..983206032df3d 100644 --- a/cmake/arm64-windows-llvm.cmake +++ b/cmake/arm64-windows-llvm.cmake @@ -9,8 +9,8 @@ set( CMAKE_CXX_COMPILER clang++ ) set( CMAKE_C_COMPILER_TARGET ${target} ) set( CMAKE_CXX_COMPILER_TARGET ${target} ) -set( arch_c_flags "-march=armv8.7-a -fvectorize -ffp-model=fast -fno-finite-math-only" ) -set( warn_c_flags "-Wno-format -Wno-unused-variable -Wno-unused-function -Wno-gnu-zero-variadic-macro-arguments" ) +#set( arch_c_flags "-march=armv8.7-a -fvectorize -ffp-model=fast -fno-finite-math-only" ) +#set( warn_c_flags "-Wno-format -Wno-unused-variable -Wno-unused-function -Wno-gnu-zero-variadic-macro-arguments" ) set( CMAKE_C_FLAGS_INIT "${arch_c_flags} ${warn_c_flags}" ) set( CMAKE_CXX_FLAGS_INIT "${arch_c_flags} ${warn_c_flags}" ) diff --git a/scripts/build-run-windows.sh b/scripts/build-run-windows.sh index 8221293a431d4..c9a5b13d71d4c 100755 --- a/scripts/build-run-windows.sh +++ b/scripts/build-run-windows.sh @@ -67,9 +67,9 @@ function build_windows_x86_qnn function build_windows_arm64_qnn { echo "build_windows_arm64 not supported now" - return 0 echo "cmake source dir:${PROJECT_HOME_PATH}" cmake -H. -B./out/windows_arm64_qnn -DCMAKE_BUILD_TYPE=Release -DGGML_OPENMP=OFF -DGGML_QNN=ON -DCMAKE_TOOLCHAIN_FILE=${PROJECT_HOME_PATH}/cmake/arm64-windows-llvm.cmake -DCMAKE_C_FLAGS=-march=armv8.7-a -DGGML_QNN_SDK_PATH=${QNN_SDK_PATH} + #cmake -H. -B./out/windows_arm64_qnn -DCMAKE_BUILD_TYPE=Release -DGGML_OPENMP=OFF -DGGML_QNN=ON -DCMAKE_TOOLCHAIN_FILE=${PROJECT_HOME_PATH}/cmake/arm64-windows-cygwin.cmake -DCMAKE_C_FLAGS=-march=armv8.7-a -DGGML_QNN_SDK_PATH=${QNN_SDK_PATH} cd out/windows_arm64_qnn make -j16 show_pwd From 5dbc27ca49fae8a552dea2a3d9e43c499992ca17 Mon Sep 17 00:00:00 2001 From: zhouwg Date: Sun, 9 Mar 2025 14:22:50 +0800 Subject: [PATCH 62/76] ggml-qnn: rebase to upstream --- common/console.cpp | 4 +- examples/export-lora/export-lora.cpp | 2 +- ggml/src/ggml-qnn/ggml-qnn-impl.h | 617 --------- ggml/src/ggml-qnn/ggml-qnn-ops.cpp | 687 ---------- ggml/src/ggml-qnn/ggml-qnn-ops.h | 52 - ggml/src/ggml-qnn/ggml-qnn.cpp | 1751 ++++++++++++++++++++++---- scripts/build-run-android.sh | 54 +- scripts/build-run-windows.sh | 222 ---- src/llama-mmap.cpp | 8 +- 9 files changed, 1522 insertions(+), 1875 deletions(-) delete mode 100644 ggml/src/ggml-qnn/ggml-qnn-impl.h delete mode 100644 ggml/src/ggml-qnn/ggml-qnn-ops.cpp delete mode 100644 ggml/src/ggml-qnn/ggml-qnn-ops.h delete mode 100755 scripts/build-run-windows.sh diff --git a/common/console.cpp b/common/console.cpp index 73b00aa95de9f..078a8d678d933 100644 --- a/common/console.cpp +++ b/common/console.cpp @@ -241,9 +241,7 @@ namespace console { (void)codepoint; return 1; #else - //return wcwidth(codepoint); - (void)codepoint; - return 1; + return wcwidth(codepoint); #endif } diff --git a/examples/export-lora/export-lora.cpp b/examples/export-lora/export-lora.cpp index 14ac107e761db..e7d0fbfffedb0 100644 --- a/examples/export-lora/export-lora.cpp +++ b/examples/export-lora/export-lora.cpp @@ -148,7 +148,7 @@ struct lora_merge_ctx { ctx_out = gguf_init_empty(); struct ggml_init_params params = { - /*.mem_size =*/ static_cast(gguf_get_n_tensors(base_model.ctx_gguf)*ggml_tensor_overhead()), + /*.mem_size =*/ gguf_get_n_tensors(base_model.ctx_gguf)*ggml_tensor_overhead(), /*.mem_buffer =*/ NULL, /*.no_alloc =*/ true, }; diff --git a/ggml/src/ggml-qnn/ggml-qnn-impl.h b/ggml/src/ggml-qnn/ggml-qnn-impl.h deleted file mode 100644 index 9d0bf559dd7e2..0000000000000 --- a/ggml/src/ggml-qnn/ggml-qnn-impl.h +++ /dev/null @@ -1,617 +0,0 @@ -/* -* Copyright (c) 2023-2024 The ggml authors -* -* Permission is hereby granted, free of charge, to any person obtaining a copy -* of this software and associated documentation files (the "Software"), to -* deal in the Software without restriction, including without limitation the -* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -* sell copies of the Software, and to permit persons to whom the Software is -* furnished to do so, subject to the following conditions: -* -* The above copyright notice and this permission notice shall be included in -* all copies or substantial portions of the Software. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -* IN THE SOFTWARE. -*/ -#pragma once -#include -#include -#include -#include -#include -#include -#include -#include -#if defined(__ANDROID__) || defined(__linux__) -#include -#include -#include -#include -#include -#include -#endif - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#if (defined __ANDROID__) || (defined ANDROID) -#include "android/log.h" -#endif - -#if !defined(__ANDROID__) && !defined(__linux__) -#include -#include -#include -#endif - -#include "QnnTypes.h" -#include "QnnCommon.h" -#include "QnnContext.h" -#include "QnnBackend.h" -#include "QnnGraph.h" -#include "QnnProperty.h" -#include "QnnTensor.h" -#include "QnnInterface.h" -#include "Saver/QnnSaver.h" -#include "System/QnnSystemInterface.h" -#include "HTP/QnnHtpDevice.h" -#include "HTP/QnnHtpGraph.h" - -#include "ggml-qnn.h" -#include "ggml-impl.h" -#include "ggml-backend-impl.h" - -class qnn_instance; -struct ggml_backend_qnn_context; -void ggmlqnn_log_internal(ggml_log_level level, const char * file, const char * func, int line, const char * format, ...); - -#if 0//def NDEBUG -#define GGMLQNN_DEBUG 0 -#define ENABLE_QNNBACKEND_PERF 0 // enable/disable op's perf info -#define GGMLQNN_PRINT_QNN_INTERNAL_LOG 0 // enable/disable QNN's internal log -#define GGMLQNN_PRINT_OP_ADD_LOG 0 // GGML_OP_ADD already verified with QNN-CPU / QNN-GPU / QNN-NPU -#define GGMLQNN_PRINT_OP_MUL_MAT_LOG 0 -#else -#define GGMLQNN_DEBUG 1 // for troubleshooting QNN backend -#define ENABLE_QNNBACKEND_PERF 0 // enable/disable op's perf info -#define GGMLQNN_PRINT_QNN_INTERNAL_LOG 0 // enable/disable QNN's internal log -#define GGMLQNN_PRINT_OP_ADD_LOG 0 // GGML_OP_ADD already verified with QNN-CPU / QNN-GPU / QNN-NPU -#define GGMLQNN_PRINT_OP_MUL_MAT_LOG 1 -#endif -#define GGML_QNN_LOGBUF_LEN 4096 - -#define GGMLQNN_LOG_ERROR(...) ggmlqnn_log_internal(GGML_LOG_LEVEL_ERROR, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) -#define GGMLQNN_LOG_WARN(...) ggmlqnn_log_internal(GGML_LOG_LEVEL_WARN , __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) -#define GGMLQNN_LOG_INFO(...) ggmlqnn_log_internal(GGML_LOG_LEVEL_INFO , __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) - -#if GGMLQNN_DEBUG -#define GGMLQNN_LOG_DEBUG(...) ggmlqnn_log_internal(GGML_LOG_LEVEL_DEBUG, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) -#else -#define GGMLQNN_LOG_DEBUG(...) -#endif - -#define CHECK_QNN_API(error, result) \ - do { \ - error = (result); \ - if (QNN_SUCCESS != error) { \ - if (error == QNN_COMMON_ERROR_NOT_SUPPORTED) { \ - GGMLQNN_LOG_WARN("WARNING: QNN feature/API not supported\n"); \ - } else { \ - GGMLQNN_LOG_INFO("QNN API error = %d(%s)\n", error, ggmlqnn_get_error_string(error)); \ - } \ - } \ - } while (0) - -#define QNN_VER_PTR(x) (&((x).v1)) -#define RPCMEM_DEFAULT_FLAGS 1 -#define RPCMEM_HEAP_ID_SYSTEM 25 - -#define DISABLE_COPY(class_name) \ - class_name(const class_name &) = delete; \ - void operator=(const class_name &) = delete - -#define DISABLE_MOVE(class_name) \ - class_name(class_name &&) = delete; \ - void operator=(class_name &&) = delete - -#define GQCGT ggmlqnn_create_general_tensor - -//#if defined(_WIN32) -#if !defined(__ANDROID__) && !defined(__linux__) -#define RTLD_GLOBAL 0x100 -#define RTLD_LOCAL 0x000 -#define RTLD_LAZY 0x000 -#define RTLD_NOW 0x001 -void * dlopen(const char * filename, int flag); -int dlclose(void * handle); -void * dlsym(void* handle, const char* name); -const char * dlerror(void); -#endif - -using pfn_rpc_mem_init = void (*)(void); -using pfn_rpc_mem_deinit = void (*)(void); -using pfn_rpc_mem_alloc = void *(*)(int, uint32_t, int); -using pfn_rpc_mem_free = void (*)(void *); -using pfn_rpc_mem_to_fd = int (*)(void *); -using _pfn_QnnSaver_initialize = decltype(QnnSaver_initialize); -using _pfn_QnnInterface_getProviders = decltype(QnnInterface_getProviders); -using _pfn_QnnSystemInterface_getProviders = decltype(QnnSystemInterface_getProviders); - -using qnn_res_t = std::tuple>; -using qnn_tensors_t = std::vector< Qnn_Tensor_t *>; - -enum class ggml_qnn_profile_level { - profile_off = 0, - profile_basic = 1, - profile_detail = 2 -}; - -enum qcom_htp_arch { - NONE = 0, - V68 = 68, - V69 = 69, - V73 = 73, - V75 = 75, - V79 = 79, -}; - -enum qcom_chipset_soc_model { - UNKNOWN_SM = 0, - SM7450 = 41, // v69, 7 Gen1 - SM8350 = 30, // v68, 888 - SM8450 = 36, // v69, SD 8 Gen 1 - SM8475 = 42, // v69, SD 8+ Gen 1 - SM8550 = 43, // v73, SD 8 Gen 2 - SM8650 = 57, // v75, SD 8 Gen 3 - SM8750 = 69, // v79, SD 8 Gen 4 -#if !defined(__ANDROID__) && !defined(__linux__) - SC7280X = 44, - SC8280X = 37, - SC8380XP = 60, -#endif -}; - -struct qcom_socinfo { - uint32_t soc_model; - size_t htp_arch; - size_t vtcm_size_in_mb; - char soc_desc[GGML_MAX_NAME]; -}; - -struct ggml_backend_qnn_context { - int device; - int threads; - char name[GGML_MAX_NAME]; - char desc[GGML_MAX_NAME]; - char lib[GGML_MAX_NAME]; - qnn_instance * instance; - struct ggml_backend * backend; - QNN_INTERFACE_VER_TYPE raw_interface; - QNN_SYSTEM_INTERFACE_VER_TYPE raw_system_interface; - struct qcom_socinfo socinfo; - - std::unique_ptr work_data; - std::vector> tasks; - size_t work_size = 0; - size_t desired_size = 0; - int n_threads = GGML_DEFAULT_N_THREADS; -}; - -struct qnn_op_caps_t { - const char * qnn_op_name = nullptr; - const size_t input_param_count = 0; - const char * qnn_param_name = nullptr; -}; -extern const qnn_op_caps_t ggmlqnn_k_op_caps[]; - -#if ENABLE_QNNBACKEND_PERF -class qnn_perf { -public: - qnn_perf(const std::string & perf_name) : _perf_name(std::move(perf_name)) {}; - qnn_perf() = delete; - qnn_perf(const qnn_perf & ) = delete; - qnn_perf & operator= (const qnn_perf & ) = delete; - - void start() { - _begin_time = ggml_time_us(); - } - - void info() { - _end_time = ggml_time_us(); - _duration = (_end_time - _begin_time); - GGMLQNN_LOG_DEBUG("duration of %s : %lld microseconds\n", _perf_name.c_str(), _duration); - } - -private: - int64_t _begin_time = 0LL; - int64_t _end_time = 0LL; - int64_t _duration = 0LL; - std::string _perf_name; -}; -#else -class qnn_perf { -public: - qnn_perf(const std::string & perf_name) { - GGML_UNUSED(perf_name); - } - qnn_perf() = delete; - qnn_perf(const qnn_perf & ) = delete; - qnn_perf & operator= (const qnn_perf & ) = delete; - - void start() {} - void info() {} -}; -#endif - -class qnn_interface { -#define DEFINE_SHIM_FUNCTION_INTERFACE(F, pointer_name) \ - template \ - inline auto qnn_##F(Args... args) const { \ - return (_qnn_interface->QNN_INTERFACE_VER_NAME.pointer_name)( \ - std::forward(args)...); \ - } - - -#define DEFINE_SHIM_FUNCTION_SYS_INTERFACE(F, pointer_name) \ - template \ - inline auto qnn_##F(Args... args) const { \ - return (_qnn_sys_interface->QNN_SYSTEM_INTERFACE_VER_NAME.pointer_name)( \ - std::forward(args)...); \ - } - - friend class qnn_instance; - -public: - qnn_interface() = default; - - // QnnBackend - DEFINE_SHIM_FUNCTION_INTERFACE(backend_create, backendCreate) - - DEFINE_SHIM_FUNCTION_INTERFACE(backend_free, backendFree) - - DEFINE_SHIM_FUNCTION_INTERFACE(backend_register_op_package, backendRegisterOpPackage) - - DEFINE_SHIM_FUNCTION_INTERFACE(backend_validate_op_config, backendValidateOpConfig) - - DEFINE_SHIM_FUNCTION_INTERFACE(backend_get_api_version, backendGetApiVersion) - - // QnnDevice - DEFINE_SHIM_FUNCTION_INTERFACE(device_create, deviceCreate) - - DEFINE_SHIM_FUNCTION_INTERFACE(device_free, deviceFree) - - DEFINE_SHIM_FUNCTION_INTERFACE(device_get_infrastructure, deviceGetInfrastructure) - - DEFINE_SHIM_FUNCTION_INTERFACE(device_get_platform_info, deviceGetPlatformInfo) - - DEFINE_SHIM_FUNCTION_INTERFACE(device_get_info, deviceGetInfo) - - // QnnContext - DEFINE_SHIM_FUNCTION_INTERFACE(context_create, contextCreate) - - DEFINE_SHIM_FUNCTION_INTERFACE(context_get_binary_size, contextGetBinarySize) - - DEFINE_SHIM_FUNCTION_INTERFACE(context_get_binary, contextGetBinary) - - DEFINE_SHIM_FUNCTION_INTERFACE(context_create_from_binary, contextCreateFromBinary) - - DEFINE_SHIM_FUNCTION_INTERFACE(context_free, contextFree) - - // QnnGraph - DEFINE_SHIM_FUNCTION_INTERFACE(graph_create, graphCreate) - - DEFINE_SHIM_FUNCTION_INTERFACE(graph_add_node, graphAddNode) - - DEFINE_SHIM_FUNCTION_INTERFACE(graph_finalize, graphFinalize) - - DEFINE_SHIM_FUNCTION_INTERFACE(graph_execute, graphExecute) - - DEFINE_SHIM_FUNCTION_INTERFACE(graph_retrieve, graphRetrieve) - - // QnnLog - DEFINE_SHIM_FUNCTION_INTERFACE(log_create, logCreate) - - DEFINE_SHIM_FUNCTION_INTERFACE(log_free, logFree) - - DEFINE_SHIM_FUNCTION_INTERFACE(log_set_log_level, logSetLogLevel) - - // QnnProfile - DEFINE_SHIM_FUNCTION_INTERFACE(profile_create, profileCreate) - - DEFINE_SHIM_FUNCTION_INTERFACE(profile_get_events, profileGetEvents) - - DEFINE_SHIM_FUNCTION_INTERFACE(profile_get_sub_events, profileGetSubEvents) - - DEFINE_SHIM_FUNCTION_INTERFACE(profile_get_event_data, profileGetEventData) - - DEFINE_SHIM_FUNCTION_INTERFACE(profile_free, profileFree) - - // QnnMem - DEFINE_SHIM_FUNCTION_INTERFACE(mem_register, memRegister) - - DEFINE_SHIM_FUNCTION_INTERFACE(mem_de_register, memDeRegister) - - // QnnProperty - DEFINE_SHIM_FUNCTION_INTERFACE(property_has_capability, propertyHasCapability) - - // QnnTensor - DEFINE_SHIM_FUNCTION_INTERFACE(tensor_create_context_tensor, tensorCreateContextTensor) - - DEFINE_SHIM_FUNCTION_INTERFACE(tensor_create_graph_tensor, tensorCreateGraphTensor) - - // QnnSystem - DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_create, systemContextCreate) - - DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_get_binary_info, systemContextGetBinaryInfo) - - DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_free, systemContextFree) - - void set_qnn_interface(const QnnInterface_t * qnn_interface) { - _qnn_interface = qnn_interface; - } - - void set_qnn_system_interface(const QnnSystemInterface_t * qnn_sys_interface) { - _qnn_sys_interface = qnn_sys_interface; - } - - uint32_t get_backend_id() const { - return _qnn_interface->backendId; - } - - bool is_loaded() const { - return ((_qnn_sys_interface != nullptr) && (_qnn_interface != nullptr)); - } - -private: - const QnnInterface_t * _qnn_interface = nullptr; - - const QnnSystemInterface_t * _qnn_sys_interface = nullptr; -}; - -class qnn_instance { -public: - using BackendIdType = decltype(QnnInterface_t{}.backendId); - - explicit qnn_instance(const std::string & lib_path, const std::string & backend_name, - const std::string & model_name) : - _lib_path(std::move(lib_path)), - _backend_name(std::move(backend_name)), - _model_name(std::move(model_name)) {} - - ~qnn_instance() { - } - - int qnn_init(const QnnSaver_Config_t ** saver_config); - - int qnn_finalize(); - - const qnn_interface & get_qnn_interface() { - if (!_qnn_interface.is_loaded()) { - GGMLQNN_LOG_WARN("pls check why _qnn_interface is not loaded\n"); - } - return _qnn_interface; - } - - const QNN_INTERFACE_VER_TYPE & get_qnn_raw_interface() { - if (!_qnn_interface.is_loaded()) { - GGMLQNN_LOG_WARN("pls check why _qnn_interface is not loaded\n"); - } - return _qnn_raw_interface; - } - - const QNN_SYSTEM_INTERFACE_VER_TYPE & get_qnn_raw_system_interface() { - if (!_qnn_interface.is_loaded()) { - GGMLQNN_LOG_WARN("pls check why _qnn_interface is not loaded\n"); - } - return _qnn_raw_system_interface; - } - - Qnn_LogHandle_t get_qnn_log_handle() { return _qnn_log_handle; } - - Qnn_ProfileHandle_t get_qnn_profile_handle() { return _qnn_profile_handle; } - - Qnn_DeviceHandle_t get_qnn_device_handle() { return _qnn_device_handle; } - - Qnn_BackendHandle_t get_qnn_backend_handle() { return _qnn_backend_handle; } - - Qnn_ContextHandle_t get_qnn_context_handle() { return _qnn_context_handle; } - - QnnSystemContext_Handle_t get_qnn_system_handle() { return _qnn_system_handle; } - - Qnn_GraphHandle_t get_qnn_graph_handle() { return _qnn_graph_handle; } - - int init_qnn_graph(const char * graph_name, - bool debug, - uint8_t do_node_validation = 1, - const QnnGraph_Config_t ** graph_configs = nullptr - ); - int init_qnn_graph(const std::string & graph_name, QNNBackend device, size_t vtcm_size_in_mb = 8, size_t hvx_threads = 8); - - int finalize_qnn_graph(); - - bool is_valid_graph() const { return _qnn_graph_handle != nullptr; } - - int init_htp_perfinfra(); - - int set_rpc_polling(); - - int set_high_performance_mode(); - - std::string & get_qnn_graph_name() { return _graph_name; } - - bool is_rpcmem_initialized() { - return _rpcmem_initialized; - } - - void set_rpcmem_initialized(bool initialized) { - _rpcmem_initialized = initialized; - } - - size_t get_rpcmem_capacity() { return _rpcmem_capacity; } - size_t get_rpcmem_usage() { return _rpcmem_usage; } - - int32_t rpcmem_to_fd(void * buf); - - int register_rpcmem(void * p_data, Qnn_Tensor_t * p_tensor); - Qnn_MemHandle_t register_rpcmem(void * p_data, const uint32_t rank, uint32_t * dimensions, Qnn_DataType_t data_type); - - void unregister_rpcmem(); - void unregister_rpcmem(Qnn_MemHandle_t mem_handle); - - void * alloc_rpcmem(size_t bytes, size_t alignment); - void * get_rpcmem_from_memhandle(Qnn_MemHandle_t mem_handle); - - void free_rpcmem(void * buf); - void free_rpcmem(); - - bool is_rpcmem_allocated(void * buf); - - bool is_rpcmem_registered(Qnn_MemHandle_t handle) { - return _qnn_mem_set.count(handle) != 0U; - } - - bool enable_qnn_rpc() { - return _enable_qnn_rpc; - } - - QNNBackend get_device_id() { - return _device_id; - } - -public: - std::map>> _qnn_graph_map; - -private: - int load_system(); - - int unload_system(); - - int load_backend(std::string & lib_path, const QnnSaver_Config_t ** saver_config); - - int unload_backend(); - - void set_qnn_raw_interface(QNN_INTERFACE_VER_TYPE & raw_interface) { - _qnn_raw_interface = raw_interface; - } - - void set_qnn_raw_system_interface(QNN_SYSTEM_INTERFACE_VER_TYPE & raw_interface) { - _qnn_raw_system_interface = raw_interface; - } - - void * alloc_rpcmem_internal(size_t bytes, size_t alignment); - - void probe_device_meminfo(); - -private: - static constexpr const int _required_num_providers = 1; - -private: - std::string _lib_path; - std::string _backend_name; - std::string _model_name; // name of prebuilt QNN model, might be used in the future - BackendIdType _backend_id; - - bool _debug_tensor = false; // flag to indicate if requested graph is to be run in debug mode - bool _do_node_validations = true; // flag to indicate whether all add_node calls need to be validated - QnnLog_Level_t _qnn_log_level = QNN_LOG_LEVEL_DEBUG; - - ggml_qnn_profile_level _profile_level = ggml_qnn_profile_level::profile_detail; - - void * _system_lib_handle = nullptr; - - Qnn_GraphHandle_t _qnn_graph_handle = nullptr; - - Qnn_LogHandle_t _qnn_log_handle = nullptr; - - Qnn_ProfileHandle_t _qnn_profile_handle = nullptr; - - Qnn_DeviceHandle_t _qnn_device_handle = nullptr; - - Qnn_BackendHandle_t _qnn_backend_handle = nullptr; - - Qnn_ContextHandle_t _qnn_context_handle = nullptr; - - QnnSystemContext_Handle_t _qnn_system_handle = nullptr; - - QnnHtpDevice_PerfInfrastructure_t * _qnn_htp_perfinfra = nullptr; - uint32_t _qnn_power_configid = 1; - uint32_t _qnn_rpc_pollingtime = 9999; // 0-10000 us for high performing - - qnn_interface _qnn_interface; - QNN_INTERFACE_VER_TYPE _qnn_raw_interface; - QNN_SYSTEM_INTERFACE_VER_TYPE _qnn_raw_system_interface; - - std::unordered_map _qnn_mem_set; - std::unordered_map _qnn_rpc_buffer_to_handles; - - static std::mutex _init_mutex; - static std::unordered_map _loaded_lib_handle; - static std::unordered_map _lib_path_to_backend_id; - static std::unordered_map _loaded_backend; - - std::atomic_bool _rpcmem_initialized{false}; - pfn_rpc_mem_alloc _pfn_rpc_mem_alloc; - pfn_rpc_mem_free _pfn_rpc_mem_free; - pfn_rpc_mem_to_fd _pfn_rpc_mem_to_fd; - pfn_rpc_mem_init _pfn_rpc_mem_init; - pfn_rpc_mem_deinit _pfn_rpc_mem_deinit; - std::unordered_map _rpcmem_store_map; - std::unordered_map _rpcmem_usage_map; - size_t _rpcmem_usage = 0; // mempool usage in Mbytes - size_t _rpcmem_capacity = 512; // mempool size in Mbytes - - std::string _graph_name; - QNNBackend _device_id; - void * _rpc_lib_handle = nullptr; - bool _enable_qnn_rpc = false; //TODO:unknown issue with QNN RPC feature - - DISABLE_COPY(qnn_instance); - DISABLE_MOVE(qnn_instance); -}; - -size_t ggmlqnn_get_opcaps_size(void); -size_t ggmlqnn_get_op_index(const ggml_tensor * tensor); -const char * ggmlqnn_get_error_string(Qnn_ErrorHandle_t qnn_error_code); -Qnn_DataType_t ggmlqnn_datatype_from_ggml_datatype(enum ggml_type ggmltype); -void * ggmlqnn_type_trait(ggml_backend_qnn_context * ctx, ggml_tensor * op); -void ggmlqnn_get_graphkey_from_op(const ggml_tensor * op, std::string & output); -uint8_t * ggmlqnn_create_rpc_buffer(qnn_instance * instance, const ggml_tensor * ggml_tensor, Qnn_Tensor_t * qnn_tensor, bool b_copydata); -Qnn_Tensor_t * ggmlqnn_create_compute_tensor(qnn_instance * instance, Qnn_GraphHandle_t handle, const ggml_tensor * tensor, Qnn_TensorType_t tensor_type); -void ggmlqnn_print_tensors_info(const char * func_name, ggml_backend_qnn_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); - -Qnn_OpConfig_t ggmlqnn_create_op_config(const char * name, const char * package, const char * type, - Qnn_Param_t * params, uint32_t num_params, - Qnn_Tensor_t * inputs, uint32_t num_inputs, - Qnn_Tensor_t * outputs, uint32_t num_outputs); -Qnn_Tensor_t * ggmlqnn_create_general_tensor(const ggml_tensor * tensor, const char * name, - Qnn_TensorType_t qnn_tensor_type, - Qnn_DataType_t qnn_data_type, - uint32_t rank, uint32_t * dims, - void * data, uint32_t data_size, - bool b_transpose = false); diff --git a/ggml/src/ggml-qnn/ggml-qnn-ops.cpp b/ggml/src/ggml-qnn/ggml-qnn-ops.cpp deleted file mode 100644 index 6ade24315f99a..0000000000000 --- a/ggml/src/ggml-qnn/ggml-qnn-ops.cpp +++ /dev/null @@ -1,687 +0,0 @@ -/* - * Copyright (c) 2023-2024 The ggml authors - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ -#include "ggml-impl.h" -#include "ggml-common.h" -#include "ggml-qnn-ops.h" - -static inline uint32_t ggmlqnn_get_tensor_data_size(const ggml_tensor * tensor) { - /* - size_t data_size = ggml_row_size(tensor->type, tensor->ne[0]); - size_t n_dims = ggml_get_tensor_rank(tensor); - for (int i = 1; i < n_dims; i++) { - data_size *= tensor->ne[i]; - } - - return data_size; - */ - return ggml_nbytes(tensor); -} - -static inline bool ggmlqnn_is_valid_params(ggml_backend_qnn_context * ctx, const ggml_tensor * src0, - const ggml_tensor * src1, ggml_tensor * dst) { - if ((nullptr == ctx) || (nullptr == src0) || (nullptr == src1) || (nullptr == dst)) { - GGMLQNN_LOG_WARN("invalid params\n"); - return false; - } - - qnn_instance * instance = ctx->instance; - if (nullptr == instance) { - GGMLQNN_LOG_WARN("invalid params\n"); - return false; - } - - return true; -} - -#define GGMLQNN_CHECK_PARAMS(ctx, src0, src1, dst) \ - do { \ - if (!ggmlqnn_is_valid_params((ctx), (src0), (src1), (dst))) { \ - return; \ - } \ - } while (0) - -/* - * provide a general skeleton to offload ggml op to QNN backend: a single node contains 2 input - * tensor and 1 output tensor -*/ -void ggml_qnn_general_node(ggml_backend_qnn_context * ctx, ggml_tensor * op) { - Qnn_ErrorHandle_t error = QNN_SUCCESS; - qnn_instance * instance = nullptr; - Qnn_GraphHandle_t graph_handle = nullptr; - Qnn_Tensor_t * p_tensor0 = nullptr; - Qnn_Tensor_t * p_tensor1 = nullptr; - Qnn_Tensor_t * p_tensor2 = nullptr; - const ggml_tensor * src0 = op->src[0]; - const ggml_tensor * src1 = op->src[1]; - ggml_tensor * dst = op; - - GGMLQNN_CHECK_PARAMS(ctx, src0, src1, dst); - instance = ctx->instance; - QNN_INTERFACE_VER_TYPE qnn_raw_interface = ctx->raw_interface; - size_t qnn_op_index = ggmlqnn_get_op_index(op); - GGML_ASSERT(qnn_op_index < ggmlqnn_get_opcaps_size()); - const char * qnn_op_name = ggmlqnn_k_op_caps[qnn_op_index].qnn_op_name; - std::string ggml_op_name_string = std::string("ggml_") + ggml_op_name(op->op); - const char * ggml_op_name = ggml_op_name_string.c_str(); - - qnn_perf op_perf = qnn_perf(ggml_op_name); - op_perf.start(); - - //ggmlqnn_print_tensors_info(__func__, ctx, src0, src1, dst); - bool enable_npu_rpc = instance->enable_qnn_rpc() && ctx->device == QNN_BACKEND_NPU; - - std::string graph_name; - ggmlqnn_get_graphkey_from_op(op, graph_name); - if (instance->_qnn_graph_map.find(graph_name) != instance->_qnn_graph_map.end()) { - //retrieve computational resource from cached QNN graph - qnn_res_t & graph_item = instance->_qnn_graph_map[graph_name]; - graph_handle = std::get<0>(graph_item); - qnn_tensors_t & tensor = std::get<1>(graph_item); - p_tensor0 = tensor[0]; - p_tensor1 = tensor[1]; - p_tensor2 = tensor[2]; - } else { - GGMLQNN_LOG_DEBUG("graph name %s", graph_name.c_str()); - GGML_ASSERT(instance->get_device_id() == ctx->device); - //create QNN graph - error = instance->init_qnn_graph(graph_name, static_cast(ctx->device), 8); - if (QNN_SUCCESS != error) { - GGMLQNN_LOG_WARN("can't create qnn graph handle with graph name %s, error = %d\n", graph_name.c_str(), error); - return; - } - graph_handle = instance->get_qnn_graph_handle(); - - //create computational tensor - p_tensor0 = ggmlqnn_create_compute_tensor(instance, graph_handle, src0, QNN_TENSOR_TYPE_APP_WRITE); - p_tensor1 = ggmlqnn_create_compute_tensor(instance, graph_handle, src1, QNN_TENSOR_TYPE_APP_WRITE); - p_tensor2 = ggmlqnn_create_compute_tensor(instance, graph_handle, dst, QNN_TENSOR_TYPE_APP_READ); - - //compose QNN graph - Qnn_Tensor_t tensor_inputs[] = { - *p_tensor0, - *p_tensor1 - }; - Qnn_Tensor_t tensor_outputs[] = { - *p_tensor2 - }; - Qnn_OpConfig_t op_config = { - QNN_OPCONFIG_VERSION_1, { - ggml_op_name, - QNN_OP_PACKAGE_NAME_QTI_AISW, - qnn_op_name, - 0, - nullptr, - 2, - tensor_inputs, - 1, - tensor_outputs - } - }; - CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, op_config)); - //finalize QNN graph - CHECK_QNN_API(error, qnn_raw_interface.graphFinalize(graph_handle, nullptr, nullptr)); - - //cache QNN graph - qnn_tensors_t ggml_op_add_tensors; - ggml_op_add_tensors.reserve(3); - ggml_op_add_tensors.push_back(p_tensor0); - ggml_op_add_tensors.push_back(p_tensor1); - ggml_op_add_tensors.push_back(p_tensor2); - auto graph_item = std::make_tuple(graph_handle, ggml_op_add_tensors); - instance->_qnn_graph_map[graph_name] = graph_item; - } - - if (enable_npu_rpc) { - uint8_t * qnn_buffer_0 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*p_tensor0)->memHandle)); - GGMLQNN_LOG_INFO("qnn_rpcbuffer_0 = %p\n", qnn_buffer_0); - if (nullptr != qnn_buffer_0) { - memcpy(qnn_buffer_0, src0->data, ggml_nbytes(src0)); - } - - uint8_t * qnn_buffer_1 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*p_tensor1)->memHandle)); - GGMLQNN_LOG_INFO("qnn_rpcbuffer_1 = %p\n", qnn_buffer_1); - if (nullptr != qnn_buffer_1) { - memcpy(qnn_buffer_1, src1->data, ggml_nbytes(src1)); - } - } else { - QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggmlqnn_get_tensor_data_size(src0)}; - QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggmlqnn_get_tensor_data_size(src1)}; - QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggmlqnn_get_tensor_data_size(dst)}; - } - - Qnn_Tensor_t tensor_inputs[] = { - *p_tensor0, - *p_tensor1 - }; - Qnn_Tensor_t tensor_outputs[] = { - *p_tensor2 - }; - CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, - tensor_inputs, 2, - tensor_outputs, 1, - nullptr, nullptr)); - if (enable_npu_rpc) { - //TODO:NPU RPC feature will failed with test-backend-ops - uint8_t * qnn_buffer_2 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*p_tensor2)->memHandle)); - if (nullptr != qnn_buffer_2) { - memcpy(dst->data, qnn_buffer_2, ggml_nbytes(dst)); - } - } - -#if GGMLQNN_PRINT_OP_ADD_LOG - op_perf.info(); -#endif -} - -/* - * this function is AI-assisted code from Grok 3 for purpose of offload 4d matrix mulmat to QNN backend - * UT in ggml-qnn-ut.cpp passed: - * ./scripts/build-run-android.sh run_ut_mulmat 0 - * ./scripts/build-run-android.sh run_ut_mulmat 1 - * ./scripts/build-run-android.sh run_ut_mulmat 2 - * - * the logic of ggml_qnn_mul_mat_4d is similar to ggml_qnn_mul_mat but much more complicated - * than ggml_qnn_mul_mat, so it's a standalone function. - * it will be combined with ggml_qnn_mul_mat in the future - */ -static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) { - Qnn_ErrorHandle_t error = QNN_SUCCESS; - bool graph_initialized = false; - qnn_perf op_perf = qnn_perf("ggml_qnn_mul_mat_4d"); - qnn_instance *instance = ctx->instance; - QNN_INTERFACE_VER_TYPE qnn_raw_interface = ctx->raw_interface; - - const ggml_tensor *src0 = op->src[0]; - const ggml_tensor *src1 = op->src[1]; - ggml_tensor *dst = op; - - GGMLQNN_CHECK_PARAMS(ctx, src0, src1, dst); - GGML_ASSERT(ggml_n_dims(src0) == 4 && ggml_n_dims(src1) == 4); - op_perf.start(); - - std::string graph_name; - ggmlqnn_get_graphkey_from_op(op, graph_name); - GGMLQNN_LOG_DEBUG("graph name %s\n", graph_name.c_str()); - - ggmlqnn_print_tensors_info(__func__, ctx, src0, src1, dst); - - Qnn_GraphHandle_t graph_handle = nullptr; - Qnn_Tensor_t *p_tensor0 = nullptr; - Qnn_Tensor_t *p_reshape0_out = nullptr; - Qnn_Tensor_t *p_tile0_out = nullptr; - Qnn_Tensor_t *p_tensor1 = nullptr; - Qnn_Tensor_t *p_permute1_out = nullptr; - Qnn_Tensor_t *p_reshape1_out = nullptr; - Qnn_Tensor_t *p_matmul_out = nullptr; - Qnn_Tensor_t *p_reshape2_out = nullptr; - - if (instance->_qnn_graph_map.find(graph_name) != instance->_qnn_graph_map.end()) { - graph_initialized = true; - qnn_res_t &graph_item = instance->_qnn_graph_map[graph_name]; - graph_handle = std::get<0>(graph_item); - qnn_tensors_t &tensors = std::get<1>(graph_item); - p_tensor0 = tensors[0]; - p_reshape0_out = tensors[1]; - p_tile0_out = tensors[2]; - p_tensor1 = tensors[3]; - p_permute1_out = tensors[4]; - p_reshape1_out = tensors[5]; - p_matmul_out = tensors[6]; - p_reshape2_out = tensors[7]; - } else { - CHECK_QNN_API(error, qnn_raw_interface.graphCreate(instance->get_qnn_context_handle(), - graph_name.c_str(), NULL, &graph_handle)); - - // Define dimensions - uint32_t K = src0->ne[0]; // Inner dimension - uint32_t M = src0->ne[1]; // Rows of src0 - uint32_t N = src1->ne[1]; // Columns of src1 - uint32_t B0 = src0->ne[2] * src0->ne[3]; // src0 batch - uint32_t B1 = src1->ne[2] * src1->ne[3]; // src1 batch (drives output) - - // Validate K only - GGML_ASSERT(src0->ne[0] == src1->ne[0]); // K must match - - // src0: [K, M, H0, B0] -> QNN: [B0, H0, M, K] - uint32_t src0_dims[] = {static_cast(src0->ne[3]), static_cast(src0->ne[2]), static_cast(src0->ne[1]), static_cast(src0->ne[0])}; - p_tensor0 = GQCGT(src0, "input0", QNN_TENSOR_TYPE_APP_WRITE, QNN_DATATYPE_FLOAT_32, 4, - src0_dims, nullptr, 0); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor0)); - - // Reshape src0 to [B0, M, K] - uint32_t reshape0_out_dims[] = {B0, M, K}; - p_reshape0_out = GQCGT(nullptr, "reshape0_out", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, 3, - reshape0_out_dims, nullptr, 0); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_reshape0_out)); - Qnn_Tensor_t reshape0_inputs[] = {*p_tensor0}; - Qnn_Tensor_t reshape0_outputs[] = {*p_reshape0_out}; - Qnn_OpConfig_t reshape0_op = ggmlqnn_create_op_config("reshape0", QNN_OP_PACKAGE_NAME_QTI_AISW, - QNN_OP_RESHAPE, nullptr, 0, - reshape0_inputs, 1, reshape0_outputs, 1); - CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, reshape0_op)); - - // Tile src0 to match B1: [B0, M, K] -> [B1, M, K] - uint32_t tile0_out_dims[] = {B1, M, K}; - p_tile0_out = GQCGT(nullptr, "tile0_out", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, 3, - tile0_out_dims, nullptr, 0); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tile0_out)); - uint32_t tile_multiples[] = {B1 / B0, 1, 1}; - uint32_t tile_dims[] = {3}; - Qnn_Tensor_t *p_tile_multiples = GQCGT(nullptr, "tile_multiples", QNN_TENSOR_TYPE_STATIC, QNN_DATATYPE_UINT_32, 1, - tile_dims, tile_multiples, sizeof(tile_multiples)); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tile_multiples)); - Qnn_Param_t tile_params[] = {{QNN_PARAMTYPE_TENSOR, "multiples", .tensorParam = *p_tile_multiples}}; - Qnn_Tensor_t tile0_inputs[] = {*p_reshape0_out}; - Qnn_Tensor_t tile0_outputs[] = {*p_tile0_out}; - Qnn_OpConfig_t tile0_op = ggmlqnn_create_op_config("tile0", QNN_OP_PACKAGE_NAME_QTI_AISW, - QNN_OP_TILE, tile_params, 1, - tile0_inputs, 1, tile0_outputs, 1); - CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, tile0_op)); - - // src1: [N, K, H1, B1] -> QNN: [B1, H1, N, K] - uint32_t src1_dims[] = {static_cast(src1->ne[3]), static_cast(src1->ne[2]), static_cast(src1->ne[1]), static_cast(src1->ne[0])}; - p_tensor1 = GQCGT(src1, "input1", QNN_TENSOR_TYPE_APP_WRITE, QNN_DATATYPE_FLOAT_32, 4, - src1_dims, nullptr, 0); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor1)); - - // Permute src1 to [B1, H1, K, N] - uint32_t perm_data[] = {0, 1, 3, 2}; - uint32_t perm_dims[] = {4}; - Qnn_Tensor_t *p_perm = GQCGT(nullptr, "perm", QNN_TENSOR_TYPE_STATIC, QNN_DATATYPE_UINT_32, 1, - perm_dims, perm_data, sizeof(perm_data)); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_perm)); - uint32_t permute1_out_dims[] = {static_cast(src1->ne[3]), static_cast(src1->ne[2]), static_cast(src1->ne[0]), static_cast(src1->ne[1])}; - p_permute1_out = GQCGT(nullptr, "permute1_out", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, 4, - permute1_out_dims, nullptr, 0); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_permute1_out)); - Qnn_Param_t permute1_params[] = {{QNN_PARAMTYPE_TENSOR, "perm", .tensorParam = *p_perm}}; - Qnn_Tensor_t permute1_inputs[] = {*p_tensor1}; - Qnn_Tensor_t permute1_outputs[] = {*p_permute1_out}; - Qnn_OpConfig_t permute1_op = ggmlqnn_create_op_config("permute1", QNN_OP_PACKAGE_NAME_QTI_AISW, - QNN_OP_TRANSPOSE, permute1_params, 1, - permute1_inputs, 1, permute1_outputs, 1); - CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, permute1_op)); - - // Reshape src1 to [B1, K, N] - uint32_t reshape1_out_dims[] = {B1, K, N}; - p_reshape1_out = GQCGT(nullptr, "reshape1_out", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, 3, - reshape1_out_dims, nullptr, 0); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_reshape1_out)); - Qnn_Tensor_t reshape1_inputs[] = {*p_permute1_out}; - Qnn_Tensor_t reshape1_outputs[] = {*p_reshape1_out}; - Qnn_OpConfig_t reshape1_op = ggmlqnn_create_op_config("reshape1", QNN_OP_PACKAGE_NAME_QTI_AISW, - QNN_OP_RESHAPE, nullptr, 0, - reshape1_inputs, 1, reshape1_outputs, 1); - CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, reshape1_op)); - - // MatMul: [B1, M, K] x [B1, K, N] -> [B1, M, N] - uint32_t matmul_out_dims[] = {B1, M, N}; - p_matmul_out = GQCGT(nullptr, "matmul_out", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, 3, - matmul_out_dims, nullptr, 0); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_matmul_out)); - Qnn_Tensor_t matmul_inputs[] = {*p_tile0_out, *p_reshape1_out}; - Qnn_Tensor_t matmul_outputs[] = {*p_matmul_out}; - Qnn_OpConfig_t matmul_op = ggmlqnn_create_op_config("matmul", QNN_OP_PACKAGE_NAME_QTI_AISW, - QNN_OP_MAT_MUL, nullptr, 0, - matmul_inputs, 2, matmul_outputs, 1); - CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, matmul_op)); - - // Output: [N, M, H1, B1] -> QNN: [B1, H1, M, N] - uint32_t reshape2_out_dims[] = {static_cast(dst->ne[3]), static_cast(dst->ne[2]), static_cast(dst->ne[1]), static_cast(dst->ne[0])}; - p_reshape2_out = GQCGT(dst, "output", QNN_TENSOR_TYPE_APP_READ, QNN_DATATYPE_FLOAT_32, 4, - reshape2_out_dims, nullptr, 0); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_reshape2_out)); - Qnn_Tensor_t reshape2_inputs[] = {*p_matmul_out}; - Qnn_Tensor_t reshape2_outputs[] = {*p_reshape2_out}; - Qnn_OpConfig_t reshape2_op = ggmlqnn_create_op_config("reshape2", QNN_OP_PACKAGE_NAME_QTI_AISW, - QNN_OP_RESHAPE, nullptr, 0, - reshape2_inputs, 1, reshape2_outputs, 1); - CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, reshape2_op)); - - // Finalize - CHECK_QNN_API(error, qnn_raw_interface.graphFinalize(graph_handle, NULL, NULL)); - - // Cache - qnn_tensors_t ggml_op_mulmat_tensors = {p_tensor0, p_reshape0_out, p_tile0_out, p_tensor1, p_permute1_out, p_reshape1_out, p_matmul_out, p_reshape2_out}; - instance->_qnn_graph_map[graph_name] = std::make_tuple(graph_handle, ggml_op_mulmat_tensors); - } - - // Execute - QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, static_cast(ggml_nbytes(src0))}; - QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, static_cast(ggml_nbytes(src1))}; - QNN_VER_PTR(*p_reshape2_out)->clientBuf = {dst->data, static_cast(ggml_nbytes(dst))}; - - Qnn_Tensor_t input_tensors[] = {*p_tensor0, *p_tensor1}; - Qnn_Tensor_t output_tensors[] = {*p_reshape2_out}; - CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, input_tensors, 2, - output_tensors, 1, NULL, NULL)); - -#if 0 - // Log dst for debugging - float *dst_data = (float *)dst->data; - GGMLQNN_LOG_DEBUG("dst shape: [%d, %d, %d, %d]\n", dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]); - for (int i = 0; i < dst->ne[0] * dst->ne[1] * dst->ne[2] * dst->ne[3]; i++) { - GGMLQNN_LOG_DEBUG("dst[%d] = %f\n", i, dst_data[i]); - } -#endif - - op_perf.info(); -} - -/* - * @brief performs matrix multiplication with FP32 & quantized weights and floating-point inputs - * using the QNN backend. this function performs matrix multiplication of the input tensor - * `src1` and the weight tensor `src0`, handling transposing, and quantization as needed, - * and stores the result in the destination tensor `dst`. - * - there are two key-points in properly handling how to offload mulmat to the QNN backend in ggml-qnn - 1. transpose - a 3x2 f32 matrix which means 3 rows and 2 columns. in ggml, it could be created from: - struct ggml_tensor* matrix = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 2, 3); - which like this: - +---+---+ - | 0 | 1 | - +---+---+ - | 2 | 3 | - +---+---+ - | 4 | 5 | - +---+---+ - with - ne[0] = 2 - ne[1] = 3 - there are different dimension order between ggml tensor and qnn tensor - - 2. QNN's MatMul can only support input tensors with rank >= 2 - - in the all, there is gap between ggml mulmat and QNN mulmat,we need to perform a transpose - operation when offloading mulmat to QNN backend. this implementation will handle transpose - in func ggml_qnn_create_general_tensor() - * - * this function is a good example to illustrated the second technical approach "mapping the - * entire ggml computational graph to QNN graph" without complex C++ encapsulation. or another - * pipeline of "how to utilize the Hexagon NPU maximally through QNN SDK", details could be found at - * https://github.com/ggml-org/llama.cpp/pull/12049#issuecomment-2678308360 - * - * @param ctx the context of ggml-qnn backend - * @param op the destination tensor where the result of the matrix multiplication will be stored. - * - * @note the logic of ggml_qnn_mul_mat is similar to ggml_qnn_general_node but much more complicated - * than ggml_qnn_general_node. so it's a standalone function. accordingly, this is another - * typical skeleton for offload other ggml ops to QNN backend. MUL_MAT take most of the compute - * time (about 95%).so to speed up llama inference, should focus on this func. there are three kinds - * of MUL_MAT to compute: - * mul_mat_f32: both src0 and src1 are F32, this will be naturally handled in QNN backend - * mul_mat_f16_f32: src0 is F16 and src1 is F32, f16 in src0 -> f32 in src0', then src0' * src1 - * mul_mat_q_f32: src0 is quantized (Q4_0, Q4_1, Q6_K...) - * and src1 is F32, src0 -> f32 in src0', then src0' * src1 -*/ -void ggml_qnn_mul_mat(ggml_backend_qnn_context * ctx, ggml_tensor * op) { - Qnn_ErrorHandle_t error = QNN_SUCCESS; - qnn_perf op_perf = qnn_perf("ggml_qnn_mul_mat"); - qnn_instance * instance = nullptr; - Qnn_GraphHandle_t graph_handle = nullptr; - Qnn_Tensor_t * p_tensor0 = nullptr; - Qnn_Tensor_t * p_tensor1 = nullptr; - Qnn_Tensor_t * p_tensor2 = nullptr; - Qnn_Tensor_t * p_param_tensor = nullptr; - Qnn_Tensor_t * p_tensor2_transpose = nullptr; - const ggml_tensor * src0 = op->src[0]; - const ggml_tensor * src1 = op->src[1]; - ggml_tensor * dst = op; - - GGMLQNN_CHECK_PARAMS(ctx, src0, src1, dst); - instance = ctx->instance; - QNN_INTERFACE_VER_TYPE qnn_raw_interface = ctx->raw_interface; - op_perf.start(); - - const enum ggml_type src0_type = src0->type; - const uint32_t src0_rank = ggml_n_dims(src0); - const uint32_t src1_rank = ggml_n_dims(src1); - GGML_ASSERT(src0_rank == src1_rank); - GGML_ASSERT(src0_rank >= 2); //QNN SDK's limitation, make QNN SDK happy - if (4 == src0_rank) { - return ggml_qnn_mul_mat_4d(ctx, op); - } - void * wdata = ggmlqnn_type_trait(ctx, op); - const size_t desired_size = ctx->desired_size; - - ggmlqnn_print_tensors_info(__func__, ctx, src0, src1, dst); - - std::string graph_name; - ggmlqnn_get_graphkey_from_op(op, graph_name); - if (instance->_qnn_graph_map.find(graph_name) != instance->_qnn_graph_map.end()) { - //retrieve computational resource from cached QNN graph - qnn_res_t & graph_item = instance->_qnn_graph_map[graph_name]; - graph_handle = std::get<0>(graph_item); - qnn_tensors_t & tensors = std::get<1>(graph_item); - p_tensor0 = tensors[0]; - p_tensor1 = tensors[1]; - p_tensor2 = tensors[2]; - p_param_tensor = tensors[3]; - p_tensor2_transpose = tensors[4]; - } else { - //create QNN graph - GGMLQNN_LOG_DEBUG("graph name %s", graph_name.c_str()); - error = qnn_raw_interface.graphCreate(instance->get_qnn_context_handle(), graph_name.c_str(), nullptr, &graph_handle); - if (QNN_SUCCESS != error) { - GGMLQNN_LOG_WARN("can't create qnn graph handle with graph name %s, error = %d\n", graph_name.c_str(), error); - return; - } - - //create computational tensor - p_tensor0 = GQCGT(src0, nullptr, QNN_TENSOR_TYPE_APP_WRITE, QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0); - p_tensor1 = GQCGT(src1, nullptr, QNN_TENSOR_TYPE_APP_WRITE, QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0); - p_tensor2 = GQCGT(dst, nullptr, QNN_TENSOR_TYPE_APP_READ, QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor0)); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor1)); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor2)); - - //create param tensor for offload 2d/3d/4d matrix multiplication - const uint32_t param_tensor_data[GGML_MAX_DIMS][GGML_MAX_DIMS] = { - {0}, - {1, 0}, - {0, 2, 1}, - {0, 1, 3, 2}, - }; - uint32_t param_tensor_dims[1] = {src0_rank}; - p_param_tensor = GQCGT(nullptr, "param", QNN_TENSOR_TYPE_STATIC, QNN_DATATYPE_UINT_32, 1, param_tensor_dims, (void *)(param_tensor_data[src0_rank - 1]), src0_rank * sizeof(uint32_t)); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_param_tensor)); - - //create transpose tensor - p_tensor2_transpose = GQCGT(dst, "transpose", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0, true); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor2_transpose)); - - //compose QNN graph: add mulmat node - Qnn_Param_t out_0_params[] = {{QNN_PARAMTYPE_SCALAR, QNN_OP_MAT_MUL_PARAM_TRANSPOSE_IN1, .scalarParam = {QNN_DATATYPE_BOOL_8, .bool8Value = 1}}}; - Qnn_Tensor_t out_0_inputs[] = {*p_tensor0, *p_tensor1}; - Qnn_Tensor_t out_0_outputs[] = {*p_tensor2_transpose}; - Qnn_OpConfig_t out_0 = ggmlqnn_create_op_config("mulmat_opconfig", QNN_OP_PACKAGE_NAME_QTI_AISW, QNN_OP_MAT_MUL, out_0_params, 1, out_0_inputs, 2, out_0_outputs, 1); - CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle,out_0)); - - //compose QNN graph: add transpose node - Qnn_Param_t out_trans1_0_params[] = { {QNN_PARAMTYPE_TENSOR, "perm", .tensorParam = *p_param_tensor}}; - Qnn_Tensor_t out_trans1_0_inputs[] = {*p_tensor2_transpose}; - Qnn_Tensor_t out_trans1_0_outputs[] = {*p_tensor2}; - Qnn_OpConfig_t out_trans1_0 = ggmlqnn_create_op_config("mulmat_transpose_opconfig", QNN_OP_PACKAGE_NAME_QTI_AISW, QNN_OP_TRANSPOSE, out_trans1_0_params, 1, out_trans1_0_inputs, 1, out_trans1_0_outputs, 1); - CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle,out_trans1_0)); - - //finalize QNN graph - CHECK_QNN_API(error, qnn_raw_interface.graphFinalize(graph_handle, nullptr, nullptr)); - - //cache QNN graph - qnn_tensors_t ggml_op_mulmat_tensors; - ggml_op_mulmat_tensors.reserve(5); - ggml_op_mulmat_tensors.push_back(p_tensor0); - ggml_op_mulmat_tensors.push_back(p_tensor1); - ggml_op_mulmat_tensors.push_back(p_tensor2); - ggml_op_mulmat_tensors.push_back(p_param_tensor); - ggml_op_mulmat_tensors.push_back(p_tensor2_transpose); - auto graph_item = std::make_tuple(graph_handle, ggml_op_mulmat_tensors); - instance->_qnn_graph_map[graph_name] = graph_item; - } - - if (src0_type != GGML_TYPE_F32) { - QNN_VER_PTR(*p_tensor0)->clientBuf = {wdata, static_cast(desired_size)}; - } else { - QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggmlqnn_get_tensor_data_size(src0)}; - } - QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggmlqnn_get_tensor_data_size(src1)}; - QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggmlqnn_get_tensor_data_size(dst)}; - - Qnn_Tensor_t tensor_inputs[] = { - *p_tensor0, - *p_tensor1 - }; - Qnn_Tensor_t tensor_outputs[] = { - *p_tensor2 - }; - CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, - tensor_inputs, 2, - tensor_outputs, 1, - nullptr, nullptr)); - op_perf.info(); -} - -void ggml_qnn_repeat(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { - GGML_UNUSED(ctx); - GGML_UNUSED(dst); -} - -void ggml_qnn_div(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { - GGML_UNUSED(ctx); - GGML_UNUSED(dst); -} - -void ggml_qnn_leaky_relu(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { - GGML_UNUSED(ctx); - GGML_UNUSED(dst); -} - -void ggml_qnn_concat(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { - GGML_UNUSED(ctx); - GGML_UNUSED(dst); -} - -void ggml_qnn_arange(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { - GGML_UNUSED(ctx); - GGML_UNUSED(dst); -} - -void ggml_qnn_sqr(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { - GGML_UNUSED(ctx); - GGML_UNUSED(dst); -} - -void ggml_qnn_clamp(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { - GGML_UNUSED(ctx); - GGML_UNUSED(dst); -} - -void ggml_qnn_scale(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { - GGML_UNUSED(ctx); - GGML_UNUSED(dst); -} - -void ggml_qnn_argsort(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { - GGML_UNUSED(ctx); - GGML_UNUSED(dst); -} - -void ggml_qnn_norm(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { - GGML_UNUSED(ctx); - GGML_UNUSED(dst); -} - -void ggml_qnn_group_norm(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { - GGML_UNUSED(ctx); - GGML_UNUSED(dst); -} - -void ggml_qnn_acc(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { - GGML_UNUSED(ctx); - GGML_UNUSED(dst); -} - -void ggml_qnn_sum_rows(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { - GGML_UNUSED(ctx); - GGML_UNUSED(dst); -} - -void ggml_qnn_upsample_nearest2d(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { - GGML_UNUSED(ctx); - GGML_UNUSED(dst); -} - -void ggml_qnn_pad(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { - GGML_UNUSED(ctx); - GGML_UNUSED(dst); -} - -void ggml_qnn_pool2d(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { - GGML_UNUSED(ctx); - GGML_UNUSED(dst); -} - -void ggml_qnn_dup(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { - GGML_UNUSED(ctx); - GGML_UNUSED(dst); -} - -void ggml_qnn_rms_norm(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { - GGML_UNUSED(ctx); - GGML_UNUSED(dst); -} - -void ggml_qnn_diag_mask(ggml_backend_qnn_context * ctx, ggml_tensor * dst, float value) { - GGML_UNUSED(ctx); - GGML_UNUSED(dst); - GGML_UNUSED(value); -} - -void ggml_qnn_im2col(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { - GGML_UNUSED(ctx); - GGML_UNUSED(dst); -} - -void ggml_qnn_timestep_embedding(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { - GGML_UNUSED(ctx); - GGML_UNUSED(dst); -} - -void ggml_qnn_cpy(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { - ggml_qnn_dup(ctx, dst); -} - -void ggml_qnn_softmax(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { - GGML_UNUSED(ctx); - GGML_UNUSED(dst); -} - -void ggml_qnn_get_rows(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { - GGML_UNUSED(ctx); - GGML_UNUSED(dst); -} - -void ggml_qnn_rope(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { - GGML_UNUSED(ctx); - GGML_UNUSED(dst); -} diff --git a/ggml/src/ggml-qnn/ggml-qnn-ops.h b/ggml/src/ggml-qnn/ggml-qnn-ops.h deleted file mode 100644 index b1c388a32a87a..0000000000000 --- a/ggml/src/ggml-qnn/ggml-qnn-ops.h +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (c) 2023-2024 The ggml authors - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ -#pragma once - -#include "ggml-qnn-impl.h" -void ggml_qnn_general_node(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -void ggml_qnn_mul_mat(ggml_backend_qnn_context * ctx, ggml_tensor * dst); - -void ggml_qnn_repeat(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -void ggml_qnn_div(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -void ggml_qnn_leaky_relu(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -void ggml_qnn_concat(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -void ggml_qnn_arange(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -void ggml_qnn_sqr(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -void ggml_qnn_clamp(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -void ggml_qnn_scale(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -void ggml_qnn_argsort(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -void ggml_qnn_norm(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -void ggml_qnn_group_norm(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -void ggml_qnn_acc(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -void ggml_qnn_sum_rows(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -void ggml_qnn_upsample_nearest2d(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -void ggml_qnn_pad(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -void ggml_qnn_pool2d(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -void ggml_qnn_dup(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -void ggml_qnn_rms_norm(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -void ggml_qnn_diag_mask(ggml_backend_qnn_context * ctx, ggml_tensor * dst, float value); -void ggml_qnn_im2col(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -void ggml_qnn_timestep_embedding(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -void ggml_qnn_cpy(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -void ggml_qnn_softmax(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -void ggml_qnn_get_rows(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -void ggml_qnn_rope(ggml_backend_qnn_context * ctx, ggml_tensor * dst); diff --git a/ggml/src/ggml-qnn/ggml-qnn.cpp b/ggml/src/ggml-qnn/ggml-qnn.cpp index 35b565c7d7669..083f3ec466528 100644 --- a/ggml/src/ggml-qnn/ggml-qnn.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn.cpp @@ -1,17 +1,18 @@ /* - * Copyright (c) 2023-2024 The ggml authors + * Copyright (c) 2024- KanTV authors * * Qualcomm QNN SDK and reference tech guides could be found at: * https://www.qualcomm.com/developer/software/qualcomm-ai-engine-direct-sdk * https://developer.qualcomm.com/software/hexagon-dsp-sdk/tools * - * the implementation of ggml-qnn backend has six sections: + * this single-source-file or self-contained implementation of ggml-qnn backend has seven sections: * section-1 does forward/external declaration, * section-2 defines ggml-qnn internal log function * section-3 does general helper macro / data structure / function * section-4 does QNN helper macro / data structure / function * section-5 does ggml-qnn backend helper macro / data structure / function / class * section-6 does implementation of ggml-qnn backend according to ggml's backend subsystem + * section-7 does implementation of offload ggml op to QNN backend * * currently provide following ggml ops' QNN backend implementation in ggml-qnn-ops.cpp: * - GGML_OP_ADD: this is a simple skeleton, can expand other ggml ops according to expertise @@ -36,19 +37,144 @@ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS * IN THE SOFTWARE. */ -#include "ggml-qnn-impl.h" -#include "ggml-qnn-ops.h" +#include +#include +#include +#include +#include +#include +#include +#include +#if defined(__ANDROID__) || defined(__linux__) +#include +#include +#include +#include +#include +#include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if (defined __ANDROID__) || (defined ANDROID) +#include "android/log.h" +#endif + +#if !defined(__ANDROID__) && !defined(__linux__) +#include +#include +#include +#endif + +#include "QnnTypes.h" +#include "QnnCommon.h" +#include "QnnContext.h" +#include "QnnBackend.h" +#include "QnnGraph.h" +#include "QnnProperty.h" +#include "QnnTensor.h" +#include "QnnInterface.h" +#include "Saver/QnnSaver.h" +#include "System/QnnSystemInterface.h" +#include "HTP/QnnHtpDevice.h" +#include "HTP/QnnHtpGraph.h" + +#include "ggml-qnn.h" +#include "ggml-impl.h" +#include "ggml-backend-impl.h" + // ================================================================================================= // section-1: forward/external declaration // ================================================================================================= -static int free_qnn_tensor(Qnn_Tensor_t * tensor); -static enum ggml_status ggml_backend_qnn_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph); +class qnn_instance; +struct ggml_backend_qnn_context; typedef void (* ggmlqnn_op_func_t)(ggml_backend_qnn_context * ctx, ggml_tensor * op); +static Qnn_Tensor_t * ggmlqnn_create_general_tensor(const ggml_tensor * tensor, const char * name, + Qnn_TensorType_t qnn_tensor_type, + Qnn_DataType_t qnn_data_type, + uint32_t rank, uint32_t * dims, + void * data, uint32_t data_size, + bool b_transpose = false); + +static void ggml_qnn_general_node(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggml_qnn_mul_mat(ggml_backend_qnn_context * ctx, ggml_tensor * dst); + +static void ggml_qnn_repeat(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggml_qnn_div(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggml_qnn_leaky_relu(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggml_qnn_concat(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggml_qnn_arange(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggml_qnn_sqr(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggml_qnn_clamp(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggml_qnn_scale(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggml_qnn_argsort(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggml_qnn_norm(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggml_qnn_group_norm(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggml_qnn_acc(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggml_qnn_sum_rows(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggml_qnn_upsample_nearest2d(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggml_qnn_pad(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggml_qnn_pool2d(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggml_qnn_dup(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggml_qnn_rms_norm(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggml_qnn_diag_mask(ggml_backend_qnn_context * ctx, ggml_tensor * dst, float value); +static void ggml_qnn_im2col(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggml_qnn_timestep_embedding(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggml_qnn_cpy(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggml_qnn_softmax(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggml_qnn_get_rows(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggml_qnn_rope(ggml_backend_qnn_context * ctx, ggml_tensor * dst); + // ================================================================================================= -// section-2: ggml-qnn internal troubleshooting function +// section-2: ggml-qnn internal troubleshooting function/class // ================================================================================================= -void ggmlqnn_log_internal(ggml_log_level level, const char * file, const char * func, int line, const char * format, ...) { +#if 0//def NDEBUG +#define GGMLQNN_DEBUG 0 +#define ENABLE_QNNBACKEND_PERF 0 // enable/disable op's perf info +#define GGMLQNN_PRINT_QNN_INTERNAL_LOG 0 // enable/disable QNN's internal log +#define GGMLQNN_PRINT_OP_ADD_LOG 0 // GGML_OP_ADD already verified with QNN-CPU / QNN-GPU / QNN-NPU +#define GGMLQNN_PRINT_OP_MUL_MAT_LOG 0 +#else +#define GGMLQNN_DEBUG 1 // for troubleshooting QNN backend +#define ENABLE_QNNBACKEND_PERF 0 // enable/disable op's perf info +#define GGMLQNN_PRINT_QNN_INTERNAL_LOG 0 // enable/disable QNN's internal log +#define GGMLQNN_PRINT_OP_ADD_LOG 0 // GGML_OP_ADD already verified with QNN-CPU / QNN-GPU / QNN-NPU +#define GGMLQNN_PRINT_OP_MUL_MAT_LOG 1 +#endif +#define GGML_QNN_LOGBUF_LEN 4096 + +#define GGMLQNN_LOG_ERROR(...) ggmlqnn_log_internal(GGML_LOG_LEVEL_ERROR, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) +#define GGMLQNN_LOG_WARN(...) ggmlqnn_log_internal(GGML_LOG_LEVEL_WARN , __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) +#define GGMLQNN_LOG_INFO(...) ggmlqnn_log_internal(GGML_LOG_LEVEL_INFO , __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) + +#if GGMLQNN_DEBUG +#define GGMLQNN_LOG_DEBUG(...) ggmlqnn_log_internal(GGML_LOG_LEVEL_DEBUG, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) +#else +#define GGMLQNN_LOG_DEBUG(...) +#endif +static void ggmlqnn_log_internal(ggml_log_level level, const char * file, const char * func, int line, const char * format, ...) { static std::mutex ggmlqnn_log_internal_mutex; static char s_ggmlqnn_log_internal_buf[GGML_QNN_LOGBUF_LEN]; @@ -78,13 +204,72 @@ void ggmlqnn_log_internal(ggml_log_level level, const char * file, const char * } } +#if ENABLE_QNNBACKEND_PERF +class qnn_perf { +public: + qnn_perf(const std::string & perf_name) : _perf_name(std::move(perf_name)) {}; + qnn_perf() = delete; + qnn_perf(const qnn_perf & ) = delete; + qnn_perf & operator= (const qnn_perf & ) = delete; + + void start() { + _begin_time = ggml_time_us(); + } + + void info() { + _end_time = ggml_time_us(); + _duration = (_end_time - _begin_time); + GGMLQNN_LOG_DEBUG("duration of %s : %lld microseconds\n", _perf_name.c_str(), _duration); + } + +private: + int64_t _begin_time = 0LL; + int64_t _end_time = 0LL; + int64_t _duration = 0LL; + std::string _perf_name; +}; +#else +class qnn_perf { +public: + qnn_perf(const std::string & perf_name) { + GGML_UNUSED(perf_name); + } + qnn_perf() = delete; + qnn_perf(const qnn_perf & ) = delete; + qnn_perf & operator= (const qnn_perf & ) = delete; + + void start() {} + void info() {} +}; +#endif + // ================================================================================================= // section-3: general helper macro / data structure / function // ================================================================================================= +#define DISABLE_COPY(class_name) \ + class_name(const class_name &) = delete; \ + void operator=(const class_name &) = delete + +#define DISABLE_MOVE(class_name) \ + class_name(class_name &&) = delete; \ + void operator=(class_name &&) = delete + +#define GQCGT ggmlqnn_create_general_tensor + +//#if defined(_WIN32) #if !defined(__ANDROID__) && !defined(__linux__) +#define RTLD_GLOBAL 0x100 +#define RTLD_LOCAL 0x000 +#define RTLD_LAZY 0x000 +#define RTLD_NOW 0x001 +static void * dlopen(const char * filename, int flag); +static int dlclose(void * handle); +static void * dlsym(void* handle, const char* name); +static const char * dlerror(void); + static const char * last_func = nullptr; static long last_err; -void * dlopen(const char * dll, int flags) { +static void * dlopen(const char * dll, int flags) { HINSTANCE h = LoadLibraryA(dll); GGML_UNUSED(flags); if (h == NULL) { @@ -94,7 +279,7 @@ void * dlopen(const char * dll, int flags) { return h; } -int dlclose(void * h) { +static int dlclose(void * h) { if (!FreeLibrary((HINSTANCE)h)) { last_err = GetLastError(); last_func = "dlclose"; @@ -103,7 +288,7 @@ int dlclose(void * h) { return 0; } -void * dlsym(void * h, const char * name) { +static void * dlsym(void * h, const char * name) { FARPROC p = GetProcAddress((HINSTANCE)h, name); if (!p) { last_err = GetLastError(); @@ -112,7 +297,7 @@ void * dlsym(void * h, const char * name) { return (void*)(intptr_t)p; } -const char * dlerror(void) { +static const char * dlerror(void) { static char str[512]; if (!last_err) return nullptr; @@ -256,6 +441,22 @@ static void * ggmlqnn_host_malloc(size_t buffer_size, size_t page_size) { // ================================================================================================= // section-4: QNN helper macro / data structure / function // ================================================================================================= +#define CHECK_QNN_API(error, result) \ + do { \ + error = (result); \ + if (QNN_SUCCESS != error) { \ + if (error == QNN_COMMON_ERROR_NOT_SUPPORTED) { \ + GGMLQNN_LOG_WARN("WARNING: QNN feature/API not supported\n"); \ + } else { \ + GGMLQNN_LOG_INFO("QNN API error = %d(%s)\n", error, ggmlqnn_get_error_string(error)); \ + } \ + } \ + } while (0) + +#define QNN_VER_PTR(x) (&((x).v1)) +#define RPCMEM_DEFAULT_FLAGS 1 +#define RPCMEM_HEAP_ID_SYSTEM 25 + #define QNN_TENSOR_GET_ID(tensor) get_qnn_tensorid(tensor) #define QNN_TENSOR_GET_NAME(tensor) get_qnn_tensorname(tensor) #define QNN_TENSOR_GET_TYPE(tensor) get_qnn_tensortype(tensor) @@ -498,7 +699,7 @@ static int free_qnn_tensor(Qnn_Tensor_t * tensor) { return err; } -const char * ggmlqnn_get_error_string(Qnn_ErrorHandle_t qnn_error_code) { +static const char * ggmlqnn_get_error_string(Qnn_ErrorHandle_t qnn_error_code) { // file:///opt/qcom/aistack/qairt/2.31.0.250130/docs/QNN/general/api_error_codes.html switch (qnn_error_code) { case QNN_SUCCESS: @@ -601,8 +802,7 @@ const char * ggmlqnn_get_error_string(Qnn_ErrorHandle_t qnn_error_code) { } } -// helper function to create an operation config -Qnn_OpConfig_t ggmlqnn_create_op_config(const char * name, const char * package, const char * type, +static Qnn_OpConfig_t ggmlqnn_create_op_config(const char * name, const char * package, const char * type, Qnn_Param_t * params, uint32_t num_params, Qnn_Tensor_t * inputs, uint32_t num_inputs, Qnn_Tensor_t * outputs, uint32_t num_outputs) { @@ -619,6 +819,81 @@ Qnn_OpConfig_t ggmlqnn_create_op_config(const char * name, const char * package, // ================================================================================================= // section-5:ggml-qnn backend helper macro / data structure / function / class // ================================================================================================= +using pfn_rpc_mem_init = void (*)(void); +using pfn_rpc_mem_deinit = void (*)(void); +using pfn_rpc_mem_alloc = void *(*)(int, uint32_t, int); +using pfn_rpc_mem_free = void (*)(void *); +using pfn_rpc_mem_to_fd = int (*)(void *); +using _pfn_QnnSaver_initialize = decltype(QnnSaver_initialize); +using _pfn_QnnInterface_getProviders = decltype(QnnInterface_getProviders); +using _pfn_QnnSystemInterface_getProviders = decltype(QnnSystemInterface_getProviders); + +using qnn_res_t = std::tuple>; +using qnn_tensors_t = std::vector< Qnn_Tensor_t *>; + +enum class ggml_qnn_profile_level { + profile_off = 0, + profile_basic = 1, + profile_detail = 2 +}; + +enum qcom_htp_arch { + NONE = 0, + V68 = 68, + V69 = 69, + V73 = 73, + V75 = 75, + V79 = 79, +}; + +enum qcom_chipset_soc_model { + UNKNOWN_SM = 0, + SM7450 = 41, // v69, 7 Gen1 + SM8350 = 30, // v68, 888 + SM8450 = 36, // v69, SD 8 Gen 1 + SM8475 = 42, // v69, SD 8+ Gen 1 + SM8550 = 43, // v73, SD 8 Gen 2 + SM8650 = 57, // v75, SD 8 Gen 3 + SM8750 = 69, // v79, SD 8 Gen 4 +#if !defined(__ANDROID__) && !defined(__linux__) + SC7280X = 44, + SC8280X = 37, + SC8380XP = 60, +#endif +}; + +struct qcom_socinfo { + uint32_t soc_model; + size_t htp_arch; + size_t vtcm_size_in_mb; + char soc_desc[GGML_MAX_NAME]; +}; + +struct ggml_backend_qnn_context { + int device; + int threads; + char name[GGML_MAX_NAME]; + char desc[GGML_MAX_NAME]; + char lib[GGML_MAX_NAME]; + qnn_instance * instance; + struct ggml_backend * backend; + QNN_INTERFACE_VER_TYPE raw_interface; + QNN_SYSTEM_INTERFACE_VER_TYPE raw_system_interface; + struct qcom_socinfo socinfo; + + std::unique_ptr work_data; + std::vector> tasks; + size_t work_size = 0; + size_t desired_size = 0; + int n_threads = GGML_DEFAULT_N_THREADS; +}; + +struct qnn_op_caps_t { + const char * qnn_op_name = nullptr; + const size_t input_param_count = 0; + const char * qnn_param_name = nullptr; +}; + //file:///opt/qcom/aistack/qairt/2.31.0.250130/docs/QNN/general/overview.html#tbl-supported-snapdragon-devices static struct qcom_socinfo g_qnn_soc_info_table[] = { /* Qualcomm SnapDragon 7 Gen 1 */ @@ -772,7 +1047,7 @@ static struct ggml_backend_qnn_context g_qnn_mgr[GGML_QNN_MAX_DEVICES] = { .socinfo = {}}, }; -const qnn_op_caps_t ggmlqnn_k_op_caps[] = { +static const qnn_op_caps_t ggmlqnn_k_op_caps[] = { {}, // GGML_OP_NONE {}, // GGML_OP_DUP { @@ -944,7 +1219,7 @@ static const char * get_ggml_type_name(ggml_type type) { } // ref:explanation of k-quants, https://github.com/ggerganov/llama.cpp/pull/1684 -Qnn_DataType_t ggmlqnn_datatype_from_ggml_datatype(enum ggml_type ggmltype) { +static Qnn_DataType_t ggmlqnn_datatype_from_ggml_datatype(enum ggml_type ggmltype) { switch (ggmltype) { case GGML_TYPE_F16: return QNN_DATATYPE_FLOAT_16; @@ -1003,127 +1278,8 @@ static void get_qnn_dimensions_from_ggml_dimensions(uint32_t * qnn_dimensions, c } } -Qnn_Tensor_t * ggmlqnn_create_general_tensor(const ggml_tensor * tensor, const char * name, - Qnn_TensorType_t qnn_tensor_type, - Qnn_DataType_t qnn_data_type, - uint32_t rank, uint32_t * dims, - void * data, uint32_t data_size, - bool b_transpose) { - Qnn_ErrorHandle_t error = QNN_SUCCESS; - char tensor_name[GGML_MAX_NAME] = {}; - - //ensure the tensor name is unique - if (nullptr == name) { - snprintf(tensor_name, GGML_MAX_NAME, "tensor_%-8d", get_idx()); - } else { - snprintf(tensor_name, GGML_MAX_NAME, "tensor_%s%-8d", name, get_idx()); - } - GGMLQNN_LOG_DEBUG("init_tensor %d", get_idx()); - inc_idx(); - - uint32_t reverse_dims[GGML_MAX_DIMS] = {}; - uint32_t transpose_dims[GGML_MAX_DIMS] = {}; - uint32_t * tensor_dims = nullptr; - //case 1:use dims info from ggml tensor - if (nullptr != tensor) { - //there are different dimension order between ggml tensor and qnn tensor - for (size_t idx = 0; idx < rank; idx++) { - reverse_dims[idx] = (uint32_t)tensor->ne[rank - 1 - idx]; - } - tensor_dims = reverse_dims; - } - //case 2: use user's specified tensor_dims - if (nullptr != dims) { - tensor_dims = dims; - } - //case 3: transpose for dst tensor - if (b_transpose) { - GGML_ASSERT(tensor != nullptr); //ensure ggml_tensor is not nullptr for this special case - - get_qnn_dimensions_from_ggml_dimensions(transpose_dims, reverse_dims, ggml_n_dims(tensor)); - tensor_dims = transpose_dims; -#if 0 - for (size_t idx = 0; idx < 4; idx++) { - GGMLQNN_LOG_DEBUG("origin dim[%d]=%d\n", idx, reverse_dims[idx]); - } - for (size_t idx = 0; idx < 4; idx++) { - GGMLQNN_LOG_DEBUG("trans dim[%d]=%d\n", idx, transpose_dims[idx]); - } -#endif - } - - Qnn_Tensor_t qnn_tensor = { - .version= QNN_TENSOR_VERSION_1, - {.v1= { - .id = 0, - .name = tensor_name, - .type = qnn_tensor_type, - .dataFormat = QNN_TENSOR_DATA_FORMAT_FLAT_BUFFER, - .dataType = qnn_data_type, - .quantizeParams = {.encodingDefinition = QNN_DEFINITION_UNDEFINED, - .quantizationEncoding = QNN_QUANTIZATION_ENCODING_UNDEFINED, - {.scaleOffsetEncoding = {.scale = 0.0000000000000000f, .offset = 0}}}, - .rank = rank, - .dimensions = tensor_dims, - .memType = QNN_TENSORMEMTYPE_RAW, - .clientBuf = {.data = nullptr, .dataSize = 0} - } - } - }; - if (nullptr != name) { - QNN_VER_PTR(qnn_tensor)->name = name; - } - Qnn_Tensor_t * p_qnn_tensor = (Qnn_Tensor_t *)calloc(1, sizeof(Qnn_Tensor_t)); - if (nullptr == p_qnn_tensor) { - GGMLQNN_LOG_WARN("calloc failed"); - return nullptr; - } - error = deep_copy_qnn_tensors(qnn_tensor, * p_qnn_tensor); - if (error != QNN_SUCCESS) { - free(p_qnn_tensor); - GGMLQNN_LOG_WARN("init tensor failed"); - return nullptr; - } - QNN_VER_PTR(*p_qnn_tensor)->clientBuf = {data, data_size}; - - return p_qnn_tensor; -} - -Qnn_Tensor_t * ggmlqnn_create_compute_tensor(qnn_instance * instance, Qnn_GraphHandle_t graph_handle, const ggml_tensor * tensor, Qnn_TensorType_t tensor_type) { - Qnn_ErrorHandle_t error = QNN_SUCCESS; - uint32_t dimensions[] = {(uint32_t) tensor->ne[0], (uint32_t) tensor->ne[1], - (uint32_t) tensor->ne[2], (uint32_t) tensor->ne[3]}; - Qnn_DataType_t qnn_data_type = QNN_DATATYPE_FLOAT_32; - Qnn_TensorType_t qnn_tensor_type = QNN_TENSOR_TYPE_APP_WRITE; - - if (0 == tensor->flags) { - qnn_tensor_type = tensor_type; - } else { - if (tensor->flags & GGML_TENSOR_FLAG_INPUT) { - qnn_tensor_type = QNN_TENSOR_TYPE_APP_WRITE; - } else if (tensor->flags & GGML_TENSOR_FLAG_OUTPUT) { - qnn_tensor_type = QNN_TENSOR_TYPE_APP_READ; - } - } - - qnn_data_type = ggmlqnn_datatype_from_ggml_datatype(tensor->type); - Qnn_Tensor_t * p_qnn_tensor = ggmlqnn_create_general_tensor(tensor, nullptr, - qnn_tensor_type, qnn_data_type, - ggml_n_dims(tensor), dimensions, - nullptr, 0); - - bool enable_npu_rpc = (instance->enable_qnn_rpc() && instance->get_device_id() == QNN_BACKEND_NPU); - if (enable_npu_rpc) { - QNN_VER_PTR(*p_qnn_tensor)->memType = QNN_TENSORMEMTYPE_MEMHANDLE; - QNN_VER_PTR(*p_qnn_tensor)->clientBuf = {.data=nullptr, .dataSize=0}; - } - QNN_INTERFACE_VER_TYPE qnn_raw_interface = instance->get_qnn_raw_interface(); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_qnn_tensor)); - - return p_qnn_tensor; -} -void * ggmlqnn_type_trait(ggml_backend_qnn_context * ctx, ggml_tensor * op) { +static void * ggmlqnn_type_trait(ggml_backend_qnn_context * ctx, ggml_tensor * op) { const ggml_tensor * src0 = op->src[0]; const ggml_tensor * src1 = op->src[1]; ggml_tensor * dst = op; @@ -1216,11 +1372,11 @@ static void append_tensor_dimensions(const ggml_tensor * tensor, std::string & o output.append(buffer, len); } -size_t ggmlqnn_get_opcaps_size() { +static size_t ggmlqnn_get_opcaps_size() { return std::size(ggmlqnn_k_op_caps); } -size_t ggmlqnn_get_op_index(const ggml_tensor * tensor) { +static size_t ggmlqnn_get_op_index(const ggml_tensor * tensor) { if (tensor->op == GGML_OP_UNARY) { return static_cast(GGML_OP_COUNT) + static_cast(ggml_get_unary_op(tensor)); } @@ -1234,7 +1390,7 @@ static size_t ggmlqnn_get_op_input_param_count(const ggml_tensor * op) { return ggmlqnn_k_op_caps[op_index].input_param_count; } -void ggmlqnn_get_graphkey_from_op(const ggml_tensor * op, std::string & output) { +static void ggmlqnn_get_graphkey_from_op(const ggml_tensor * op, std::string & output) { GGML_ASSERT(op->op != GGML_OP_NONE); output += ggml_op_desc(op); output += get_ggml_type_name(op->type); @@ -1254,118 +1410,445 @@ Fn load_qnn_functionpointers(void * handle, const char * function_name) { return reinterpret_cast(dlsym(handle, function_name)); } -std::mutex qnn_instance::_init_mutex; -std::unordered_map qnn_instance::_loaded_lib_handle; -std::unordered_map qnn_instance::_lib_path_to_backend_id; -std::unordered_map qnn_instance::_loaded_backend; +class qnn_interface { +#define DEFINE_SHIM_FUNCTION_INTERFACE(F, pointer_name) \ + template \ + inline auto qnn_##F(Args... args) const { \ + return (_qnn_interface->QNN_INTERFACE_VER_NAME.pointer_name)( \ + std::forward(args)...); \ + } -void * qnn_instance::alloc_rpcmem_internal(size_t bytes, size_t alignment) { - if (!_rpcmem_initialized) { - GGMLQNN_LOG_WARN("rpc memory not initialized\n"); - return nullptr; - } - auto allocate_bytes = static_cast(bytes + alignment); - void * buf = _pfn_rpc_mem_alloc(RPCMEM_HEAP_ID_SYSTEM, RPCMEM_DEFAULT_FLAGS, allocate_bytes); - if (nullptr == buf) { - GGMLQNN_LOG_WARN("failed to allocate rpc memory\n"); - return nullptr; - } +#define DEFINE_SHIM_FUNCTION_SYS_INTERFACE(F, pointer_name) \ + template \ + inline auto qnn_##F(Args... args) const { \ + return (_qnn_sys_interface->QNN_SYSTEM_INTERFACE_VER_NAME.pointer_name)( \ + std::forward(args)...); \ + } - auto aligned_buf = reinterpret_cast(ggmlqnn_align_to(alignment, - reinterpret_cast(buf))); - bool status = _rpcmem_store_map.insert(std::pair(aligned_buf, buf)).second; - if (!status) { - GGMLQNN_LOG_WARN("failed to allocate rpc memory\n"); - _pfn_rpc_mem_free(buf); - } - return aligned_buf; -} + friend class qnn_instance; -void * qnn_instance::alloc_rpcmem(size_t bytes, size_t alignment) { - if (_rpcmem_usage > (_rpcmem_capacity - 8)) { // reserve 8Mbytes in rpc mempool - GGMLQNN_LOG_WARN("rpc mempool capcaity: %d MB, usage: %d MB", _rpcmem_capacity, _rpcmem_usage); - return nullptr; - } +public: + qnn_interface() = default; - auto aligned_buf = alloc_rpcmem_internal(bytes, alignment); - if (nullptr == aligned_buf) - return nullptr; - _rpcmem_usage_map.insert(std::pair(aligned_buf, bytes)); + // QnnBackend + DEFINE_SHIM_FUNCTION_INTERFACE(backend_create, backendCreate) - size_t rpcmem_usage_in_bytes = _rpcmem_usage * (1 << 20); - rpcmem_usage_in_bytes += bytes; - _rpcmem_usage = rpcmem_usage_in_bytes / ( 1 << 20); - return aligned_buf; -} + DEFINE_SHIM_FUNCTION_INTERFACE(backend_free, backendFree) -void qnn_instance::free_rpcmem(void * buf) { - size_t rpcbuffer_size = 0; - if (!_rpcmem_initialized) { - GGMLQNN_LOG_WARN("rpc memory not initialized\n"); - } else if (0 == _rpcmem_store_map.count(buf)) { - GGMLQNN_LOG_WARN("no allocated tensor\n"); - } else { - GGMLQNN_LOG_DEBUG("free rpc mem %p", _rpcmem_store_map[buf]); - for (std::unordered_map::iterator it = _rpcmem_usage_map.begin(); - it != _rpcmem_usage_map.end(); - it++) { - void * rpcbuffer = it->first; - if (buf == rpcbuffer) { - rpcbuffer_size = it->second; - size_t rpcmem_usage_in_bytes = _rpcmem_usage * (1 << 20); - rpcmem_usage_in_bytes -= rpcbuffer_size; - _rpcmem_usage = rpcmem_usage_in_bytes / ( 1 << 20); - } - } - if (rpcbuffer_size != 0) { - _rpcmem_usage_map.erase(buf); - } else { - GGMLQNN_LOG_WARN("it shouldn't happen, pls check why?"); - } - _pfn_rpc_mem_free(_rpcmem_store_map[buf]); - _rpcmem_store_map.erase(buf); - } -} + DEFINE_SHIM_FUNCTION_INTERFACE(backend_register_op_package, backendRegisterOpPackage) -void qnn_instance::free_rpcmem() { - if (_rpcmem_store_map.empty()) { - GGMLQNN_LOG_WARN("no rpcmem allocated\n"); - return; - } + DEFINE_SHIM_FUNCTION_INTERFACE(backend_validate_op_config, backendValidateOpConfig) - for (std::unordered_map::iterator it = _rpcmem_store_map.begin(); - it != _qnn_mem_set.end(); - it++) { - void * rpcbuffer = it->second; - GGMLQNN_LOG_DEBUG("free rpc buffer %p", rpcbuffer); - _pfn_rpc_mem_free(rpcbuffer); - } - _rpcmem_store_map.clear(); - _rpcmem_usage_map.clear(); - _rpcmem_usage = 0; -} + DEFINE_SHIM_FUNCTION_INTERFACE(backend_get_api_version, backendGetApiVersion) -int32_t qnn_instance::rpcmem_to_fd(void * buf) { - int32_t mem_fd = -1; - if (!is_rpcmem_initialized()) { - GGMLQNN_LOG_WARN("rpc memory not initialized\n"); - } else { - mem_fd = _pfn_rpc_mem_to_fd(buf); - } + // QnnDevice + DEFINE_SHIM_FUNCTION_INTERFACE(device_create, deviceCreate) - return mem_fd; -} + DEFINE_SHIM_FUNCTION_INTERFACE(device_free, deviceFree) -int qnn_instance::register_rpcmem(void * p_data, Qnn_Tensor_t * p_tensor) { - if (nullptr == p_data || (nullptr == p_tensor)) { - GGMLQNN_LOG_WARN("invalid param\n"); - return 1; - } + DEFINE_SHIM_FUNCTION_INTERFACE(device_get_infrastructure, deviceGetInfrastructure) - if (!is_rpcmem_initialized()) { - GGMLQNN_LOG_WARN("rpc memory not initialized\n"); - return 2; + DEFINE_SHIM_FUNCTION_INTERFACE(device_get_platform_info, deviceGetPlatformInfo) + + DEFINE_SHIM_FUNCTION_INTERFACE(device_get_info, deviceGetInfo) + + // QnnContext + DEFINE_SHIM_FUNCTION_INTERFACE(context_create, contextCreate) + + DEFINE_SHIM_FUNCTION_INTERFACE(context_get_binary_size, contextGetBinarySize) + + DEFINE_SHIM_FUNCTION_INTERFACE(context_get_binary, contextGetBinary) + + DEFINE_SHIM_FUNCTION_INTERFACE(context_create_from_binary, contextCreateFromBinary) + + DEFINE_SHIM_FUNCTION_INTERFACE(context_free, contextFree) + + // QnnGraph + DEFINE_SHIM_FUNCTION_INTERFACE(graph_create, graphCreate) + + DEFINE_SHIM_FUNCTION_INTERFACE(graph_add_node, graphAddNode) + + DEFINE_SHIM_FUNCTION_INTERFACE(graph_finalize, graphFinalize) + + DEFINE_SHIM_FUNCTION_INTERFACE(graph_execute, graphExecute) + + DEFINE_SHIM_FUNCTION_INTERFACE(graph_retrieve, graphRetrieve) + + // QnnLog + DEFINE_SHIM_FUNCTION_INTERFACE(log_create, logCreate) + + DEFINE_SHIM_FUNCTION_INTERFACE(log_free, logFree) + + DEFINE_SHIM_FUNCTION_INTERFACE(log_set_log_level, logSetLogLevel) + + // QnnProfile + DEFINE_SHIM_FUNCTION_INTERFACE(profile_create, profileCreate) + + DEFINE_SHIM_FUNCTION_INTERFACE(profile_get_events, profileGetEvents) + + DEFINE_SHIM_FUNCTION_INTERFACE(profile_get_sub_events, profileGetSubEvents) + + DEFINE_SHIM_FUNCTION_INTERFACE(profile_get_event_data, profileGetEventData) + + DEFINE_SHIM_FUNCTION_INTERFACE(profile_free, profileFree) + + // QnnMem + DEFINE_SHIM_FUNCTION_INTERFACE(mem_register, memRegister) + + DEFINE_SHIM_FUNCTION_INTERFACE(mem_de_register, memDeRegister) + + // QnnProperty + DEFINE_SHIM_FUNCTION_INTERFACE(property_has_capability, propertyHasCapability) + + // QnnTensor + DEFINE_SHIM_FUNCTION_INTERFACE(tensor_create_context_tensor, tensorCreateContextTensor) + + DEFINE_SHIM_FUNCTION_INTERFACE(tensor_create_graph_tensor, tensorCreateGraphTensor) + + // QnnSystem + DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_create, systemContextCreate) + + DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_get_binary_info, systemContextGetBinaryInfo) + + DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_free, systemContextFree) + + void set_qnn_interface(const QnnInterface_t * qnn_interface) { + _qnn_interface = qnn_interface; + } + + void set_qnn_system_interface(const QnnSystemInterface_t * qnn_sys_interface) { + _qnn_sys_interface = qnn_sys_interface; + } + + uint32_t get_backend_id() const { + return _qnn_interface->backendId; + } + + bool is_loaded() const { + return ((_qnn_sys_interface != nullptr) && (_qnn_interface != nullptr)); + } + +private: + const QnnInterface_t * _qnn_interface = nullptr; + + const QnnSystemInterface_t * _qnn_sys_interface = nullptr; +}; + +class qnn_instance { +public: + using BackendIdType = decltype(QnnInterface_t{}.backendId); + + explicit qnn_instance(const std::string & lib_path, const std::string & backend_name, + const std::string & model_name) : + _lib_path(std::move(lib_path)), + _backend_name(std::move(backend_name)), + _model_name(std::move(model_name)) {} + + ~qnn_instance() { + } + + int qnn_init(const QnnSaver_Config_t ** saver_config); + + int qnn_finalize(); + + const qnn_interface & get_qnn_interface() { + if (!_qnn_interface.is_loaded()) { + GGMLQNN_LOG_WARN("pls check why _qnn_interface is not loaded\n"); + } + return _qnn_interface; + } + + const QNN_INTERFACE_VER_TYPE & get_qnn_raw_interface() { + if (!_qnn_interface.is_loaded()) { + GGMLQNN_LOG_WARN("pls check why _qnn_interface is not loaded\n"); + } + return _qnn_raw_interface; + } + + const QNN_SYSTEM_INTERFACE_VER_TYPE & get_qnn_raw_system_interface() { + if (!_qnn_interface.is_loaded()) { + GGMLQNN_LOG_WARN("pls check why _qnn_interface is not loaded\n"); + } + return _qnn_raw_system_interface; + } + + Qnn_LogHandle_t get_qnn_log_handle() { return _qnn_log_handle; } + + Qnn_ProfileHandle_t get_qnn_profile_handle() { return _qnn_profile_handle; } + + Qnn_DeviceHandle_t get_qnn_device_handle() { return _qnn_device_handle; } + + Qnn_BackendHandle_t get_qnn_backend_handle() { return _qnn_backend_handle; } + + Qnn_ContextHandle_t get_qnn_context_handle() { return _qnn_context_handle; } + + QnnSystemContext_Handle_t get_qnn_system_handle() { return _qnn_system_handle; } + + Qnn_GraphHandle_t get_qnn_graph_handle() { return _qnn_graph_handle; } + + int init_qnn_graph(const char * graph_name, + bool debug, + uint8_t do_node_validation = 1, + const QnnGraph_Config_t ** graph_configs = nullptr + ); + int init_qnn_graph(const std::string & graph_name, QNNBackend device, size_t vtcm_size_in_mb = 8, size_t hvx_threads = 8); + + int finalize_qnn_graph(); + + bool is_valid_graph() const { return _qnn_graph_handle != nullptr; } + + int init_htp_perfinfra(); + + int set_rpc_polling(); + + int set_high_performance_mode(); + + std::string & get_qnn_graph_name() { return _graph_name; } + + bool is_rpcmem_initialized() { + return _rpcmem_initialized; + } + + void set_rpcmem_initialized(bool initialized) { + _rpcmem_initialized = initialized; + } + + size_t get_rpcmem_capacity() { return _rpcmem_capacity; } + size_t get_rpcmem_usage() { return _rpcmem_usage; } + + int32_t rpcmem_to_fd(void * buf); + + int register_rpcmem(void * p_data, Qnn_Tensor_t * p_tensor); + Qnn_MemHandle_t register_rpcmem(void * p_data, const uint32_t rank, uint32_t * dimensions, Qnn_DataType_t data_type); + + void unregister_rpcmem(); + void unregister_rpcmem(Qnn_MemHandle_t mem_handle); + + void * alloc_rpcmem(size_t bytes, size_t alignment); + void * get_rpcmem_from_memhandle(Qnn_MemHandle_t mem_handle); + + void free_rpcmem(void * buf); + void free_rpcmem(); + + bool is_rpcmem_allocated(void * buf); + + bool is_rpcmem_registered(Qnn_MemHandle_t handle) { + return _qnn_mem_set.count(handle) != 0U; + } + + bool enable_qnn_rpc() { + return _enable_qnn_rpc; + } + + QNNBackend get_device_id() { + return _device_id; + } + +public: + std::map>> _qnn_graph_map; + +private: + int load_system(); + + int unload_system(); + + int load_backend(std::string & lib_path, const QnnSaver_Config_t ** saver_config); + + int unload_backend(); + + void set_qnn_raw_interface(QNN_INTERFACE_VER_TYPE & raw_interface) { + _qnn_raw_interface = raw_interface; + } + + void set_qnn_raw_system_interface(QNN_SYSTEM_INTERFACE_VER_TYPE & raw_interface) { + _qnn_raw_system_interface = raw_interface; + } + + void * alloc_rpcmem_internal(size_t bytes, size_t alignment); + + void probe_device_meminfo(); + +private: + static constexpr const int _required_num_providers = 1; + +private: + std::string _lib_path; + std::string _backend_name; + std::string _model_name; // name of prebuilt QNN model, might be used in the future + BackendIdType _backend_id; + + bool _debug_tensor = false; // flag to indicate if requested graph is to be run in debug mode + bool _do_node_validations = true; // flag to indicate whether all add_node calls need to be validated + QnnLog_Level_t _qnn_log_level = QNN_LOG_LEVEL_DEBUG; + + ggml_qnn_profile_level _profile_level = ggml_qnn_profile_level::profile_detail; + + void * _system_lib_handle = nullptr; + + Qnn_GraphHandle_t _qnn_graph_handle = nullptr; + + Qnn_LogHandle_t _qnn_log_handle = nullptr; + + Qnn_ProfileHandle_t _qnn_profile_handle = nullptr; + + Qnn_DeviceHandle_t _qnn_device_handle = nullptr; + + Qnn_BackendHandle_t _qnn_backend_handle = nullptr; + + Qnn_ContextHandle_t _qnn_context_handle = nullptr; + + QnnSystemContext_Handle_t _qnn_system_handle = nullptr; + + QnnHtpDevice_PerfInfrastructure_t * _qnn_htp_perfinfra = nullptr; + uint32_t _qnn_power_configid = 1; + uint32_t _qnn_rpc_pollingtime = 9999; // 0-10000 us for high performing + + qnn_interface _qnn_interface; + QNN_INTERFACE_VER_TYPE _qnn_raw_interface; + QNN_SYSTEM_INTERFACE_VER_TYPE _qnn_raw_system_interface; + + std::unordered_map _qnn_mem_set; + std::unordered_map _qnn_rpc_buffer_to_handles; + + static std::mutex _init_mutex; + static std::unordered_map _loaded_lib_handle; + static std::unordered_map _lib_path_to_backend_id; + static std::unordered_map _loaded_backend; + + std::atomic_bool _rpcmem_initialized{false}; + pfn_rpc_mem_alloc _pfn_rpc_mem_alloc; + pfn_rpc_mem_free _pfn_rpc_mem_free; + pfn_rpc_mem_to_fd _pfn_rpc_mem_to_fd; + pfn_rpc_mem_init _pfn_rpc_mem_init; + pfn_rpc_mem_deinit _pfn_rpc_mem_deinit; + std::unordered_map _rpcmem_store_map; + std::unordered_map _rpcmem_usage_map; + size_t _rpcmem_usage = 0; // mempool usage in Mbytes + size_t _rpcmem_capacity = 512; // mempool size in Mbytes + + std::string _graph_name; + QNNBackend _device_id; + void * _rpc_lib_handle = nullptr; + bool _enable_qnn_rpc = false; //TODO:unknown issue with QNN RPC feature + + DISABLE_COPY(qnn_instance); + DISABLE_MOVE(qnn_instance); +}; + + +std::mutex qnn_instance::_init_mutex; +std::unordered_map qnn_instance::_loaded_lib_handle; +std::unordered_map qnn_instance::_lib_path_to_backend_id; +std::unordered_map qnn_instance::_loaded_backend; + +void * qnn_instance::alloc_rpcmem_internal(size_t bytes, size_t alignment) { + if (!_rpcmem_initialized) { + GGMLQNN_LOG_WARN("rpc memory not initialized\n"); + return nullptr; + } + + auto allocate_bytes = static_cast(bytes + alignment); + void * buf = _pfn_rpc_mem_alloc(RPCMEM_HEAP_ID_SYSTEM, RPCMEM_DEFAULT_FLAGS, allocate_bytes); + if (nullptr == buf) { + GGMLQNN_LOG_WARN("failed to allocate rpc memory\n"); + return nullptr; + } + + auto aligned_buf = reinterpret_cast(ggmlqnn_align_to(alignment, + reinterpret_cast(buf))); + bool status = _rpcmem_store_map.insert(std::pair(aligned_buf, buf)).second; + if (!status) { + GGMLQNN_LOG_WARN("failed to allocate rpc memory\n"); + _pfn_rpc_mem_free(buf); + } + return aligned_buf; +} + +void * qnn_instance::alloc_rpcmem(size_t bytes, size_t alignment) { + if (_rpcmem_usage > (_rpcmem_capacity - 8)) { // reserve 8Mbytes in rpc mempool + GGMLQNN_LOG_WARN("rpc mempool capcaity: %d MB, usage: %d MB", _rpcmem_capacity, _rpcmem_usage); + return nullptr; + } + + auto aligned_buf = alloc_rpcmem_internal(bytes, alignment); + if (nullptr == aligned_buf) + return nullptr; + _rpcmem_usage_map.insert(std::pair(aligned_buf, bytes)); + + size_t rpcmem_usage_in_bytes = _rpcmem_usage * (1 << 20); + rpcmem_usage_in_bytes += bytes; + _rpcmem_usage = rpcmem_usage_in_bytes / ( 1 << 20); + return aligned_buf; +} + +void qnn_instance::free_rpcmem(void * buf) { + size_t rpcbuffer_size = 0; + if (!_rpcmem_initialized) { + GGMLQNN_LOG_WARN("rpc memory not initialized\n"); + } else if (0 == _rpcmem_store_map.count(buf)) { + GGMLQNN_LOG_WARN("no allocated tensor\n"); + } else { + GGMLQNN_LOG_DEBUG("free rpc mem %p", _rpcmem_store_map[buf]); + for (std::unordered_map::iterator it = _rpcmem_usage_map.begin(); + it != _rpcmem_usage_map.end(); + it++) { + void * rpcbuffer = it->first; + if (buf == rpcbuffer) { + rpcbuffer_size = it->second; + size_t rpcmem_usage_in_bytes = _rpcmem_usage * (1 << 20); + rpcmem_usage_in_bytes -= rpcbuffer_size; + _rpcmem_usage = rpcmem_usage_in_bytes / ( 1 << 20); + } + } + if (rpcbuffer_size != 0) { + _rpcmem_usage_map.erase(buf); + } else { + GGMLQNN_LOG_WARN("it shouldn't happen, pls check why?"); + } + _pfn_rpc_mem_free(_rpcmem_store_map[buf]); + _rpcmem_store_map.erase(buf); + } +} + +void qnn_instance::free_rpcmem() { + if (_rpcmem_store_map.empty()) { + GGMLQNN_LOG_WARN("no rpcmem allocated\n"); + return; + } + + for (std::unordered_map::iterator it = _rpcmem_store_map.begin(); + it != _qnn_mem_set.end(); + it++) { + void * rpcbuffer = it->second; + GGMLQNN_LOG_DEBUG("free rpc buffer %p", rpcbuffer); + _pfn_rpc_mem_free(rpcbuffer); + } + _rpcmem_store_map.clear(); + _rpcmem_usage_map.clear(); + _rpcmem_usage = 0; +} + +int32_t qnn_instance::rpcmem_to_fd(void * buf) { + int32_t mem_fd = -1; + if (!is_rpcmem_initialized()) { + GGMLQNN_LOG_WARN("rpc memory not initialized\n"); + } else { + mem_fd = _pfn_rpc_mem_to_fd(buf); + } + + return mem_fd; +} + +int qnn_instance::register_rpcmem(void * p_data, Qnn_Tensor_t * p_tensor) { + if (nullptr == p_data || (nullptr == p_tensor)) { + GGMLQNN_LOG_WARN("invalid param\n"); + return 1; + } + + if (!is_rpcmem_initialized()) { + GGMLQNN_LOG_WARN("rpc memory not initialized\n"); + return 2; } if (is_rpcmem_registered((QNN_VER_PTR(*p_tensor)->memHandle))) { @@ -2238,7 +2721,7 @@ void qnn_instance::probe_device_meminfo() { GGMLQNN_LOG_INFO("capacity of rpc ion memory %d MB\n", _rpcmem_capacity); } -uint8_t * ggmlqnn_create_rpc_buffer(qnn_instance * instance, const ggml_tensor * ggml_tensor, Qnn_Tensor_t * qnn_tensor, bool b_copydata) { +static uint8_t * ggmlqnn_create_rpc_buffer(qnn_instance * instance, const ggml_tensor * ggml_tensor, Qnn_Tensor_t * qnn_tensor, bool b_copydata) { if (nullptr == instance || nullptr == ggml_tensor || nullptr == qnn_tensor) { GGMLQNN_LOG_WARN("invalid params\n"); return nullptr; @@ -2257,7 +2740,7 @@ uint8_t * ggmlqnn_create_rpc_buffer(qnn_instance * instance, const ggml_tensor * return qnn_rpcbuffer; } -void ggmlqnn_print_tensors_info(const char * func_name, ggml_backend_qnn_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggmlqnn_print_tensors_info(const char * func_name, ggml_backend_qnn_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { //skip sanity check of params if (nullptr != func_name && nullptr != ctx) { GGMLQNN_LOG_DEBUG("call %s in dev %s\n", func_name, ctx->name); @@ -2286,6 +2769,126 @@ static void dump_op_info(const struct ggml_tensor * tensor) { ggmlqnn_print_tensors_info(nullptr, nullptr, src0, src1, dst); } +static Qnn_Tensor_t * ggmlqnn_create_general_tensor(const ggml_tensor * tensor, const char * name, + Qnn_TensorType_t qnn_tensor_type, + Qnn_DataType_t qnn_data_type, + uint32_t rank, uint32_t * dims, + void * data, uint32_t data_size, + bool b_transpose) { + Qnn_ErrorHandle_t error = QNN_SUCCESS; + char tensor_name[GGML_MAX_NAME] = {}; + + //ensure the tensor name is unique + if (nullptr == name) { + snprintf(tensor_name, GGML_MAX_NAME, "tensor_%-8d", get_idx()); + } else { + snprintf(tensor_name, GGML_MAX_NAME, "tensor_%s%-8d", name, get_idx()); + } + GGMLQNN_LOG_DEBUG("init_tensor %d", get_idx()); + inc_idx(); + + uint32_t reverse_dims[GGML_MAX_DIMS] = {}; + uint32_t transpose_dims[GGML_MAX_DIMS] = {}; + uint32_t * tensor_dims = nullptr; + //case 1:use dims info from ggml tensor + if (nullptr != tensor) { + //there are different dimension order between ggml tensor and qnn tensor + for (size_t idx = 0; idx < rank; idx++) { + reverse_dims[idx] = (uint32_t)tensor->ne[rank - 1 - idx]; + } + tensor_dims = reverse_dims; + } + //case 2: use user's specified tensor_dims + if (nullptr != dims) { + tensor_dims = dims; + } + //case 3: transpose for dst tensor + if (b_transpose) { + GGML_ASSERT(tensor != nullptr); //ensure ggml_tensor is not nullptr for this special case + + get_qnn_dimensions_from_ggml_dimensions(transpose_dims, reverse_dims, ggml_n_dims(tensor)); + tensor_dims = transpose_dims; +#if 0 + for (size_t idx = 0; idx < 4; idx++) { + GGMLQNN_LOG_DEBUG("origin dim[%d]=%d\n", idx, reverse_dims[idx]); + } + for (size_t idx = 0; idx < 4; idx++) { + GGMLQNN_LOG_DEBUG("trans dim[%d]=%d\n", idx, transpose_dims[idx]); + } +#endif + } + + Qnn_Tensor_t qnn_tensor = { + .version= QNN_TENSOR_VERSION_1, + {.v1= { + .id = 0, + .name = tensor_name, + .type = qnn_tensor_type, + .dataFormat = QNN_TENSOR_DATA_FORMAT_FLAT_BUFFER, + .dataType = qnn_data_type, + .quantizeParams = {.encodingDefinition = QNN_DEFINITION_UNDEFINED, + .quantizationEncoding = QNN_QUANTIZATION_ENCODING_UNDEFINED, + {.scaleOffsetEncoding = {.scale = 0.0000000000000000f, .offset = 0}}}, + .rank = rank, + .dimensions = tensor_dims, + .memType = QNN_TENSORMEMTYPE_RAW, + .clientBuf = {.data = nullptr, .dataSize = 0} + } + } + }; + if (nullptr != name) { + QNN_VER_PTR(qnn_tensor)->name = name; + } + Qnn_Tensor_t * p_qnn_tensor = (Qnn_Tensor_t *)calloc(1, sizeof(Qnn_Tensor_t)); + if (nullptr == p_qnn_tensor) { + GGMLQNN_LOG_WARN("calloc failed"); + return nullptr; + } + error = deep_copy_qnn_tensors(qnn_tensor, * p_qnn_tensor); + if (error != QNN_SUCCESS) { + free(p_qnn_tensor); + GGMLQNN_LOG_WARN("init tensor failed"); + return nullptr; + } + QNN_VER_PTR(*p_qnn_tensor)->clientBuf = {data, data_size}; + + return p_qnn_tensor; +} + +static Qnn_Tensor_t * ggmlqnn_create_compute_tensor(qnn_instance * instance, Qnn_GraphHandle_t graph_handle, const ggml_tensor * tensor, Qnn_TensorType_t tensor_type) { + Qnn_ErrorHandle_t error = QNN_SUCCESS; + uint32_t dimensions[] = {(uint32_t) tensor->ne[0], (uint32_t) tensor->ne[1], + (uint32_t) tensor->ne[2], (uint32_t) tensor->ne[3]}; + Qnn_DataType_t qnn_data_type = QNN_DATATYPE_FLOAT_32; + Qnn_TensorType_t qnn_tensor_type = QNN_TENSOR_TYPE_APP_WRITE; + + if (0 == tensor->flags) { + qnn_tensor_type = tensor_type; + } else { + if (tensor->flags & GGML_TENSOR_FLAG_INPUT) { + qnn_tensor_type = QNN_TENSOR_TYPE_APP_WRITE; + } else if (tensor->flags & GGML_TENSOR_FLAG_OUTPUT) { + qnn_tensor_type = QNN_TENSOR_TYPE_APP_READ; + } + } + + qnn_data_type = ggmlqnn_datatype_from_ggml_datatype(tensor->type); + Qnn_Tensor_t * p_qnn_tensor = ggmlqnn_create_general_tensor(tensor, nullptr, + qnn_tensor_type, qnn_data_type, + ggml_n_dims(tensor), dimensions, + nullptr, 0); + + bool enable_npu_rpc = (instance->enable_qnn_rpc() && instance->get_device_id() == QNN_BACKEND_NPU); + if (enable_npu_rpc) { + QNN_VER_PTR(*p_qnn_tensor)->memType = QNN_TENSORMEMTYPE_MEMHANDLE; + QNN_VER_PTR(*p_qnn_tensor)->clientBuf = {.data=nullptr, .dataSize=0}; + } + QNN_INTERFACE_VER_TYPE qnn_raw_interface = instance->get_qnn_raw_interface(); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_qnn_tensor)); + + return p_qnn_tensor; +} + // ================================================================================================= // section-6: implementation of ggml-qnn backend // ================================================================================================= @@ -3066,3 +3669,669 @@ ggml_backend_t ggml_backend_qnn_init(size_t device, const char * qnn_lib_path) { } GGML_BACKEND_DL_IMPL(ggml_backend_qnn_reg) + +// ================================================================================================= +// section-7: offload GGML op to QNN backend +// ================================================================================================= +static inline uint32_t ggmlqnn_get_tensor_data_size(const ggml_tensor * tensor) { + /* + size_t data_size = ggml_row_size(tensor->type, tensor->ne[0]); + size_t n_dims = ggml_get_tensor_rank(tensor); + for (int i = 1; i < n_dims; i++) { + data_size *= tensor->ne[i]; + } + + return data_size; + */ + return ggml_nbytes(tensor); +} + +static inline bool ggmlqnn_is_valid_params(ggml_backend_qnn_context * ctx, const ggml_tensor * src0, + const ggml_tensor * src1, ggml_tensor * dst) { + if ((nullptr == ctx) || (nullptr == src0) || (nullptr == src1) || (nullptr == dst)) { + GGMLQNN_LOG_WARN("invalid params\n"); + return false; + } + + qnn_instance * instance = ctx->instance; + if (nullptr == instance) { + GGMLQNN_LOG_WARN("invalid params\n"); + return false; + } + + return true; +} + +#define GGMLQNN_CHECK_PARAMS(ctx, src0, src1, dst) \ + do { \ + if (!ggmlqnn_is_valid_params((ctx), (src0), (src1), (dst))) { \ + return; \ + } \ + } while (0) + +/* + * provide a general skeleton to offload ggml op to QNN backend: a single node contains 2 input + * tensor and 1 output tensor +*/ +void ggml_qnn_general_node(ggml_backend_qnn_context * ctx, ggml_tensor * op) { + Qnn_ErrorHandle_t error = QNN_SUCCESS; + qnn_instance * instance = nullptr; + Qnn_GraphHandle_t graph_handle = nullptr; + Qnn_Tensor_t * p_tensor0 = nullptr; + Qnn_Tensor_t * p_tensor1 = nullptr; + Qnn_Tensor_t * p_tensor2 = nullptr; + const ggml_tensor * src0 = op->src[0]; + const ggml_tensor * src1 = op->src[1]; + ggml_tensor * dst = op; + + GGMLQNN_CHECK_PARAMS(ctx, src0, src1, dst); + instance = ctx->instance; + QNN_INTERFACE_VER_TYPE qnn_raw_interface = ctx->raw_interface; + size_t qnn_op_index = ggmlqnn_get_op_index(op); + GGML_ASSERT(qnn_op_index < ggmlqnn_get_opcaps_size()); + const char * qnn_op_name = ggmlqnn_k_op_caps[qnn_op_index].qnn_op_name; + std::string ggml_op_name_string = std::string("ggml_") + ggml_op_name(op->op); + const char * ggml_op_name = ggml_op_name_string.c_str(); + + qnn_perf op_perf = qnn_perf(ggml_op_name); + op_perf.start(); + + //ggmlqnn_print_tensors_info(__func__, ctx, src0, src1, dst); + bool enable_npu_rpc = instance->enable_qnn_rpc() && ctx->device == QNN_BACKEND_NPU; + + std::string graph_name; + ggmlqnn_get_graphkey_from_op(op, graph_name); + if (instance->_qnn_graph_map.find(graph_name) != instance->_qnn_graph_map.end()) { + //retrieve computational resource from cached QNN graph + qnn_res_t & graph_item = instance->_qnn_graph_map[graph_name]; + graph_handle = std::get<0>(graph_item); + qnn_tensors_t & tensor = std::get<1>(graph_item); + p_tensor0 = tensor[0]; + p_tensor1 = tensor[1]; + p_tensor2 = tensor[2]; + } else { + GGMLQNN_LOG_DEBUG("graph name %s", graph_name.c_str()); + GGML_ASSERT(instance->get_device_id() == ctx->device); + //create QNN graph + error = instance->init_qnn_graph(graph_name, static_cast(ctx->device), 8); + if (QNN_SUCCESS != error) { + GGMLQNN_LOG_WARN("can't create qnn graph handle with graph name %s, error = %d\n", graph_name.c_str(), error); + return; + } + graph_handle = instance->get_qnn_graph_handle(); + + //create computational tensor + p_tensor0 = ggmlqnn_create_compute_tensor(instance, graph_handle, src0, QNN_TENSOR_TYPE_APP_WRITE); + p_tensor1 = ggmlqnn_create_compute_tensor(instance, graph_handle, src1, QNN_TENSOR_TYPE_APP_WRITE); + p_tensor2 = ggmlqnn_create_compute_tensor(instance, graph_handle, dst, QNN_TENSOR_TYPE_APP_READ); + + //compose QNN graph + Qnn_Tensor_t tensor_inputs[] = { + *p_tensor0, + *p_tensor1 + }; + Qnn_Tensor_t tensor_outputs[] = { + *p_tensor2 + }; + Qnn_OpConfig_t op_config = { + QNN_OPCONFIG_VERSION_1, { + ggml_op_name, + QNN_OP_PACKAGE_NAME_QTI_AISW, + qnn_op_name, + 0, + nullptr, + 2, + tensor_inputs, + 1, + tensor_outputs + } + }; + CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, op_config)); + //finalize QNN graph + CHECK_QNN_API(error, qnn_raw_interface.graphFinalize(graph_handle, nullptr, nullptr)); + + //cache QNN graph + qnn_tensors_t ggml_op_add_tensors; + ggml_op_add_tensors.reserve(3); + ggml_op_add_tensors.push_back(p_tensor0); + ggml_op_add_tensors.push_back(p_tensor1); + ggml_op_add_tensors.push_back(p_tensor2); + auto graph_item = std::make_tuple(graph_handle, ggml_op_add_tensors); + instance->_qnn_graph_map[graph_name] = graph_item; + } + + if (enable_npu_rpc) { + uint8_t * qnn_buffer_0 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*p_tensor0)->memHandle)); + GGMLQNN_LOG_INFO("qnn_rpcbuffer_0 = %p\n", qnn_buffer_0); + if (nullptr != qnn_buffer_0) { + memcpy(qnn_buffer_0, src0->data, ggml_nbytes(src0)); + } + + uint8_t * qnn_buffer_1 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*p_tensor1)->memHandle)); + GGMLQNN_LOG_INFO("qnn_rpcbuffer_1 = %p\n", qnn_buffer_1); + if (nullptr != qnn_buffer_1) { + memcpy(qnn_buffer_1, src1->data, ggml_nbytes(src1)); + } + } else { + QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggmlqnn_get_tensor_data_size(src0)}; + QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggmlqnn_get_tensor_data_size(src1)}; + QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggmlqnn_get_tensor_data_size(dst)}; + } + + Qnn_Tensor_t tensor_inputs[] = { + *p_tensor0, + *p_tensor1 + }; + Qnn_Tensor_t tensor_outputs[] = { + *p_tensor2 + }; + CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, + tensor_inputs, 2, + tensor_outputs, 1, + nullptr, nullptr)); + if (enable_npu_rpc) { + //TODO:NPU RPC feature will failed with test-backend-ops + uint8_t * qnn_buffer_2 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*p_tensor2)->memHandle)); + if (nullptr != qnn_buffer_2) { + memcpy(dst->data, qnn_buffer_2, ggml_nbytes(dst)); + } + } + +#if GGMLQNN_PRINT_OP_ADD_LOG + op_perf.info(); +#endif +} + +/* + * this function is AI-assisted code from Grok 3 for purpose of offload 4d matrix mulmat to QNN backend + * UT in ggml-qnn-ut.cpp passed: + * ./scripts/build-run-android.sh run_ut_mulmat 0 + * ./scripts/build-run-android.sh run_ut_mulmat 1 + * ./scripts/build-run-android.sh run_ut_mulmat 2 + * + * the logic of ggml_qnn_mul_mat_4d is similar to ggml_qnn_mul_mat but much more complicated + * than ggml_qnn_mul_mat, so it's a standalone function. + * it will be combined with ggml_qnn_mul_mat in the future + */ +static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) { + Qnn_ErrorHandle_t error = QNN_SUCCESS; + bool graph_initialized = false; + qnn_perf op_perf = qnn_perf("ggml_qnn_mul_mat_4d"); + qnn_instance *instance = ctx->instance; + QNN_INTERFACE_VER_TYPE qnn_raw_interface = ctx->raw_interface; + + const ggml_tensor *src0 = op->src[0]; + const ggml_tensor *src1 = op->src[1]; + ggml_tensor *dst = op; + + GGMLQNN_CHECK_PARAMS(ctx, src0, src1, dst); + GGML_ASSERT(ggml_n_dims(src0) == 4 && ggml_n_dims(src1) == 4); + op_perf.start(); + + std::string graph_name; + ggmlqnn_get_graphkey_from_op(op, graph_name); + GGMLQNN_LOG_DEBUG("graph name %s\n", graph_name.c_str()); + + ggmlqnn_print_tensors_info(__func__, ctx, src0, src1, dst); + + Qnn_GraphHandle_t graph_handle = nullptr; + Qnn_Tensor_t *p_tensor0 = nullptr; + Qnn_Tensor_t *p_reshape0_out = nullptr; + Qnn_Tensor_t *p_tile0_out = nullptr; + Qnn_Tensor_t *p_tensor1 = nullptr; + Qnn_Tensor_t *p_permute1_out = nullptr; + Qnn_Tensor_t *p_reshape1_out = nullptr; + Qnn_Tensor_t *p_matmul_out = nullptr; + Qnn_Tensor_t *p_reshape2_out = nullptr; + + if (instance->_qnn_graph_map.find(graph_name) != instance->_qnn_graph_map.end()) { + graph_initialized = true; + qnn_res_t &graph_item = instance->_qnn_graph_map[graph_name]; + graph_handle = std::get<0>(graph_item); + qnn_tensors_t &tensors = std::get<1>(graph_item); + p_tensor0 = tensors[0]; + p_reshape0_out = tensors[1]; + p_tile0_out = tensors[2]; + p_tensor1 = tensors[3]; + p_permute1_out = tensors[4]; + p_reshape1_out = tensors[5]; + p_matmul_out = tensors[6]; + p_reshape2_out = tensors[7]; + } else { + CHECK_QNN_API(error, qnn_raw_interface.graphCreate(instance->get_qnn_context_handle(), + graph_name.c_str(), NULL, &graph_handle)); + + // Define dimensions + uint32_t K = src0->ne[0]; // Inner dimension + uint32_t M = src0->ne[1]; // Rows of src0 + uint32_t N = src1->ne[1]; // Columns of src1 + uint32_t B0 = src0->ne[2] * src0->ne[3]; // src0 batch + uint32_t B1 = src1->ne[2] * src1->ne[3]; // src1 batch (drives output) + + // Validate K only + GGML_ASSERT(src0->ne[0] == src1->ne[0]); // K must match + + // src0: [K, M, H0, B0] -> QNN: [B0, H0, M, K] + uint32_t src0_dims[] = {static_cast(src0->ne[3]), static_cast(src0->ne[2]), static_cast(src0->ne[1]), static_cast(src0->ne[0])}; + p_tensor0 = GQCGT(src0, "input0", QNN_TENSOR_TYPE_APP_WRITE, QNN_DATATYPE_FLOAT_32, 4, + src0_dims, nullptr, 0); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor0)); + + // Reshape src0 to [B0, M, K] + uint32_t reshape0_out_dims[] = {B0, M, K}; + p_reshape0_out = GQCGT(nullptr, "reshape0_out", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, 3, + reshape0_out_dims, nullptr, 0); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_reshape0_out)); + Qnn_Tensor_t reshape0_inputs[] = {*p_tensor0}; + Qnn_Tensor_t reshape0_outputs[] = {*p_reshape0_out}; + Qnn_OpConfig_t reshape0_op = ggmlqnn_create_op_config("reshape0", QNN_OP_PACKAGE_NAME_QTI_AISW, + QNN_OP_RESHAPE, nullptr, 0, + reshape0_inputs, 1, reshape0_outputs, 1); + CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, reshape0_op)); + + // Tile src0 to match B1: [B0, M, K] -> [B1, M, K] + uint32_t tile0_out_dims[] = {B1, M, K}; + p_tile0_out = GQCGT(nullptr, "tile0_out", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, 3, + tile0_out_dims, nullptr, 0); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tile0_out)); + uint32_t tile_multiples[] = {B1 / B0, 1, 1}; + uint32_t tile_dims[] = {3}; + Qnn_Tensor_t *p_tile_multiples = GQCGT(nullptr, "tile_multiples", QNN_TENSOR_TYPE_STATIC, QNN_DATATYPE_UINT_32, 1, + tile_dims, tile_multiples, sizeof(tile_multiples)); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tile_multiples)); + Qnn_Param_t tile_params[] = {{QNN_PARAMTYPE_TENSOR, "multiples", .tensorParam = *p_tile_multiples}}; + Qnn_Tensor_t tile0_inputs[] = {*p_reshape0_out}; + Qnn_Tensor_t tile0_outputs[] = {*p_tile0_out}; + Qnn_OpConfig_t tile0_op = ggmlqnn_create_op_config("tile0", QNN_OP_PACKAGE_NAME_QTI_AISW, + QNN_OP_TILE, tile_params, 1, + tile0_inputs, 1, tile0_outputs, 1); + CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, tile0_op)); + + // src1: [N, K, H1, B1] -> QNN: [B1, H1, N, K] + uint32_t src1_dims[] = {static_cast(src1->ne[3]), static_cast(src1->ne[2]), static_cast(src1->ne[1]), static_cast(src1->ne[0])}; + p_tensor1 = GQCGT(src1, "input1", QNN_TENSOR_TYPE_APP_WRITE, QNN_DATATYPE_FLOAT_32, 4, + src1_dims, nullptr, 0); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor1)); + + // Permute src1 to [B1, H1, K, N] + uint32_t perm_data[] = {0, 1, 3, 2}; + uint32_t perm_dims[] = {4}; + Qnn_Tensor_t *p_perm = GQCGT(nullptr, "perm", QNN_TENSOR_TYPE_STATIC, QNN_DATATYPE_UINT_32, 1, + perm_dims, perm_data, sizeof(perm_data)); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_perm)); + uint32_t permute1_out_dims[] = {static_cast(src1->ne[3]), static_cast(src1->ne[2]), static_cast(src1->ne[0]), static_cast(src1->ne[1])}; + p_permute1_out = GQCGT(nullptr, "permute1_out", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, 4, + permute1_out_dims, nullptr, 0); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_permute1_out)); + Qnn_Param_t permute1_params[] = {{QNN_PARAMTYPE_TENSOR, "perm", .tensorParam = *p_perm}}; + Qnn_Tensor_t permute1_inputs[] = {*p_tensor1}; + Qnn_Tensor_t permute1_outputs[] = {*p_permute1_out}; + Qnn_OpConfig_t permute1_op = ggmlqnn_create_op_config("permute1", QNN_OP_PACKAGE_NAME_QTI_AISW, + QNN_OP_TRANSPOSE, permute1_params, 1, + permute1_inputs, 1, permute1_outputs, 1); + CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, permute1_op)); + + // Reshape src1 to [B1, K, N] + uint32_t reshape1_out_dims[] = {B1, K, N}; + p_reshape1_out = GQCGT(nullptr, "reshape1_out", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, 3, + reshape1_out_dims, nullptr, 0); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_reshape1_out)); + Qnn_Tensor_t reshape1_inputs[] = {*p_permute1_out}; + Qnn_Tensor_t reshape1_outputs[] = {*p_reshape1_out}; + Qnn_OpConfig_t reshape1_op = ggmlqnn_create_op_config("reshape1", QNN_OP_PACKAGE_NAME_QTI_AISW, + QNN_OP_RESHAPE, nullptr, 0, + reshape1_inputs, 1, reshape1_outputs, 1); + CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, reshape1_op)); + + // MatMul: [B1, M, K] x [B1, K, N] -> [B1, M, N] + uint32_t matmul_out_dims[] = {B1, M, N}; + p_matmul_out = GQCGT(nullptr, "matmul_out", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, 3, + matmul_out_dims, nullptr, 0); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_matmul_out)); + Qnn_Tensor_t matmul_inputs[] = {*p_tile0_out, *p_reshape1_out}; + Qnn_Tensor_t matmul_outputs[] = {*p_matmul_out}; + Qnn_OpConfig_t matmul_op = ggmlqnn_create_op_config("matmul", QNN_OP_PACKAGE_NAME_QTI_AISW, + QNN_OP_MAT_MUL, nullptr, 0, + matmul_inputs, 2, matmul_outputs, 1); + CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, matmul_op)); + + // Output: [N, M, H1, B1] -> QNN: [B1, H1, M, N] + uint32_t reshape2_out_dims[] = {static_cast(dst->ne[3]), static_cast(dst->ne[2]), static_cast(dst->ne[1]), static_cast(dst->ne[0])}; + p_reshape2_out = GQCGT(dst, "output", QNN_TENSOR_TYPE_APP_READ, QNN_DATATYPE_FLOAT_32, 4, + reshape2_out_dims, nullptr, 0); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_reshape2_out)); + Qnn_Tensor_t reshape2_inputs[] = {*p_matmul_out}; + Qnn_Tensor_t reshape2_outputs[] = {*p_reshape2_out}; + Qnn_OpConfig_t reshape2_op = ggmlqnn_create_op_config("reshape2", QNN_OP_PACKAGE_NAME_QTI_AISW, + QNN_OP_RESHAPE, nullptr, 0, + reshape2_inputs, 1, reshape2_outputs, 1); + CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, reshape2_op)); + + // Finalize + CHECK_QNN_API(error, qnn_raw_interface.graphFinalize(graph_handle, NULL, NULL)); + + // Cache + qnn_tensors_t ggml_op_mulmat_tensors = {p_tensor0, p_reshape0_out, p_tile0_out, p_tensor1, p_permute1_out, p_reshape1_out, p_matmul_out, p_reshape2_out}; + instance->_qnn_graph_map[graph_name] = std::make_tuple(graph_handle, ggml_op_mulmat_tensors); + } + + // Execute + QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, static_cast(ggml_nbytes(src0))}; + QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, static_cast(ggml_nbytes(src1))}; + QNN_VER_PTR(*p_reshape2_out)->clientBuf = {dst->data, static_cast(ggml_nbytes(dst))}; + + Qnn_Tensor_t input_tensors[] = {*p_tensor0, *p_tensor1}; + Qnn_Tensor_t output_tensors[] = {*p_reshape2_out}; + CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, input_tensors, 2, + output_tensors, 1, NULL, NULL)); + +#if 0 + // Log dst for debugging + float *dst_data = (float *)dst->data; + GGMLQNN_LOG_DEBUG("dst shape: [%d, %d, %d, %d]\n", dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]); + for (int i = 0; i < dst->ne[0] * dst->ne[1] * dst->ne[2] * dst->ne[3]; i++) { + GGMLQNN_LOG_DEBUG("dst[%d] = %f\n", i, dst_data[i]); + } +#endif + + op_perf.info(); +} + +/* + * @brief performs matrix multiplication with FP32 & quantized weights and floating-point inputs + * using the QNN backend. this function performs matrix multiplication of the input tensor + * `src1` and the weight tensor `src0`, handling transposing, and quantization as needed, + * and stores the result in the destination tensor `dst`. + * + there are two key-points in properly handling how to offload mulmat to the QNN backend in ggml-qnn + 1. transpose + a 3x2 f32 matrix which means 3 rows and 2 columns. in ggml, it could be created from: + struct ggml_tensor* matrix = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 2, 3); + which like this: + +---+---+ + | 0 | 1 | + +---+---+ + | 2 | 3 | + +---+---+ + | 4 | 5 | + +---+---+ + with + ne[0] = 2 + ne[1] = 3 + there are different dimension order between ggml tensor and qnn tensor + + 2. QNN's MatMul can only support input tensors with rank >= 2 + + in the all, there is gap between ggml mulmat and QNN mulmat,we need to perform a transpose + operation when offloading mulmat to QNN backend. this implementation will handle transpose + in func ggml_qnn_create_general_tensor() + * + * this function is a good example to illustrated the second technical approach "mapping the + * entire ggml computational graph to QNN graph" without complex C++ encapsulation. or another + * pipeline of "how to utilize the Hexagon NPU maximally through QNN SDK", details could be found at + * https://github.com/ggml-org/llama.cpp/pull/12049#issuecomment-2678308360 + * + * @param ctx the context of ggml-qnn backend + * @param op the destination tensor where the result of the matrix multiplication will be stored. + * + * @note the logic of ggml_qnn_mul_mat is similar to ggml_qnn_general_node but much more complicated + * than ggml_qnn_general_node. so it's a standalone function. accordingly, this is another + * typical skeleton for offload other ggml ops to QNN backend. MUL_MAT take most of the compute + * time (about 95%).so to speed up llama inference, should focus on this func. there are three kinds + * of MUL_MAT to compute: + * mul_mat_f32: both src0 and src1 are F32, this will be naturally handled in QNN backend + * mul_mat_f16_f32: src0 is F16 and src1 is F32, f16 in src0 -> f32 in src0', then src0' * src1 + * mul_mat_q_f32: src0 is quantized (Q4_0, Q4_1, Q6_K...) + * and src1 is F32, src0 -> f32 in src0', then src0' * src1 +*/ +void ggml_qnn_mul_mat(ggml_backend_qnn_context * ctx, ggml_tensor * op) { + Qnn_ErrorHandle_t error = QNN_SUCCESS; + qnn_perf op_perf = qnn_perf("ggml_qnn_mul_mat"); + qnn_instance * instance = nullptr; + Qnn_GraphHandle_t graph_handle = nullptr; + Qnn_Tensor_t * p_tensor0 = nullptr; + Qnn_Tensor_t * p_tensor1 = nullptr; + Qnn_Tensor_t * p_tensor2 = nullptr; + Qnn_Tensor_t * p_param_tensor = nullptr; + Qnn_Tensor_t * p_tensor2_transpose = nullptr; + const ggml_tensor * src0 = op->src[0]; + const ggml_tensor * src1 = op->src[1]; + ggml_tensor * dst = op; + + GGMLQNN_CHECK_PARAMS(ctx, src0, src1, dst); + instance = ctx->instance; + QNN_INTERFACE_VER_TYPE qnn_raw_interface = ctx->raw_interface; + op_perf.start(); + + const enum ggml_type src0_type = src0->type; + const uint32_t src0_rank = ggml_n_dims(src0); + const uint32_t src1_rank = ggml_n_dims(src1); + GGML_ASSERT(src0_rank == src1_rank); + GGML_ASSERT(src0_rank >= 2); //QNN SDK's limitation, make QNN SDK happy + if (4 == src0_rank) { + return ggml_qnn_mul_mat_4d(ctx, op); + } + void * wdata = ggmlqnn_type_trait(ctx, op); + const size_t desired_size = ctx->desired_size; + + ggmlqnn_print_tensors_info(__func__, ctx, src0, src1, dst); + + std::string graph_name; + ggmlqnn_get_graphkey_from_op(op, graph_name); + if (instance->_qnn_graph_map.find(graph_name) != instance->_qnn_graph_map.end()) { + //retrieve computational resource from cached QNN graph + qnn_res_t & graph_item = instance->_qnn_graph_map[graph_name]; + graph_handle = std::get<0>(graph_item); + qnn_tensors_t & tensors = std::get<1>(graph_item); + p_tensor0 = tensors[0]; + p_tensor1 = tensors[1]; + p_tensor2 = tensors[2]; + p_param_tensor = tensors[3]; + p_tensor2_transpose = tensors[4]; + } else { + //create QNN graph + GGMLQNN_LOG_DEBUG("graph name %s", graph_name.c_str()); + error = qnn_raw_interface.graphCreate(instance->get_qnn_context_handle(), graph_name.c_str(), nullptr, &graph_handle); + if (QNN_SUCCESS != error) { + GGMLQNN_LOG_WARN("can't create qnn graph handle with graph name %s, error = %d\n", graph_name.c_str(), error); + return; + } + + //create computational tensor + p_tensor0 = GQCGT(src0, nullptr, QNN_TENSOR_TYPE_APP_WRITE, QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0); + p_tensor1 = GQCGT(src1, nullptr, QNN_TENSOR_TYPE_APP_WRITE, QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0); + p_tensor2 = GQCGT(dst, nullptr, QNN_TENSOR_TYPE_APP_READ, QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor0)); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor1)); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor2)); + + //create param tensor for offload 2d/3d/4d matrix multiplication + const uint32_t param_tensor_data[GGML_MAX_DIMS][GGML_MAX_DIMS] = { + {0}, + {1, 0}, + {0, 2, 1}, + {0, 1, 3, 2}, + }; + uint32_t param_tensor_dims[1] = {src0_rank}; + p_param_tensor = GQCGT(nullptr, "param", QNN_TENSOR_TYPE_STATIC, QNN_DATATYPE_UINT_32, 1, param_tensor_dims, (void *)(param_tensor_data[src0_rank - 1]), src0_rank * sizeof(uint32_t)); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_param_tensor)); + + //create transpose tensor + p_tensor2_transpose = GQCGT(dst, "transpose", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0, true); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor2_transpose)); + + //compose QNN graph: add mulmat node + Qnn_Param_t out_0_params[] = {{QNN_PARAMTYPE_SCALAR, QNN_OP_MAT_MUL_PARAM_TRANSPOSE_IN1, .scalarParam = {QNN_DATATYPE_BOOL_8, .bool8Value = 1}}}; + Qnn_Tensor_t out_0_inputs[] = {*p_tensor0, *p_tensor1}; + Qnn_Tensor_t out_0_outputs[] = {*p_tensor2_transpose}; + Qnn_OpConfig_t out_0 = ggmlqnn_create_op_config("mulmat_opconfig", QNN_OP_PACKAGE_NAME_QTI_AISW, QNN_OP_MAT_MUL, out_0_params, 1, out_0_inputs, 2, out_0_outputs, 1); + CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle,out_0)); + + //compose QNN graph: add transpose node + Qnn_Param_t out_trans1_0_params[] = { {QNN_PARAMTYPE_TENSOR, "perm", .tensorParam = *p_param_tensor}}; + Qnn_Tensor_t out_trans1_0_inputs[] = {*p_tensor2_transpose}; + Qnn_Tensor_t out_trans1_0_outputs[] = {*p_tensor2}; + Qnn_OpConfig_t out_trans1_0 = ggmlqnn_create_op_config("mulmat_transpose_opconfig", QNN_OP_PACKAGE_NAME_QTI_AISW, QNN_OP_TRANSPOSE, out_trans1_0_params, 1, out_trans1_0_inputs, 1, out_trans1_0_outputs, 1); + CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle,out_trans1_0)); + + //finalize QNN graph + CHECK_QNN_API(error, qnn_raw_interface.graphFinalize(graph_handle, nullptr, nullptr)); + + //cache QNN graph + qnn_tensors_t ggml_op_mulmat_tensors; + ggml_op_mulmat_tensors.reserve(5); + ggml_op_mulmat_tensors.push_back(p_tensor0); + ggml_op_mulmat_tensors.push_back(p_tensor1); + ggml_op_mulmat_tensors.push_back(p_tensor2); + ggml_op_mulmat_tensors.push_back(p_param_tensor); + ggml_op_mulmat_tensors.push_back(p_tensor2_transpose); + auto graph_item = std::make_tuple(graph_handle, ggml_op_mulmat_tensors); + instance->_qnn_graph_map[graph_name] = graph_item; + } + + if (src0_type != GGML_TYPE_F32) { + QNN_VER_PTR(*p_tensor0)->clientBuf = {wdata, static_cast(desired_size)}; + } else { + QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggmlqnn_get_tensor_data_size(src0)}; + } + QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggmlqnn_get_tensor_data_size(src1)}; + QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggmlqnn_get_tensor_data_size(dst)}; + + Qnn_Tensor_t tensor_inputs[] = { + *p_tensor0, + *p_tensor1 + }; + Qnn_Tensor_t tensor_outputs[] = { + *p_tensor2 + }; + CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, + tensor_inputs, 2, + tensor_outputs, 1, + nullptr, nullptr)); + op_perf.info(); +} + +void ggml_qnn_repeat(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); +} + +void ggml_qnn_div(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); +} + +void ggml_qnn_leaky_relu(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); +} + +void ggml_qnn_concat(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); +} + +void ggml_qnn_arange(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); +} + +void ggml_qnn_sqr(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); +} + +void ggml_qnn_clamp(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); +} + +void ggml_qnn_scale(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); +} + +void ggml_qnn_argsort(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); +} + +void ggml_qnn_norm(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); +} + +void ggml_qnn_group_norm(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); +} + +void ggml_qnn_acc(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); +} + +void ggml_qnn_sum_rows(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); +} + +void ggml_qnn_upsample_nearest2d(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); +} + +void ggml_qnn_pad(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); +} + +void ggml_qnn_pool2d(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); +} + +void ggml_qnn_dup(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); +} + +void ggml_qnn_rms_norm(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); +} + +void ggml_qnn_diag_mask(ggml_backend_qnn_context * ctx, ggml_tensor * dst, float value) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); + GGML_UNUSED(value); +} + +void ggml_qnn_im2col(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); +} + +void ggml_qnn_timestep_embedding(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); +} + +void ggml_qnn_cpy(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + ggml_qnn_dup(ctx, dst); +} + +void ggml_qnn_softmax(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); +} + +void ggml_qnn_get_rows(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); +} + +void ggml_qnn_rope(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); +} diff --git a/scripts/build-run-android.sh b/scripts/build-run-android.sh index 5e69024298dbe..5b5e55aa2f7b6 100755 --- a/scripts/build-run-android.sh +++ b/scripts/build-run-android.sh @@ -147,6 +147,7 @@ function prepare_run_on_phone() adb shell chmod +x ${REMOTE_PATH}/${program} } + function run_llamacli() { prepare_run_on_phone llama-cli @@ -212,35 +213,6 @@ function run_test-op() } -function run_ut_add() -{ - prepare_run_on_phone ggml-qnn-ut - - adb shell "cd ${REMOTE_PATH} \ - && export LD_LIBRARY_PATH=${REMOTE_PATH} \ - && ${REMOTE_PATH}/ggml-qnn-ut -t GGML_OP_ADD -b $qnnbackend" - -} - -function run_ut_mulmat() -{ - prepare_run_on_phone ggml-qnn-ut - - adb shell "cd ${REMOTE_PATH} \ - && export LD_LIBRARY_PATH=${REMOTE_PATH} \ - && ${REMOTE_PATH}/ggml-qnn-ut -t GGML_OP_MUL_MAT -b $qnnbackend" - -} - -function run_ut_mul() -{ - prepare_run_on_phone ggml-qnn-ut - - adb shell "cd ${REMOTE_PATH} \ - && export LD_LIBRARY_PATH=${REMOTE_PATH} \ - && ${REMOTE_PATH}/ggml-qnn-ut -t GGML_OP_MUL -b $qnnbackend" - -} function print_oplist() { @@ -330,10 +302,7 @@ function show_usage() echo " $0 build" echo " $0 updateqnnlib" echo " $0 run_testops" - echo " $0 run_testop [ADD/MUL/MUL_MAT] [0 (QNN_CPU) / 1 (QNN_GPU) / 2 (QNN_NPU)]" - echo " $0 run_ut_add 0 (QNN_CPU) / 1 (QNN_GPU) / 2 (QNN_NPU) / 3 (ggml)" - echo " $0 run_ut_mulmat 0 (QNN_CPU) / 1 (QNN_GPU) / 2 (QNN_NPU) / 3 (ggml)" - echo " $0 run_ut_mul 0 (QNN_CPU) / 1 (QNN_GPU) / 2 (QNN_NPU) / 3 (ggml)" + echo " $0 run_testop [ADD/MUL/MUL_MAT/...(op from print_oplist)] [0 (QNN_CPU) / 1 (QNN_GPU) / 2 (QNN_NPU)]" echo " $0 run_llamacli 0 (QNN_CPU) / 1 (QNN_GPU) / 2 (QNN_NPU) / 3 (ggml)" echo " $0 run_llamabench 0 (QNN_CPU) / 1 (QNN_GPU) / 2 (QNN_NPU) / 3 (ggml)" @@ -374,31 +343,20 @@ elif [ $# == 1 ]; then fi elif [ $# == 2 ]; then qnnbackend=$2 - if [ ${qnnbackend} -gt 3 ]; then - show_usage - exit 1 - fi - if [ "$1" == "run_llamacli" ]; then run_llamacli exit 0 elif [ "$1" == "run_llamabench" ]; then run_llamabench exit 0 - elif [ "$1" == "run_ut_add" ]; then - run_ut_add - exit 0 - elif [ "$1" == "run_ut_mulmat" ]; then - run_ut_mulmat - exit 0 - elif [ "$1" == "run_ut_mul" ]; then - run_ut_mul exit 0 + else + show_usage + exit 1 fi elif [ $# == 3 ]; then + #opname can be found via print_oplist: opname=$2 -#TODO: check opname in oplist -#opname can be found via print_oplist: qnnbackend=$3 if [ ${qnnbackend} -gt 3 ]; then diff --git a/scripts/build-run-windows.sh b/scripts/build-run-windows.sh deleted file mode 100755 index c9a5b13d71d4c..0000000000000 --- a/scripts/build-run-windows.sh +++ /dev/null @@ -1,222 +0,0 @@ -#!/bin/bash -# build llama.cpp or llama.cpp + ggml-qnn for Windows with cygwin on Windows -# build llama.cpp + ggml-qnn for Snapdragon desktop SoC equipped WoA(Windows on ARM) with cygwin on Windows - -# items marked TODO has not verified yet - -set -e - - -PWD=`pwd` -PREFIX_PATH=/cygdrive/c -GGUF_MODEL_NAME=${PREFIX_PATH}/qwen1_5-1_8b-chat-q4_0.gguf -PROJECT_HOME_PATH=`pwd` - -#QNN SDK could be found at: -#https://www.qualcomm.com/developer/software/qualcomm-ai-engine-direct-sdk -#https://developer.qualcomm.com/software/hexagon-dsp-sdk/tools -QNN_SDK_URL=https://www.qualcomm.com/developer/software/qualcomm-ai-engine-direct-sdk -QNN_SDK_PATH=${PREFIX_PATH}/qairt/2.31.0.250130/ - -#default is QNN NPU -qnnbackend=2 - -function dump_vars() -{ - echo -e "QNN_SDK_PATH: ${QNN_SDK_PATH}" -} - - -function show_pwd() -{ - echo -e "current working path:$(pwd)\n" -} - - -function check_qnn_sdk() -{ - if [ ! -d ${QNN_SDK_PATH} ]; then - echo -e "QNN_SDK_PATH ${QNN_SDK_PATH} not exist, pls check or download it from ${QNN_SDK_URL}...\n" - exit 1 - fi -} - -function build_windows_x86 -{ - echo "build_windows_x86-without-qnn" - cmake -H. -B./out/windows_x86 -DCMAKE_BUILD_TYPE=Release -DGGML_OPENMP=OFF - cd out/windows_x86 - make -j16 - show_pwd - - cd - -} - -function build_windows_x86_qnn -{ - echo "build_windows_x86-with-qnn" - cmake -H. -B./out/windows_x86_qnn -DCMAKE_BUILD_TYPE=Release -DGGML_OPENMP=OFF -DGGML_QNN=ON -DGGML_QNN_SDK_PATH=${QNN_SDK_PATH} - cd out/windows_x86_qnn - make -j16 - show_pwd - - cd - -} - -#TODO -function build_windows_arm64_qnn -{ - echo "build_windows_arm64 not supported now" - echo "cmake source dir:${PROJECT_HOME_PATH}" - cmake -H. -B./out/windows_arm64_qnn -DCMAKE_BUILD_TYPE=Release -DGGML_OPENMP=OFF -DGGML_QNN=ON -DCMAKE_TOOLCHAIN_FILE=${PROJECT_HOME_PATH}/cmake/arm64-windows-llvm.cmake -DCMAKE_C_FLAGS=-march=armv8.7-a -DGGML_QNN_SDK_PATH=${QNN_SDK_PATH} - #cmake -H. -B./out/windows_arm64_qnn -DCMAKE_BUILD_TYPE=Release -DGGML_OPENMP=OFF -DGGML_QNN=ON -DCMAKE_TOOLCHAIN_FILE=${PROJECT_HOME_PATH}/cmake/arm64-windows-cygwin.cmake -DCMAKE_C_FLAGS=-march=armv8.7-a -DGGML_QNN_SDK_PATH=${QNN_SDK_PATH} - cd out/windows_arm64_qnn - make -j16 - show_pwd - - cd - -} - - -function remove_temp_dir() -{ - if [ -d out/windows_x86 ]; then - echo "remove out/windows_x86 directory in `pwd`" - rm -rf out/windows_x86 - fi -} - - -function check_qnn_libs() -{ - echo "do nothing" -} - - -function update_qnn_libs() -{ - echo "do nothing" -} - -function build_x86() -{ - show_pwd - check_qnn_sdk - dump_vars - #some unexpected behaviour on Windows - #remove_temp_dir - build_windows_x86 -} - -function build_x86_qnn() -{ - show_pwd - check_qnn_sdk - dump_vars - #some unexpected behaviour on Windows - #remove_temp_dir - build_windows_x86_qnn -} - -function build_arm64_qnn() -{ - show_pwd - check_qnn_sdk - dump_vars - #some unexpected behaviour on Windows - #remove_temp_dir - build_windows_arm64_qnn -} - -function run_llamacli() -{ - check_qnn_libs - echo "not supported on Windows now" - - #llama-cli -mg ${qnnbackend} -no-cnv -m ${GGUF_MODEL_NAME} -p \"introduce the movie Once Upon a Time in America briefly.\n\" - -} - - -function run_llamabench() -{ - check_qnn_libs - echo "not supported on Windows now" - - #llama-bench -mg ${qnnbackend} -m ${GGUF_MODEL_NAME}" - -} - - -function run_test-backend-ops() -{ - check_qnn_libs - echo "not supported on Windows now" - - #test-backend-ops test" - -} - - -function show_usage() -{ - echo "Usage:" - echo " $0 build_x86" - echo " $0 build_x86_qnn" - echo " $0 build_arm64_qnn" - echo " $0 run_testop" - echo " $0 run_llamacli 0 (QNN_CPU) / 1 (QNN_GPU) / 2 (QNN_NPU) / 3 (ggml)" - echo " $0 run_llamabench 0 (QNN_CPU) / 1 (QNN_GPU) / 2 (QNN_NPU) / 3 (ggml)" - echo -e "\n\n\n" -} - - -show_pwd - -check_qnn_sdk - -if [ $# == 0 ]; then - show_usage - exit 1 -elif [ $# == 1 ]; then - if [ "$1" == "-h" ]; then - show_usage - exit 1 - elif [ "$1" == "help" ]; then - show_usage - exit 1 - elif [ "$1" == "build_x86" ]; then - build_x86 - exit 0 - elif [ "$1" == "build_x86_qnn" ]; then - build_x86_qnn - exit 0 - elif [ "$1" == "build_arm64_qnn" ]; then - build_arm64_qnn - exit 0 - - elif [ "$1" == "run_testop" ]; then - run_test-backend-ops - exit 0 - else - show_usage - exit 1 - fi -elif [ $# == 2 ]; then - qnnbackend=$2 - if [ ${qnnbackend} -gt 3 ]; then - show_usage - exit 1 - fi - - if [ "$1" == "run_llamacli" ]; then - run_llamacli - exit 0 - elif [ "$1" == "run_llamabench" ]; then - run_llamabench - exit 0 - fi -else - show_usage - exit 1 -fi diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpp index 7345eee2ea989..9da97f1bc5057 100644 --- a/src/llama-mmap.cpp +++ b/src/llama-mmap.cpp @@ -481,10 +481,10 @@ struct llama_mlock::impl { // Skip resource limit checks on visionOS/tvOS suggest = false; #else - struct rlimit lock_limit = {}; - //if (suggest && getrlimit(RLIMIT_MEMLOCK, &lock_limit)) { - // suggest = false; - //} + struct rlimit lock_limit; + if (suggest && getrlimit(RLIMIT_MEMLOCK, &lock_limit)) { + suggest = false; + } if (suggest && (lock_limit.rlim_max > lock_limit.rlim_cur + size)) { suggest = false; } From 79fb362ff22277bd3707a767cbd2253c8c1e326f Mon Sep 17 00:00:00 2001 From: zhouwg Date: Tue, 11 Mar 2025 12:52:54 +0800 Subject: [PATCH 63/76] ggml-qnn: pr to upstream --- CMakeLists.txt | 1 - cmake/aarch64-w64-mingw32.cmake | 18 - cmake/arm64-windows-cygwin.cmake | 16 - cmake/arm64-windows-llvm.cmake | 4 +- ggml/src/ggml-qnn/CMakeLists.txt | 3 +- ggml/src/ggml-qnn/ggml-qnn.cpp | 2606 +++++++++++++++++------------- scripts/build-run-android.sh | 70 +- scripts/ggml-qnn.cfg | 9 + tests/CMakeLists.txt | 1 - tests/ggml-qnn-ut.cpp | 480 ------ 10 files changed, 1583 insertions(+), 1625 deletions(-) delete mode 100644 cmake/aarch64-w64-mingw32.cmake delete mode 100644 cmake/arm64-windows-cygwin.cmake create mode 100644 scripts/ggml-qnn.cfg delete mode 100644 tests/ggml-qnn-ut.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 73a9c554f651e..23cfbce5ae566 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,7 +6,6 @@ include(CheckIncludeFileCXX) set(CMAKE_WARN_UNUSED_CLI YES) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) -set(CMAKE_VERBOSE_MAKEFILE on) if (NOT XCODE AND NOT MSVC AND NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE Release CACHE STRING "Build type" FORCE) diff --git a/cmake/aarch64-w64-mingw32.cmake b/cmake/aarch64-w64-mingw32.cmake deleted file mode 100644 index 775fa46337628..0000000000000 --- a/cmake/aarch64-w64-mingw32.cmake +++ /dev/null @@ -1,18 +0,0 @@ -#TODO -#not work on Linux -set( CMAKE_SYSTEM_NAME mingw ) -set( CMAKE_SYSTEM_PROCESSOR arm64 ) - -set( target aarch64-w64-mingw32 ) - -set( CMAKE_C_COMPILER aarch64-w64-mingw32-gcc ) -set( CMAKE_CXX_COMPILER aarch64-w64-mingw32-g++ ) - -set( CMAKE_C_COMPILER_TARGET ${target} ) -set( CMAKE_CXX_COMPILER_TARGET ${target} ) - -#set( arch_c_flags "-march=armv8.7-a -fvectorize -ffp-model=fast -fno-finite-math-only" ) -#set( warn_c_flags "-Wno-format -Wno-unused-variable -Wno-unused-function -Wno-gnu-zero-variadic-macro-arguments" ) - -set( CMAKE_C_FLAGS_INIT "${arch_c_flags} ${warn_c_flags}" ) -set( CMAKE_CXX_FLAGS_INIT "${arch_c_flags} ${warn_c_flags}" ) diff --git a/cmake/arm64-windows-cygwin.cmake b/cmake/arm64-windows-cygwin.cmake deleted file mode 100644 index c7a313bb77adf..0000000000000 --- a/cmake/arm64-windows-cygwin.cmake +++ /dev/null @@ -1,16 +0,0 @@ -set( CMAKE_SYSTEM_NAME CYGWIN) -set( CMAKE_SYSTEM_PROCESSOR arm64 ) - -set( target aarch64-w64-cygwin) - -set( CMAKE_C_COMPILER clang ) -set( CMAKE_CXX_COMPILER clang++ ) - -set( CMAKE_C_COMPILER_TARGET ${target} ) -set( CMAKE_CXX_COMPILER_TARGET ${target} ) - -set( arch_c_flags "-march=armv8.7-a -fvectorize -ffp-model=fast -fno-finite-math-only" ) -set( warn_c_flags "-Wno-format -Wno-unused-variable -Wno-unused-function -Wno-gnu-zero-variadic-macro-arguments" ) - -set( CMAKE_C_FLAGS_INIT "${arch_c_flags} ${warn_c_flags}" ) -set( CMAKE_CXX_FLAGS_INIT "${arch_c_flags} ${warn_c_flags}" ) diff --git a/cmake/arm64-windows-llvm.cmake b/cmake/arm64-windows-llvm.cmake index 983206032df3d..8023796800683 100644 --- a/cmake/arm64-windows-llvm.cmake +++ b/cmake/arm64-windows-llvm.cmake @@ -9,8 +9,8 @@ set( CMAKE_CXX_COMPILER clang++ ) set( CMAKE_C_COMPILER_TARGET ${target} ) set( CMAKE_CXX_COMPILER_TARGET ${target} ) -#set( arch_c_flags "-march=armv8.7-a -fvectorize -ffp-model=fast -fno-finite-math-only" ) -#set( warn_c_flags "-Wno-format -Wno-unused-variable -Wno-unused-function -Wno-gnu-zero-variadic-macro-arguments" ) +set( arch_c_flags "-march=armv8.7-a -fvectorize -ffp-model=fast -fno-finite-math-only" ) +set( warn_c_flags "-Wno-format -Wno-unused-variable -Wno-unused-function -Wno-gnu-zero-variadic-macro-arguments" ) set( CMAKE_C_FLAGS_INIT "${arch_c_flags} ${warn_c_flags}" ) set( CMAKE_CXX_FLAGS_INIT "${arch_c_flags} ${warn_c_flags}" ) diff --git a/ggml/src/ggml-qnn/CMakeLists.txt b/ggml/src/ggml-qnn/CMakeLists.txt index c11e2f82fa92b..fcbbc33a9b136 100644 --- a/ggml/src/ggml-qnn/CMakeLists.txt +++ b/ggml/src/ggml-qnn/CMakeLists.txt @@ -1,4 +1,5 @@ message(STATUS "Using QNN backend") +message("CMAKE_SYSTEM_NAME : ${CMAKE_SYSTEM_NAME}") if(CMAKE_SYSTEM_NAME STREQUAL "Android") find_library(LOG_LIB log) @@ -6,8 +7,6 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Android") set(QNN_DEFAULT_LIB_SEARCH_PATH "/data/local/tmp/" CACHE STRING "customized library search path for QNN backend") elseif(CMAKE_SYSTEM_NAME STREQUAL "Windows") set(QNN_DEFAULT_LIB_SEARCH_PATH "C:\\" CACHE STRING "customized library search path for QNN backend") -elseif(CMAKE_SYSTEM_NAME STREQUAL "CYGWIN") - set(QNN_DEFAULT_LIB_SEARCH_PATH "/cygdrive/c/qairt/2.31.0.250130/" CACHE STRING "customized library search path for QNN backend") else() message(FATAL_ERROR "QNN now only available on Android and Windows(Windows on ARM)") endif() diff --git a/ggml/src/ggml-qnn/ggml-qnn.cpp b/ggml/src/ggml-qnn/ggml-qnn.cpp index 083f3ec466528..7c3477094ea9f 100644 --- a/ggml/src/ggml-qnn/ggml-qnn.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn.cpp @@ -1,20 +1,23 @@ /* - * Copyright (c) 2024- KanTV authors + * Copyright (c) 2023-2024 The ggml authors * * Qualcomm QNN SDK and reference tech guides could be found at: * https://www.qualcomm.com/developer/software/qualcomm-ai-engine-direct-sdk * https://developer.qualcomm.com/software/hexagon-dsp-sdk/tools * - * this single-source-file or self-contained implementation of ggml-qnn backend has seven sections: - * section-1 does forward/external declaration, - * section-2 defines ggml-qnn internal log function - * section-3 does general helper macro / data structure / function - * section-4 does QNN helper macro / data structure / function - * section-5 does ggml-qnn backend helper macro / data structure / function / class - * section-6 does implementation of ggml-qnn backend according to ggml's backend subsystem - * section-7 does implementation of offload ggml op to QNN backend + * this single-source-file or self-contained implementation of ggml-qnn backend has 10 sections: + * section-1 forward/prototype declaration + * section-2 global vars, macros, data structures + * section-3 ggml-qnn internal troubleshooting function/class + * section-4 helper function for WoA(Windows on ARM) + * section-5 general helper function + * section-6 QNN helper function + * section-7 ggml-qnn backend helper function / class + * section-8 implementation of ggml-qnn backend according to ggml's backend subsystem + * section-9 implementation of offload ggml op to QNN backend + * section-10 illustrate why the second approach is actual an fake at the moment * - * currently provide following ggml ops' QNN backend implementation in ggml-qnn-ops.cpp: + * currently provide following ggml op' QNN backend implementation: * - GGML_OP_ADD: this is a simple skeleton, can expand other ggml ops according to expertise * - GGML_OP_MUL: this is a simple skeleton, can expand other ggml ops according to expertise * - GGML_OP_MUL_MAT:this is a complicated skeleton, can expand other complex ggml ops accordingly @@ -52,6 +55,7 @@ #include #include #include +#include #endif #include @@ -65,6 +69,7 @@ #include #include #include +#include #include #include #include @@ -72,10 +77,8 @@ #include #include #include -#include #include #include -#include #include #if (defined __ANDROID__) || (defined ANDROID) #include "android/log.h" @@ -105,22 +108,26 @@ #include "ggml-backend-impl.h" // ================================================================================================= -// section-1: forward/external declaration +// section-1: forward/prototype declaration // ================================================================================================= class qnn_instance; struct ggml_backend_qnn_context; typedef void (* ggmlqnn_op_func_t)(ggml_backend_qnn_context * ctx, ggml_tensor * op); - static Qnn_Tensor_t * ggmlqnn_create_general_tensor(const ggml_tensor * tensor, const char * name, Qnn_TensorType_t qnn_tensor_type, Qnn_DataType_t qnn_data_type, uint32_t rank, uint32_t * dims, void * data, uint32_t data_size, bool b_transpose = false); +static enum ggml_status ggmlqnn_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph); +static void ggmlqnn_log_internal(ggml_log_level level, const char * file, const char * func, int line, const char * format, ...); +static inline bool ggmlqnn_is_valid_params(ggml_backend_qnn_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); +//op functions: +//done static void ggml_qnn_general_node(ggml_backend_qnn_context * ctx, ggml_tensor * dst); static void ggml_qnn_mul_mat(ggml_backend_qnn_context * ctx, ggml_tensor * dst); - +//todo static void ggml_qnn_repeat(ggml_backend_qnn_context * ctx, ggml_tensor * dst); static void ggml_qnn_div(ggml_backend_qnn_context * ctx, ggml_tensor * dst); static void ggml_qnn_leaky_relu(ggml_backend_qnn_context * ctx, ggml_tensor * dst); @@ -134,1032 +141,1215 @@ static void ggml_qnn_norm(ggml_backend_qnn_context * ctx, ggml_tensor * dst); static void ggml_qnn_group_norm(ggml_backend_qnn_context * ctx, ggml_tensor * dst); static void ggml_qnn_acc(ggml_backend_qnn_context * ctx, ggml_tensor * dst); static void ggml_qnn_sum_rows(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -static void ggml_qnn_upsample_nearest2d(ggml_backend_qnn_context * ctx, ggml_tensor * dst); static void ggml_qnn_pad(ggml_backend_qnn_context * ctx, ggml_tensor * dst); static void ggml_qnn_pool2d(ggml_backend_qnn_context * ctx, ggml_tensor * dst); static void ggml_qnn_dup(ggml_backend_qnn_context * ctx, ggml_tensor * dst); static void ggml_qnn_rms_norm(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -static void ggml_qnn_diag_mask(ggml_backend_qnn_context * ctx, ggml_tensor * dst, float value); -static void ggml_qnn_im2col(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -static void ggml_qnn_timestep_embedding(ggml_backend_qnn_context * ctx, ggml_tensor * dst); static void ggml_qnn_cpy(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggml_qnn_rope(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggml_qnn_im2col(ggml_backend_qnn_context * ctx, ggml_tensor * dst); static void ggml_qnn_softmax(ggml_backend_qnn_context * ctx, ggml_tensor * dst); static void ggml_qnn_get_rows(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -static void ggml_qnn_rope(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggml_qnn_upsample_nearest2d(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggml_qnn_timestep_embedding(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggml_qnn_diag_mask(ggml_backend_qnn_context * ctx, ggml_tensor * dst, float value); // ================================================================================================= -// section-2: ggml-qnn internal troubleshooting function/class +// section-2: global var, macro, data structure // ================================================================================================= +// the following two vars can be fetched from [qnn_runtimelib_path]/ggml-qnn.cfg +// [general] +// print_qnn_internal_log=0 +// inference_approach=0 +static int g_print_qnn_internal_log = 0; // enable/disable QNN's internal log +static int g_inference_approach = 0; // 0: general approach,similar to ggml-sycl or ggml-cann 1: mapping entire ggml cgraph to QNN graph +static const char * g_qnn_cfgfilename = "ggml-qnn.cfg"; + +#if defined(__ANDROID__) +//Android command line program +static const char * g_qnn_runtimelib_path = "/data/local/tmp/"; +#elif defined(__linux__) +static const char * g_qnn_runtimelib_path = "/tmp/"; +#elif defined(_WIN32) +static const char * g_qnn_runtimelib_path = "C:\\"; +#endif + +#if !defined(__ANDROID__) && !defined(__linux__) +static std::atomic g_ggmltensor_idx(0); //ensure every QNN tensor name is unique +#else +static int32_t g_ggmltensor_idx = 0; //ensure every QNN tensor name is unique +#endif + #if 0//def NDEBUG -#define GGMLQNN_DEBUG 0 -#define ENABLE_QNNBACKEND_PERF 0 // enable/disable op's perf info -#define GGMLQNN_PRINT_QNN_INTERNAL_LOG 0 // enable/disable QNN's internal log -#define GGMLQNN_PRINT_OP_ADD_LOG 0 // GGML_OP_ADD already verified with QNN-CPU / QNN-GPU / QNN-NPU -#define GGMLQNN_PRINT_OP_MUL_MAT_LOG 0 +#define GGMLQNN_DEBUG 0 +#define ENABLE_QNNBACKEND_PERF 0 +#define GGMLQNN_PRINT_OP_ADD_LOG 0 // GGML_OP_ADD already verified with QNN-CPU / QNN-GPU / QNN-NPU +#define GGMLQNN_PRINT_OP_MUL_MAT_LOG 0 #else -#define GGMLQNN_DEBUG 1 // for troubleshooting QNN backend -#define ENABLE_QNNBACKEND_PERF 0 // enable/disable op's perf info -#define GGMLQNN_PRINT_QNN_INTERNAL_LOG 0 // enable/disable QNN's internal log -#define GGMLQNN_PRINT_OP_ADD_LOG 0 // GGML_OP_ADD already verified with QNN-CPU / QNN-GPU / QNN-NPU -#define GGMLQNN_PRINT_OP_MUL_MAT_LOG 1 +#define GGMLQNN_DEBUG 1 // for troubleshooting QNN backend +#define ENABLE_QNNBACKEND_PERF 0 +#define GGMLQNN_PRINT_OP_ADD_LOG 0 // GGML_OP_ADD already verified with QNN-CPU / QNN-GPU / QNN-NPU +#define GGMLQNN_PRINT_OP_MUL_MAT_LOG 1 #endif -#define GGML_QNN_LOGBUF_LEN 4096 +#define GGML_QNN_LOGBUF_LEN 4096 -#define GGMLQNN_LOG_ERROR(...) ggmlqnn_log_internal(GGML_LOG_LEVEL_ERROR, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) -#define GGMLQNN_LOG_WARN(...) ggmlqnn_log_internal(GGML_LOG_LEVEL_WARN , __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) -#define GGMLQNN_LOG_INFO(...) ggmlqnn_log_internal(GGML_LOG_LEVEL_INFO , __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) +#define GGMLQNN_LOG_ERROR(...) ggmlqnn_log_internal(GGML_LOG_LEVEL_ERROR, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) +#define GGMLQNN_LOG_WARN(...) ggmlqnn_log_internal(GGML_LOG_LEVEL_WARN , __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) +#define GGMLQNN_LOG_INFO(...) ggmlqnn_log_internal(GGML_LOG_LEVEL_INFO , __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) #if GGMLQNN_DEBUG -#define GGMLQNN_LOG_DEBUG(...) ggmlqnn_log_internal(GGML_LOG_LEVEL_DEBUG, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) +#define GGMLQNN_LOG_DEBUG(...) ggmlqnn_log_internal(GGML_LOG_LEVEL_DEBUG, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) #else #define GGMLQNN_LOG_DEBUG(...) #endif -static void ggmlqnn_log_internal(ggml_log_level level, const char * file, const char * func, int line, const char * format, ...) { - static std::mutex ggmlqnn_log_internal_mutex; - static char s_ggmlqnn_log_internal_buf[GGML_QNN_LOGBUF_LEN]; - GGML_UNUSED(file); -#if !(defined __ANDROID__) || !(defined ANDROID) - GGML_UNUSED(level); -#endif - { - std::lock_guard lock(ggmlqnn_log_internal_mutex); - va_list args; - va_start(args, format); - int len_prefix = snprintf(s_ggmlqnn_log_internal_buf, GGML_QNN_LOGBUF_LEN, "[%s, %d]: ", func, line); - int len = vsnprintf(s_ggmlqnn_log_internal_buf + len_prefix, GGML_QNN_LOGBUF_LEN - len_prefix, format, args); - if (len < (GGML_QNN_LOGBUF_LEN - len_prefix)) { -#if (defined __ANDROID__) || (defined ANDROID) - //for Android application(standard APP or command line tool) - __android_log_print(ANDROID_LOG_INFO, "ggml-qnn", "%s\n", s_ggmlqnn_log_internal_buf); - if (GGML_LOG_LEVEL_INFO == level) { - printf("%s\n", s_ggmlqnn_log_internal_buf); - } -#else - //for Snapdragon based WoA(Windows on ARM) device or Linux - printf("%s\n", s_ggmlqnn_log_internal_buf); -#endif - } - va_end(args); - } -} +#define GGMLQNN_MEM_ADD(alignment) (sizeof (size_t) + alignment) +#define GGMLQNN_MEM_MASK(alignment) ((uintptr_t)alignment - 1) +#define GQCGT ggmlqnn_create_general_tensor +#define QNN_VER_PTR(x) (&((x).v1)) +#define RPCMEM_DEFAULT_FLAGS 1 +#define RPCMEM_HEAP_ID_SYSTEM 25 -#if ENABLE_QNNBACKEND_PERF -class qnn_perf { -public: - qnn_perf(const std::string & perf_name) : _perf_name(std::move(perf_name)) {}; - qnn_perf() = delete; - qnn_perf(const qnn_perf & ) = delete; - qnn_perf & operator= (const qnn_perf & ) = delete; +#define QNN_TENSOR_GET_ID(tensor) get_qnn_tensorid(tensor) +#define QNN_TENSOR_GET_NAME(tensor) get_qnn_tensorname(tensor) +#define QNN_TENSOR_GET_TYPE(tensor) get_qnn_tensortype(tensor) +#define QNN_TENSOR_GET_DATA_FORMAT(tensor) get_qnn_tensor_dataformat(tensor) +#define QNN_TENSOR_GET_DATA_TYPE(tensor) get_qnn_tensor_datatype(tensor) +#define QNN_TENSOR_GET_QUANT_PARAMS(tensor) get_qnn_tensor_quantparams(tensor) +#define QNN_TENSOR_GET_RANK(tensor) get_qnn_tensor_rank(tensor) +#define QNN_TENSOR_GET_DIMENSIONS(tensor) get_qnn_tensor_dimensions(tensor) +#define QNN_TENSOR_GET_MEM_TYPE(tensor) get_qnn_tensor_memtype(tensor) +#define QNN_TENSOR_GET_CLIENT_BUF(tensor) get_qnn_tensor_clientbuf(tensor) +#define QNN_TENSOR_GET_MEM_HANDLE(tensor) get_qnn_tensor_memhandle(tensor) - void start() { - _begin_time = ggml_time_us(); - } +#define QNN_TENSOR_SET_ID(tensor, value) set_qnn_tensor_id(tensor, value) +#define QNN_TENSOR_SET_NAME(tensor, value) set_qnn_tensor_name(tensor, value) +#define QNN_TENSOR_SET_TYPE(tensor, value) set_qnn_tensor_type(tensor, value) +#define QNN_TENSOR_SET_DATA_FORMAT(tensor, value) set_qnn_tensor_dataformat(tensor, value) +#define QNN_TENSOR_SET_DATA_TYPE(tensor, value) set_qnn_tensor_datatype(tensor, value) +#define QNN_TENSOR_SET_QUANT_PARAMS(tensor, value) set_qnn_tensor_quantparams(tensor, value) +#define QNN_TENSOR_SET_RANK(tensor, value) set_qnn_tensor_rank(tensor, value) +#define QNN_TENSOR_SET_DIMENSIONS(tensor, value) set_qnn_tensor_dimensions(tensor, value) +#define QNN_TENSOR_SET_MEM_TYPE(tensor, value) set_qnn_tensor_memtype(tensor, value) +#define QNN_TENSOR_SET_CLIENT_BUF(tensor, value) set_qnn_tensor_clientbuf(tensor, value) +#define QNN_TENSOR_SET_MEM_HANDLE(tensor, value) set_qnn_tensor_memhandle(tensor, value) - void info() { - _end_time = ggml_time_us(); - _duration = (_end_time - _begin_time); - GGMLQNN_LOG_DEBUG("duration of %s : %lld microseconds\n", _perf_name.c_str(), _duration); - } +#define DISABLE_COPY(class_name) \ + class_name(const class_name &) = delete; \ + void operator=(const class_name &) = delete -private: - int64_t _begin_time = 0LL; - int64_t _end_time = 0LL; - int64_t _duration = 0LL; - std::string _perf_name; -}; -#else -class qnn_perf { -public: - qnn_perf(const std::string & perf_name) { - GGML_UNUSED(perf_name); - } - qnn_perf() = delete; - qnn_perf(const qnn_perf & ) = delete; - qnn_perf & operator= (const qnn_perf & ) = delete; +#define DISABLE_MOVE(class_name) \ + class_name(class_name &&) = delete; \ + void operator=(class_name &&) = delete - void start() {} - void info() {} -}; -#endif +#define CHECK_QNN_API(error, result) \ + do { \ + error = (result); \ + if (QNN_SUCCESS != error) { \ + if (error == QNN_COMMON_ERROR_NOT_SUPPORTED) { \ + GGMLQNN_LOG_WARN("WARNING: QNN feature/API not supported\n"); \ + } else { \ + GGMLQNN_LOG_INFO("QNN API error = %d(%s)\n", error, ggmlqnn_get_qnnerror_string(error)); \ + } \ + } \ + } while (0) -// ================================================================================================= -// section-3: general helper macro / data structure / function -// ================================================================================================= -#define DISABLE_COPY(class_name) \ - class_name(const class_name &) = delete; \ - void operator=(const class_name &) = delete +#define GGMLQNN_CHECK_PARAMS(ctx, src0, src1, dst) \ + do { \ + if (!ggmlqnn_is_valid_params((ctx), (src0), (src1), (dst))) { \ + return; \ + } \ + } while (0) -#define DISABLE_MOVE(class_name) \ - class_name(class_name &&) = delete; \ - void operator=(class_name &&) = delete +using pfn_rpc_mem_init = void (*)(void); +using pfn_rpc_mem_deinit = void (*)(void); +using pfn_rpc_mem_alloc = void *(*)(int, uint32_t, int); +using pfn_rpc_mem_free = void (*)(void *); +using pfn_rpc_mem_to_fd = int (*)(void *); +using _pfn_QnnSaver_initialize = decltype(QnnSaver_initialize); +using _pfn_QnnInterface_getProviders = decltype(QnnInterface_getProviders); +using _pfn_QnnSystemInterface_getProviders = decltype(QnnSystemInterface_getProviders); +using qnn_res_t = std::tuple>; +using qnn_tensors_t = std::vector< Qnn_Tensor_t *>; + +enum class ggml_qnn_profile_level { + profile_off = 0, + profile_basic = 1, + profile_detail = 2 +}; -#define GQCGT ggmlqnn_create_general_tensor +enum qcom_htp_arch { + NONE = 0, + V68 = 68, + V69 = 69, + V73 = 73, + V75 = 75, + V79 = 79, +}; -//#if defined(_WIN32) +enum qcom_chipset_soc_model { + UNKNOWN_SM = 0, + SM7450 = 41, // v69, 7 Gen1 + SM8350 = 30, // v68, 888 + SM8450 = 36, // v69, SD 8 Gen 1 + SM8475 = 42, // v69, SD 8+ Gen 1 + SM8550 = 43, // v73, SD 8 Gen 2 + SM8650 = 57, // v75, SD 8 Gen 3 + SM8750 = 69, // v79, SD 8 Gen 4 #if !defined(__ANDROID__) && !defined(__linux__) -#define RTLD_GLOBAL 0x100 -#define RTLD_LOCAL 0x000 -#define RTLD_LAZY 0x000 -#define RTLD_NOW 0x001 -static void * dlopen(const char * filename, int flag); -static int dlclose(void * handle); -static void * dlsym(void* handle, const char* name); -static const char * dlerror(void); + SC7280X = 44, + SC8280X = 37, + SC8380XP = 60, +#endif +}; -static const char * last_func = nullptr; -static long last_err; -static void * dlopen(const char * dll, int flags) { - HINSTANCE h = LoadLibraryA(dll); - GGML_UNUSED(flags); - if (h == NULL) { - last_err = GetLastError(); - last_func = "dlopen"; - } - return h; -} +struct qcom_socinfo { + uint32_t soc_model; + size_t htp_arch; + size_t vtcm_size_in_mb; + char soc_desc[GGML_MAX_NAME]; +}; -static int dlclose(void * h) { - if (!FreeLibrary((HINSTANCE)h)) { - last_err = GetLastError(); - last_func = "dlclose"; - return -1; - } - return 0; -} +struct ggml_backend_qnn_context { + int device; + int threads; + char name[GGML_MAX_NAME]; + char desc[GGML_MAX_NAME]; + char lib[GGML_MAX_NAME]; + qnn_instance * instance; + struct ggml_backend * backend; + QNN_INTERFACE_VER_TYPE raw_interface; + QNN_SYSTEM_INTERFACE_VER_TYPE raw_system_interface; + struct qcom_socinfo socinfo; -static void * dlsym(void * h, const char * name) { - FARPROC p = GetProcAddress((HINSTANCE)h, name); - if (!p) { - last_err = GetLastError(); - last_func = "dlsym"; - } - return (void*)(intptr_t)p; -} + std::unique_ptr work_data; + std::vector> tasks; + size_t work_size = 0; + size_t desired_size = 0; + int n_threads = GGML_DEFAULT_N_THREADS; +}; -static const char * dlerror(void) { - static char str[512]; - if (!last_err) return nullptr; +struct qnn_op_caps_t { + const char * qnn_op_name = nullptr; + const size_t input_param_count = 0; + const char * qnn_param_name = nullptr; +}; - snprintf(str, 512, "%s error #%ld", last_func, last_err); - last_err = 0; - last_func = NULL; +//file:///opt/qcom/aistack/qairt/2.31.0.250130/docs/QNN/general/overview.html#tbl-supported-snapdragon-devices +static struct qcom_socinfo g_qnn_soc_info_table[] = { + /* Qualcomm SnapDragon 7 Gen 1 */ + { + .soc_model = SM7450, + .htp_arch = V69, + .vtcm_size_in_mb = 8, + .soc_desc = "Qualcomm SnapDragon 7 Gen 1"}, - return str; -} -#endif + /* Qualcomm SnapDragon 888 */ + { + .soc_model = SM8350, + .htp_arch = V68, + .vtcm_size_in_mb = 8, + .soc_desc = "Qualcomm SnapDragon 888 "}, -#define GGMLQNN_MEM_ADD(alignment) (sizeof (size_t) + alignment) -#define GGMLQNN_MEM_MASK(alignment) ((uintptr_t)alignment - 1) -static void * ggmlqnn_mallocz_aligned(size_t size, size_t alignment) { - uint8_t * buffer = NULL; - size_t * sp = NULL; - buffer = static_cast(calloc(1, size + GGMLQNN_MEM_ADD(alignment))); - if (!buffer) - return NULL; - sp = (size_t *)buffer; - *sp = size; - buffer = (uint8_t *)(((uintptr_t) buffer + GGMLQNN_MEM_ADD(alignment)) & ~GGMLQNN_MEM_MASK(alignment)); - buffer[-1] = buffer - (uint8_t *)sp; - return buffer; -} + /* Qualcomm SnapDragon 8 Gen 1 */ + { + .soc_model = SM8450, + .htp_arch = V69, + .vtcm_size_in_mb = 8, + .soc_desc = "Qualcomm SnapDragon 8 Gen 1"}, -static void * ggmlqnn_malloc_aligned(size_t size, size_t alignment) { - uint8_t * buffer = NULL; - size_t * sp = NULL; - buffer = static_cast(malloc(size + GGMLQNN_MEM_ADD(alignment))); - if (!buffer) - return NULL; - sp = (size_t *)buffer; - *sp = size; - buffer = (uint8_t *)(((uintptr_t) buffer + GGMLQNN_MEM_ADD(alignment)) & ~GGMLQNN_MEM_MASK(alignment)); - buffer[-1] = buffer - (uint8_t *)sp; - return buffer; -} + /* Qualcomm SnapDragon 8 Gen 1+ */ + { + .soc_model = SM8475, + .htp_arch = V69, + .vtcm_size_in_mb = 8, + .soc_desc = "Qualcomm SnapDragon 8 Gen 1+"}, -static void ggmqnn_free_aligned(void * ptr) { - uint8_t * old = (uint8_t *)ptr; - if (!old) - return; - old -= old[-1]; - free(old); -} + /* Qualcomm SnapDragon 8 Gen 2 */ + { + .soc_model = SM8550, + .htp_arch = V73, + .vtcm_size_in_mb = 8, + .soc_desc = "Qualcomm SnapDragon 8 Gen 2"}, -static intptr_t ggmlqnn_align_to(size_t alignment, intptr_t offset) { - return offset % alignment == 0 ? offset - : offset + - (static_cast(alignment) - - offset % static_cast(alignment)); -} + /* Qualcomm SnapDragon 8 Gen 3 */ + { + .soc_model = SM8650, + .htp_arch = V75, + .vtcm_size_in_mb = 8, + .soc_desc = "Qualcomm SnapDragon 8 Gen 3 "}, -static size_t get_system_total_memory_in_bytes() { -#if defined(__ANDROID__) || defined(__linux__) - struct sysinfo info = {}; - if (0 == sysinfo(&info)) { - return (info.totalram + info.totalswap) * info.mem_unit; - } - size_t pages = (size_t)sysconf(_SC_PHYS_PAGES); - size_t page_size = (size_t)sysconf(_SC_PAGE_SIZE); + /* Qualcomm SnapDragon 8 Gen 4 */ + { + .soc_model = SM8750, + .htp_arch = V79, + .vtcm_size_in_mb = 8, + .soc_desc = "Qualcomm SnapDragon 8 Gen 4"}, - return pages * page_size; -#else - //FIXME: Snapdragon based WoA(Windows on ARM) - MEMORYSTATUSEX statex; - statex.dwLength = sizeof(statex); - if (GlobalMemoryStatusEx(&statex)) { - GGMLQNN_LOG_INFO("total physical mem:%llu Mb", statex.ullTotalPhys >> 20); - GGMLQNN_LOG_INFO("avail physical mem:%llu Mb", statex.ullAvailPhys >> 20); - return statex.ullTotalPhys; - } - return 0; -#endif -} +#if !defined(__ANDROID__) && !defined(__linux__) + /* Qualcomm SnapDragon 7c Gen 2 */ + { + .soc_model = SC7280X, + .htp_arch = V68, + .vtcm_size_in_mb = 8, + .soc_desc = "Qualcomm SnapDragon 7c Gen 2"}, -static size_t get_system_free_memory_in_bytes() { -#if defined(__ANDROID__) || defined(__linux__) - struct sysinfo info = {}; - if (0 == sysinfo(&info)) { - return (info.freeram + info.freeswap) * info.mem_unit; - } - size_t avail_pages = (size_t)sysconf(_SC_AVPHYS_PAGES); - size_t page_size = (size_t)sysconf(_SC_PAGE_SIZE); + /* Qualcomm SnapDragon 8cx Gen 3 */ + { + .soc_model = SC8280X, + .htp_arch = V68, + .vtcm_size_in_mb = 8, + .soc_desc = "Qualcomm SnapDragon 8cx Gen 3"}, - return avail_pages * page_size; -#else - //FIXME: Snapdragon based WoA(Windows on ARM) - MEMORYSTATUSEX statex; - statex.dwLength = sizeof(statex); - if (GlobalMemoryStatusEx(&statex)) { - GGMLQNN_LOG_INFO("total physical mem:%llu Mb", statex.ullTotalPhys >> 20); - GGMLQNN_LOG_INFO("avail physical mem:%llu Mb", statex.ullAvailPhys >> 20); - return statex.ullAvailPhys; - } - return 0; + /* Qualcomm SnapDragon 8cx Gen 4 */ + { + .soc_model = SC8380XP, + .htp_arch = V73, + .vtcm_size_in_mb = 8, + .soc_desc = "Qualcomm SnapDragon 8cx Gen 4"}, #endif -} - -static size_t ggmlqnn_memscpy(void * dst, size_t dst_size, const void * src, size_t copy_size) { - if (!dst || !src || !dst_size || !copy_size) - return 0; - - size_t min_size = dst_size < copy_size ? dst_size : copy_size; - memcpy(dst, src, min_size); - - return min_size; -} +}; -static char * ggmlqnn_strndup(const char * source, size_t maxlen) { -#if defined(__ANDROID__) || defined(__linux__) - return strndup(source, maxlen); +// file:///opt/qcom/aistack/qairt/2.31.0.250130/docs/QNN/general/quantization.html +// CPU - Choose a non-quantized model.Quantized models are currently incompatible with the CPU backend +// GPU - Choose a non-quantized model.Quantized models are currently incompatible with the GPU backend +// HTP - Choose a quantized model. Quantized models are required when running on the HTP backend +// DSP - Choose a quantized model. Quantized models are required when running on the DSP backend +// HTA - Choose a quantized model. Quantized models are required when running on the HTA backend +static struct ggml_backend_qnn_context g_qnn_mgr[GGML_QNN_MAX_DEVICES] = { + [QNN_BACKEND_CPU] = {.device = 0, + .threads = 1, + .name = "qnn-cpu", + .desc = "Qualcomm Kryo CPU", +#if !defined(__ANDROID__) && !defined(__linux__) + .lib = "QnnCpu.dll", #else - //FIXME:behaviour is not exactly same to Android&Linux - GGML_UNUSED(maxlen); - return strdup(source); + .lib = "libQnnCpu.so", #endif -} + .instance = nullptr, + .backend = nullptr, + .raw_interface = {}, + .raw_system_interface = {}, + .socinfo = {}}, -static void * ggmlqnn_host_malloc(size_t buffer_size, size_t page_size) { - void * data = nullptr; -#if defined(__ANDROID__) || defined(__linux__) - int result = posix_memalign((void **)&data, page_size, buffer_size); - if (result != 0) { - GGMLQNN_LOG_WARN("%s: error: posix_memalign failed\n", __func__); - return nullptr; - } + [QNN_BACKEND_GPU] = {.device = 1, + .threads = 1, + .name = "qnn-gpu", + .desc = "Qualcomm Adreno GPU", +#if !defined(__ANDROID__) && !defined(__linux__) + .lib = "QnnGpu.dll", #else - //GGMLQNN_LOG_DEBUG("buffer_size %d, page_size %d\n", buffer_size, page_size); - data = ggmlqnn_malloc_aligned(buffer_size, page_size); - if (nullptr == data) { - GGMLQNN_LOG_WARN("%s: error: host_malloc failed\n", __func__); - } + .lib = "libQnnGpu.so", #endif + .instance = nullptr, + .backend = nullptr, + .raw_interface = {}, + .raw_system_interface = {}, + .socinfo = {}}, - return data; -} - -// ================================================================================================= -// section-4: QNN helper macro / data structure / function -// ================================================================================================= -#define CHECK_QNN_API(error, result) \ - do { \ - error = (result); \ - if (QNN_SUCCESS != error) { \ - if (error == QNN_COMMON_ERROR_NOT_SUPPORTED) { \ - GGMLQNN_LOG_WARN("WARNING: QNN feature/API not supported\n"); \ - } else { \ - GGMLQNN_LOG_INFO("QNN API error = %d(%s)\n", error, ggmlqnn_get_error_string(error)); \ - } \ - } \ - } while (0) - -#define QNN_VER_PTR(x) (&((x).v1)) -#define RPCMEM_DEFAULT_FLAGS 1 -#define RPCMEM_HEAP_ID_SYSTEM 25 - -#define QNN_TENSOR_GET_ID(tensor) get_qnn_tensorid(tensor) -#define QNN_TENSOR_GET_NAME(tensor) get_qnn_tensorname(tensor) -#define QNN_TENSOR_GET_TYPE(tensor) get_qnn_tensortype(tensor) -#define QNN_TENSOR_GET_DATA_FORMAT(tensor) get_qnn_tensor_dataformat(tensor) -#define QNN_TENSOR_GET_DATA_TYPE(tensor) get_qnn_tensor_datatype(tensor) -#define QNN_TENSOR_GET_QUANT_PARAMS(tensor) get_qnn_tensor_quantparams(tensor) -#define QNN_TENSOR_GET_RANK(tensor) get_qnn_tensor_rank(tensor) -#define QNN_TENSOR_GET_DIMENSIONS(tensor) get_qnn_tensor_dimensions(tensor) -#define QNN_TENSOR_GET_MEM_TYPE(tensor) get_qnn_tensor_memtype(tensor) -#define QNN_TENSOR_GET_CLIENT_BUF(tensor) get_qnn_tensor_clientbuf(tensor) -#define QNN_TENSOR_GET_MEM_HANDLE(tensor) get_qnn_tensor_memhandle(tensor) - -#define QNN_TENSOR_SET_ID(tensor, value) set_qnn_tensor_id(tensor, value) -#define QNN_TENSOR_SET_NAME(tensor, value) set_qnn_tensor_name(tensor, value) -#define QNN_TENSOR_SET_TYPE(tensor, value) set_qnn_tensor_type(tensor, value) -#define QNN_TENSOR_SET_DATA_FORMAT(tensor, value) set_qnn_tensor_dataformat(tensor, value) -#define QNN_TENSOR_SET_DATA_TYPE(tensor, value) set_qnn_tensor_datatype(tensor, value) -#define QNN_TENSOR_SET_QUANT_PARAMS(tensor, value) set_qnn_tensor_quantparams(tensor, value) -#define QNN_TENSOR_SET_RANK(tensor, value) set_qnn_tensor_rank(tensor, value) -#define QNN_TENSOR_SET_DIMENSIONS(tensor, value) set_qnn_tensor_dimensions(tensor, value) -#define QNN_TENSOR_SET_MEM_TYPE(tensor, value) set_qnn_tensor_memtype(tensor, value) -#define QNN_TENSOR_SET_CLIENT_BUF(tensor, value) set_qnn_tensor_clientbuf(tensor, value) -#define QNN_TENSOR_SET_MEM_HANDLE(tensor, value) set_qnn_tensor_memhandle(tensor, value) - -static inline uint32_t get_qnn_tensorid(const Qnn_Tensor_t & tensor) { - if (tensor.version == QNN_TENSOR_VERSION_1) { - return tensor.v1.id; - } - - return 0u; -} - -static inline const char * get_qnn_tensorname(const Qnn_Tensor_t & tensor) { - if (tensor.version == QNN_TENSOR_VERSION_1) { - return tensor.v1.name; - } - return nullptr; -} + [QNN_BACKEND_NPU] = {.device = 2, + .threads = 1, + .name = "qnn-npu", + .desc = "Qualcomm NPU(Hexagon Tensor Processor)", +#if !defined(__ANDROID__) && !defined(__linux__) + .lib = "QnnHtp.dll", +#else + .lib = "libQnnHtp.so", +#endif + .instance = nullptr, + .backend = nullptr, + .raw_interface = {}, + .raw_system_interface = {}, + .socinfo = {}}, +}; -static inline Qnn_TensorType_t get_qnn_tensortype(const Qnn_Tensor_t & tensor) { - if (tensor.version == QNN_TENSOR_VERSION_1) { - return tensor.v1.type; - } - return QNN_TENSOR_TYPE_UNDEFINED; -} +static const qnn_op_caps_t ggmlqnn_k_op_caps[] = { + {}, // GGML_OP_NONE + {}, // GGML_OP_DUP + { + // GGML_OP_ADD + QNN_OP_ELEMENT_WISE_ADD, + 2, + }, + {}, // GGML_OP_ADD1 + {}, // GGML_OP_ACC + {}, // GGML_OP_SUB + { + // GGML_OP_MUL + QNN_OP_ELEMENT_WISE_MULTIPLY, + 2, + }, + {}, // GGML_OP_DIV + {}, // GGML_OP_SQR + {}, // GGML_OP_SQRT + {}, // GGML_OP_LOG + {}, // GGML_OP_SIN + {}, // GGML_OP_COS + {}, // GGML_OP_SUM + {}, // GGML_OP_SUM_ROWS + {}, // GGML_OP_MEAN + {}, // GGML_OP_ARGMAX + {}, // GGML_OP_COUNT_EQUAL + {}, // GGML_OP_REPEAT + {}, // GGML_OP_REPEAT_BACK + {}, // GGML_OP_CONCAT + {}, // GGML_OP_SILU_BACK + {}, // GGML_OP_NORM + {}, // GGML_OP_RMS_NORM + {}, // GGML_OP_RMS_NORM_BACK + {}, // GGML_OP_GROUP_NORM + { + // GGML_OP_MUL_MAT + QNN_OP_MAT_MUL, + 2, + }, + {}, // GGML_OP_MUL_MAT_ID + {}, // GGML_OP_OUT_PROD + {}, // GGML_OP_SCALE + {}, // GGML_OP_SET + {}, // GGML_OP_CPY + {}, // GGML_OP_CONT + {}, // GGML_OP_RESHAPE + {}, // GGML_OP_VIEW + {}, // GGML_OP_PERMUTE + {}, // GGML_OP_TRANSPOSE + {}, // GGML_OP_GET_ROWS + {}, // GGML_OP_GET_ROWS_BACK + {}, // GGML_OP_DIAG + {}, // GGML_OP_DIAG_MASK_INF + {}, // GGML_OP_DIAG_MASK_ZERO + {}, // GGML_OP_SOFT_MAX + {}, // GGML_OP_SOFT_MAX_BACK + {}, // GGML_OP_ROPE + {}, // GGML_OP_ROPE_BACK + {}, // GGML_OP_CLAMP + {}, // GGML_OP_CONV_TRANSPOSE_1D + {}, // GGML_OP_IM2COL + {}, // GGML_OP_IM2COL_BACK + {}, // GGML_OP_CONV_TRANSPOSE_2D + {}, // GGML_OP_POOL_1D + {}, // GGML_OP_POOL_2D + {}, // GGML_OP_POOL_2D_BACK + {}, // GGML_OP_UPSCALE + {}, // GGML_OP_PAD + {}, // GGML_OP_PAD_REFLECT_1D + {}, // GGML_OP_ARANGE + {}, // GGML_OP_TIMESTEP_EMBEDDING + {}, // GGML_OP_ARGSORT + {}, // GGML_OP_LEAKY_RELU + {}, // GGML_OP_FLASH_ATTN_EXT + {}, // GGML_OP_FLASH_ATTN_BACK + {}, // GGML_OP_SSM_CONV + {}, // GGML_OP_SSM_SCAN + {}, // GGML_OP_WIN_PART + {}, // GGML_OP_WIN_UNPART + {}, // GGML_OP_GET_REL_POS + {}, // GGML_OP_ADD_REL_POS + {}, // GGML_OP_RWKV_WKV6 + {}, // GGML_OP_GATED_LINEAR_ATTN + {}, // GGML_OP_UNARY + {}, // GGML_OP_MAP_UNARY + {}, // GGML_OP_MAP_BINARY + {}, // GGML_OP_MAP_CUSTOM1_F32 + {}, // GGML_OP_MAP_CUSTOM2_F32 + {}, // GGML_OP_MAP_CUSTOM3_F32 + {}, // GGML_OP_MAP_CUSTOM1 + {}, // GGML_OP_MAP_CUSTOM2 + {}, // GGML_OP_MAP_CUSTOM3 + {}, // GGML_OP_CROSS_ENTROPY_LOSS + {}, // GGML_OP_CROSS_ENTROPY_LOSS_BACK + {}, // GGML_OP_OPT_STEP_ADAMW + {}, // GGML_UNARY_OP_ABS + {}, // GGML_UNARY_OP_SGN + {}, // GGML_UNARY_OP_NEG + {}, // GGML_UNARY_OP_STEP + {}, // GGML_UNARY_OP_TANH + {}, // GGML_UNARY_OP_ELU + {}, // GGML_UNARY_OP_RELU + {}, // GGML_UNARY_OP_SIGMOID + {}, // GGML_UNARY_OP_GELU + {}, // GGML_UNARY_OP_GELU_QUICK + {}, // GGML_UNARY_OP_SILU + {}, // GGML_UNARY_OP_HARDSWISH + {}, // GGML_UNARY_OP_HARDSIGMOID + {}, // GGML_UNARY_OP_EXP +}; -static inline Qnn_TensorDataFormat_t get_qnn_tensor_dataformat(const Qnn_Tensor_t & tensor) { - if (tensor.version == QNN_TENSOR_VERSION_1) { - return tensor.v1.dataFormat; - } - return QNN_TENSOR_DATA_FORMAT_FLAT_BUFFER; -} +// ================================================================================================= +// section-3: ggml-qnn internal troubleshooting function/class +// ================================================================================================= +static void ggmlqnn_log_internal(ggml_log_level level, const char * file, const char * func, int line, const char * format, ...) { + static std::mutex ggmlqnn_log_internal_mutex; + static char s_ggmlqnn_log_internal_buf[GGML_QNN_LOGBUF_LEN]; -static inline Qnn_DataType_t get_qnn_tensor_datatype(const Qnn_Tensor_t & tensor) { - if (tensor.version == QNN_TENSOR_VERSION_1) { - return tensor.v1.dataType; + GGML_UNUSED(file); +#if !(defined __ANDROID__) || !(defined ANDROID) + GGML_UNUSED(level); +#endif + { + std::lock_guard lock(ggmlqnn_log_internal_mutex); + va_list args; + va_start(args, format); + int len_prefix = snprintf(s_ggmlqnn_log_internal_buf, GGML_QNN_LOGBUF_LEN, "[%s, %d]: ", func, line); + int len = vsnprintf(s_ggmlqnn_log_internal_buf + len_prefix, GGML_QNN_LOGBUF_LEN - len_prefix, format, args); + if (len < (GGML_QNN_LOGBUF_LEN - len_prefix)) { +#if (defined __ANDROID__) || (defined ANDROID) + __android_log_print(ANDROID_LOG_INFO, "ggml-qnn", "%s\n", s_ggmlqnn_log_internal_buf); + if (GGML_LOG_LEVEL_INFO == level) { + printf("%s\n", s_ggmlqnn_log_internal_buf); + } +#else + //for Snapdragon based WoA(Windows on ARM) device or Linux + printf("%s\n", s_ggmlqnn_log_internal_buf); +#endif + } + va_end(args); } - return QNN_DATATYPE_UNDEFINED; } -static inline Qnn_QuantizeParams_t get_qnn_tensor_quantparams(const Qnn_Tensor_t & tensor) { - if (tensor.version == QNN_TENSOR_VERSION_1) { - return tensor.v1.quantizeParams; - } - return QNN_QUANTIZE_PARAMS_INIT; -} +#if ENABLE_QNNBACKEND_PERF +class qnn_perf { +public: + qnn_perf(const std::string & perf_name) : _perf_name(std::move(perf_name)) {}; + qnn_perf() = delete; + qnn_perf(const qnn_perf & ) = delete; + qnn_perf & operator= (const qnn_perf & ) = delete; -static inline uint32_t get_qnn_tensor_rank(const Qnn_Tensor_t & tensor) { - if (tensor.version == QNN_TENSOR_VERSION_1) { - return tensor.v1.rank; + void start() { + _begin_time = ggml_time_us(); } - return 0u; -} -static inline uint32_t * get_qnn_tensor_dimensions(const Qnn_Tensor_t & tensor) { - if (tensor.version == QNN_TENSOR_VERSION_1) { - return tensor.v1.dimensions; + void info() { + _end_time = ggml_time_us(); + _duration = (_end_time - _begin_time); + GGMLQNN_LOG_DEBUG("duration of %s : %lld microseconds\n", _perf_name.c_str(), _duration); } - return nullptr; -} -static inline Qnn_TensorMemType_t get_qnn_tensor_memtype(const Qnn_Tensor_t & tensor) { - if (tensor.version == QNN_TENSOR_VERSION_1) { - return tensor.v1.memType; +private: + int64_t _begin_time = 0LL; + int64_t _end_time = 0LL; + int64_t _duration = 0LL; + std::string _perf_name; +}; +#else +class qnn_perf { +public: + qnn_perf(const std::string & perf_name) { + GGML_UNUSED(perf_name); } - return QNN_TENSORMEMTYPE_UNDEFINED; -} + qnn_perf() = delete; + qnn_perf(const qnn_perf & ) = delete; + qnn_perf & operator= (const qnn_perf & ) = delete; -static inline void set_qnn_tensor_id(Qnn_Tensor_t & tensor, uint32_t id) { - if (tensor.version == QNN_TENSOR_VERSION_1) { - tensor.v1.id = id; + void start() {} + void info() {} +}; +#endif + +class qnn_cfg { +public: + void dump(std::function worker) { + if (!_load_success) { + GGMLQNN_LOG_INFO("qnn cfg file %s not loadded", _cfg_filename.c_str()); + return; + } + auto iter = _qnn_cfg.begin(); + while (iter != _qnn_cfg.end()) { + auto kv_iter = iter->second.begin(); + while (kv_iter != iter->second.end()) { + worker(iter->first, kv_iter->first, kv_iter->second); + ++kv_iter; + } + ++iter; + } } -} -static inline void set_qnn_tensor_name(Qnn_Tensor_t & tensor, const char * name) { - if (tensor.version == QNN_TENSOR_VERSION_1) { - tensor.v1.name = name; + bool load(const std::string & file_name) { + if (file_name == "") { + return false; + } + _cfg_filename = file_name; + std::ifstream in; + std::string line; + in.open(file_name.c_str()); + if (not in.is_open()) { + GGMLQNN_LOG_WARN("can't open file %s", file_name.c_str()); + return false; + } + while (getline(in, line)) { + std::string section, key, value; + if (not parse_line(line, section, key, value)) { + continue; + } + set_section_keyvalue(section, key, value); + } + _load_success = true; + return true; } -} -static inline void set_qnn_tensor_type(Qnn_Tensor_t & tensor, Qnn_TensorType_t type) { - if (tensor.version == QNN_TENSOR_VERSION_1) { - tensor.v1.type = type; + void get_stringvalue(const std::string & section, const std::string & key, std::string & value, std::string default_value) { + value = default_value; + if (_qnn_cfg.find(section) == _qnn_cfg.end()) { + return; + } + if (_qnn_cfg[section].find(key) == _qnn_cfg[section].end()) { + return; + } + value = _qnn_cfg[section][key]; } -} -static inline void set_qnn_tensor_dataformat(Qnn_Tensor_t & tensor, Qnn_TensorDataFormat_t format) { - if (tensor.version == QNN_TENSOR_VERSION_1) { - tensor.v1.dataFormat = format; + void get_intvalue(const std::string & section, const std::string & key, int & value, int default_value) { + value = default_value; + if (_qnn_cfg.find(section) == _qnn_cfg.end()) { + return; + } + if (_qnn_cfg[section].find(key) == _qnn_cfg[section].end()) { + return; + } + value = atol(_qnn_cfg[section][key].c_str()); } -} -static inline void set_qnn_tensor_datatype(Qnn_Tensor_t & tensor, Qnn_DataType_t dataType) { - if (tensor.version == QNN_TENSOR_VERSION_1) { - tensor.v1.dataType = dataType; +private: + void ltrim(std::string & str) { + if (str.empty()) return; + size_t len = 0; + char* temp = (char*)str.c_str(); + while (*temp && isblank(*temp)) { + ++len; + ++temp; + } + if (len > 0) str.erase(0, len); } -} -static inline void set_qnn_tensor_quantparams(Qnn_Tensor_t & tensor, Qnn_QuantizeParams_t params) { - if (tensor.version == QNN_TENSOR_VERSION_1) { - tensor.v1.quantizeParams = params; + void rtrim(std::string & str) { + if (str.empty()) return; + size_t len = str.length(); + size_t pos = len; + while (pos > 0) { + if (not isblank(str[pos - 1])) { + break; + } + --pos; + } + if (pos != len) str.erase(pos); } -} -static inline void set_qnn_tensor_rank(Qnn_Tensor_t & tensor, uint32_t rank) { - if (tensor.version == QNN_TENSOR_VERSION_1) { - tensor.v1.rank = rank; + void trim(std::string& str) { + ltrim(str); + rtrim(str); } -} -static inline void set_qnn_tensor_dimensions(Qnn_Tensor_t & tensor, uint32_t * dims) { - if (tensor.version == QNN_TENSOR_VERSION_1) { - tensor.v1.dimensions = dims; + void set_section_keyvalue(std::string & section, std::string & key, std::string & value) { + if (_qnn_cfg.find(section) == _qnn_cfg.end()) { + std::unordered_map kv_map; + _qnn_cfg[section] = kv_map; + } + if (key != "" && value != "") _qnn_cfg[section][key] = value; } -} -static inline void set_qnn_tensor_memtype(Qnn_Tensor_t & tensor, Qnn_TensorMemType_t memType) { - if (tensor.version == QNN_TENSOR_VERSION_1) { - tensor.v1.memType = memType; + bool parse_line(std::string & line, std::string & section, std::string & key, std::string & value) { + static std::string cur_section = ""; + std::string nodes[2] = {"#", ";"}; + for (int i = 0; i < 2; ++i) { + std::string::size_type pos = line.find(nodes[i]); + if (pos != std::string::npos) line.erase(pos); + } + trim(line); + if (line == "") return false; + if (line[0] == '[' && line[line.size() - 1] == ']') { + section = line.substr(1, line.size() - 2); + trim(section); + cur_section = section; + return false; + } + if (cur_section == "") return false; + bool is_key = true; + for (size_t i = 0; i < line.size(); ++i) { + if (line[i] == '=') { + is_key = false; + continue; + } + if (is_key) { + key += line[i]; + } else { + value += line[i]; + } + } + section = cur_section; + trim(key); + trim(value); + return true; } +private: + std::unordered_map> _qnn_cfg; + bool _load_success = false; + std::string _cfg_filename; +}; + +// ================================================================================================= +// section-4: helper function for WoA(Window on ARM) +// ================================================================================================= +#if !defined(__ANDROID__) && !defined(__linux__) +#define RTLD_GLOBAL 0x100 +#define RTLD_LOCAL 0x000 +#define RTLD_LAZY 0x000 +#define RTLD_NOW 0x001 +static void * dlopen(const char * filename, int flag); +static int dlclose(void * handle); +static void * dlsym(void* handle, const char* name); +static const char * dlerror(void); + +static const char * last_func = nullptr; +static long last_err; +static void * dlopen(const char * dll, int flags) { + HINSTANCE h = LoadLibraryA(dll); + GGML_UNUSED(flags); + if (h == NULL) { + last_err = GetLastError(); + last_func = "dlopen"; + } + return h; } -static inline void set_qnn_tensor_clientbuf(Qnn_Tensor_t & tensor, Qnn_ClientBuffer_t clientBuf) { - if (tensor.version == QNN_TENSOR_VERSION_1) { - tensor.v1.clientBuf = clientBuf; - } +static int dlclose(void * h) { + if (!FreeLibrary((HINSTANCE)h)) { + last_err = GetLastError(); + last_func = "dlclose"; + return -1; + } + return 0; } -static inline void set_qnn_tensor_memhandle(Qnn_Tensor_t & tensor, Qnn_MemHandle_t handle) { - if (tensor.version == QNN_TENSOR_VERSION_1) { - tensor.v1.memHandle = handle; - } +static void * dlsym(void * h, const char * name) { + FARPROC p = GetProcAddress((HINSTANCE)h, name); + if (!p) { + last_err = GetLastError(); + last_func = "dlsym"; + } + return (void*)(intptr_t)p; } -static int deep_copy_qnn_tensors(Qnn_Tensor_t & src, Qnn_Tensor_t & dst) { - int err = 0; +static const char * dlerror(void) { + static char str[512]; + if (!last_err) return nullptr; - dst.version = src.version; - QNN_TENSOR_SET_NAME(dst, ggmlqnn_strndup(QNN_TENSOR_GET_NAME(src), std::string(QNN_TENSOR_GET_NAME(src)).size())); - if (nullptr == QNN_TENSOR_GET_NAME(dst)) { - return 1; - } - QNN_TENSOR_SET_ID(dst, QNN_TENSOR_GET_ID(src)); - QNN_TENSOR_SET_TYPE(dst, QNN_TENSOR_GET_TYPE(src)); - QNN_TENSOR_SET_DATA_FORMAT(dst, QNN_TENSOR_GET_DATA_FORMAT(src)); - QNN_TENSOR_SET_DATA_TYPE(dst, QNN_TENSOR_GET_DATA_TYPE(src)); - QNN_TENSOR_SET_MEM_TYPE(dst, QNN_TENSOR_GET_MEM_TYPE(src)); + snprintf(str, 512, "%s error #%ld", last_func, last_err); + last_err = 0; + last_func = NULL; - if (QNN_TENSOR_GET_MEM_TYPE(src) == QNN_TENSORMEMTYPE_RAW) { - Qnn_ClientBuffer_t client_buf = {nullptr, 0}; - QNN_TENSOR_SET_CLIENT_BUF(dst, client_buf); - } else if (QNN_TENSOR_GET_MEM_TYPE(src) == QNN_TENSORMEMTYPE_MEMHANDLE) { - QNN_TENSOR_SET_MEM_HANDLE(dst, nullptr); - } else { - return 1; - } + return str; +} +#endif - Qnn_QuantizeParams_t src_qparam = QNN_TENSOR_GET_QUANT_PARAMS(src); - Qnn_QuantizationEncoding_t encoding = src_qparam.quantizationEncoding; - if (encoding == QNN_QUANTIZATION_ENCODING_AXIS_SCALE_OFFSET) { - Qnn_QuantizeParams_t src_qparam_cpy = src_qparam; - Qnn_AxisScaleOffset_t & axis_scale_offset = src_qparam_cpy.axisScaleOffsetEncoding; - Qnn_ScaleOffset_t ** scale_offset = &axis_scale_offset.scaleOffset; - size_t scale_offset_size = axis_scale_offset.numScaleOffsets * sizeof(Qnn_ScaleOffset_t); - *scale_offset = (Qnn_ScaleOffset_t *)malloc(scale_offset_size); - ggmlqnn_memscpy(*scale_offset, - scale_offset_size, - src_qparam.axisScaleOffsetEncoding.scaleOffset, - scale_offset_size); - QNN_TENSOR_SET_QUANT_PARAMS(dst, src_qparam_cpy); - } else if (encoding == QNN_QUANTIZATION_ENCODING_BW_AXIS_SCALE_OFFSET) { - Qnn_QuantizeParams_t src_qparam_cpy = src_qparam; - Qnn_BwAxisScaleOffset_t & bwaxis_scale_offset = src_qparam_cpy.bwAxisScaleOffsetEncoding; - size_t scale_size = bwaxis_scale_offset.numElements * sizeof(float); - float ** scales = &bwaxis_scale_offset.scales; - int32_t ** offsets = &bwaxis_scale_offset.offsets; - *scales = (float *)malloc(scale_size); - ggmlqnn_memscpy(*scales, scale_size, src_qparam.bwAxisScaleOffsetEncoding.scales, scale_size); +// ================================================================================================= +// section-5: general helper function +// ================================================================================================= +//the following 3 helper funcs are used to ensure every QNN tensor name is unique +static void ggmqnn_reset_tensoridx() { + g_ggmltensor_idx = 0; +} - if (bwaxis_scale_offset.offsets != nullptr) { - size_t offset_size = bwaxis_scale_offset.numElements * sizeof(int32_t); - *offsets = (int32_t *)malloc(offset_size); - ggmlqnn_memscpy(*offsets, offset_size, src_qparam.bwAxisScaleOffsetEncoding.offsets, offset_size); - } - QNN_TENSOR_SET_QUANT_PARAMS(dst, src_qparam_cpy); - } else { - QNN_TENSOR_SET_QUANT_PARAMS(dst, src_qparam); - } +static void ggmqnn_inc_tensoridx() { + g_ggmltensor_idx++; +} - uint32_t rank = QNN_TENSOR_GET_RANK(src); - QNN_TENSOR_SET_RANK(dst, rank); - size_t dim_size = GGML_MAX_DIMS * sizeof(uint32_t); - uint32_t * dimensions = (uint32_t *)malloc(dim_size); - if (nullptr == dimensions) { - GGMLQNN_LOG_WARN("deep_copy_qnn_tensors() allocation error while copying tensor %s\n", QNN_TENSOR_GET_NAME(src)); - return 1; +static int32_t ggmqnn_get_tensoridx() { +#if !defined(__ANDROID__) && !defined(__linux__) + return g_ggmltensor_idx.load(); +#else + return g_ggmltensor_idx; +#endif +} + +static void * ggmlqnn_mallocz_aligned(size_t size, size_t alignment) { + uint8_t * buffer = NULL; + size_t * sp = NULL; + buffer = static_cast(calloc(1, size + GGMLQNN_MEM_ADD(alignment))); + if (!buffer) + return NULL; + sp = (size_t *)buffer; + *sp = size; + buffer = (uint8_t *)(((uintptr_t) buffer + GGMLQNN_MEM_ADD(alignment)) & ~GGMLQNN_MEM_MASK(alignment)); + buffer[-1] = buffer - (uint8_t *)sp; + return buffer; +} + +static void * ggmlqnn_malloc_aligned(size_t size, size_t alignment) { + uint8_t * buffer = NULL; + size_t * sp = NULL; + buffer = static_cast(malloc(size + GGMLQNN_MEM_ADD(alignment))); + if (!buffer) + return NULL; + sp = (size_t *)buffer; + *sp = size; + buffer = (uint8_t *)(((uintptr_t) buffer + GGMLQNN_MEM_ADD(alignment)) & ~GGMLQNN_MEM_MASK(alignment)); + buffer[-1] = buffer - (uint8_t *)sp; + return buffer; +} + +static void ggmqnn_free_aligned(void * ptr) { + uint8_t * old = (uint8_t *)ptr; + if (!old) + return; + old -= old[-1]; + free(old); +} + +static intptr_t ggmlqnn_align_to(size_t alignment, intptr_t offset) { + return offset % alignment == 0 ? offset + : offset + + (static_cast(alignment) - + offset % static_cast(alignment)); +} + +static size_t ggmlqnn_get_system_total_memory_in_bytes() { +#if defined(__ANDROID__) || defined(__linux__) + struct sysinfo info = {}; + if (0 == sysinfo(&info)) { + return (info.totalram + info.totalswap) * info.mem_unit; } - ggmlqnn_memscpy(dimensions, dim_size, QNN_TENSOR_GET_DIMENSIONS(src), dim_size); - QNN_TENSOR_SET_DIMENSIONS(dst, dimensions); + size_t pages = (size_t)sysconf(_SC_PHYS_PAGES); + size_t page_size = (size_t)sysconf(_SC_PAGE_SIZE); - return err; + return pages * page_size; +#else + //FIXME: Snapdragon based WoA(Windows on ARM) + MEMORYSTATUSEX statex; + statex.dwLength = sizeof(statex); + if (GlobalMemoryStatusEx(&statex)) { + GGMLQNN_LOG_INFO("total physical mem:%llu Mb", statex.ullTotalPhys >> 20); + GGMLQNN_LOG_INFO("avail physical mem:%llu Mb", statex.ullAvailPhys >> 20); + return statex.ullTotalPhys; + } + return 0; +#endif } -static int free_qnn_tensor(Qnn_Tensor_t * tensor) { - int err = 0; - free((void *) QNN_TENSOR_GET_NAME(*tensor)); - Qnn_QuantizeParams_t src_qparam = QNN_TENSOR_GET_QUANT_PARAMS(*tensor); - Qnn_QuantizationEncoding_t encoding = src_qparam.quantizationEncoding; - if (encoding == QNN_QUANTIZATION_ENCODING_AXIS_SCALE_OFFSET) { - free(src_qparam.axisScaleOffsetEncoding.scaleOffset); - } else if (encoding == QNN_QUANTIZATION_ENCODING_BW_AXIS_SCALE_OFFSET) { - free(src_qparam.bwAxisScaleOffsetEncoding.scales); - if (src_qparam.bwAxisScaleOffsetEncoding.offsets != nullptr) { - free(src_qparam.bwAxisScaleOffsetEncoding.offsets); - } +static size_t ggmlqnn_get_system_free_memory_in_bytes() { +#if defined(__ANDROID__) || defined(__linux__) + struct sysinfo info = {}; + if (0 == sysinfo(&info)) { + return (info.freeram + info.freeswap) * info.mem_unit; } - free(QNN_TENSOR_GET_DIMENSIONS(*tensor)); - free(tensor); + size_t avail_pages = (size_t)sysconf(_SC_AVPHYS_PAGES); + size_t page_size = (size_t)sysconf(_SC_PAGE_SIZE); - return err; + return avail_pages * page_size; +#else + //FIXME: Snapdragon based WoA(Windows on ARM) + MEMORYSTATUSEX statex; + statex.dwLength = sizeof(statex); + if (GlobalMemoryStatusEx(&statex)) { + GGMLQNN_LOG_INFO("total physical mem:%llu Mb", statex.ullTotalPhys >> 20); + GGMLQNN_LOG_INFO("avail physical mem:%llu Mb", statex.ullAvailPhys >> 20); + return statex.ullAvailPhys; + } + return 0; +#endif } -static const char * ggmlqnn_get_error_string(Qnn_ErrorHandle_t qnn_error_code) { - // file:///opt/qcom/aistack/qairt/2.31.0.250130/docs/QNN/general/api_error_codes.html - switch (qnn_error_code) { - case QNN_SUCCESS: - return "QNN_SUCCESS"; - case QNN_COMMON_ERROR_GENERAL: - return "QNN_COMMON_ERROR_GENERAL"; +static size_t ggmlqnn_memscpy(void * dst, size_t dst_size, const void * src, size_t copy_size) { + if (!dst || !src || !dst_size || !copy_size) + return 0; - // QnnGraph_Error_t - case QNN_GRAPH_ERROR_UNSUPPORTED_FEATURE: - return "QNN_GRAPH_ERROR_UNSUPPORTED_FEATURE"; - case QNN_GRAPH_ERROR_MEM_ALLOC: - return "QNN_GRAPH_ERROR_MEM_ALLOC"; - case QNN_GRAPH_ERROR_INVALID_ARGUMENT: - return "QNN_GRAPH_ERROR_INVALID_ARGUMENT"; - case QNN_GRAPH_ERROR_INVALID_HANDLE: - return "QNN_GRAPH_ERROR_INVALID_HANDLE"; - case QNN_GRAPH_ERROR_GRAPH_DOES_NOT_EXIST: - return "QNN_GRAPH_ERROR_GRAPH_DOES_NOT_EXIST"; - case QNN_GRAPH_ERROR_INVALID_NAME: - return "QNN_GRAPH_ERROR_INVALID_NAME"; - case QNN_GRAPH_ERROR_INVALID_TENSOR: - return "QNN_GRAPH_ERROR_INVALID_TENSOR"; - case QNN_GRAPH_ERROR_INVALID_OP_CONFIG: - return "QNN_GRAPH_ERROR_INVALID_OP_CONFIG"; - case QNN_GRAPH_ERROR_SET_PROFILE: - return "QNN_GRAPH_ERROR_SET_PROFILE"; - case QNN_GRAPH_ERROR_UNCONNECTED_NODE: - return "QNN_GRAPH_ERROR_UNCONNECTED_NODE"; - case QNN_GRAPH_ERROR_CREATE_FAILED: - return "QNN_GRAPH_ERROR_CREATE_FAILED"; - case QNN_GRAPH_ERROR_OPTIMIZATION_FAILED: - return "QNN_GRAPH_ERROR_OPTIMIZATION_FAILED"; - case QNN_GRAPH_ERROR_FINALIZE_FAILED: - return "QNN_GRAPH_ERROR_FINALIZE_FAILED"; - case QNN_GRAPH_ERROR_GRAPH_NOT_FINALIZED: - return "QNN_GRAPH_ERROR_GRAPH_NOT_FINALIZED"; - case QNN_GRAPH_ERROR_GRAPH_FINALIZED: - return "QNN_GRAPH_ERROR_GRAPH_FINALIZED"; - case QNN_GRAPH_ERROR_EXECUTION_ASYNC_FIFO_FULL: - return "QNN_GRAPH_ERROR_EXECUTION_ASYNC_FIFO_FULL"; - case QNN_GRAPH_ERROR_SIGNAL_IN_USE: - return "QNN_GRAPH_ERROR_SIGNAL_IN_USE"; - case QNN_GRAPH_ERROR_ABORTED: - return "QNN_GRAPH_ERROR_ABORTED"; - case QNN_GRAPH_ERROR_PROFILE_IN_USE: - return "QNN_GRAPH_ERROR_PROFILE_IN_USE"; - case QNN_GRAPH_ERROR_TIMED_OUT: - return "QNN_GRAPH_ERROR_TIMED_OUT"; - case QNN_GRAPH_ERROR_SUBGRAPH: - return "QNN_GRAPH_ERROR_SUBGRAPH"; - case QNN_GRAPH_ERROR_DISABLED: - return "QNN_GRAPH_ERROR_DISABLED"; - case QNN_GRAPH_ERROR_DYNAMIC_TENSOR_SHAPE: - return "QNN_GRAPH_ERROR_DYNAMIC_TENSOR_SHAPE"; - case QNN_GRAPH_ERROR_TENSOR_SPARSITY: - return "QNN_GRAPH_ERROR_TENSOR_SPARSITY"; - case QNN_GRAPH_ERROR_EARLY_TERMINATION: - return "QNN_GRAPH_ERROR_EARLY_TERMINATION"; - case QNN_GRAPH_ERROR_INVALID_CONTEXT: - return "QNN_GRAPH_ERROR_INVALID_CONTEXT"; + size_t min_size = dst_size < copy_size ? dst_size : copy_size; - //QQnnTensor_Error_t - //Invalid context/graph handle in creating tensor - case QNN_TENSOR_ERROR_INVALID_HANDLE: - return "QNN_TENSOR_ERROR_INVALID_HANDLE"; - //Tensor with specified credentials not registered with a context/graph - case QNN_TENSOR_ERROR_DOES_NOT_EXIST: - return "QNN_TENSOR_ERROR_DOES_NOT_EXIST"; - // (deprecated) Tensor has already been registered with backend - case QNN_TENSOR_ERROR_ALREADY_EXISTS: - return "QNN_TENSOR_ERROR_ALREADY_EXISTS"; - // Invalid tensor param. - case QNN_TENSOR_ERROR_INVALID_TENSOR_PARAM: - return "QNN_TENSOR_ERROR_INVALID_TENSOR_PARAM"; - // This tensor param is currently unsupported - case QNN_TENSOR_ERROR_UNSUPPORTED_TENSOR_PARAM: - return "QNN_TENSOR_ERROR_UNSUPPORTED_TENSOR_PARAM"; - // Tensor provided for update is invalid - case QNN_TENSOR_ERROR_INCOMPATIBLE_TENSOR_UPDATE: - return "QNN_TENSOR_ERROR_INCOMPATIBLE_TENSOR_UPDATE"; + memcpy(dst, src, min_size); - // QnnOpPackage_Error_t - case QNN_OP_PACKAGE_ERROR_LIBRARY_ALREADY_INITIALIZED: - return "QNN_OP_PACKAGE_ERROR_LIBRARY_ALREADY_INITIALIZED"; - case QNN_OP_PACKAGE_ERROR_LIBRARY_NOT_INITIALIZED: - return "QNN_OP_PACKAGE_ERROR_LIBRARY_NOT_INITIALIZED"; - case QNN_OP_PACKAGE_ERROR_INVALID_HANDLE: - return "QNN_OP_PACKAGE_ERROR_INVALID_HANDLE"; - case QNN_OP_PACKAGE_ERROR_INVALID_INFRASTRUCTURE: - return "QNN_OP_PACKAGE_ERROR_INVALID_INFRASTRUCTURE"; - case QNN_OP_PACKAGE_ERROR_INVALID_INFO: - return "QNN_OP_PACKAGE_ERROR_INVALID_INFO"; - case QNN_OP_PACKAGE_ERROR_VALIDATION_FAILURE: - return "QNN_OP_PACKAGE_ERROR_VALIDATION_FAILURE"; - case QNN_OP_PACKAGE_ERROR_INVALID_ARGUMENT: - return "QNN_OP_PACKAGE_ERROR_INVALID_ARGUMENT"; + return min_size; +} - default: - return "unknown QNN error"; - } +static char * ggmlqnn_strndup(const char * source, size_t maxlen) { +#if defined(__ANDROID__) || defined(__linux__) + return strndup(source, maxlen); +#else + //FIXME:behaviour is not exactly same to Android&Linux + GGML_UNUSED(maxlen); + return strdup(source); +#endif } -static Qnn_OpConfig_t ggmlqnn_create_op_config(const char * name, const char * package, const char * type, - Qnn_Param_t * params, uint32_t num_params, - Qnn_Tensor_t * inputs, uint32_t num_inputs, - Qnn_Tensor_t * outputs, uint32_t num_outputs) { - Qnn_OpConfigV1_t v1 = {name, package, type, - num_params, params, - num_inputs, inputs, - num_outputs, outputs - }; - Qnn_OpConfig_t opcfg = {QNN_OPCONFIG_VERSION_1, {v1}}; +static void * ggmlqnn_host_malloc(size_t buffer_size, size_t page_size) { + void * data = nullptr; +#if defined(__ANDROID__) || defined(__linux__) + int result = posix_memalign((void **)&data, page_size, buffer_size); + if (result != 0) { + GGMLQNN_LOG_WARN("%s: error: posix_memalign failed\n", __func__); + return nullptr; + } +#else + //GGMLQNN_LOG_DEBUG("buffer_size %d, page_size %d\n", buffer_size, page_size); + data = ggmlqnn_malloc_aligned(buffer_size, page_size); + if (nullptr == data) { + GGMLQNN_LOG_WARN("%s: error: host_malloc failed\n", __func__); + } +#endif - return opcfg; + return data; +} + +static void ggmlqnn_load_cfg() { + std::string cfg_filename = std::string(g_qnn_runtimelib_path) + std::string(g_qnn_cfgfilename); + GGMLQNN_LOG_INFO("load ggml-qnn config from %s", cfg_filename.c_str()); + qnn_cfg qnncfg_instance; + qnncfg_instance.load(cfg_filename); + qnncfg_instance.dump([](const std::string & section, const std::string & key, const std::string value) { + std::ostringstream tmposs; + tmposs << "section[" << section << "],[" << key << "] = [" << value << "]" << std::endl; + GGMLQNN_LOG_INFO("%s", tmposs.str().c_str()); + }); + std::string npu_inference_datatype; + qnncfg_instance.get_intvalue("general", "print_qnn_internal_log", g_print_qnn_internal_log, 0); + qnncfg_instance.get_intvalue("general", "inference_approach", g_inference_approach, 0); + qnncfg_instance.get_stringvalue("npu", "npu_inference_datatype", npu_inference_datatype, "fp32"); + GGMLQNN_LOG_INFO("print_qnn_internal_log=%d", g_print_qnn_internal_log); + GGMLQNN_LOG_INFO("inference_approach=%d", g_inference_approach); + GGMLQNN_LOG_INFO("npu inference data type=%s", npu_inference_datatype.c_str()); } // ================================================================================================= -// section-5:ggml-qnn backend helper macro / data structure / function / class +// section-6: QNN helper function // ================================================================================================= -using pfn_rpc_mem_init = void (*)(void); -using pfn_rpc_mem_deinit = void (*)(void); -using pfn_rpc_mem_alloc = void *(*)(int, uint32_t, int); -using pfn_rpc_mem_free = void (*)(void *); -using pfn_rpc_mem_to_fd = int (*)(void *); -using _pfn_QnnSaver_initialize = decltype(QnnSaver_initialize); -using _pfn_QnnInterface_getProviders = decltype(QnnInterface_getProviders); -using _pfn_QnnSystemInterface_getProviders = decltype(QnnSystemInterface_getProviders); +static inline uint32_t get_qnn_tensorid(const Qnn_Tensor_t & tensor) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + return tensor.v1.id; + } -using qnn_res_t = std::tuple>; -using qnn_tensors_t = std::vector< Qnn_Tensor_t *>; + return 0u; +} -enum class ggml_qnn_profile_level { - profile_off = 0, - profile_basic = 1, - profile_detail = 2 -}; +static inline const char * get_qnn_tensorname(const Qnn_Tensor_t & tensor) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + return tensor.v1.name; + } + return nullptr; +} -enum qcom_htp_arch { - NONE = 0, - V68 = 68, - V69 = 69, - V73 = 73, - V75 = 75, - V79 = 79, -}; +static inline Qnn_TensorType_t get_qnn_tensortype(const Qnn_Tensor_t & tensor) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + return tensor.v1.type; + } + return QNN_TENSOR_TYPE_UNDEFINED; +} -enum qcom_chipset_soc_model { - UNKNOWN_SM = 0, - SM7450 = 41, // v69, 7 Gen1 - SM8350 = 30, // v68, 888 - SM8450 = 36, // v69, SD 8 Gen 1 - SM8475 = 42, // v69, SD 8+ Gen 1 - SM8550 = 43, // v73, SD 8 Gen 2 - SM8650 = 57, // v75, SD 8 Gen 3 - SM8750 = 69, // v79, SD 8 Gen 4 -#if !defined(__ANDROID__) && !defined(__linux__) - SC7280X = 44, - SC8280X = 37, - SC8380XP = 60, -#endif -}; +static inline Qnn_TensorDataFormat_t get_qnn_tensor_dataformat(const Qnn_Tensor_t & tensor) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + return tensor.v1.dataFormat; + } + return QNN_TENSOR_DATA_FORMAT_FLAT_BUFFER; +} -struct qcom_socinfo { - uint32_t soc_model; - size_t htp_arch; - size_t vtcm_size_in_mb; - char soc_desc[GGML_MAX_NAME]; -}; +static inline Qnn_DataType_t get_qnn_tensor_datatype(const Qnn_Tensor_t & tensor) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + return tensor.v1.dataType; + } + return QNN_DATATYPE_UNDEFINED; +} -struct ggml_backend_qnn_context { - int device; - int threads; - char name[GGML_MAX_NAME]; - char desc[GGML_MAX_NAME]; - char lib[GGML_MAX_NAME]; - qnn_instance * instance; - struct ggml_backend * backend; - QNN_INTERFACE_VER_TYPE raw_interface; - QNN_SYSTEM_INTERFACE_VER_TYPE raw_system_interface; - struct qcom_socinfo socinfo; +static inline Qnn_QuantizeParams_t get_qnn_tensor_quantparams(const Qnn_Tensor_t & tensor) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + return tensor.v1.quantizeParams; + } + return QNN_QUANTIZE_PARAMS_INIT; +} - std::unique_ptr work_data; - std::vector> tasks; - size_t work_size = 0; - size_t desired_size = 0; - int n_threads = GGML_DEFAULT_N_THREADS; -}; +static inline uint32_t get_qnn_tensor_rank(const Qnn_Tensor_t & tensor) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + return tensor.v1.rank; + } + return 0u; +} -struct qnn_op_caps_t { - const char * qnn_op_name = nullptr; - const size_t input_param_count = 0; - const char * qnn_param_name = nullptr; -}; +static inline uint32_t * get_qnn_tensor_dimensions(const Qnn_Tensor_t & tensor) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + return tensor.v1.dimensions; + } + return nullptr; +} -//file:///opt/qcom/aistack/qairt/2.31.0.250130/docs/QNN/general/overview.html#tbl-supported-snapdragon-devices -static struct qcom_socinfo g_qnn_soc_info_table[] = { - /* Qualcomm SnapDragon 7 Gen 1 */ - { - .soc_model = SM7450, - .htp_arch = V69, - .vtcm_size_in_mb = 8, - .soc_desc = "Qualcomm SnapDragon 7 Gen 1"}, +static inline Qnn_TensorMemType_t get_qnn_tensor_memtype(const Qnn_Tensor_t & tensor) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + return tensor.v1.memType; + } + return QNN_TENSORMEMTYPE_UNDEFINED; +} - /* Qualcomm SnapDragon 888 */ - { - .soc_model = SM8350, - .htp_arch = V68, - .vtcm_size_in_mb = 8, - .soc_desc = "Qualcomm SnapDragon 888 "}, +static inline void set_qnn_tensor_id(Qnn_Tensor_t & tensor, uint32_t id) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + tensor.v1.id = id; + } +} - /* Qualcomm SnapDragon 8 Gen 1 */ - { - .soc_model = SM8450, - .htp_arch = V69, - .vtcm_size_in_mb = 8, - .soc_desc = "Qualcomm SnapDragon 8 Gen 1"}, - - /* Qualcomm SnapDragon 8 Gen 1+ */ - { - .soc_model = SM8475, - .htp_arch = V69, - .vtcm_size_in_mb = 8, - .soc_desc = "Qualcomm SnapDragon 8 Gen 1+"}, - - /* Qualcomm SnapDragon 8 Gen 2 */ - { - .soc_model = SM8550, - .htp_arch = V73, - .vtcm_size_in_mb = 8, - .soc_desc = "Qualcomm SnapDragon 8 Gen 2"}, +static inline void set_qnn_tensor_name(Qnn_Tensor_t & tensor, const char * name) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + tensor.v1.name = name; + } +} - /* Qualcomm SnapDragon 8 Gen 3 */ - { - .soc_model = SM8650, - .htp_arch = V75, - .vtcm_size_in_mb = 8, - .soc_desc = "Qualcomm SnapDragon 8 Gen 3 "}, +static inline void set_qnn_tensor_type(Qnn_Tensor_t & tensor, Qnn_TensorType_t type) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + tensor.v1.type = type; + } +} - /* Qualcomm SnapDragon 8 Gen 4 */ - { - .soc_model = SM8750, - .htp_arch = V79, - .vtcm_size_in_mb = 8, - .soc_desc = "Qualcomm SnapDragon 8 Gen 4"}, +static inline void set_qnn_tensor_dataformat(Qnn_Tensor_t & tensor, Qnn_TensorDataFormat_t format) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + tensor.v1.dataFormat = format; + } +} -#if !defined(__ANDROID__) && !defined(__linux__) - /* Qualcomm SnapDragon 7c Gen 2 */ - { - .soc_model = SC7280X, - .htp_arch = V68, - .vtcm_size_in_mb = 8, - .soc_desc = "Qualcomm SnapDragon 7c Gen 2"}, +static inline void set_qnn_tensor_datatype(Qnn_Tensor_t & tensor, Qnn_DataType_t dataType) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + tensor.v1.dataType = dataType; + } +} - /* Qualcomm SnapDragon 8cx Gen 3 */ - { - .soc_model = SC8280X, - .htp_arch = V68, - .vtcm_size_in_mb = 8, - .soc_desc = "Qualcomm SnapDragon 8cx Gen 3"}, +static inline void set_qnn_tensor_quantparams(Qnn_Tensor_t & tensor, Qnn_QuantizeParams_t params) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + tensor.v1.quantizeParams = params; + } +} - /* Qualcomm SnapDragon 8cx Gen 4 */ - { - .soc_model = SC8380XP, - .htp_arch = V73, - .vtcm_size_in_mb = 8, - .soc_desc = "Qualcomm SnapDragon 8cx Gen 4"}, -#endif +static inline void set_qnn_tensor_rank(Qnn_Tensor_t & tensor, uint32_t rank) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + tensor.v1.rank = rank; + } +} -}; +static inline void set_qnn_tensor_dimensions(Qnn_Tensor_t & tensor, uint32_t * dims) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + tensor.v1.dimensions = dims; + } +} +static inline void set_qnn_tensor_memtype(Qnn_Tensor_t & tensor, Qnn_TensorMemType_t memType) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + tensor.v1.memType = memType; + } +} -#if defined(__ANDROID__) -static const char * g_qnn_runtimelib_path = "/data/local/tmp/"; -#elif defined(__linux__) -static const char * g_qnn_runtimelib_path = "/tmp/"; -#elif defined(_WIN32) -static const char * g_qnn_runtimelib_path = "C:\\"; -#else //cygwin on Windows -static const char * g_qnn_runtimelib_path = "/cygdrive/c/"; -#endif -//the following helper funcs are used to ensure every QNN tensor name is unique -static std::atomic g_ggmltensor_idx(0); -static void reset_idx() { - g_ggmltensor_idx = 0; +static inline void set_qnn_tensor_clientbuf(Qnn_Tensor_t & tensor, Qnn_ClientBuffer_t clientBuf) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + tensor.v1.clientBuf = clientBuf; + } } -static void inc_idx() { - g_ggmltensor_idx++; +static inline void set_qnn_tensor_memhandle(Qnn_Tensor_t & tensor, Qnn_MemHandle_t handle) { + if (tensor.version == QNN_TENSOR_VERSION_1) { + tensor.v1.memHandle = handle; + } } -static int32_t get_idx() { - return g_ggmltensor_idx.load(); +static int deep_copy_qnn_tensors(Qnn_Tensor_t & src, Qnn_Tensor_t & dst) { + int err = 0; + + dst.version = src.version; + QNN_TENSOR_SET_NAME(dst, ggmlqnn_strndup(QNN_TENSOR_GET_NAME(src), std::string(QNN_TENSOR_GET_NAME(src)).size())); + if (nullptr == QNN_TENSOR_GET_NAME(dst)) { + return 1; + } + QNN_TENSOR_SET_ID(dst, QNN_TENSOR_GET_ID(src)); + QNN_TENSOR_SET_TYPE(dst, QNN_TENSOR_GET_TYPE(src)); + QNN_TENSOR_SET_DATA_FORMAT(dst, QNN_TENSOR_GET_DATA_FORMAT(src)); + QNN_TENSOR_SET_DATA_TYPE(dst, QNN_TENSOR_GET_DATA_TYPE(src)); + QNN_TENSOR_SET_MEM_TYPE(dst, QNN_TENSOR_GET_MEM_TYPE(src)); + + if (QNN_TENSOR_GET_MEM_TYPE(src) == QNN_TENSORMEMTYPE_RAW) { + Qnn_ClientBuffer_t client_buf = {nullptr, 0}; + QNN_TENSOR_SET_CLIENT_BUF(dst, client_buf); + } else if (QNN_TENSOR_GET_MEM_TYPE(src) == QNN_TENSORMEMTYPE_MEMHANDLE) { + QNN_TENSOR_SET_MEM_HANDLE(dst, nullptr); + } else { + return 1; + } + + Qnn_QuantizeParams_t src_qparam = QNN_TENSOR_GET_QUANT_PARAMS(src); + Qnn_QuantizationEncoding_t encoding = src_qparam.quantizationEncoding; + if (encoding == QNN_QUANTIZATION_ENCODING_AXIS_SCALE_OFFSET) { + Qnn_QuantizeParams_t src_qparam_cpy = src_qparam; + Qnn_AxisScaleOffset_t & axis_scale_offset = src_qparam_cpy.axisScaleOffsetEncoding; + Qnn_ScaleOffset_t ** scale_offset = &axis_scale_offset.scaleOffset; + size_t scale_offset_size = axis_scale_offset.numScaleOffsets * sizeof(Qnn_ScaleOffset_t); + *scale_offset = (Qnn_ScaleOffset_t *)malloc(scale_offset_size); + ggmlqnn_memscpy(*scale_offset, + scale_offset_size, + src_qparam.axisScaleOffsetEncoding.scaleOffset, + scale_offset_size); + QNN_TENSOR_SET_QUANT_PARAMS(dst, src_qparam_cpy); + } else if (encoding == QNN_QUANTIZATION_ENCODING_BW_AXIS_SCALE_OFFSET) { + Qnn_QuantizeParams_t src_qparam_cpy = src_qparam; + Qnn_BwAxisScaleOffset_t & bwaxis_scale_offset = src_qparam_cpy.bwAxisScaleOffsetEncoding; + size_t scale_size = bwaxis_scale_offset.numElements * sizeof(float); + float ** scales = &bwaxis_scale_offset.scales; + int32_t ** offsets = &bwaxis_scale_offset.offsets; + *scales = (float *)malloc(scale_size); + ggmlqnn_memscpy(*scales, scale_size, src_qparam.bwAxisScaleOffsetEncoding.scales, scale_size); + + if (bwaxis_scale_offset.offsets != nullptr) { + size_t offset_size = bwaxis_scale_offset.numElements * sizeof(int32_t); + *offsets = (int32_t *)malloc(offset_size); + ggmlqnn_memscpy(*offsets, offset_size, src_qparam.bwAxisScaleOffsetEncoding.offsets, offset_size); + } + QNN_TENSOR_SET_QUANT_PARAMS(dst, src_qparam_cpy); + } else { + QNN_TENSOR_SET_QUANT_PARAMS(dst, src_qparam); + } + + uint32_t rank = QNN_TENSOR_GET_RANK(src); + QNN_TENSOR_SET_RANK(dst, rank); + size_t dim_size = GGML_MAX_DIMS * sizeof(uint32_t); + uint32_t * dimensions = (uint32_t *)malloc(dim_size); + if (nullptr == dimensions) { + GGMLQNN_LOG_WARN("deep_copy_qnn_tensors() allocation error while copying tensor %s\n", QNN_TENSOR_GET_NAME(src)); + return 1; + } + ggmlqnn_memscpy(dimensions, dim_size, QNN_TENSOR_GET_DIMENSIONS(src), dim_size); + QNN_TENSOR_SET_DIMENSIONS(dst, dimensions); + + return err; } -// file:///opt/qcom/aistack/qairt/2.31.0.250130/docs/QNN/general/quantization.html -// CPU - Choose a non-quantized model.Quantized models are currently incompatible with the CPU backend -// GPU - Choose a non-quantized model.Quantized models are currently incompatible with the GPU backend -// HTP - Choose a quantized model. Quantized models are required when running on the HTP backend -// DSP - Choose a quantized model. Quantized models are required when running on the DSP backend -// HTA - Choose a quantized model. Quantized models are required when running on the HTA backend -static struct ggml_backend_qnn_context g_qnn_mgr[GGML_QNN_MAX_DEVICES] = { - [QNN_BACKEND_CPU] = {.device = 0, - .threads = 1, - .name = "qnn-cpu", - .desc = "Qualcomm Kryo CPU", -#if !defined(__ANDROID__) && !defined(__linux__) - .lib = "QnnCpu.dll", -#else - .lib = "libQnnCpu.so", -#endif - .instance = nullptr, - .backend = nullptr, - .raw_interface = {}, - .raw_system_interface = {}, - .socinfo = {}}, +static int free_qnn_tensor(Qnn_Tensor_t * tensor) { + int err = 0; + free((void *) QNN_TENSOR_GET_NAME(*tensor)); + Qnn_QuantizeParams_t src_qparam = QNN_TENSOR_GET_QUANT_PARAMS(*tensor); + Qnn_QuantizationEncoding_t encoding = src_qparam.quantizationEncoding; + if (encoding == QNN_QUANTIZATION_ENCODING_AXIS_SCALE_OFFSET) { + free(src_qparam.axisScaleOffsetEncoding.scaleOffset); + } else if (encoding == QNN_QUANTIZATION_ENCODING_BW_AXIS_SCALE_OFFSET) { + free(src_qparam.bwAxisScaleOffsetEncoding.scales); + if (src_qparam.bwAxisScaleOffsetEncoding.offsets != nullptr) { + free(src_qparam.bwAxisScaleOffsetEncoding.offsets); + } + } + free(QNN_TENSOR_GET_DIMENSIONS(*tensor)); + free(tensor); - [QNN_BACKEND_GPU] = {.device = 1, - .threads = 1, - .name = "qnn-gpu", - .desc = "Qualcomm Adreno GPU", -#if !defined(__ANDROID__) && !defined(__linux__) - .lib = "QnnGpu.dll", -#else - .lib = "libQnnGpu.so", -#endif - .instance = nullptr, - .backend = nullptr, - .raw_interface = {}, - .raw_system_interface = {}, - .socinfo = {}}, + return err; +} - [QNN_BACKEND_NPU] = {.device = 2, - .threads = 1, - .name = "qnn-npu", - .desc = "Qualcomm NPU(Hexagon Tensor Processor)", -#if !defined(__ANDROID__) && !defined(__linux__) - .lib = "QnnHtp.dll", -#else - .lib = "libQnnHtp.so", -#endif - .instance = nullptr, - .backend = nullptr, - .raw_interface = {}, - .raw_system_interface = {}, - .socinfo = {}}, -}; +static const char * ggmlqnn_get_qnnerror_string(Qnn_ErrorHandle_t qnn_error_code) { + // file:///opt/qcom/aistack/qairt/2.31.0.250130/docs/QNN/general/api_error_codes.html + switch (qnn_error_code) { + case QNN_SUCCESS: + return "QNN_SUCCESS"; + case QNN_COMMON_ERROR_GENERAL: + return "QNN_COMMON_ERROR_GENERAL"; -static const qnn_op_caps_t ggmlqnn_k_op_caps[] = { - {}, // GGML_OP_NONE - {}, // GGML_OP_DUP - { - // GGML_OP_ADD - QNN_OP_ELEMENT_WISE_ADD, - 2, - }, - {}, // GGML_OP_ADD1 - {}, // GGML_OP_ACC - {}, // GGML_OP_SUB - { - // GGML_OP_MUL - QNN_OP_ELEMENT_WISE_MULTIPLY, - 2, - }, - {}, // GGML_OP_DIV - {}, // GGML_OP_SQR - {}, // GGML_OP_SQRT - {}, // GGML_OP_LOG - {}, // GGML_OP_SIN - {}, // GGML_OP_COS - {}, // GGML_OP_SUM - {}, // GGML_OP_SUM_ROWS - {}, // GGML_OP_MEAN - {}, // GGML_OP_ARGMAX - {}, // GGML_OP_COUNT_EQUAL - {}, // GGML_OP_REPEAT - {}, // GGML_OP_REPEAT_BACK - {}, // GGML_OP_CONCAT - {}, // GGML_OP_SILU_BACK - {}, // GGML_OP_NORM - {}, // GGML_OP_RMS_NORM - {}, // GGML_OP_RMS_NORM_BACK - {}, // GGML_OP_GROUP_NORM - { - // GGML_OP_MUL_MAT - QNN_OP_MAT_MUL, - 2, - }, - {}, // GGML_OP_MUL_MAT_ID - {}, // GGML_OP_OUT_PROD - {}, // GGML_OP_SCALE - {}, // GGML_OP_SET - {}, // GGML_OP_CPY - {}, // GGML_OP_CONT - {}, // GGML_OP_RESHAPE - {}, // GGML_OP_VIEW - {}, // GGML_OP_PERMUTE - {}, // GGML_OP_TRANSPOSE - {}, // GGML_OP_GET_ROWS - {}, // GGML_OP_GET_ROWS_BACK - {}, // GGML_OP_DIAG - {}, // GGML_OP_DIAG_MASK_INF - {}, // GGML_OP_DIAG_MASK_ZERO - {}, // GGML_OP_SOFT_MAX - {}, // GGML_OP_SOFT_MAX_BACK - {}, // GGML_OP_ROPE - {}, // GGML_OP_ROPE_BACK - {}, // GGML_OP_CLAMP - {}, // GGML_OP_CONV_TRANSPOSE_1D - {}, // GGML_OP_IM2COL - {}, // GGML_OP_IM2COL_BACK - {}, // GGML_OP_CONV_TRANSPOSE_2D - {}, // GGML_OP_POOL_1D - {}, // GGML_OP_POOL_2D - {}, // GGML_OP_POOL_2D_BACK - {}, // GGML_OP_UPSCALE - {}, // GGML_OP_PAD - {}, // GGML_OP_PAD_REFLECT_1D - {}, // GGML_OP_ARANGE - {}, // GGML_OP_TIMESTEP_EMBEDDING - {}, // GGML_OP_ARGSORT - {}, // GGML_OP_LEAKY_RELU - {}, // GGML_OP_FLASH_ATTN_EXT - {}, // GGML_OP_FLASH_ATTN_BACK - {}, // GGML_OP_SSM_CONV - {}, // GGML_OP_SSM_SCAN - {}, // GGML_OP_WIN_PART - {}, // GGML_OP_WIN_UNPART - {}, // GGML_OP_GET_REL_POS - {}, // GGML_OP_ADD_REL_POS - {}, // GGML_OP_RWKV_WKV6 - {}, // GGML_OP_GATED_LINEAR_ATTN - {}, // GGML_OP_UNARY - {}, // GGML_OP_MAP_UNARY - {}, // GGML_OP_MAP_BINARY - {}, // GGML_OP_MAP_CUSTOM1_F32 - {}, // GGML_OP_MAP_CUSTOM2_F32 - {}, // GGML_OP_MAP_CUSTOM3_F32 - {}, // GGML_OP_MAP_CUSTOM1 - {}, // GGML_OP_MAP_CUSTOM2 - {}, // GGML_OP_MAP_CUSTOM3 - {}, // GGML_OP_CROSS_ENTROPY_LOSS - {}, // GGML_OP_CROSS_ENTROPY_LOSS_BACK - {}, // GGML_OP_OPT_STEP_ADAMW - {}, // GGML_UNARY_OP_ABS - {}, // GGML_UNARY_OP_SGN - {}, // GGML_UNARY_OP_NEG - {}, // GGML_UNARY_OP_STEP - {}, // GGML_UNARY_OP_TANH - {}, // GGML_UNARY_OP_ELU - {}, // GGML_UNARY_OP_RELU - {}, // GGML_UNARY_OP_SIGMOID - {}, // GGML_UNARY_OP_GELU - {}, // GGML_UNARY_OP_GELU_QUICK - {}, // GGML_UNARY_OP_SILU - {}, // GGML_UNARY_OP_HARDSWISH - {}, // GGML_UNARY_OP_HARDSIGMOID - {}, // GGML_UNARY_OP_EXP -}; + // QnnGraph_Error_t + case QNN_GRAPH_ERROR_UNSUPPORTED_FEATURE: + return "QNN_GRAPH_ERROR_UNSUPPORTED_FEATURE"; + case QNN_GRAPH_ERROR_MEM_ALLOC: + return "QNN_GRAPH_ERROR_MEM_ALLOC"; + case QNN_GRAPH_ERROR_INVALID_ARGUMENT: + return "QNN_GRAPH_ERROR_INVALID_ARGUMENT"; + case QNN_GRAPH_ERROR_INVALID_HANDLE: + return "QNN_GRAPH_ERROR_INVALID_HANDLE"; + case QNN_GRAPH_ERROR_GRAPH_DOES_NOT_EXIST: + return "QNN_GRAPH_ERROR_GRAPH_DOES_NOT_EXIST"; + case QNN_GRAPH_ERROR_INVALID_NAME: + return "QNN_GRAPH_ERROR_INVALID_NAME"; + case QNN_GRAPH_ERROR_INVALID_TENSOR: + return "QNN_GRAPH_ERROR_INVALID_TENSOR"; + case QNN_GRAPH_ERROR_INVALID_OP_CONFIG: + return "QNN_GRAPH_ERROR_INVALID_OP_CONFIG"; + case QNN_GRAPH_ERROR_SET_PROFILE: + return "QNN_GRAPH_ERROR_SET_PROFILE"; + case QNN_GRAPH_ERROR_UNCONNECTED_NODE: + return "QNN_GRAPH_ERROR_UNCONNECTED_NODE"; + case QNN_GRAPH_ERROR_CREATE_FAILED: + return "QNN_GRAPH_ERROR_CREATE_FAILED"; + case QNN_GRAPH_ERROR_OPTIMIZATION_FAILED: + return "QNN_GRAPH_ERROR_OPTIMIZATION_FAILED"; + case QNN_GRAPH_ERROR_FINALIZE_FAILED: + return "QNN_GRAPH_ERROR_FINALIZE_FAILED"; + case QNN_GRAPH_ERROR_GRAPH_NOT_FINALIZED: + return "QNN_GRAPH_ERROR_GRAPH_NOT_FINALIZED"; + case QNN_GRAPH_ERROR_GRAPH_FINALIZED: + return "QNN_GRAPH_ERROR_GRAPH_FINALIZED"; + case QNN_GRAPH_ERROR_EXECUTION_ASYNC_FIFO_FULL: + return "QNN_GRAPH_ERROR_EXECUTION_ASYNC_FIFO_FULL"; + case QNN_GRAPH_ERROR_SIGNAL_IN_USE: + return "QNN_GRAPH_ERROR_SIGNAL_IN_USE"; + case QNN_GRAPH_ERROR_ABORTED: + return "QNN_GRAPH_ERROR_ABORTED"; + case QNN_GRAPH_ERROR_PROFILE_IN_USE: + return "QNN_GRAPH_ERROR_PROFILE_IN_USE"; + case QNN_GRAPH_ERROR_TIMED_OUT: + return "QNN_GRAPH_ERROR_TIMED_OUT"; + case QNN_GRAPH_ERROR_SUBGRAPH: + return "QNN_GRAPH_ERROR_SUBGRAPH"; + case QNN_GRAPH_ERROR_DISABLED: + return "QNN_GRAPH_ERROR_DISABLED"; + case QNN_GRAPH_ERROR_DYNAMIC_TENSOR_SHAPE: + return "QNN_GRAPH_ERROR_DYNAMIC_TENSOR_SHAPE"; + case QNN_GRAPH_ERROR_TENSOR_SPARSITY: + return "QNN_GRAPH_ERROR_TENSOR_SPARSITY"; + case QNN_GRAPH_ERROR_EARLY_TERMINATION: + return "QNN_GRAPH_ERROR_EARLY_TERMINATION"; + case QNN_GRAPH_ERROR_INVALID_CONTEXT: + return "QNN_GRAPH_ERROR_INVALID_CONTEXT"; + + //QQnnTensor_Error_t + //Invalid context/graph handle in creating tensor + case QNN_TENSOR_ERROR_INVALID_HANDLE: + return "QNN_TENSOR_ERROR_INVALID_HANDLE"; + //Tensor with specified credentials not registered with a context/graph + case QNN_TENSOR_ERROR_DOES_NOT_EXIST: + return "QNN_TENSOR_ERROR_DOES_NOT_EXIST"; + // (deprecated) Tensor has already been registered with backend + case QNN_TENSOR_ERROR_ALREADY_EXISTS: + return "QNN_TENSOR_ERROR_ALREADY_EXISTS"; + // Invalid tensor param. + case QNN_TENSOR_ERROR_INVALID_TENSOR_PARAM: + return "QNN_TENSOR_ERROR_INVALID_TENSOR_PARAM"; + // This tensor param is currently unsupported + case QNN_TENSOR_ERROR_UNSUPPORTED_TENSOR_PARAM: + return "QNN_TENSOR_ERROR_UNSUPPORTED_TENSOR_PARAM"; + // Tensor provided for update is invalid + case QNN_TENSOR_ERROR_INCOMPATIBLE_TENSOR_UPDATE: + return "QNN_TENSOR_ERROR_INCOMPATIBLE_TENSOR_UPDATE"; + + // QnnOpPackage_Error_t + case QNN_OP_PACKAGE_ERROR_LIBRARY_ALREADY_INITIALIZED: + return "QNN_OP_PACKAGE_ERROR_LIBRARY_ALREADY_INITIALIZED"; + case QNN_OP_PACKAGE_ERROR_LIBRARY_NOT_INITIALIZED: + return "QNN_OP_PACKAGE_ERROR_LIBRARY_NOT_INITIALIZED"; + case QNN_OP_PACKAGE_ERROR_INVALID_HANDLE: + return "QNN_OP_PACKAGE_ERROR_INVALID_HANDLE"; + case QNN_OP_PACKAGE_ERROR_INVALID_INFRASTRUCTURE: + return "QNN_OP_PACKAGE_ERROR_INVALID_INFRASTRUCTURE"; + case QNN_OP_PACKAGE_ERROR_INVALID_INFO: + return "QNN_OP_PACKAGE_ERROR_INVALID_INFO"; + case QNN_OP_PACKAGE_ERROR_VALIDATION_FAILURE: + return "QNN_OP_PACKAGE_ERROR_VALIDATION_FAILURE"; + case QNN_OP_PACKAGE_ERROR_INVALID_ARGUMENT: + return "QNN_OP_PACKAGE_ERROR_INVALID_ARGUMENT"; + + default: + return "unknown QNN error"; + } +} + +static Qnn_OpConfig_t ggmlqnn_create_op_config(const char * name, const char * package, const char * type, + Qnn_Param_t * params, uint32_t num_params, + Qnn_Tensor_t * inputs, uint32_t num_inputs, + Qnn_Tensor_t * outputs, uint32_t num_outputs) { + Qnn_OpConfigV1_t v1 = {name, package, type, + num_params, params, + num_inputs, inputs, + num_outputs, outputs + }; + Qnn_OpConfig_t opcfg = {QNN_OPCONFIG_VERSION_1, {v1}}; + + return opcfg; +} -static const char * qnn_get_socmodel_desc(uint32_t soc_model) { +// ================================================================================================= +// section-7:ggml-qnn backend helper function / class +// ================================================================================================= +static const char * ggmlqnn_get_socmodel_desc(uint32_t soc_model) { switch (soc_model) { case SM7450: return "SM7450"; @@ -1180,7 +1370,7 @@ static const char * qnn_get_socmodel_desc(uint32_t soc_model) { } } -static const char * qnn_get_htparch_desc(size_t htp_arch) { +static const char * ggmlqnn_get_htparch_desc(size_t htp_arch) { switch (htp_arch) { case V68: return "QCOM_HTP_V68"; @@ -1207,13 +1397,7 @@ static struct qcom_socinfo * ggmlqnn_get_socinfo_from_socmodel(uint32_t soc_mode return nullptr; } - -static const char * ggml_get_type_name(ggml_type type) { - const struct ggml_type_traits * traits = ggml_get_type_traits(type); - return traits->type_name; -} - -static const char * get_ggml_type_name(ggml_type type) { +static const char * ggmlqnn_get_ggml_type_name(ggml_type type) { const auto * traits = ggml_get_type_traits(type); return traits->type_name; } @@ -1260,7 +1444,7 @@ static ggml_type ggml_datatype_from_qnn_datatype(Qnn_DataType_t qnn_type) { return GGML_TYPE_COUNT; } -static void get_qnn_dimensions_from_ggml_dimensions(uint32_t * qnn_dimensions, const uint32_t * ggml_dimensions, uint32_t rank) { +static void ggmlqnn_get_qnn_dimensions_from_ggml_dimensions(uint32_t * qnn_dimensions, const uint32_t * ggml_dimensions, uint32_t rank) { if (rank > GGML_MAX_DIMS) { GGMLQNN_LOG_WARN("invalid params"); return; @@ -1278,7 +1462,6 @@ static void get_qnn_dimensions_from_ggml_dimensions(uint32_t * qnn_dimensions, c } } - static void * ggmlqnn_type_trait(ggml_backend_qnn_context * ctx, ggml_tensor * op) { const ggml_tensor * src0 = op->src[0]; const ggml_tensor * src1 = op->src[1]; @@ -1347,9 +1530,9 @@ static void * ggmlqnn_type_trait(ggml_backend_qnn_context * ctx, ggml_tensor * o return wdata; } -static void append_tensor_dimensions(const ggml_tensor * tensor, std::string & output) { +static void ggmlqnn_append_tensor_dimensions(const ggml_tensor * tensor, std::string & output) { char buffer[256] = {}; - const char * type_name = get_ggml_type_name(tensor->type); + const char * type_name = ggmlqnn_get_ggml_type_name(tensor->type); int len = 0; switch (ggml_n_dims(tensor)) { case 1: @@ -1393,7 +1576,7 @@ static size_t ggmlqnn_get_op_input_param_count(const ggml_tensor * op) { static void ggmlqnn_get_graphkey_from_op(const ggml_tensor * op, std::string & output) { GGML_ASSERT(op->op != GGML_OP_NONE); output += ggml_op_desc(op); - output += get_ggml_type_name(op->type); + output += ggmlqnn_get_ggml_type_name(op->type); size_t param_count = ggmlqnn_get_op_input_param_count(op); for (size_t i = 0; i < param_count; ++i) { auto * input = op->src[i]; @@ -1401,12 +1584,61 @@ static void ggmlqnn_get_graphkey_from_op(const ggml_tensor * op, std::string & o break; } output += '_'; - append_tensor_dimensions(input, output); + ggmlqnn_append_tensor_dimensions(input, output); + } +} + +static void ggmlqnn_get_opkey_with_srcop_desc(const ggml_tensor * op, std::string & output) { + output += ggml_op_desc(op); + output += '('; + if (op->src[0]) { + output += ggml_op_desc(op->src[0]); + } + for (size_t i = 1; i < GGML_MAX_DIMS && op->src[i]; ++i) { + output += ','; + output += ggml_op_desc(op->src[i]); + } + output += ')'; +} + +static void ggmlqnn_get_graphkey_from_cgraph(const ggml_cgraph * cgraph, std::string & output) { + if (nullptr == cgraph || 0 == cgraph->n_nodes) { + GGMLQNN_LOG_WARN("empty ggml computational graph"); + return; + } + + bool is_start = true; + for (int i = 0; i < cgraph->n_nodes; ++i) { + auto * op = cgraph->nodes[i]; + if (ggml_is_empty(op)) { + GGMLQNN_LOG_WARN("empty op in graph, skipping"); + continue; + } + + if (op->op == GGML_OP_NONE) { + GGMLQNN_LOG_WARN("GGML_OP_NONE in graph, skipping"); + continue; + } + + if (is_start) { + ggmlqnn_get_graphkey_from_op(cgraph->nodes[0], output); + is_start = false; + } else { + output += '#'; + ggmlqnn_get_opkey_with_srcop_desc(op, output); + } + } + + if (cgraph->n_nodes > 1) { + auto * last_op = cgraph->nodes[cgraph->n_nodes - 1]; + output += ggmlqnn_get_ggml_type_name(last_op->type); + output += '_'; + ggmlqnn_append_tensor_dimensions(last_op, output); } } template -Fn load_qnn_functionpointers(void * handle, const char * function_name) { +Fn ggmlqnn_load_qnn_functionpointers(void * handle, const char * function_name) { return reinterpret_cast(dlsym(handle, function_name)); } @@ -1667,7 +1899,17 @@ class qnn_instance { void * alloc_rpcmem_internal(size_t bytes, size_t alignment); - void probe_device_meminfo(); + void htp_print_info(); + + void htp_probe_device_meminfo(); + + void print_backend_info(); + + void htp_set_memory_grow_size(size_t size = 1ul * 1024 * 1024); + + void htp_enter_performance_mode(); + + void htp_set_n_hvx_threads(size_t n_threads); private: static constexpr const int _required_num_providers = 1; @@ -1685,6 +1927,8 @@ class qnn_instance { ggml_qnn_profile_level _profile_level = ggml_qnn_profile_level::profile_detail; void * _system_lib_handle = nullptr; + void * _loaded_lib_handle = nullptr; + const QnnInterface_t * _loaded_backend = nullptr; Qnn_GraphHandle_t _qnn_graph_handle = nullptr; @@ -1701,8 +1945,11 @@ class qnn_instance { QnnSystemContext_Handle_t _qnn_system_handle = nullptr; QnnHtpDevice_PerfInfrastructure_t * _qnn_htp_perfinfra = nullptr; - uint32_t _qnn_power_configid = 1; - uint32_t _qnn_rpc_pollingtime = 9999; // 0-10000 us for high performing + uint32_t _qnn_htp_powerconfig_id = 1; + uint32_t _qnn_htp_device_id = 0; + uint32_t _qnn_htp_core_id = 0; + + uint32_t _qnn_rpc_pollingtime = 9999; // 0-10000 us for high performing qnn_interface _qnn_interface; QNN_INTERFACE_VER_TYPE _qnn_raw_interface; @@ -1711,11 +1958,6 @@ class qnn_instance { std::unordered_map _qnn_mem_set; std::unordered_map _qnn_rpc_buffer_to_handles; - static std::mutex _init_mutex; - static std::unordered_map _loaded_lib_handle; - static std::unordered_map _lib_path_to_backend_id; - static std::unordered_map _loaded_backend; - std::atomic_bool _rpcmem_initialized{false}; pfn_rpc_mem_alloc _pfn_rpc_mem_alloc; pfn_rpc_mem_free _pfn_rpc_mem_free; @@ -1736,12 +1978,6 @@ class qnn_instance { DISABLE_MOVE(qnn_instance); }; - -std::mutex qnn_instance::_init_mutex; -std::unordered_map qnn_instance::_loaded_lib_handle; -std::unordered_map qnn_instance::_lib_path_to_backend_id; -std::unordered_map qnn_instance::_loaded_backend; - void * qnn_instance::alloc_rpcmem_internal(size_t bytes, size_t alignment) { if (!_rpcmem_initialized) { GGMLQNN_LOG_WARN("rpc memory not initialized\n"); @@ -1990,7 +2226,7 @@ int qnn_instance::load_backend(std::string & lib_path, const QnnSaver_Config_t * return 1; } - auto get_providers = load_qnn_functionpointers<_pfn_QnnInterface_getProviders *>( + auto get_providers = ggmlqnn_load_qnn_functionpointers<_pfn_QnnInterface_getProviders *>( lib_handle, "QnnInterface_getProviders"); if (nullptr == get_providers) { @@ -1998,7 +2234,6 @@ int qnn_instance::load_backend(std::string & lib_path, const QnnSaver_Config_t * return 2; } - // get QnnInterface Providers std::uint32_t num_providers = 0; const QnnInterface_t ** provider_list = nullptr; error = get_providers(&provider_list, &num_providers); @@ -2036,25 +2271,12 @@ int qnn_instance::load_backend(std::string & lib_path, const QnnSaver_Config_t * set_qnn_raw_interface(qnn_interface); BackendIdType backend_id = provider_list[0]->backendId; - _lib_path_to_backend_id[lib_path] = backend_id; - if (_loaded_backend.count(backend_id) > 0) { - GGMLQNN_LOG_WARN("lib_path %s is loaded, but backend %d already exists\n", - lib_path.c_str(), backend_id); - } - _loaded_backend[backend_id] = provider_list[0]; - if (_loaded_lib_handle.count(backend_id) > 0) { - GGMLQNN_LOG_WARN("closing %p\n", _loaded_lib_handle[backend_id]); - int dlclose_error = dlclose(_loaded_lib_handle[backend_id]); - if (dlclose_error != 0) { - GGMLQNN_LOG_WARN("fail to close %p with error %s\n", _loaded_lib_handle[backend_id], dlerror()); - } - } - _loaded_lib_handle[backend_id] = lib_handle; - _backend_id = backend_id; + _loaded_backend = provider_list[0]; + _loaded_lib_handle = lib_handle; + _backend_id = backend_id; auto saver_initialize = - load_qnn_functionpointers<_pfn_QnnSaver_initialize *>( - _loaded_lib_handle[backend_id], "QnnSaver_initialize"); + ggmlqnn_load_qnn_functionpointers<_pfn_QnnSaver_initialize *>(_loaded_lib_handle, "QnnSaver_initialize"); if (nullptr != saver_initialize) { error = saver_initialize(saver_config); if (error != QNN_SUCCESS) { @@ -2070,17 +2292,11 @@ int qnn_instance::load_backend(std::string & lib_path, const QnnSaver_Config_t * int qnn_instance::unload_backend() { int dlclose_error = 0; - for (auto & it : _loaded_lib_handle) { - dlclose_error = dlclose(it.second); - if (dlclose_error != 0) { - GGMLQNN_LOG_WARN("failed to close QNN backend %d, error %s\n", it.first, dlerror()); - } + dlclose_error = dlclose(_loaded_lib_handle); + if (dlclose_error != 0) { + GGMLQNN_LOG_WARN("failed to close QNN backend %d, error %s\n", _backend_id, dlerror()); } - _loaded_lib_handle.clear(); - _lib_path_to_backend_id.clear(); - _loaded_backend.clear(); - return 0; } @@ -2195,12 +2411,14 @@ int qnn_instance::unload_system() { return result; } -#if GGMLQNN_PRINT_QNN_INTERNAL_LOG static void ggml_qnn_logcallback(const char * fmt, QnnLog_Level_t level, uint64_t timestamp, va_list argp) { + if (0 == g_print_qnn_internal_log) + return; + static std::mutex log_mutex; static unsigned char s_ggml_qnn_logbuf[GGML_QNN_LOGBUF_LEN]; @@ -2234,39 +2452,12 @@ static void ggml_qnn_logcallback(const char * fmt, GGMLQNN_LOG_DEBUG("%8.1fms [%-7s] %s\n", ms, log_level_desc, s_ggml_qnn_logbuf); } } -#else -static void ggml_qnn_logcallback(const char * fmt, - QnnLog_Level_t level, - uint64_t timestamp, - va_list argp) { - GGML_UNUSED(fmt); - GGML_UNUSED(level); - GGML_UNUSED(timestamp); - GGML_UNUSED(argp); -} -#endif int qnn_instance::qnn_init(const QnnSaver_Config_t ** saver_config) { BackendIdType backend_id = QNN_BACKEND_ID_NULL; GGMLQNN_LOG_DEBUG("enter qni_init\n"); - const std::lock_guard lock(_init_mutex); - if (0 != load_system()) { - GGMLQNN_LOG_WARN("can not load QNN system lib, pls check why?\n"); - return 1; - } else { - GGMLQNN_LOG_DEBUG("load QNN system lib successfully\n"); - } - - std::string backend_lib_path = _lib_path + _backend_name; - if (0 == _lib_path_to_backend_id.count(backend_lib_path)) { - int is_load_ok = load_backend(backend_lib_path, saver_config); - if (0 != is_load_ok) { - GGMLQNN_LOG_WARN("failed to load QNN backend\n"); - return 2; - } - } - _device_id = QNN_BACKEND_CPU; + _device_id = QNN_BACKEND_GGML; if (_backend_name.find("QnnCpu") != std::string::npos) { _device_id = QNN_BACKEND_CPU; } @@ -2276,17 +2467,27 @@ int qnn_instance::qnn_init(const QnnSaver_Config_t ** saver_config) { if (_backend_name.find("QnnHtp") != std::string::npos) { _device_id = QNN_BACKEND_NPU; } + if (QNN_BACKEND_GGML == _device_id) { + GGMLQNN_LOG_INFO("user specified qnn backend is ggml, skip QNN initialize"); + return 0; + } - backend_id = _lib_path_to_backend_id[backend_lib_path]; - if (0 == _loaded_backend.count(backend_id) || - 0 == _loaded_lib_handle.count(backend_id)) { - GGMLQNN_LOG_WARN("library %s is loaded but loaded backend count=%zu, loaded lib_handle count=%zu\n", - backend_lib_path.c_str(), - _loaded_backend.count(backend_id), - _loaded_lib_handle.count(backend_id)); - return 3; + if (0 != load_system()) { + GGMLQNN_LOG_WARN("can not load QNN system lib, pls check why?\n"); + return 1; + } else { + GGMLQNN_LOG_DEBUG("load QNN system lib successfully\n"); + } + + std::string backend_lib_path = _lib_path + _backend_name; + + int is_load_ok = load_backend(backend_lib_path, saver_config); + if (0 != is_load_ok) { + GGMLQNN_LOG_WARN("failed to load QNN backend\n"); + return 2; } - _qnn_interface.set_qnn_interface(_loaded_backend[backend_id]); + + _qnn_interface.set_qnn_interface(_loaded_backend); #if 1 _qnn_interface.qnn_log_create(ggml_qnn_logcallback, _qnn_log_level, &_qnn_log_handle); #else @@ -2294,7 +2495,7 @@ int qnn_instance::qnn_init(const QnnSaver_Config_t ** saver_config) { #endif if (nullptr == _qnn_log_handle) { GGMLQNN_LOG_WARN("why failed to initialize qnn log\n"); //NPU backend not work on Qualcomm SoC based low-end phone - return 4; + return 3; } else { GGMLQNN_LOG_DEBUG("initialize qnn log successfully\n"); } @@ -2305,7 +2506,7 @@ int qnn_instance::qnn_init(const QnnSaver_Config_t ** saver_config) { &_qnn_backend_handle); if (nullptr == _qnn_backend_handle) { GGMLQNN_LOG_WARN("why failed to initialize qnn backend\n"); - return 5; + return 4; } else { GGMLQNN_LOG_DEBUG("initialize qnn backend successfully\n"); } @@ -2335,7 +2536,7 @@ int qnn_instance::qnn_init(const QnnSaver_Config_t ** saver_config) { if (QNN_PROFILE_NO_ERROR != _qnn_raw_interface.profileCreate( _qnn_backend_handle, QNN_PROFILE_LEVEL_BASIC, &_qnn_profile_handle)) { GGMLQNN_LOG_WARN("unable to create profile handle in the backend\n"); - return 6; + return 5; } else { GGMLQNN_LOG_DEBUG("initialize qnn profile successfully\n"); } @@ -2344,7 +2545,7 @@ int qnn_instance::qnn_init(const QnnSaver_Config_t ** saver_config) { if (QNN_PROFILE_NO_ERROR != _qnn_raw_interface.profileCreate( _qnn_backend_handle, QNN_PROFILE_LEVEL_DETAILED, &_qnn_profile_handle)) { GGMLQNN_LOG_WARN("unable to create profile handle in the backend\n"); - return 7; + return 6; } else { GGMLQNN_LOG_DEBUG("initialize qnn profile successfully\n"); } @@ -2358,7 +2559,7 @@ int qnn_instance::qnn_init(const QnnSaver_Config_t ** saver_config) { #endif if (nullptr == _rpc_lib_handle) { GGMLQNN_LOG_WARN("failed to load qualcomm's rpc lib, error:%s\n", dlerror()); - return 8; + return 7; } else { GGMLQNN_LOG_DEBUG("load rpcmem lib successfully\n"); set_rpcmem_initialized(true); @@ -2372,7 +2573,7 @@ int qnn_instance::qnn_init(const QnnSaver_Config_t ** saver_config) { || nullptr == _pfn_rpc_mem_to_fd) { GGMLQNN_LOG_WARN("unable to access symbols in QNN RPC lib. dlerror(): %s", dlerror()); dlclose(_rpc_lib_handle); - return 9; + return 8; } if (nullptr != _pfn_rpc_mem_init) // make Qualcomm's SoC based low-end phone happy @@ -2384,51 +2585,30 @@ int qnn_instance::qnn_init(const QnnSaver_Config_t ** saver_config) { &_qnn_context_handle); if (nullptr == _qnn_context_handle) { GGMLQNN_LOG_WARN("why failed to initialize qnn context, error:%s\n", strerror(errno)); - return 10; + return 9; } else { GGMLQNN_LOG_DEBUG("initialize qnn context successfully\n"); } if (_backend_name.find("Htp") != std::string::npos) { - const QnnDevice_PlatformInfo_t * p_info = nullptr; - _qnn_raw_interface.deviceGetPlatformInfo(nullptr, &p_info); - GGMLQNN_LOG_INFO("device counts %d", p_info->v1.numHwDevices); - QnnDevice_HardwareDeviceInfo_t * infos = p_info->v1.hwDevices; - for (size_t i = 0; i < p_info->v1.numHwDevices; i++) { - GGMLQNN_LOG_INFO("deviceID:%d, deviceType:%d, numCores %d", infos[i].v1.deviceId, - infos[i].v1.deviceType, infos[i].v1.numCores); - QnnDevice_DeviceInfoExtension_t devinfo = infos[i].v1.deviceInfoExtension; - QnnHtpDevice_OnChipDeviceInfoExtension_t chipinfo = devinfo->onChipDevice; - QnnHtpDevice_Arch_t htp_arch = chipinfo.arch; - GGMLQNN_LOG_INFO("htp_type:%d(%s)", devinfo->devType, - (devinfo->devType == QNN_HTP_DEVICE_TYPE_ON_CHIP) ? "QNN_HTP_DEVICE_TYPE_ON_CHIP" : "QNN_HTP_DEVICE_TYPE_UNKNOWN"); - GGMLQNN_LOG_INFO("qualcomm soc_model:%d(%s), htp_arch:%d(%s), vtcm_size:%d MB", \ - chipinfo.socModel, qnn_get_socmodel_desc(chipinfo.socModel), \ - htp_arch, qnn_get_htparch_desc(htp_arch), chipinfo.vtcmSize); - struct qcom_socinfo * socinfo = ggmlqnn_get_socinfo_from_socmodel(chipinfo.socModel); - g_qnn_mgr[QNN_BACKEND_NPU].socinfo = { chipinfo.socModel, htp_arch, chipinfo.vtcmSize, {}}; - if (nullptr != socinfo) { - memcpy(g_qnn_mgr[QNN_BACKEND_NPU].socinfo.soc_desc, socinfo->soc_desc, sizeof(socinfo->soc_desc)); - GGMLQNN_LOG_INFO("soc info:%s", socinfo->soc_desc); - } else { - memcpy(g_qnn_mgr[QNN_BACKEND_NPU].socinfo.soc_desc, "unknown", 7); - GGMLQNN_LOG_INFO("soc info:unknown"); - } - } - _qnn_raw_interface.deviceFreePlatformInfo(nullptr, p_info); + htp_print_info(); - probe_device_meminfo(); + htp_probe_device_meminfo(); if (0 != init_htp_perfinfra()) { GGMLQNN_LOG_WARN("initialize HTP performance failure"); } +#if 0 if (0 != set_rpc_polling()) { GGMLQNN_LOG_WARN("set RPC polling failure"); } if (0 != set_high_performance_mode()) { GGMLQNN_LOG_WARN("set HTP high performance mode failure"); } - +#else + htp_set_memory_grow_size(); + htp_enter_performance_mode(); +#endif if (enable_qnn_rpc()) { GGMLQNN_LOG_INFO("NPU RPC feature enabled"); } else { @@ -2436,6 +2616,8 @@ int qnn_instance::qnn_init(const QnnSaver_Config_t ** saver_config) { } } + print_backend_info(); + GGMLQNN_LOG_DEBUG("leave qni_init\n"); return 0; @@ -2446,7 +2628,7 @@ int qnn_instance::qnn_finalize() { Qnn_ErrorHandle_t error = QNN_SUCCESS; GGMLQNN_LOG_DEBUG("enter %s\n", __func__); - reset_idx(); + ggmqnn_reset_tensoridx(); free_rpcmem(); unregister_rpcmem(); @@ -2555,8 +2737,15 @@ int qnn_instance::init_qnn_graph(const std::string & graph_name, QNNBackend devi graph_vtcm_config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM; graph_vtcm_config.customConfig = &vtcm_config; + QnnHtpGraph_CustomConfig_t fp16_config; + fp16_config.option = QNN_HTP_GRAPH_CONFIG_OPTION_PRECISION; + fp16_config.precision = QNN_PRECISION_FLOAT16; + QnnGraph_Config_t graph_fp16_config; + graph_fp16_config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM; + graph_fp16_config.customConfig = &fp16_config; + const QnnGraph_Config_t * graph_configs[] = {&graph_hvx_config, &graph_dlbc_config, &graph_vtcm_config, - &graph_opt_config, nullptr}; + &graph_opt_config, &graph_fp16_config, nullptr}; error = _qnn_interface.qnn_graph_create(_qnn_context_handle, graph_name.c_str(), graph_configs, &graph_handle); } else { error = _qnn_interface.qnn_graph_create(_qnn_context_handle, graph_name.c_str(), nullptr, &graph_handle); @@ -2565,12 +2754,15 @@ int qnn_instance::init_qnn_graph(const std::string & graph_name, QNNBackend devi if (error != QNN_SUCCESS) { GGMLQNN_LOG_ERROR("[%s][%s]failed to create qnn graph, error: %s", ggml_backend_qnn_get_devname(device), graph_name.c_str(), - ggmlqnn_get_error_string(error)); + ggmlqnn_get_qnnerror_string(error)); return error; } GGMLQNN_LOG_DEBUG("[%s]create graph %s succeed", ggml_backend_qnn_get_devname(device), graph_name.c_str()); _qnn_graph_handle = graph_handle; + if (device == QNN_BACKEND_NPU) { + htp_set_n_hvx_threads(hvx_threads); + } return QNN_SUCCESS; } @@ -2636,11 +2828,14 @@ int qnn_instance::init_htp_perfinfra() { QnnHtpDevice_Infrastructure_t * htp_infra = static_cast(device_infra); QnnHtpDevice_PerfInfrastructure_t * htp_perfinfra = &htp_infra->perfInfra; uint32_t power_configid = 1; - uint32_t device_id = 0; - uint32_t core_id = 0; + uint32_t device_id = 0; + uint32_t core_id = 0; htp_perfinfra->createPowerConfigId(device_id, core_id, &power_configid); - _qnn_htp_perfinfra = htp_perfinfra; - _qnn_power_configid = power_configid; + _qnn_htp_perfinfra = htp_perfinfra; + _qnn_htp_powerconfig_id = power_configid; + //FIXME:hardcode to 0 and 0 although it's correct + _qnn_htp_device_id = device_id; + _qnn_htp_core_id = core_id; return 0; } @@ -2653,7 +2848,7 @@ int qnn_instance::set_rpc_polling() { rpc_pollingtime.rpcPollingTimeConfig = _qnn_rpc_pollingtime; const QnnHtpPerfInfrastructure_PowerConfig_t * power_configs[] = {&rpc_pollingtime, nullptr}; if (_qnn_htp_perfinfra) { - _qnn_htp_perfinfra->setPowerConfig(_qnn_power_configid, power_configs); + _qnn_htp_perfinfra->setPowerConfig(_qnn_htp_powerconfig_id, power_configs); } } return 0; @@ -2670,7 +2865,7 @@ int qnn_instance::set_high_performance_mode() { power_config.option = QNN_HTP_PERF_INFRASTRUCTURE_POWER_CONFIGOPTION_DCVS_V3; power_config.dcvsV3Config.dcvsEnable = 0; power_config.dcvsV3Config.setDcvsEnable = 1; - power_config.dcvsV3Config.contextId = _qnn_power_configid; + power_config.dcvsV3Config.contextId = _qnn_htp_powerconfig_id; power_config.dcvsV3Config.powerMode = QNN_HTP_PERF_INFRASTRUCTURE_POWERMODE_PERFORMANCE_MODE; power_config.dcvsV3Config.setSleepLatency = 1; // True to consider Latency parameter otherwise False power_config.dcvsV3Config.setBusParams = 1; // True to consider Bus parameter otherwise False @@ -2691,12 +2886,43 @@ int qnn_instance::set_high_performance_mode() { // set power config with different performance parameters const QnnHtpPerfInfrastructure_PowerConfig_t * power_configs[] = {&power_config, nullptr}; - _qnn_htp_perfinfra->setPowerConfig(_qnn_power_configid, power_configs); + _qnn_htp_perfinfra->setPowerConfig(_qnn_htp_powerconfig_id, power_configs); return 0; } -void qnn_instance::probe_device_meminfo() { +void qnn_instance::htp_print_info() { + const QnnDevice_PlatformInfo_t * p_info = nullptr; + _qnn_raw_interface.deviceGetPlatformInfo(nullptr, &p_info); + GGMLQNN_LOG_INFO("HTP device counts %d", p_info->v1.numHwDevices); + QnnDevice_HardwareDeviceInfo_t * infos = p_info->v1.hwDevices; + for (size_t i = 0; i < p_info->v1.numHwDevices; i++) { + GGMLQNN_LOG_INFO("HTP deviceID:%d, deviceType:%d, numCores %d", infos[i].v1.deviceId, + infos[i].v1.deviceType, infos[i].v1.numCores); + QnnDevice_DeviceInfoExtension_t devinfo = infos[i].v1.deviceInfoExtension; + QnnHtpDevice_OnChipDeviceInfoExtension_t chipinfo = devinfo->onChipDevice; + QnnHtpDevice_Arch_t htp_arch = chipinfo.arch; + GGMLQNN_LOG_INFO("HTP_TYPE:%d(%s)", devinfo->devType, + (devinfo->devType == QNN_HTP_DEVICE_TYPE_ON_CHIP) ? "QNN_HTP_DEVICE_TYPE_ON_CHIP" : "QNN_HTP_DEVICE_TYPE_UNKNOWN"); + GGMLQNN_LOG_INFO("qualcomm soc_model:%d(%s), htp_arch:%d(%s), vtcm_size:%d MB，" \ + "dlbc_support:%d, signedpd_support:%d", \ + chipinfo.socModel, ggmlqnn_get_socmodel_desc(chipinfo.socModel), \ + htp_arch, ggmlqnn_get_htparch_desc(htp_arch), chipinfo.vtcmSize, \ + chipinfo.dlbcSupport, chipinfo.signedPdSupport); + struct qcom_socinfo * socinfo = ggmlqnn_get_socinfo_from_socmodel(chipinfo.socModel); + g_qnn_mgr[QNN_BACKEND_NPU].socinfo = { chipinfo.socModel, htp_arch, chipinfo.vtcmSize, {}}; + if (nullptr != socinfo) { + memcpy(g_qnn_mgr[QNN_BACKEND_NPU].socinfo.soc_desc, socinfo->soc_desc, sizeof(socinfo->soc_desc)); + GGMLQNN_LOG_INFO("soc info:%s", socinfo->soc_desc); + } else { + memcpy(g_qnn_mgr[QNN_BACKEND_NPU].socinfo.soc_desc, "unknown", 7); + GGMLQNN_LOG_INFO("soc info:unknown"); + } + } + _qnn_raw_interface.deviceFreePlatformInfo(nullptr, p_info); +} + +void qnn_instance::htp_probe_device_meminfo() { size_t candidate_size = 0; uint8_t * rpc_buffer = nullptr; const int SIZE_IN_MB = (1 << 20); @@ -2721,6 +2947,140 @@ void qnn_instance::probe_device_meminfo() { GGMLQNN_LOG_INFO("capacity of rpc ion memory %d MB\n", _rpcmem_capacity); } +void qnn_instance::print_backend_info() { + auto print_property = [&](const char * name, QnnProperty_Key_t property) { + auto ret = _qnn_raw_interface.propertyHasCapability(property); + + const char * status = "Unknown"; + if (ret == QNN_PROPERTY_SUPPORTED) { + status = "Yes"; + } else if (ret == QNN_PROPERTY_NOT_SUPPORTED) { + status = "No"; + } + + GGMLQNN_LOG_INFO("%s: %s", name, status); + }; + + GGMLQNN_LOG_INFO("QNN backend properties:"); + print_property("Create context from binary list", QNN_PROPERTY_CONTEXT_SUPPORT_CREATE_FROM_BINARY_LIST_ASYNC); + print_property("Dynamic batch", QNN_PROPERTY_GRAPH_SUPPORT_BATCH_MULTIPLE); + print_property("Early termination", QNN_PROPERTY_GRAPH_SUPPORT_EARLY_TERMINATION); + print_property("Dynamic dimensions", QNN_PROPERTY_TENSOR_SUPPORT_DYNAMIC_DIMENSIONS); + print_property("Blockwise quantization", QNN_PROPERTY_TENSOR_SUPPORT_QUANTIZATION_ENCODING_BLOCK); + print_property("Blockwise quantization with expansion", QNN_PROPERTY_TENSOR_SUPPORT_QUANTIZATION_ENCODING_BLOCKWISE_EXPANSION); + print_property("Vector quantization", QNN_PROPERTY_TENSOR_SUPPORT_QUANTIZATION_ENCODING_VECTOR); + print_property("Tensor sparsity", QNN_PROPERTY_TENSOR_SUPPORT_SPARSITY); + print_property("Updateable application tensor", QNN_PROPERTY_TENSOR_SUPPORT_UPDATEABLE_APP_TENSORS); + print_property("Updateable native tensor", QNN_PROPERTY_TENSOR_SUPPORT_UPDATEABLE_NATIVE_TENSORS); + print_property("Updateable static tensor", QNN_PROPERTY_TENSOR_SUPPORT_UPDATEABLE_STATIC_TENSORS); + print_property("Qnn group device", QNN_PROPERTY_GROUP_DEVICE); +} + +void qnn_instance::htp_set_memory_grow_size(size_t size) { + QnnHtpPerfInfrastructure_MemoryConfig_t grow_size_config = { + .option = QNN_HTP_PERF_INFRASTRUCTURE_MEMORY_CONFIGOPTION_GROW_SIZE, + .memGrowSizeConfig = (uint32_t)size, + }; + + const QnnHtpPerfInfrastructure_MemoryConfig_t *memory_config[] = { + &grow_size_config, + nullptr, + }; + Qnn_ErrorHandle_t ret = _qnn_htp_perfinfra->setMemoryConfig(_qnn_htp_device_id, _qnn_htp_core_id, memory_config); + if (ret != QNN_SUCCESS) { + GGMLQNN_LOG_WARN("failed to set HTP memory config"); + } else { + GGMLQNN_LOG_INFO("succeed to set HTP memory config"); + } +} + +void qnn_instance::htp_enter_performance_mode() { + QnnHtpPerfInfrastructure_PowerConfig_t dcvs_v3_config = { + .option = QNN_HTP_PERF_INFRASTRUCTURE_POWER_CONFIGOPTION_DCVS_V3, + .dcvsV3Config = + { + .contextId = _qnn_htp_powerconfig_id, + + .setDcvsEnable = 1, + .dcvsEnable = 0, + + .powerMode = QNN_HTP_PERF_INFRASTRUCTURE_POWERMODE_PERFORMANCE_MODE, + + .setSleepLatency = 1, + .sleepLatency = 40, + + .setSleepDisable = 1, + .sleepDisable = 1, + + .setBusParams = 1, + .busVoltageCornerMin = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER, + .busVoltageCornerTarget = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER, + .busVoltageCornerMax = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER, + + .setCoreParams = 1, + .coreVoltageCornerMin = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER, + .coreVoltageCornerTarget = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER, + .coreVoltageCornerMax = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER, + }, + }; + + QnnHtpPerfInfrastructure_PowerConfig_t hmx_config = { + .option = QNN_HTP_PERF_INFRASTRUCTURE_POWER_CONFIGOPTION_HMX_V2, + .hmxV2Config = + { + .hmxPickDefault = 0, + .hmxVoltageCornerMin = DCVS_EXP_VCORNER_MAX, + .hmxVoltageCornerTarget = DCVS_EXP_VCORNER_MAX, + .hmxVoltageCornerMax = DCVS_EXP_VCORNER_MAX, + .hmxPerfMode = QNN_HTP_PERF_INFRASTRUCTURE_CLK_PERF_HIGH, + }, + }; + + QnnHtpPerfInfrastructure_PowerConfig_t rpc_ctrl_config = { + .option = QNN_HTP_PERF_INFRASTRUCTURE_POWER_CONFIGOPTION_RPC_CONTROL_LATENCY, + .rpcControlLatencyConfig = 100, + }; + + QnnHtpPerfInfrastructure_PowerConfig_t rpc_poll_config = { + .option = QNN_HTP_PERF_INFRASTRUCTURE_POWER_CONFIGOPTION_RPC_POLLING_TIME, + .rpcPollingTimeConfig = 9999, + }; + + const QnnHtpPerfInfrastructure_PowerConfig_t *power_configs[] = { + &dcvs_v3_config, + &hmx_config, + &rpc_ctrl_config, + &rpc_poll_config, + nullptr, + }; + Qnn_ErrorHandle_t ret = _qnn_htp_perfinfra->setPowerConfig(_qnn_htp_powerconfig_id, power_configs); + if (ret != QNN_SUCCESS) { + GGMLQNN_LOG_WARN("failed to set HTP power config"); + } else { + GGMLQNN_LOG_INFO("succeed to set HTP power config"); + } +} + +void qnn_instance::htp_set_n_hvx_threads(size_t n_threads) { + QnnHtpGraph_CustomConfig_t htp_hvx_thread_config = { + .option = QNN_HTP_GRAPH_CONFIG_OPTION_NUM_HVX_THREADS, + .numHvxThreads = n_threads, + }; + + QnnGraph_Config_t hvx_thread_config = { + .option = QNN_GRAPH_CONFIG_OPTION_CUSTOM, + .customConfig = &htp_hvx_thread_config, + }; + + const QnnGraph_Config_t * graph_configs[] = {&hvx_thread_config, nullptr}; + Qnn_ErrorHandle_t ret = _qnn_raw_interface.graphSetConfig(_qnn_graph_handle, graph_configs); + if (ret != QNN_SUCCESS) { + GGMLQNN_LOG_WARN("failed to set QNN graph config: set hvx threads %d", n_threads); + } else { + GGMLQNN_LOG_INFO("succeed to set QNN graph config: set hvx threads %d", n_threads); + } +} + static uint8_t * ggmlqnn_create_rpc_buffer(qnn_instance * instance, const ggml_tensor * ggml_tensor, Qnn_Tensor_t * qnn_tensor, bool b_copydata) { if (nullptr == instance || nullptr == ggml_tensor || nullptr == qnn_tensor) { GGMLQNN_LOG_WARN("invalid params\n"); @@ -2741,7 +3101,7 @@ static uint8_t * ggmlqnn_create_rpc_buffer(qnn_instance * instance, const ggml_t } static void ggmlqnn_print_tensors_info(const char * func_name, ggml_backend_qnn_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - //skip sanity check of params + //skip sanity check of params because of performance concern if (nullptr != func_name && nullptr != ctx) { GGMLQNN_LOG_DEBUG("call %s in dev %s\n", func_name, ctx->name); } @@ -2760,8 +3120,8 @@ static void ggmlqnn_print_tensors_info(const char * func_name, ggml_backend_qnn_ GGMLQNN_LOG_DEBUG("\n"); } -static void dump_op_info(const struct ggml_tensor * tensor) { - //skip sanity check of params +static void ggmlqnn_dump_op_info(const struct ggml_tensor * tensor) { + //skip sanity check of params because of performance concern const struct ggml_tensor * src0 = tensor->src[0]; struct ggml_tensor * src1 = tensor->src[1]; struct ggml_tensor * dst = const_cast(tensor); @@ -2780,12 +3140,12 @@ static Qnn_Tensor_t * ggmlqnn_create_general_tensor(const ggml_tensor * tensor, //ensure the tensor name is unique if (nullptr == name) { - snprintf(tensor_name, GGML_MAX_NAME, "tensor_%-8d", get_idx()); + snprintf(tensor_name, GGML_MAX_NAME, "tensor_%-8d", ggmqnn_get_tensoridx()); } else { - snprintf(tensor_name, GGML_MAX_NAME, "tensor_%s%-8d", name, get_idx()); + snprintf(tensor_name, GGML_MAX_NAME, "tensor_%s%-8d", name, ggmqnn_get_tensoridx()); } - GGMLQNN_LOG_DEBUG("init_tensor %d", get_idx()); - inc_idx(); + GGMLQNN_LOG_DEBUG("init_tensor %d", ggmqnn_get_tensoridx()); + ggmqnn_inc_tensoridx(); uint32_t reverse_dims[GGML_MAX_DIMS] = {}; uint32_t transpose_dims[GGML_MAX_DIMS] = {}; @@ -2806,7 +3166,7 @@ static Qnn_Tensor_t * ggmlqnn_create_general_tensor(const ggml_tensor * tensor, if (b_transpose) { GGML_ASSERT(tensor != nullptr); //ensure ggml_tensor is not nullptr for this special case - get_qnn_dimensions_from_ggml_dimensions(transpose_dims, reverse_dims, ggml_n_dims(tensor)); + ggmlqnn_get_qnn_dimensions_from_ggml_dimensions(transpose_dims, reverse_dims, ggml_n_dims(tensor)); tensor_dims = transpose_dims; #if 0 for (size_t idx = 0; idx < 4; idx++) { @@ -2890,7 +3250,7 @@ static Qnn_Tensor_t * ggmlqnn_create_compute_tensor(qnn_instance * instance, Qnn } // ================================================================================================= -// section-6: implementation of ggml-qnn backend +// section-8: implementation of ggml-qnn backend // ================================================================================================= //TODO: refine this function as it is a performance hotspot/bottleneck function static bool ggml_qnn_can_handle_op(const ggml_backend_qnn_context * ctx, const struct ggml_tensor * tensor) { @@ -2935,7 +3295,7 @@ static bool ggml_qnn_can_handle_op(const ggml_backend_qnn_context * ctx, const s GGML_UNUSED(ne1); if (tensor->op == GGML_OP_ADD) { - //dump_op_info(tensor); + //ggmlqnn_dump_op_info(tensor); if (!ggml_are_same_shape(src0, src1)) { return false; } @@ -2945,7 +3305,7 @@ static bool ggml_qnn_can_handle_op(const ggml_backend_qnn_context * ctx, const s } if (tensor->op == GGML_OP_MUL_MAT) { - //dump_op_info(tensor); + //ggmlqnn_dump_op_info(tensor); if (src0_rank != src1_rank) // make QNN SDK happy return false; if (src0_rank < 2) // QNN's limitation, make QNN SDK happy @@ -2969,7 +3329,9 @@ static bool ggml_qnn_can_handle_op(const ggml_backend_qnn_context * ctx, const s } if (tensor->op == GGML_OP_MUL) { - //dump_op_info(tensor); + //ggmlqnn_dump_op_info(tensor); + if (ctx->device == QNN_BACKEND_NPU) + return false; if ((src0_rank != 2) || (src1_rank != 2)) //TODO: 3D and 4D matrix return false; return (src0->type == GGML_TYPE_F32) @@ -3296,19 +3658,25 @@ static enum ggml_status ggml_backend_qnn_graph_compute(ggml_backend_t backend, s enum ggml_status result = GGML_STATUS_SUCCESS; ggml_backend_qnn_context * ctx = (ggml_backend_qnn_context *) backend->context; GGML_UNUSED(ctx); - - for (int i = 0; i < cgraph->n_nodes; i++) { - ggml_tensor * node = cgraph->nodes[i]; - if (ggml_is_empty(node) || node->op == GGML_OP_RESHAPE - || node->op == GGML_OP_TRANSPOSE || node->op == GGML_OP_VIEW - || node->op == GGML_OP_PERMUTE || node->op == GGML_OP_NONE) { - continue; - } - bool ok = ggml_qnn_compute_forward(backend, node); - if (!ok) { - GGMLQNN_LOG_DEBUG("%s: error: op not supported %s (%s)\n", - __func__, node->name, ggml_op_name(node->op)); + //GGMLQNN_LOG_DEBUG("device %d", ctx->device); + //GGMLQNN_LOG_DEBUG("cgraph->n_nodes %d", cgraph->n_nodes); + + if (0 == g_inference_approach) { + for (int i = 0; i < cgraph->n_nodes; i++) { + ggml_tensor * node = cgraph->nodes[i]; + if (ggml_is_empty(node) || node->op == GGML_OP_RESHAPE + || node->op == GGML_OP_TRANSPOSE || node->op == GGML_OP_VIEW + || node->op == GGML_OP_PERMUTE || node->op == GGML_OP_NONE) { + continue; + } + bool ok = ggml_qnn_compute_forward(backend, node); + if (!ok) { + GGMLQNN_LOG_DEBUG("%s: error: op not supported %s (%s)\n", __func__, node->name, ggml_op_name(node->op)); + } } + } else { + //offload entire cgraph to QNN CPU & GPU & NPU + return ggmlqnn_graph_compute(backend, cgraph); } return result; @@ -3331,8 +3699,8 @@ static const char * ggml_backend_qnn_device_get_description(ggml_backend_dev_t d return "unknown"; } if (0 == strncmp(ctx->name, "qnn-npu", 7)) { - const char * soc_info = qnn_get_socmodel_desc(ctx->socinfo.soc_model); - const char * htp_arch = qnn_get_htparch_desc(ctx->socinfo.htp_arch); + const char * soc_info = ggmlqnn_get_socmodel_desc(ctx->socinfo.soc_model); + const char * htp_arch = ggmlqnn_get_htparch_desc(ctx->socinfo.htp_arch); std::string dev_desc = std::string(ctx->desc) + std::string(soc_info) + "_" + std::string(htp_arch) + "," + std::string(ctx->socinfo.soc_desc); @@ -3353,12 +3721,12 @@ static void ggml_backend_qnn_device_get_memory(ggml_backend_dev_t dev, size_t * } if (QNN_BACKEND_CPU == ctx->device || QNN_BACKEND_GGML == ctx->device) { - *total = get_system_total_memory_in_bytes(); - *free = get_system_free_memory_in_bytes(); + *total = ggmlqnn_get_system_total_memory_in_bytes(); + *free = ggmlqnn_get_system_free_memory_in_bytes(); } else if (QNN_BACKEND_GPU == ctx->device) { //TODO: probe GPU info in Qualcomm Adreno GPU - *total = get_system_total_memory_in_bytes(); - *free = get_system_free_memory_in_bytes(); + *total = ggmlqnn_get_system_total_memory_in_bytes(); + *free = ggmlqnn_get_system_free_memory_in_bytes(); } else if (QNN_BACKEND_NPU == ctx->device) { size_t rpc_ion_memsize = ctx->instance->get_rpcmem_capacity(); size_t rpc_ion_usage = ctx->instance->get_rpcmem_usage(); @@ -3370,8 +3738,15 @@ static void ggml_backend_qnn_device_get_memory(ggml_backend_dev_t dev, size_t * } static enum ggml_backend_dev_type ggml_backend_qnn_device_get_type(ggml_backend_dev_t dev) { - GGML_UNUSED(dev); - return GGML_BACKEND_DEVICE_TYPE_ACCEL; + struct ggml_backend_qnn_context * ctx = static_cast(dev->context); + if (QNN_BACKEND_CPU == ctx->device) + return GGML_BACKEND_DEVICE_TYPE_ACCEL; + else if (QNN_BACKEND_GPU == ctx->device) + return GGML_BACKEND_DEVICE_TYPE_GPU; + else if (QNN_BACKEND_NPU == ctx->device) + return GGML_BACKEND_DEVICE_TYPE_ACCEL; + else + return GGML_BACKEND_DEVICE_TYPE_CPU; } static void ggml_backend_qnn_device_get_props(ggml_backend_dev_t dev, @@ -3605,6 +3980,8 @@ ggml_backend_t ggml_backend_qnn_init(size_t device, const char * qnn_lib_path) { return g_qnn_mgr[device].backend; } + ggmlqnn_load_cfg(); + #if defined(__ANDROID__) std::string path = qnn_lib_path; if (QNN_BACKEND_NPU == device) { @@ -3671,7 +4048,7 @@ ggml_backend_t ggml_backend_qnn_init(size_t device, const char * qnn_lib_path) { GGML_BACKEND_DL_IMPL(ggml_backend_qnn_reg) // ================================================================================================= -// section-7: offload GGML op to QNN backend +// section-9: general approach: offload GGML op to QNN backend // ================================================================================================= static inline uint32_t ggmlqnn_get_tensor_data_size(const ggml_tensor * tensor) { /* @@ -3702,13 +4079,6 @@ static inline bool ggmlqnn_is_valid_params(ggml_backend_qnn_context * ctx, const return true; } -#define GGMLQNN_CHECK_PARAMS(ctx, src0, src1, dst) \ - do { \ - if (!ggmlqnn_is_valid_params((ctx), (src0), (src1), (dst))) { \ - return; \ - } \ - } while (0) - /* * provide a general skeleton to offload ggml op to QNN backend: a single node contains 2 input * tensor and 1 output tensor @@ -3750,10 +4120,10 @@ void ggml_qnn_general_node(ggml_backend_qnn_context * ctx, ggml_tensor * op) { p_tensor1 = tensor[1]; p_tensor2 = tensor[2]; } else { - GGMLQNN_LOG_DEBUG("graph name %s", graph_name.c_str()); + GGMLQNN_LOG_INFO("graph name %s", graph_name.c_str()); GGML_ASSERT(instance->get_device_id() == ctx->device); //create QNN graph - error = instance->init_qnn_graph(graph_name, static_cast(ctx->device), 8); + error = instance->init_qnn_graph(graph_name, static_cast(ctx->device), 8, 4); if (QNN_SUCCESS != error) { GGMLQNN_LOG_WARN("can't create qnn graph handle with graph name %s, error = %d\n", graph_name.c_str(), error); return; @@ -3844,10 +4214,7 @@ void ggml_qnn_general_node(ggml_backend_qnn_context * ctx, ggml_tensor * op) { /* * this function is AI-assisted code from Grok 3 for purpose of offload 4d matrix mulmat to QNN backend - * UT in ggml-qnn-ut.cpp passed: - * ./scripts/build-run-android.sh run_ut_mulmat 0 - * ./scripts/build-run-android.sh run_ut_mulmat 1 - * ./scripts/build-run-android.sh run_ut_mulmat 2 + * various UT has verified and succeed but failed in CT of test-backend-ops * * the logic of ggml_qnn_mul_mat_4d is similar to ggml_qnn_mul_mat but much more complicated * than ggml_qnn_mul_mat, so it's a standalone function. @@ -4130,12 +4497,13 @@ void ggml_qnn_mul_mat(ggml_backend_qnn_context * ctx, ggml_tensor * op) { p_tensor2_transpose = tensors[4]; } else { //create QNN graph - GGMLQNN_LOG_DEBUG("graph name %s", graph_name.c_str()); - error = qnn_raw_interface.graphCreate(instance->get_qnn_context_handle(), graph_name.c_str(), nullptr, &graph_handle); + GGMLQNN_LOG_INFO("graph name %s", graph_name.c_str()); + error = instance->init_qnn_graph(graph_name, static_cast(ctx->device), 8, 4); if (QNN_SUCCESS != error) { GGMLQNN_LOG_WARN("can't create qnn graph handle with graph name %s, error = %d\n", graph_name.c_str(), error); return; } + graph_handle = instance->get_qnn_graph_handle(); //create computational tensor p_tensor0 = GQCGT(src0, nullptr, QNN_TENSOR_TYPE_APP_WRITE, QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0); @@ -4335,3 +4703,65 @@ void ggml_qnn_rope(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { GGML_UNUSED(ctx); GGML_UNUSED(dst); } + +// ================================================================================================= +// section-10: second approach: mapping ggml computational cgraph to QNN graph +// ================================================================================================= +// details: https://github.com/ggml-org/llama.cpp/pull/12326#issuecomment-2712838649 +// ref: https://github.com/kantv-ai/kantv/blob/kantv-poc-with-qnn/core/ggml/jni/Inception_v3.cpp#L20634 +// TODO: mapping entire ggml cgraph to a single QNN graph +static enum ggml_status ggmlqnn_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph) { + enum ggml_status ggml_result = GGML_STATUS_SUCCESS; + Qnn_ErrorHandle_t qnn_error = QNN_SUCCESS; + qnn_perf op_perf = qnn_perf("ggmlqnn_graph_compute"); + qnn_instance * instance = nullptr; + Qnn_GraphHandle_t graph_handle = nullptr; + ggml_backend_qnn_context * ctx = (ggml_backend_qnn_context *) backend->context; + instance = ctx->instance; + QNN_INTERFACE_VER_TYPE qnn_raw_interface = ctx->raw_interface; + op_perf.start(); + + //now we got the entire ggml cgraph + GGMLQNN_LOG_DEBUG("qnn device %d(%s)", ctx->device, ggml_backend_qnn_get_devname(ctx->device)); + GGMLQNN_LOG_DEBUG("cgraph->n_nodes %d", cgraph->n_nodes); + int num_nodes = std::min(5, cgraph->n_nodes); + //for (int i = 0; i < cgraph->n_nodes; i++) { + for (int i = 0; i < num_nodes; i++) { + ggml_tensor * node = cgraph->nodes[i]; + GGMLQNN_LOG_DEBUG("%s: op %s (%s)\n", __func__, node->name, ggml_op_name(node->op)); + } + + //now we'll offload the entire ggml cgraph to a single opcfg QNN graph + std::string graph_name; + ggmlqnn_get_graphkey_from_cgraph(cgraph, graph_name); + if (instance->_qnn_graph_map.find(graph_name) != instance->_qnn_graph_map.end()) { + //retrieve computational resource from cached QNN graph + qnn_res_t & graph_item = instance->_qnn_graph_map[graph_name]; + graph_handle = std::get<0>(graph_item); + } else { + //create QNN graph + GGMLQNN_LOG_INFO("graph name %s", graph_name.c_str()); + qnn_error = instance->init_qnn_graph(graph_name, static_cast(ctx->device), 8, 4); + if (QNN_SUCCESS != qnn_error) { + GGMLQNN_LOG_WARN("can't create qnn graph handle with graph name %s, error = %d(%s)\n", graph_name.c_str(), qnn_error, + ggmlqnn_get_qnnerror_string(qnn_error)); + return ggml_result; + } + graph_handle = instance->get_qnn_graph_handle(); + //TODO: compose a single opcfg QNN graph + + //TODO: finalize QNN graph + //CHECK_QNN_API(qnn_error, qnn_raw_interface.graphFinalize(graph_handle, nullptr, nullptr)); + + //cache QNN graph + qnn_tensors_t ggml_op_mulmat_tensors; + ggml_op_mulmat_tensors.reserve(0); + auto graph_item = std::make_tuple(graph_handle, ggml_op_mulmat_tensors); + instance->_qnn_graph_map[graph_name] = graph_item; + } + //exec QNN graph + + GGMLQNN_LOG_DEBUG("the second inference approach \"mapping cgraph to QNN graph\" is actually not supported now"); + + return ggml_result; +} diff --git a/scripts/build-run-android.sh b/scripts/build-run-android.sh index 5b5e55aa2f7b6..393f4d458f41b 100755 --- a/scripts/build-run-android.sh +++ b/scripts/build-run-android.sh @@ -14,7 +14,9 @@ GGUF_MODEL_NAME=/sdcard/qwen1_5-1_8b-chat-q4_0.gguf #https://www.qualcomm.com/developer/software/qualcomm-ai-engine-direct-sdk #https://developer.qualcomm.com/software/hexagon-dsp-sdk/tools QNN_SDK_URL=https://www.qualcomm.com/developer/software/qualcomm-ai-engine-direct-sdk -QNN_SDK_PATH=/opt/qcom/aistack/qairt/2.31.0.250130/ +QNN_SDK_INSTALL_PATH=/opt/qcom/aistack/qairt/ +QNN_SDK_VERSION=2.32.0.250228 +QNN_SDK_PATH=${QNN_SDK_INSTALL_PATH}/${QNN_SDK_VERSION} #default is QNN NPU qnnbackend=2 @@ -32,11 +34,35 @@ function show_pwd() } -function check_qnn_sdk() +function check_and_download_qnn_sdk() { + is_qnn_sdk_exist=1 + if [ ! -d ${QNN_SDK_PATH} ]; then - echo -e "QNN_SDK_PATH ${QNN_SDK_PATH} not exist, pls check or download it from ${QNN_SDK_URL}...\n" - exit 1 + echo -e "QNN_SDK_PATH ${QNN_SDK_PATH} not exist, download it from ${QNN_SDK_URL}...\n" + is_qnn_sdk_exist=0 + fi + + if [ ! -f ${QNN_SDK_PATH}/sdk.yaml ]; then + is_qnn_sdk_exist=0 + fi + + if [ ${is_qnn_sdk_exist} -eq 0 ]; then + echo "sudo mkdir -p ${QNN_SDK_INSTALL_PATH}" + sudo mkdir -p ${QNN_SDK_INSTALL_PATH} + if [ ! -f v${QNN_SDK_VERSION}.zip ]; then + wget --no-config --quiet --show-progress -O v${QNN_SDK_VERSION}.zip https://softwarecenter.qualcomm.com/api/download/software/sdks/Qualcomm_AI_Runtime_Community/All/${QNN_SDK_VERSION}/v${QNN_SDK_VERSION}.zip + fi + unzip v${QNN_SDK_VERSION}.zip + if [ $? -ne 0 ]; then + printf "failed to download Qualcomm QNN SDK to %s \n" "${QNN_SDK_PATH}" + exit 1 + fi + sudo mv qairt/${QNN_SDK_VERSION} ${QNN_SDK_INSTALL_PATH}/ + printf "Qualcomm QNN SDK saved to ${QNN_SDK_PATH} \n\n" + sudo rm -rf qairt + else + printf "Qualcomm QNN SDK already exist:${QNN_SDK_PATH} \n\n" fi } @@ -75,7 +101,7 @@ function check_and_download_ndk() function build_arm64 { - cmake -H. -B./out/android -DCMAKE_BUILD_TYPE=Release -DGGML_USE_QNN=ON -DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK}/build/cmake/android.toolchain.cmake -DANDROID_ABI=arm64-v8a -DANDROID_PLATFORM=latest -DCMAKE_C_FLAGS=-march=armv8.7-a -DGGML_QNN=ON -DGGML_QNN_SDK_PATH=${QNN_SDK_PATH} + cmake -H. -B./out/android -DCMAKE_BUILD_TYPE=Release -DGGML_OPENMP=OFF -DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK}/build/cmake/android.toolchain.cmake -DANDROID_ABI=arm64-v8a -DANDROID_PLATFORM=latest -DCMAKE_C_FLAGS=-march=armv8.7-a -DGGML_QNN=ON -DGGML_QNN_SDK_PATH=${QNN_SDK_PATH} cd out/android make -j16 show_pwd @@ -97,11 +123,14 @@ function check_qnn_libs() { #reuse the cached qnn libs on Android phone adb shell ls ${REMOTE_PATH}/libQnnCpu.so + adb shell ls ${REMOTE_PATH}/libQnnGpu.so + adb shell ls ${REMOTE_PATH}/libQnnHtp.so if [ $? -eq 0 ]; then printf "QNN libs already exist on Android phone\n" else update_qnn_libs fi + update_qnn_cfg } @@ -119,11 +148,17 @@ function update_qnn_libs() } +function update_qnn_cfg() +{ + adb push ./scripts/ggml-qnn.cfg ${REMOTE_PATH}/ +} + + function build_ggml_qnn() { show_pwd check_and_download_ndk - check_qnn_sdk + check_and_download_qnn_sdk dump_vars remove_temp_dir build_arm64 @@ -140,21 +175,20 @@ function prepare_run_on_phone() check_qnn_libs - if [ -f ./out/android/bin/libggml-qnn.so ]; then + if [ -f ./out/android/bin/libggml-cpu.so ]; then adb push ./out/android/bin/*.so ${REMOTE_PATH}/ fi adb push ./out/android/bin/${program} ${REMOTE_PATH}/ adb shell chmod +x ${REMOTE_PATH}/${program} } - function run_llamacli() { prepare_run_on_phone llama-cli adb shell "cd ${REMOTE_PATH} \ && export LD_LIBRARY_PATH=${REMOTE_PATH} \ - && ${REMOTE_PATH}/llama-cli -mg ${qnnbackend} -no-cnv -m ${GGUF_MODEL_NAME} -p \"introduce the movie Once Upon a Time in America briefly.\n\"" + && ${REMOTE_PATH}/llama-cli -mg ${qnnbackend} -ngl 99 -no-cnv -m ${GGUF_MODEL_NAME} -p \"introduce the movie Once Upon a Time in America briefly.\n\"" } @@ -213,7 +247,6 @@ function run_test-op() } - function print_oplist() { oplist="DUP @@ -302,7 +335,7 @@ function show_usage() echo " $0 build" echo " $0 updateqnnlib" echo " $0 run_testops" - echo " $0 run_testop [ADD/MUL/MUL_MAT/...(op from print_oplist)] [0 (QNN_CPU) / 1 (QNN_GPU) / 2 (QNN_NPU)]" + echo " $0 run_testop [ADD/MUL/MUL_MAT......(op from print_oplist)] [0 (QNN_CPU) / 1 (QNN_GPU) / 2 (QNN_NPU)]" echo " $0 run_llamacli 0 (QNN_CPU) / 1 (QNN_GPU) / 2 (QNN_NPU) / 3 (ggml)" echo " $0 run_llamabench 0 (QNN_CPU) / 1 (QNN_GPU) / 2 (QNN_NPU) / 3 (ggml)" @@ -312,7 +345,8 @@ function show_usage() show_pwd -check_qnn_sdk +check_and_download_ndk +check_and_download_qnn_sdk if [ $# == 0 ]; then show_usage @@ -343,20 +377,22 @@ elif [ $# == 1 ]; then fi elif [ $# == 2 ]; then qnnbackend=$2 + if [ ${qnnbackend} -gt 3 ]; then + show_usage + exit 1 + fi + if [ "$1" == "run_llamacli" ]; then run_llamacli exit 0 elif [ "$1" == "run_llamabench" ]; then run_llamabench exit 0 - exit 0 - else - show_usage - exit 1 fi elif [ $# == 3 ]; then - #opname can be found via print_oplist: opname=$2 +#TODO: check opname in oplist +#opname can be found via print_oplist: qnnbackend=$3 if [ ${qnnbackend} -gt 3 ]; then diff --git a/scripts/ggml-qnn.cfg b/scripts/ggml-qnn.cfg new file mode 100644 index 0000000000000..5796e613ff2af --- /dev/null +++ b/scripts/ggml-qnn.cfg @@ -0,0 +1,9 @@ +[general] +# enable/disable QNN's internal log +print_qnn_internal_log = 0 +# 0: general approach,similar to ggml-sycl or ggml-cann +# 1: mapping entire ggml cgraph to QNN graph +inference_approach = 0 + +[npu] +npu_inference_datatype = "fp16" diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index cd7ca4310f73d..7a158d6024d78 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -137,7 +137,6 @@ llama_target_and_test(test-chat-template.cpp) # llama_target_and_test(test-opt.cpp) # SLOW llama_target_and_test(test-gguf.cpp) llama_target_and_test(test-backend-ops.cpp) -llama_target_and_test(ggml-qnn-ut.cpp) llama_target_and_test(test-model-load-cancel.cpp LABEL "model") llama_target_and_test(test-autorelease.cpp LABEL "model") diff --git a/tests/ggml-qnn-ut.cpp b/tests/ggml-qnn-ut.cpp deleted file mode 100644 index 75d941263b82c..0000000000000 --- a/tests/ggml-qnn-ut.cpp +++ /dev/null @@ -1,480 +0,0 @@ -/* - * Copyright (c) 2023-2024 The ggml authors - * - * implementation of self-made Android command line tool for verify ggml-qnn backend - * this file will help you to understand fundamental principle of ggml and ggml-qnn backend - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#if defined(__ANDROID__) || defined(__linux__) -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#endif - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "ggml.h" -#include "ggml-cpu.h" -#include "ggml-alloc.h" -#include "ggml-backend.h" -#include "ggml-qnn.h" - -static void tensor_dump(const ggml_tensor * tensor, const char * name); - -#define TENSOR_DUMP(tensor) tensor_dump(tensor, #tensor) - -static bool ggml_graph_compute_helper( - struct ggml_backend * backend, - struct ggml_cgraph * graph, - std::vector & buf, - int n_threads, - ggml_abort_callback abort_callback, - void * abort_callback_data) { - struct ggml_cplan plan = ggml_graph_plan(graph, n_threads, NULL); - - plan.abort_callback = abort_callback; - plan.abort_callback_data = abort_callback_data; - - if (plan.work_size > 0) { - buf.resize(plan.work_size); - plan.work_data = buf.data(); - } - - if (nullptr != backend) - return ggml_backend_graph_compute(backend, graph) == GGML_STATUS_SUCCESS; - else - return ggml_graph_compute(graph, &plan); -} - - -static void tensor_dump_elements(const ggml_tensor * tensor) { - float value = 0; - std::ostringstream tmposs; - if (tensor->type == GGML_TYPE_F32) { - for (int h = 0; h < tensor->ne[3]; h++) { - for (int i = 0; i < tensor->ne[2]; i++) { - for (int j = 0; j < tensor->ne[1]; j++) { - for (int k = 0; k < tensor->ne[0]; k++) { - value = ((float *) tensor->data)[h * tensor->ne[2] + i * tensor->ne[1] + - j * tensor->ne[0] + k]; - tmposs << std::setw(8) << std::fixed << std::setprecision(2) << value - << " "; - } - if (strlen(tmposs.str().c_str()) <= (4096 - 96)) { - printf("%s\n", tmposs.str().c_str()); - } - tmposs.clear(); - tmposs.str(""); - } - } - } - } - - printf("\n"); -} - - -static void tensor_dump(const ggml_tensor * tensor, const char * name) { - printf("dump ggml tensor %s(%s)\n", name, tensor->name); - printf("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64", nb = (%5zi, %5zi, %5zi, %5zi)\n", - name, - tensor->type, ggml_type_name(tensor->type), - tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3], - tensor->nb[0], tensor->nb[1], tensor->nb[2], tensor->nb[2]); - tensor_dump_elements(tensor); - - printf("\n"); -} - - -static uint32_t get_tensor_rank(const ggml_tensor * tensor) { - uint32_t rank = 0; - for (int i = 0; i < GGML_MAX_DIMS; i++) { - if ((0 != tensor->ne[i]) && (1 != tensor->ne[i])) { - rank++; - } - } - return rank; -} - - -static uint32_t get_tensor_data_size(const ggml_tensor * tensor) { - return ggml_nbytes(tensor); -} - - -//ref: https://github.com/ggerganov/llama.cpp/blob/master/tests/test-backend-ops.cpp#L20 -static void init_tensor_uniform(ggml_tensor * tensor, float min = -1.0f, float max = 1.0f) { - // static RNG initialization (revisit if n_threads stops being constant) - static const size_t n_threads = std::thread::hardware_concurrency(); - static std::vector generators = []() { - std::random_device rd; - std::vector vec; - vec.reserve(n_threads); - //for (size_t i = 0; i < n_threads; i++) { vec.emplace_back(1234 + i); } // fixed seed - for (size_t i = 0; i < n_threads; i++) { vec.emplace_back(rd()); } - return vec; - }(); - - size_t size = ggml_nelements(tensor); - std::vector data(size); - - auto init_thread = [&](size_t ith, size_t start, size_t end) { - std::uniform_real_distribution distribution(min, max); - for (size_t i = start; i < end; i++) { - data[i] = distribution(generators[ith]); - } - }; - - std::vector threads; - threads.reserve(n_threads); - for (size_t i = 0; i < n_threads; i++) { - size_t start = i*size/n_threads; - size_t end = (i+1)*size/n_threads; - threads.emplace_back(init_thread, i, start, end); - } - for (auto & t : threads) { - t.join(); - } - if (tensor->type == GGML_TYPE_F32 || tensor->type == GGML_TYPE_I32) { - ggml_backend_tensor_set(tensor, data.data(), 0, size * sizeof(float)); - } else if (ggml_is_quantized(tensor->type) || tensor->type == GGML_TYPE_F16 || tensor->type == GGML_TYPE_BF16) { - GGML_ASSERT(size % ggml_blck_size(tensor->type) == 0); - std::vector dataq(ggml_row_size(tensor->type, size)); - std::vector imatrix(tensor->ne[0], 1.0f); // dummy importance matrix - const float * im = imatrix.data(); - if (!ggml_quantize_requires_imatrix(tensor->type)) { - // when the imatrix is optional, we want to test both quantization with and without imatrix - // use one of the random numbers to decide - if (data[0] > 0.5f*(min + max)) { - im = nullptr; - } - } - ggml_quantize_chunk(tensor->type, data.data(), dataq.data(), 0, size/tensor->ne[0], tensor->ne[0], im); - GGML_ASSERT(ggml_validate_row_data(tensor->type, dataq.data(), dataq.size())); - ggml_backend_tensor_set(tensor, dataq.data(), 0, dataq.size()); - } else if (tensor->type == GGML_TYPE_I8 || tensor->type == GGML_TYPE_I16 || tensor->type == GGML_TYPE_I32) { - // This is going to create some weird integers though. - ggml_backend_tensor_set(tensor, data.data(), 0, ggml_nbytes(tensor)); - } else { - GGML_ASSERT(false); - } -} - - -//ref: https://github.com/ggerganov/llama.cpp/blob/master/tests/test-backend-ops.cpp#L310 -static void initialize_tensors(ggml_context * ctx) { - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) { - init_tensor_uniform(t); - } -} - - -static void show_usage() { - printf(" " \ - "\nUsage: ggml-qnn-ut [options]\n" \ - "\n" \ - "Options:\n" \ - " -t GGML_OP_ADD / GGML_OP_MUL / GGML_OP_MULMAT\n" \ - " -b 0(QNN_CPU) 1(QNN_GPU) 2(QNN_NPU) 3(QNN_GGML)\n" \ - " ?/h print usage information\n\n" - ); -} - - -int main(int argc, char * argv[]) { - int64_t n_begin_time = 0LL; - int64_t n_end_time = 0LL; - int64_t n_duration = 0LL; - size_t ctx_size = 0; - int sizey = 4; - int sizex = 4; - int num_threads = 4; - int n_backend_type = QNN_BACKEND_CPU; - int n_ggml_op_type = GGML_OP_ADD; - - struct ggml_context * ctx = nullptr; - struct ggml_cgraph * gf = nullptr; - struct ggml_tensor * src0 = nullptr; - struct ggml_tensor * src1 = nullptr; - struct ggml_tensor * dst = nullptr; - ggml_backend_t backend = nullptr; - ggml_backend_buffer_t buffer= nullptr; - ggml_type qtype = GGML_TYPE_F32; - //ggml_type qtype = GGML_TYPE_Q4_0; - std::vector work_buffer; - - for (int i = 1; i < argc; i++) { - if (0 == strcmp(argv[i], "-t")) { - if (i + 1 < argc) { - if (0 == memcmp(argv[i + 1], "GGML_OP_ADD", 11)) { - n_ggml_op_type = GGML_OP_ADD; - } else if (0 == memcmp(argv[i + 1], "GGML_OP_MUL_MAT", 15)) { - n_ggml_op_type = GGML_OP_MUL_MAT; - } else if (0 == memcmp(argv[i + 1], "GGML_OP_MUL", 11)) { - n_ggml_op_type = GGML_OP_MUL; - } else { - show_usage(); - return 1; - } - i++; - } - } else if (0 == strcmp(argv[i], "-b")) { - if (i + 1 < argc) { - int backend = atoi(argv[i + 1]); - if (backend <= QNN_BACKEND_GGML) - n_backend_type = backend; - else { - show_usage(); - return 1; - } - i++; - } - } else { - show_usage(); - return 1; - } - } - - printf("Testing %zu devices\n\n", ggml_backend_dev_count()); - for (size_t i = 0; i < ggml_backend_dev_count(); i++) { - ggml_backend_dev_t dev = ggml_backend_dev_get(i); - - printf("Backend %zu/%zu: %s\n", i + 1, ggml_backend_dev_count(), - ggml_backend_dev_name(dev)); - - if (ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_CPU) { - printf(" Skipping CPU backend\n"); - continue; - } - - backend = ggml_backend_dev_init(dev, reinterpret_cast(i)); - GGML_ASSERT(backend != NULL); - if (backend != nullptr) { - printf("%s: initialize %s backend\n", __func__, ggml_backend_dev_name(dev)); - } - - printf(" Device description: %s\n", ggml_backend_dev_description(dev)); - size_t free, total; - ggml_backend_dev_memory(dev, &free, &total); - printf(" Device memory: %zu MB (%zu MB free)\n", total / 1024 / 1024, free / 1024 / 1024); - printf("\n"); - } - - ggml_backend_t backend_cpu = nullptr; - backend_cpu = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr); - if (nullptr == backend_cpu) { - printf("failed to initialize cpu backend\n"); - exit(1); - } else { - printf("succeed to initialize cpu backend\n"); - } - - printf("ggml op:%d(%s)", n_ggml_op_type, ggml_op_name((enum ggml_op) n_ggml_op_type)); - - n_begin_time = ggml_time_us(); - srand(time(NULL)); - - ctx_size += 1024 * 1024 * 32; - printf("Allocating Memory of size %zi bytes, %zi MB\n", ctx_size, - (ctx_size / 1024 / 1024)); - - struct ggml_init_params params = { - /*.mem_size =*/ ctx_size, - /*.mem_buffer =*/ NULL, - /* no_alloc =*/ 0 - }; - - if (n_backend_type != QNN_BACKEND_GGML) { - params.no_alloc = true; - } - - ctx = ggml_init(params); - if (!ctx) { - printf("ggml_init() failed\n"); - return 2; - } - - printf("creating new tensors\n"); - printf("ggml_blck_size(%s) %ld\n", ggml_type_name(qtype), ggml_blck_size(qtype)); - printf("ggml_type_size(%s) %ld\n", ggml_type_name(qtype), ggml_type_size(qtype)); - if (qtype != GGML_TYPE_F32) { - sizex = ggml_blck_size(qtype); - } - - if (n_ggml_op_type == GGML_OP_ADD) { - src0 = ggml_new_tensor_2d(ctx, qtype, sizey, sizex); - src1 = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, sizey, sizex); - } else { - //verify 2D matrix - //src0 = ggml_new_tensor_2d(ctx, qtype, 128, 64); - //src1 = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 128, 2); - //verify 3D matrix - //src0 = ggml_new_tensor_3d(ctx, qtype, 128, 64, 8); - //src1 = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, 128, 2, 8); - //verify 4D matrix -#if 1 //ok - src0 = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, 256, 16, 3, 2); - src1 = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, 256, 1, 6, 4); -#else //ok - src0 = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, 256, 16, 3, 2); - src1 = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, 256, 16, 3, 2); -#endif - } - - ggml_set_input(src0); - ggml_set_input(src1); - switch (n_ggml_op_type) { - case GGML_OP_ADD: - dst = ggml_add(ctx, src0, src1); - break; - case GGML_OP_MUL: - dst = ggml_mul(ctx, src0, src1); - break; - case GGML_OP_MUL_MAT: - dst = ggml_mul_mat(ctx, src0, src1); - break; - default: - printf("ggml op %d(%s) not supported", n_ggml_op_type, - ggml_op_name((enum ggml_op) n_ggml_op_type)); - ggml_free(ctx); - ggml_backend_free(backend); - return 3; - } - - ggml_set_output(dst); - -#ifdef GGML_USE_QNN - if (n_backend_type != QNN_BACKEND_GGML) { - printf("init QNN backend %d\n", n_backend_type); - //re-init again - backend = ggml_backend_qnn_init(n_backend_type, "/data/local/tmp/"); - if (nullptr == backend) { - printf("create qnn backend %d(%s) failed\n", n_backend_type, ggml_backend_qnn_get_devname(n_backend_type)); - return 1; - } else { - printf("create qnn backend %d(%s) succeed\n", n_backend_type, ggml_backend_qnn_get_devname(n_backend_type)); - } - - //buffer = ggml_backend_alloc_ctx_tensors(ctx, backend); - ggml_backend_buffer_type_t buft = ggml_backend_get_default_buffer_type(backend); - buffer = ggml_backend_alloc_ctx_tensors_from_buft(ctx, buft); - if (!buffer) { - printf("%s: failed to allocate backend buffer\n", __func__); - ggml_free(ctx); - ggml_backend_free(backend); - return 4; - } - } else { - printf("init default cpu backend\n"); - backend = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr); - } -#endif - - printf("creating compute graph\n"); - gf = ggml_new_graph(ctx); - ggml_build_forward_expand(gf, dst); - - if (qtype == GGML_TYPE_F32) { - if (n_backend_type != QNN_BACKEND_GGML) { - initialize_tensors(ctx); - } else { - ggml_set_f32(src0, (rand() % 100 + 1)); - ggml_set_f32(src1, (rand() % 100 + 1)); - ggml_set_f32(dst, 0.0f); - } - //for compare compute result between cpu backend and QNN backend - ggml_set_f32(src0, 1.0f); - ggml_set_f32(src1, 2.0f); - ggml_set_f32(dst, 0.0f); - } else { - initialize_tensors(ctx); - } - - ggml_graph_compute_helper(backend, gf, work_buffer, num_threads, nullptr, nullptr); - if (get_tensor_data_size(dst) < (100 * 100)) { - printf("dump result tensors:\n"); - TENSOR_DUMP(src0); - TENSOR_DUMP(src1); - TENSOR_DUMP(dst); - } else { - printf("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", - src0->name, - src0->type, ggml_type_name(src0->type), src0->ne[0], src0->ne[1], src0->ne[2], - src0->nb[0], src0->nb[1], src0->nb[2]); - printf("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", - src1->name, - src1->type, ggml_type_name(src1->type), src1->ne[0], src1->ne[1], src1->ne[2], - src1->nb[0], src1->nb[1], src1->nb[2]); - printf("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", - dst->name, - dst->type, ggml_type_name(dst->type), dst->ne[0], dst->ne[1], dst->ne[2], dst->nb[0], - dst->nb[1], dst->nb[2]); - } - //TENSOR_DUMP(dst); - - ggml_free(ctx); - ggml_backend_buffer_free(buffer); - ggml_backend_free(backend); - - n_end_time = ggml_time_us(); - n_duration = (n_end_time - n_begin_time) / 1000; -#ifdef GGML_USE_QNN - printf("duration of ut GGML_OP_%s using QNN backend %s: %ld milliseconds\n", ggml_op_name((enum ggml_op)n_ggml_op_type), ggml_backend_qnn_get_devname(n_backend_type), n_duration); -#endif - - return 0; -} From bc0342f30c373860116627e3cea7a3c1854d7a31 Mon Sep 17 00:00:00 2001 From: zhouwg Date: Tue, 18 Mar 2025 22:17:00 +0800 Subject: [PATCH 64/76] ggml-qnn: rebase to upstream --- CMakeLists.txt | 11 + ggml/src/ggml-qnn/ggml-qnn.cpp | 2311 ++++++++++++++++++-------------- scripts/build-run-android.sh | 76 +- scripts/ggml-qnn.cfg | 21 +- 4 files changed, 1355 insertions(+), 1064 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 23cfbce5ae566..44202c009aef8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,6 +7,16 @@ set(CMAKE_WARN_UNUSED_CLI YES) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) +if(CMAKE_SYSTEM_NAME STREQUAL "Android") + set(TARGET_SNAPDRAGON8GEN3 ON) + if(TARGET_SNAPDRAGON8GEN3) + #works fine on Snapdragon 8Gen3 with 1.5x(45+ tokens/second)-3x(70+ tokens/second) performance gain through the default ggml backend + add_definitions(-march=armv8.7-a) + add_definitions(-mcpu=cortex-x1) + add_definitions(-mtune=cortex-x1) + endif() +endif() + if (NOT XCODE AND NOT MSVC AND NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE Release CACHE STRING "Build type" FORCE) set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo") @@ -119,6 +129,7 @@ llama_option_depr(WARNING LLAMA_RPC GGML_RPC) llama_option_depr(WARNING LLAMA_SYCL GGML_SYCL) llama_option_depr(WARNING LLAMA_SYCL_F16 GGML_SYCL_F16) llama_option_depr(WARNING LLAMA_CANN GGML_CANN) +llama_option_depr(WARNING LLAMA_QNN GGML_QNN) if (NOT MSVC) if (LLAMA_SANITIZE_THREAD) diff --git a/ggml/src/ggml-qnn/ggml-qnn.cpp b/ggml/src/ggml-qnn/ggml-qnn.cpp index 7c3477094ea9f..834af1e08e30f 100644 --- a/ggml/src/ggml-qnn/ggml-qnn.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn.cpp @@ -14,13 +14,16 @@ * section-6 QNN helper function * section-7 ggml-qnn backend helper function / class * section-8 implementation of ggml-qnn backend according to ggml's backend subsystem - * section-9 implementation of offload ggml op to QNN backend - * section-10 illustrate why the second approach is actual an fake at the moment + * section-9 implementation of general approach or the first tech approach + * section-10 implementation of the second tech approach:mapping the entire ggml cgraph to a single QNN graph * * currently provide following ggml op' QNN backend implementation: - * - GGML_OP_ADD: this is a simple skeleton, can expand other ggml ops according to expertise - * - GGML_OP_MUL: this is a simple skeleton, can expand other ggml ops according to expertise - * - GGML_OP_MUL_MAT:this is a complicated skeleton, can expand other complex ggml ops accordingly + * - GGML_OP_ADD/GGML_OP_SUB/GGML_OP_MUL/GGML_OP_DIV: + * this is a simple skeleton, can expand other ggml ops according to expertise + * - GGML_OP_LOG/GGML_OP_SQRT: + * this is a simple skeleton, can expand other ggml ops according to expertise + * - GGML_OP_MUL_MAT: + * this is a complicated skeleton, can expand other complex ggml ops accordingly * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to @@ -108,90 +111,70 @@ #include "ggml-backend-impl.h" // ================================================================================================= -// section-1: forward/prototype declaration +// section-1: forward/prototype declaration, macro // ================================================================================================= class qnn_instance; +struct qnn_parameter; struct ggml_backend_qnn_context; -typedef void (* ggmlqnn_op_func_t)(ggml_backend_qnn_context * ctx, ggml_tensor * op); -static Qnn_Tensor_t * ggmlqnn_create_general_tensor(const ggml_tensor * tensor, const char * name, - Qnn_TensorType_t qnn_tensor_type, - Qnn_DataType_t qnn_data_type, - uint32_t rank, uint32_t * dims, - void * data, uint32_t data_size, - bool b_transpose = false); -static enum ggml_status ggmlqnn_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph); -static void ggmlqnn_log_internal(ggml_log_level level, const char * file, const char * func, int line, const char * format, ...); -static inline bool ggmlqnn_is_valid_params(ggml_backend_qnn_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); - -//op functions: -//done -static void ggml_qnn_general_node(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -static void ggml_qnn_mul_mat(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -//todo -static void ggml_qnn_repeat(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -static void ggml_qnn_div(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -static void ggml_qnn_leaky_relu(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -static void ggml_qnn_concat(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -static void ggml_qnn_arange(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -static void ggml_qnn_sqr(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -static void ggml_qnn_clamp(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -static void ggml_qnn_scale(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -static void ggml_qnn_argsort(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -static void ggml_qnn_norm(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -static void ggml_qnn_group_norm(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -static void ggml_qnn_acc(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -static void ggml_qnn_sum_rows(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -static void ggml_qnn_pad(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -static void ggml_qnn_pool2d(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -static void ggml_qnn_dup(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -static void ggml_qnn_rms_norm(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -static void ggml_qnn_cpy(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -static void ggml_qnn_rope(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -static void ggml_qnn_im2col(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -static void ggml_qnn_softmax(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -static void ggml_qnn_get_rows(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -static void ggml_qnn_upsample_nearest2d(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -static void ggml_qnn_timestep_embedding(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -static void ggml_qnn_diag_mask(ggml_backend_qnn_context * ctx, ggml_tensor * dst, float value); -// ================================================================================================= -// section-2: global var, macro, data structure -// ================================================================================================= -// the following two vars can be fetched from [qnn_runtimelib_path]/ggml-qnn.cfg -// [general] -// print_qnn_internal_log=0 -// inference_approach=0 -static int g_print_qnn_internal_log = 0; // enable/disable QNN's internal log -static int g_inference_approach = 0; // 0: general approach,similar to ggml-sycl or ggml-cann 1: mapping entire ggml cgraph to QNN graph -static const char * g_qnn_cfgfilename = "ggml-qnn.cfg"; +typedef void (* ggmlqnn_op_func_t)(ggml_backend_qnn_context * ctx, ggml_tensor * op); -#if defined(__ANDROID__) -//Android command line program -static const char * g_qnn_runtimelib_path = "/data/local/tmp/"; -#elif defined(__linux__) -static const char * g_qnn_runtimelib_path = "/tmp/"; -#elif defined(_WIN32) -static const char * g_qnn_runtimelib_path = "C:\\"; -#endif - -#if !defined(__ANDROID__) && !defined(__linux__) -static std::atomic g_ggmltensor_idx(0); //ensure every QNN tensor name is unique -#else -static int32_t g_ggmltensor_idx = 0; //ensure every QNN tensor name is unique -#endif +//general function prototypes for ggml-qnn backend +static void ggmlqnn_dump_tensor(const ggml_tensor * tensor, const char * name); +static enum ggml_status ggmlqnn_backend_graph_compute_special(ggml_backend_t backend, struct ggml_cgraph * cgraph); +static void ggmlqnn_log_internal(ggml_log_level level, const char * file, const char * func, int line, const char * format, ...); +static inline bool ggmlqnn_is_valid_params(ggml_backend_qnn_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); +static Qnn_Tensor_t * ggmlqnn_create_general_tensor(qnn_instance * instance, Qnn_GraphHandle_t graph_handle, + const ggml_tensor * tensor, const char * name, + Qnn_TensorType_t qnn_tensor_type, + Qnn_DataType_t qnn_data_type, + uint32_t rank, uint32_t * dims, + void * data, uint32_t data_size, + bool b_transpose = false); + +//function prototypes for all op functions in the first tech approach(general approach in other backends) +//general op function for elment-wise operation on 1/2 input tensors and 1 output tensor +static void ggmlqnn_compute_elementwise(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggmlqnn_compute_mul_mat(ggml_backend_qnn_context * ctx, ggml_tensor * dst); + +//todo by AI experts +static void ggmlqnn_compute_repeat(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggmlqnn_compute_leaky_relu(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggmlqnn_compute_concat(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggmlqnn_compute_arange(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggmlqnn_compute_sqr(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggmlqnn_compute_clamp(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggmlqnn_compute_scale(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggmlqnn_compute_argsort(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggmlqnn_compute_norm(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggmlqnn_compute_group_norm(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggmlqnn_compute_acc(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggmlqnn_compute_sum_rows(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggmlqnn_compute_pad(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggmlqnn_compute_pool2d(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggmlqnn_compute_dup(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggmlqnn_compute_rms_norm(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggmlqnn_compute_cpy(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggmlqnn_compute_rope(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggmlqnn_compute_im2col(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggmlqnn_compute_softmax(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggmlqnn_compute_get_rows(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggmlqnn_compute_upsample_nearest2d(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggmlqnn_compute_timestep_embedding(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggmlqnn_compute_diag_mask(ggml_backend_qnn_context * ctx, ggml_tensor * dst, float value); + +//function prototypes for all op functions in the second tech approach("mapping the entire cgraph to a single QNN graph") +static void ggmlqnn_graph_addnode(ggml_backend_qnn_context * ctx, struct ggml_cgraph * cgraph, + Qnn_GraphHandle_t graph_handle, std::string & graph_name, ggml_tensor * op, bool is_reuse_graph = false); #if 0//def NDEBUG #define GGMLQNN_DEBUG 0 -#define ENABLE_QNNBACKEND_PERF 0 -#define GGMLQNN_PRINT_OP_ADD_LOG 0 // GGML_OP_ADD already verified with QNN-CPU / QNN-GPU / QNN-NPU -#define GGMLQNN_PRINT_OP_MUL_MAT_LOG 0 #else -#define GGMLQNN_DEBUG 1 // for troubleshooting QNN backend -#define ENABLE_QNNBACKEND_PERF 0 -#define GGMLQNN_PRINT_OP_ADD_LOG 0 // GGML_OP_ADD already verified with QNN-CPU / QNN-GPU / QNN-NPU -#define GGMLQNN_PRINT_OP_MUL_MAT_LOG 1 +#define GGMLQNN_DEBUG 1 #endif + #define GGML_QNN_LOGBUF_LEN 4096 +#define GGML_QNN_TMPBUF_LEN 256 #define GGMLQNN_LOG_ERROR(...) ggmlqnn_log_internal(GGML_LOG_LEVEL_ERROR, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) #define GGMLQNN_LOG_WARN(...) ggmlqnn_log_internal(GGML_LOG_LEVEL_WARN , __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) @@ -202,10 +185,10 @@ static int32_t g_ggmltensor_idx = 0; //ensure every QNN tensor name is unique #else #define GGMLQNN_LOG_DEBUG(...) #endif +#define GGMLQNN_DUMP_TENSOR(tensor) ggmlqnn_dump_tensor(tensor, #tensor) #define GGMLQNN_MEM_ADD(alignment) (sizeof (size_t) + alignment) #define GGMLQNN_MEM_MASK(alignment) ((uintptr_t)alignment - 1) -#define GQCGT ggmlqnn_create_general_tensor #define QNN_VER_PTR(x) (&((x).v1)) #define RPCMEM_DEFAULT_FLAGS 1 #define RPCMEM_HEAP_ID_SYSTEM 25 @@ -261,6 +244,9 @@ static int32_t g_ggmltensor_idx = 0; //ensure every QNN tensor name is unique } \ } while (0) +// ================================================================================================= +// section-2: data type, data structure, global vars +// ================================================================================================= using pfn_rpc_mem_init = void (*)(void); using pfn_rpc_mem_deinit = void (*)(void); using pfn_rpc_mem_alloc = void *(*)(int, uint32_t, int); @@ -269,10 +255,20 @@ using pfn_rpc_mem_to_fd = int (*)(void *); using _pfn_QnnSaver_initialize = decltype(QnnSaver_initialize); using _pfn_QnnInterface_getProviders = decltype(QnnInterface_getProviders); using _pfn_QnnSystemInterface_getProviders = decltype(QnnSystemInterface_getProviders); -using qnn_res_t = std::tuple>; -using qnn_tensors_t = std::vector< Qnn_Tensor_t *>; -enum class ggml_qnn_profile_level { +//QNN resource management for the first technical approach(general approach in ggml-sycl or ggml-cann) +using qnn_ptensors_t = std::vector< Qnn_Tensor_t *>; +using qnn_singlenode_res_t = std::tuple; + +//QNN resource management for the second technical approach(mapping the entire cgraph to a single QNN graph) +using qnn_tensors_t = std::vector< Qnn_Tensor_t >; +using qnn_tensor_pair_t = std::tuple< ggml_tensor *, Qnn_Tensor_t *>; +using qnn_tensor_pairs_t = std::vector< qnn_tensor_pair_t >; +using qnn_cgraph_node_t = std::tuple; +using qnn_cgraph_nodes_t = std::vector; +using qnn_multinode_res_t = std::tuple; + +enum class qnn_profile_level { profile_off = 0, profile_basic = 1, profile_detail = 2 @@ -322,17 +318,71 @@ struct ggml_backend_qnn_context { QNN_SYSTEM_INTERFACE_VER_TYPE raw_system_interface; struct qcom_socinfo socinfo; + //QNN resource management for the first technical approach(general approach in ggml-sycl or ggml-cann) + std::map qnn_singlenode_graph_map; + //QNN resource management for the second technical approach(mapping the entire cgraph to a single QNN graph) + std::map qnn_multinode_graph_map; + std::unique_ptr work_data; std::vector> tasks; - size_t work_size = 0; - size_t desired_size = 0; - int n_threads = GGML_DEFAULT_N_THREADS; + size_t work_size; + size_t desired_size; + int n_threads; +}; + +struct qnn_op_caps { + bool supported; + ggml_op op; + const char * qnn_op_name; + const size_t input_param_count; + const char * qnn_param_name; +}; + +struct qnn_parameter { + int print_qnn_internal_log; // enable/disable QNN's internal log + int enable_perf; // enable/disable perf of op function + int print_tensors_info; // enable/disable print tensors info in op function + int dump_op_info; // enable/disable dump op info in handle_op + int precision_mode; // 0: default 1:fp16 + int hvx_threads; + int vtcm_size_in_mb; + int enable_dlbc; + int inference_approach; // 0: general approach,similar to ggml-sycl or ggml-cann 1: mapping entire ggml cgraph to QNN graph + int qnn_backend; // 0: QNN-CPU backend, 1: QNN-GPU backend, 2: QNN-NPU backend + const char * qnn_cfgfilename; + const char * qnn_runtimelib_path; }; -struct qnn_op_caps_t { - const char * qnn_op_name = nullptr; - const size_t input_param_count = 0; - const char * qnn_param_name = nullptr; +//TODO:I don't think threadsafe is required at the moment +// so we can uniform them to avoid compiler/toolchain's complains +#if !defined(__ANDROID__) && !defined(__linux__) +static std::atomic g_qnntensor_idx(0); //ensure every QNN tensor name is unique +static std::atomic g_qnnopcfg_idx(0); //ensure every QNN opconfig name is unique +#else +static int32_t g_qnntensor_idx = 0; //ensure every QNN tensor name is unique +static int32_t g_qnnopcfg_idx = 0; //ensure every QNN opconfig name is unique +#endif + +static struct qnn_parameter g_qnn_params = { + .print_qnn_internal_log = 0, + .enable_perf = 0, + .print_tensors_info = 0, + .dump_op_info = 0, + .precision_mode = 0, + .hvx_threads = 4, + .vtcm_size_in_mb = 8, + .enable_dlbc = 1, + .inference_approach = 0, + .qnn_backend = 2, //default is QNN-NPU backend + .qnn_cfgfilename = "ggml-qnn.cfg", +#if defined(__ANDROID__) +//Android command line program + .qnn_runtimelib_path = "/data/local/tmp/", +#elif defined(__linux__) + .qnn_runtimelib_path = "/tmp/", +#elif defined(_WIN32) + .qnn_runtimelib_path = "C:\\", +#endif }; //file:///opt/qcom/aistack/qairt/2.31.0.250130/docs/QNN/general/overview.html#tbl-supported-snapdragon-devices @@ -464,118 +514,115 @@ static struct ggml_backend_qnn_context g_qnn_mgr[GGML_QNN_MAX_DEVICES] = { .socinfo = {}}, }; -static const qnn_op_caps_t ggmlqnn_k_op_caps[] = { - {}, // GGML_OP_NONE - {}, // GGML_OP_DUP - { - // GGML_OP_ADD - QNN_OP_ELEMENT_WISE_ADD, - 2, - }, - {}, // GGML_OP_ADD1 - {}, // GGML_OP_ACC - {}, // GGML_OP_SUB - { - // GGML_OP_MUL - QNN_OP_ELEMENT_WISE_MULTIPLY, - 2, - }, - {}, // GGML_OP_DIV - {}, // GGML_OP_SQR - {}, // GGML_OP_SQRT - {}, // GGML_OP_LOG - {}, // GGML_OP_SIN - {}, // GGML_OP_COS - {}, // GGML_OP_SUM - {}, // GGML_OP_SUM_ROWS - {}, // GGML_OP_MEAN - {}, // GGML_OP_ARGMAX - {}, // GGML_OP_COUNT_EQUAL - {}, // GGML_OP_REPEAT - {}, // GGML_OP_REPEAT_BACK - {}, // GGML_OP_CONCAT - {}, // GGML_OP_SILU_BACK - {}, // GGML_OP_NORM - {}, // GGML_OP_RMS_NORM - {}, // GGML_OP_RMS_NORM_BACK - {}, // GGML_OP_GROUP_NORM - { - // GGML_OP_MUL_MAT - QNN_OP_MAT_MUL, - 2, - }, - {}, // GGML_OP_MUL_MAT_ID - {}, // GGML_OP_OUT_PROD - {}, // GGML_OP_SCALE - {}, // GGML_OP_SET - {}, // GGML_OP_CPY - {}, // GGML_OP_CONT - {}, // GGML_OP_RESHAPE - {}, // GGML_OP_VIEW - {}, // GGML_OP_PERMUTE - {}, // GGML_OP_TRANSPOSE - {}, // GGML_OP_GET_ROWS - {}, // GGML_OP_GET_ROWS_BACK - {}, // GGML_OP_DIAG - {}, // GGML_OP_DIAG_MASK_INF - {}, // GGML_OP_DIAG_MASK_ZERO - {}, // GGML_OP_SOFT_MAX - {}, // GGML_OP_SOFT_MAX_BACK - {}, // GGML_OP_ROPE - {}, // GGML_OP_ROPE_BACK - {}, // GGML_OP_CLAMP - {}, // GGML_OP_CONV_TRANSPOSE_1D - {}, // GGML_OP_IM2COL - {}, // GGML_OP_IM2COL_BACK - {}, // GGML_OP_CONV_TRANSPOSE_2D - {}, // GGML_OP_POOL_1D - {}, // GGML_OP_POOL_2D - {}, // GGML_OP_POOL_2D_BACK - {}, // GGML_OP_UPSCALE - {}, // GGML_OP_PAD - {}, // GGML_OP_PAD_REFLECT_1D - {}, // GGML_OP_ARANGE - {}, // GGML_OP_TIMESTEP_EMBEDDING - {}, // GGML_OP_ARGSORT - {}, // GGML_OP_LEAKY_RELU - {}, // GGML_OP_FLASH_ATTN_EXT - {}, // GGML_OP_FLASH_ATTN_BACK - {}, // GGML_OP_SSM_CONV - {}, // GGML_OP_SSM_SCAN - {}, // GGML_OP_WIN_PART - {}, // GGML_OP_WIN_UNPART - {}, // GGML_OP_GET_REL_POS - {}, // GGML_OP_ADD_REL_POS - {}, // GGML_OP_RWKV_WKV6 - {}, // GGML_OP_GATED_LINEAR_ATTN - {}, // GGML_OP_UNARY - {}, // GGML_OP_MAP_UNARY - {}, // GGML_OP_MAP_BINARY - {}, // GGML_OP_MAP_CUSTOM1_F32 - {}, // GGML_OP_MAP_CUSTOM2_F32 - {}, // GGML_OP_MAP_CUSTOM3_F32 - {}, // GGML_OP_MAP_CUSTOM1 - {}, // GGML_OP_MAP_CUSTOM2 - {}, // GGML_OP_MAP_CUSTOM3 - {}, // GGML_OP_CROSS_ENTROPY_LOSS - {}, // GGML_OP_CROSS_ENTROPY_LOSS_BACK - {}, // GGML_OP_OPT_STEP_ADAMW - {}, // GGML_UNARY_OP_ABS - {}, // GGML_UNARY_OP_SGN - {}, // GGML_UNARY_OP_NEG - {}, // GGML_UNARY_OP_STEP - {}, // GGML_UNARY_OP_TANH - {}, // GGML_UNARY_OP_ELU - {}, // GGML_UNARY_OP_RELU - {}, // GGML_UNARY_OP_SIGMOID - {}, // GGML_UNARY_OP_GELU - {}, // GGML_UNARY_OP_GELU_QUICK - {}, // GGML_UNARY_OP_SILU - {}, // GGML_UNARY_OP_HARDSWISH - {}, // GGML_UNARY_OP_HARDSIGMOID - {}, // GGML_UNARY_OP_EXP +static constexpr const qnn_op_caps ggmlqnn_k_op_caps[] = { + {true, GGML_OP_NONE, nullptr, 0, nullptr}, + {false, GGML_OP_DUP}, + {true, GGML_OP_ADD, QNN_OP_ELEMENT_WISE_ADD, 2}, + {false, GGML_OP_ADD1}, + {false, GGML_OP_ACC}, + {true, GGML_OP_SUB, QNN_OP_ELEMENT_WISE_SUBTRACT, 2}, + {true, GGML_OP_MUL, QNN_OP_ELEMENT_WISE_MULTIPLY, 2}, + {true, GGML_OP_DIV, QNN_OP_ELEMENT_WISE_DIVIDE, 2}, + {false, GGML_OP_SQR}, + {true, GGML_OP_SQRT, QNN_OP_ELEMENT_WISE_SQUARE_ROOT, 1}, + {true, GGML_OP_LOG, QNN_OP_ELEMENT_WISE_LOG, 1}, + {false, GGML_OP_SIN}, + {false, GGML_OP_COS}, + {false, GGML_OP_SUM}, + {false, GGML_OP_SUM_ROWS}, + {false, GGML_OP_MEAN}, + {false, GGML_OP_ARGMAX}, + {false, GGML_OP_COUNT_EQUAL}, + {false, GGML_OP_REPEAT}, + {false, GGML_OP_REPEAT_BACK}, + {false, GGML_OP_CONCAT}, + {false, GGML_OP_SILU_BACK}, + {false, GGML_OP_NORM}, + {false, GGML_OP_RMS_NORM}, + {false, GGML_OP_RMS_NORM_BACK}, + {false, GGML_OP_GROUP_NORM}, + {false, GGML_OP_L2_NORM}, + {true, GGML_OP_MUL_MAT, QNN_OP_MAT_MUL, 2}, + {false, GGML_OP_MUL_MAT_ID}, + {false, GGML_OP_OUT_PROD}, + {false, GGML_OP_SCALE}, + {false, GGML_OP_SET}, + {false, GGML_OP_CPY}, + {false, GGML_OP_CONT}, + {false, GGML_OP_RESHAPE}, + {false, GGML_OP_VIEW}, + {false, GGML_OP_PERMUTE}, + {false, GGML_OP_TRANSPOSE}, + {false, GGML_OP_GET_ROWS}, + {false, GGML_OP_GET_ROWS_BACK}, + {false, GGML_OP_DIAG}, + {false, GGML_OP_DIAG_MASK_INF}, + {false, GGML_OP_DIAG_MASK_ZERO}, + {false, GGML_OP_SOFT_MAX}, + {false, GGML_OP_SOFT_MAX_BACK}, + {false, GGML_OP_ROPE}, + {false, GGML_OP_ROPE_BACK}, + {false, GGML_OP_CLAMP}, + {false, GGML_OP_CONV_TRANSPOSE_1D}, + {false, GGML_OP_IM2COL}, + {false, GGML_OP_IM2COL_BACK}, + {false, GGML_OP_CONV_TRANSPOSE_2D}, + {false, GGML_OP_POOL_1D}, + {false, GGML_OP_POOL_2D}, + {false, GGML_OP_POOL_2D_BACK}, + {false, GGML_OP_UPSCALE}, + {false, GGML_OP_PAD}, + {false, GGML_OP_PAD_REFLECT_1D}, + {false, GGML_OP_ARANGE}, + {false, GGML_OP_TIMESTEP_EMBEDDING}, + {false, GGML_OP_ARGSORT}, + {false, GGML_OP_LEAKY_RELU}, + {false, GGML_OP_FLASH_ATTN_EXT}, + {false, GGML_OP_FLASH_ATTN_BACK}, + {false, GGML_OP_SSM_CONV}, + {false, GGML_OP_SSM_SCAN}, + {false, GGML_OP_WIN_PART}, + {false, GGML_OP_WIN_UNPART}, + {false, GGML_OP_GET_REL_POS}, + {false, GGML_OP_ADD_REL_POS}, + {false, GGML_OP_RWKV_WKV6}, + {false, GGML_OP_GATED_LINEAR_ATTN}, + {false, GGML_OP_RWKV_WKV7}, + {false, GGML_OP_UNARY}, + {false, GGML_OP_MAP_UNARY}, + {false, GGML_OP_MAP_BINARY}, + {false, GGML_OP_MAP_CUSTOM1_F32}, + {false, GGML_OP_MAP_CUSTOM2_F32}, + {false, GGML_OP_MAP_CUSTOM3_F32}, + {false, GGML_OP_MAP_CUSTOM1}, + {false, GGML_OP_MAP_CUSTOM2}, + {false, GGML_OP_MAP_CUSTOM3}, + {false, GGML_OP_CROSS_ENTROPY_LOSS}, + {false, GGML_OP_CROSS_ENTROPY_LOSS_BACK}, + {false, GGML_OP_OPT_STEP_ADAMW}, + {false, static_cast(GGML_UNARY_OP_ABS)}, + {false, static_cast(GGML_UNARY_OP_SGN)}, + {false, static_cast(GGML_UNARY_OP_NEG)}, + {false, static_cast(GGML_UNARY_OP_STEP)}, + {false, static_cast(GGML_UNARY_OP_TANH)}, + {false, static_cast(GGML_UNARY_OP_ELU)}, + {false, static_cast(GGML_UNARY_OP_RELU)}, + {false, static_cast(GGML_UNARY_OP_SIGMOID)}, + {false, static_cast(GGML_UNARY_OP_GELU)}, + {false, static_cast(GGML_UNARY_OP_GELU_QUICK)}, + {false, static_cast(GGML_UNARY_OP_SILU)}, + {false, static_cast(GGML_UNARY_OP_HARDSWISH)}, + {false, static_cast(GGML_UNARY_OP_HARDSIGMOID)}, + {false, static_cast(GGML_UNARY_OP_EXP)} }; +static_assert(ggmlqnn_k_op_caps[GGML_OP_NONE].supported, "GGML_OP_NONE is not true"); +static_assert(ggmlqnn_k_op_caps[GGML_OP_ADD].supported, "GGML_OP_ADD is not true"); +static_assert(ggmlqnn_k_op_caps[GGML_OP_MUL].supported, "GGML_OP_MUL is not true"); +static_assert(ggmlqnn_k_op_caps[GGML_OP_MUL_MAT].supported, "GGML_OP_MUL_MAT is not true"); +static_assert(std::size(ggmlqnn_k_op_caps) == (GGML_OP_COUNT + GGML_UNARY_OP_COUNT), + "pls check ggmlqnn_k_op_caps and ensure is corresponding to latest ggml.h"); + // ================================================================================================= // section-3: ggml-qnn internal troubleshooting function/class // ================================================================================================= @@ -608,7 +655,84 @@ static void ggmlqnn_log_internal(ggml_log_level level, const char * file, const } } -#if ENABLE_QNNBACKEND_PERF +static void ggmlqnn_print_tensors_info(const char * func_name, const ggml_backend_qnn_context * ctx, + const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * dst) { + //skip sanity check of params because of performance concern + if (0 == g_qnn_params.print_tensors_info) + return; + + if (nullptr != func_name && nullptr != ctx) { + GGMLQNN_LOG_DEBUG("call %s in dev %s\n", func_name, ctx->name); + } + GGMLQNN_LOG_DEBUG("%-6s: type = %i (%s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi, %5zi)", + src0->name, + src0->type, ggml_type_name(src0->type), src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], + src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3]); + if (nullptr != src1) { + GGMLQNN_LOG_DEBUG( + "%-6s: type = %i (%s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi, %5zi)", + src1->name, + src1->type, ggml_type_name(src1->type), src1->ne[0], src1->ne[1], src1->ne[2], + src1->ne[3], + src1->nb[0], src1->nb[1], src1->nb[2], src1->nb[3]); + } + GGMLQNN_LOG_DEBUG("%-6s: type = %i (%s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi, %5zi)", + dst->name, + dst->type, ggml_type_name(dst->type), dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], + dst->nb[0], dst->nb[1], dst->nb[2], dst->nb[3]); + GGMLQNN_LOG_DEBUG("\n"); +} + +static void ggmlqnn_dump_op_info(const struct ggml_tensor * tensor) { + //skip sanity check of params because of performance concern + if (0 == g_qnn_params.dump_op_info) + return; + + const struct ggml_tensor * src0 = tensor->src[0]; + struct ggml_tensor * src1 = tensor->src[1]; + struct ggml_tensor * dst = const_cast(tensor); + GGMLQNN_LOG_DEBUG("op name:%s, tensor type:%s", ggml_op_name(tensor->op), ggml_type_name(tensor->type)); + ggmlqnn_print_tensors_info(nullptr, nullptr, src0, src1, dst); +} + +static void ggmlqnn_dump_tensor_elements(const ggml_tensor * tensor) { + float value = 0; + std::ostringstream tmposs; + if (tensor->type == GGML_TYPE_F32) { + for (int h = 0; h < tensor->ne[3]; h++) { + for (int i = 0; i < tensor->ne[2]; i++) { + for (int j = 0; j < tensor->ne[1]; j++) { + for (int k = 0; k < tensor->ne[0]; k++) { + value = ((float *) tensor->data)[h * tensor->ne[2] + i * tensor->ne[1] + + j * tensor->ne[0] + k]; + tmposs << std::setw(8) << std::fixed << std::setprecision(2) << value + << " "; + } + if (strlen(tmposs.str().c_str()) <= (GGML_QNN_LOGBUF_LEN - 96)) { + GGMLQNN_LOG_DEBUG("%s\n", tmposs.str().c_str()); + } + tmposs.clear(); + tmposs.str(""); + } + } + } + } + + GGMLQNN_LOG_DEBUG("\n"); +} + +static void ggmlqnn_dump_tensor(const ggml_tensor * tensor, const char * name) { + GGMLQNN_LOG_DEBUG("dump ggml tensor %s(%s)\n", name, tensor->name); + GGMLQNN_LOG_DEBUG("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64", nb = (%5zi, %5zi, %5zi, %5zi)\n", + name, + tensor->type, ggml_type_name(tensor->type), + tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3], + tensor->nb[0], tensor->nb[1], tensor->nb[2], tensor->nb[2]); + ggmlqnn_dump_tensor_elements(tensor); + + GGMLQNN_LOG_DEBUG("\n"); +} + class qnn_perf { public: qnn_perf(const std::string & perf_name) : _perf_name(std::move(perf_name)) {}; @@ -617,10 +741,14 @@ class qnn_perf { qnn_perf & operator= (const qnn_perf & ) = delete; void start() { + if (0 == g_qnn_params.enable_perf) + return; _begin_time = ggml_time_us(); } void info() { + if (0 == g_qnn_params.enable_perf) + return; _end_time = ggml_time_us(); _duration = (_end_time - _begin_time); GGMLQNN_LOG_DEBUG("duration of %s : %lld microseconds\n", _perf_name.c_str(), _duration); @@ -632,160 +760,6 @@ class qnn_perf { int64_t _duration = 0LL; std::string _perf_name; }; -#else -class qnn_perf { -public: - qnn_perf(const std::string & perf_name) { - GGML_UNUSED(perf_name); - } - qnn_perf() = delete; - qnn_perf(const qnn_perf & ) = delete; - qnn_perf & operator= (const qnn_perf & ) = delete; - - void start() {} - void info() {} -}; -#endif - -class qnn_cfg { -public: - void dump(std::function worker) { - if (!_load_success) { - GGMLQNN_LOG_INFO("qnn cfg file %s not loadded", _cfg_filename.c_str()); - return; - } - auto iter = _qnn_cfg.begin(); - while (iter != _qnn_cfg.end()) { - auto kv_iter = iter->second.begin(); - while (kv_iter != iter->second.end()) { - worker(iter->first, kv_iter->first, kv_iter->second); - ++kv_iter; - } - ++iter; - } - } - - bool load(const std::string & file_name) { - if (file_name == "") { - return false; - } - _cfg_filename = file_name; - std::ifstream in; - std::string line; - in.open(file_name.c_str()); - if (not in.is_open()) { - GGMLQNN_LOG_WARN("can't open file %s", file_name.c_str()); - return false; - } - while (getline(in, line)) { - std::string section, key, value; - if (not parse_line(line, section, key, value)) { - continue; - } - set_section_keyvalue(section, key, value); - } - _load_success = true; - return true; - } - - void get_stringvalue(const std::string & section, const std::string & key, std::string & value, std::string default_value) { - value = default_value; - if (_qnn_cfg.find(section) == _qnn_cfg.end()) { - return; - } - if (_qnn_cfg[section].find(key) == _qnn_cfg[section].end()) { - return; - } - value = _qnn_cfg[section][key]; - } - - void get_intvalue(const std::string & section, const std::string & key, int & value, int default_value) { - value = default_value; - if (_qnn_cfg.find(section) == _qnn_cfg.end()) { - return; - } - if (_qnn_cfg[section].find(key) == _qnn_cfg[section].end()) { - return; - } - value = atol(_qnn_cfg[section][key].c_str()); - } - -private: - void ltrim(std::string & str) { - if (str.empty()) return; - size_t len = 0; - char* temp = (char*)str.c_str(); - while (*temp && isblank(*temp)) { - ++len; - ++temp; - } - if (len > 0) str.erase(0, len); - } - - void rtrim(std::string & str) { - if (str.empty()) return; - size_t len = str.length(); - size_t pos = len; - while (pos > 0) { - if (not isblank(str[pos - 1])) { - break; - } - --pos; - } - if (pos != len) str.erase(pos); - } - - void trim(std::string& str) { - ltrim(str); - rtrim(str); - } - - void set_section_keyvalue(std::string & section, std::string & key, std::string & value) { - if (_qnn_cfg.find(section) == _qnn_cfg.end()) { - std::unordered_map kv_map; - _qnn_cfg[section] = kv_map; - } - if (key != "" && value != "") _qnn_cfg[section][key] = value; - } - - bool parse_line(std::string & line, std::string & section, std::string & key, std::string & value) { - static std::string cur_section = ""; - std::string nodes[2] = {"#", ";"}; - for (int i = 0; i < 2; ++i) { - std::string::size_type pos = line.find(nodes[i]); - if (pos != std::string::npos) line.erase(pos); - } - trim(line); - if (line == "") return false; - if (line[0] == '[' && line[line.size() - 1] == ']') { - section = line.substr(1, line.size() - 2); - trim(section); - cur_section = section; - return false; - } - if (cur_section == "") return false; - bool is_key = true; - for (size_t i = 0; i < line.size(); ++i) { - if (line[i] == '=') { - is_key = false; - continue; - } - if (is_key) { - key += line[i]; - } else { - value += line[i]; - } - } - section = cur_section; - trim(key); - trim(value); - return true; - } -private: - std::unordered_map> _qnn_cfg; - bool _load_success = false; - std::string _cfg_filename; -}; // ================================================================================================= // section-4: helper function for WoA(Window on ARM) @@ -845,20 +819,36 @@ static const char * dlerror(void) { // ================================================================================================= // section-5: general helper function // ================================================================================================= -//the following 3 helper funcs are used to ensure every QNN tensor name is unique -static void ggmqnn_reset_tensoridx() { - g_ggmltensor_idx = 0; +//TODO: merge the following 6 helper functions which used to ensure every QNN tensor/opcfg name is unique +static void ggmlqnn_reset_tensoridx() { + g_qnntensor_idx = 0; } -static void ggmqnn_inc_tensoridx() { - g_ggmltensor_idx++; +static void ggmlqnn_inc_tensoridx() { + g_qnntensor_idx++; } -static int32_t ggmqnn_get_tensoridx() { +static int32_t ggmlqnn_get_tensoridx() { #if !defined(__ANDROID__) && !defined(__linux__) - return g_ggmltensor_idx.load(); + return g_qnntensor_idx.load(); #else - return g_ggmltensor_idx; + return g_qnntensor_idx; +#endif +} + +static void ggmlqnn_reset_opcfgidx() { + g_qnnopcfg_idx = 0; +} + +static void ggmlqnn_inc_opcfgidx() { + g_qnnopcfg_idx++; +} + +static int32_t ggmlqnn_get_opcfgidx() { +#if !defined(__ANDROID__) && !defined(__linux__) + return g_qnnopcfg_idx.load(); +#else + return g_qnnopcfg_idx; #endif } @@ -989,23 +979,18 @@ static void * ggmlqnn_host_malloc(size_t buffer_size, size_t page_size) { return data; } -static void ggmlqnn_load_cfg() { - std::string cfg_filename = std::string(g_qnn_runtimelib_path) + std::string(g_qnn_cfgfilename); - GGMLQNN_LOG_INFO("load ggml-qnn config from %s", cfg_filename.c_str()); - qnn_cfg qnncfg_instance; - qnncfg_instance.load(cfg_filename); - qnncfg_instance.dump([](const std::string & section, const std::string & key, const std::string value) { - std::ostringstream tmposs; - tmposs << "section[" << section << "],[" << key << "] = [" << value << "]" << std::endl; - GGMLQNN_LOG_INFO("%s", tmposs.str().c_str()); - }); - std::string npu_inference_datatype; - qnncfg_instance.get_intvalue("general", "print_qnn_internal_log", g_print_qnn_internal_log, 0); - qnncfg_instance.get_intvalue("general", "inference_approach", g_inference_approach, 0); - qnncfg_instance.get_stringvalue("npu", "npu_inference_datatype", npu_inference_datatype, "fp32"); - GGMLQNN_LOG_INFO("print_qnn_internal_log=%d", g_print_qnn_internal_log); - GGMLQNN_LOG_INFO("inference_approach=%d", g_inference_approach); - GGMLQNN_LOG_INFO("npu inference data type=%s", npu_inference_datatype.c_str()); +static void ggmlqnn_get_timestring(char * p_currenttime) { + time_t n_seconds = 0; + struct tm * p_tm = nullptr; + + if (nullptr == p_currenttime) + return; + + time(&n_seconds); + p_tm = localtime(&n_seconds); + snprintf(p_currenttime, GGML_QNN_TMPBUF_LEN, "%04d-%02d-%02d-%02d-%02d-%02d", + p_tm->tm_year + 1900, p_tm->tm_mon + 1, p_tm->tm_mday, + p_tm->tm_hour, p_tm->tm_min, p_tm->tm_sec); } // ================================================================================================= @@ -1015,7 +1000,6 @@ static inline uint32_t get_qnn_tensorid(const Qnn_Tensor_t & tensor) { if (tensor.version == QNN_TENSOR_VERSION_1) { return tensor.v1.id; } - return 0u; } @@ -1336,7 +1320,19 @@ static Qnn_OpConfig_t ggmlqnn_create_op_config(const char * name, const char * p Qnn_Param_t * params, uint32_t num_params, Qnn_Tensor_t * inputs, uint32_t num_inputs, Qnn_Tensor_t * outputs, uint32_t num_outputs) { - Qnn_OpConfigV1_t v1 = {name, package, type, + + char opcfg_name[GGML_MAX_NAME] = {}; + + //ensure the opcfg name is unique + if (nullptr == name) { + snprintf(opcfg_name, GGML_MAX_NAME, "opcfg_%-8d", ggmlqnn_get_opcfgidx()); + } else { + snprintf(opcfg_name, GGML_MAX_NAME, "opcfg_%s_%-8d", name, ggmlqnn_get_opcfgidx()); + } + GGMLQNN_LOG_DEBUG("create qnn opconfig %s", opcfg_name); + ggmlqnn_inc_opcfgidx(); + + Qnn_OpConfigV1_t v1 = {opcfg_name, package, type, num_params, params, num_inputs, inputs, num_outputs, outputs @@ -1531,7 +1527,7 @@ static void * ggmlqnn_type_trait(ggml_backend_qnn_context * ctx, ggml_tensor * o } static void ggmlqnn_append_tensor_dimensions(const ggml_tensor * tensor, std::string & output) { - char buffer[256] = {}; + char buffer[GGML_QNN_TMPBUF_LEN] = {}; const char * type_name = ggmlqnn_get_ggml_type_name(tensor->type); int len = 0; switch (ggml_n_dims(tensor)) { @@ -1583,64 +1579,207 @@ static void ggmlqnn_get_graphkey_from_op(const ggml_tensor * op, std::string & o if (!input) { break; } - output += '_'; - ggmlqnn_append_tensor_dimensions(input, output); + output += '_'; + ggmlqnn_append_tensor_dimensions(input, output); + } +} + +static void ggmlqnn_get_opkey_with_srcop_desc(const ggml_tensor * op, std::string & output) { + output += ggml_op_desc(op); + output += '('; + if (op->src[0]) { + output += ggml_op_desc(op->src[0]); + } + for (size_t i = 1; i < GGML_MAX_DIMS && op->src[i]; ++i) { + output += ','; + output += ggml_op_desc(op->src[i]); + } + output += ')'; +} + +static void ggmlqnn_get_graphkey_from_cgraph(const ggml_cgraph * cgraph, std::string & output) { + if (nullptr == cgraph || 0 == cgraph->n_nodes) { + GGMLQNN_LOG_WARN("empty ggml computational graph"); + return; + } + + //output += "cgraph_" + std::to_string(ggml_time_us()); + //return; + + bool is_start = true; + for (int i = 0; i < cgraph->n_nodes; ++i) { + auto * op = cgraph->nodes[i]; + if (ggml_is_empty(op)) { + GGMLQNN_LOG_WARN("empty op in graph, skipping"); + continue; + } + + if (op->op == GGML_OP_NONE) { + GGMLQNN_LOG_WARN("GGML_OP_NONE in graph, skipping"); + continue; + } + + if (is_start) { + ggmlqnn_get_graphkey_from_op(cgraph->nodes[0], output); + is_start = false; + } else { + output += '#'; + ggmlqnn_get_opkey_with_srcop_desc(op, output); + } + } + + if (cgraph->n_nodes > 1) { + auto * last_op = cgraph->nodes[cgraph->n_nodes - 1]; + output += ggmlqnn_get_ggml_type_name(last_op->type); + output += '_'; + ggmlqnn_append_tensor_dimensions(last_op, output); + } +} + +template +Fn ggmlqnn_load_qnn_functionpointers(void * handle, const char * function_name) { + return reinterpret_cast(dlsym(handle, function_name)); +} + +class qnn_cfg { +public: + void dump(std::function worker) { + if (!_load_success) { + GGMLQNN_LOG_INFO("qnn cfg file %s not loaded", _cfg_filename.c_str()); + return; + } + auto iter = _qnn_cfg.begin(); + while (iter != _qnn_cfg.end()) { + auto kv_iter = iter->second.begin(); + while (kv_iter != iter->second.end()) { + worker(iter->first, kv_iter->first, kv_iter->second); + ++kv_iter; + } + ++iter; + } + } + + bool load(const std::string & file_name) { + if (file_name == "") { + return false; + } + _cfg_filename = file_name; + std::ifstream in; + std::string line; + in.open(file_name.c_str()); + if (not in.is_open()) { + GGMLQNN_LOG_WARN("can't open file %s", file_name.c_str()); + return false; + } + while (getline(in, line)) { + std::string section, key, value; + if (not parse_line(line, section, key, value)) { + continue; + } + set_section_keyvalue(section, key, value); + } + _load_success = true; + return true; + } + + void get_stringvalue(const std::string & section, const std::string & key, std::string & value, std::string default_value) { + value = default_value; + if (_qnn_cfg.find(section) == _qnn_cfg.end()) { + return; + } + if (_qnn_cfg[section].find(key) == _qnn_cfg[section].end()) { + return; + } + value = _qnn_cfg[section][key]; } -} -static void ggmlqnn_get_opkey_with_srcop_desc(const ggml_tensor * op, std::string & output) { - output += ggml_op_desc(op); - output += '('; - if (op->src[0]) { - output += ggml_op_desc(op->src[0]); - } - for (size_t i = 1; i < GGML_MAX_DIMS && op->src[i]; ++i) { - output += ','; - output += ggml_op_desc(op->src[i]); + void get_intvalue(const std::string & section, const std::string & key, int & value, int default_value) { + value = default_value; + if (_qnn_cfg.find(section) == _qnn_cfg.end()) { + return; + } + if (_qnn_cfg[section].find(key) == _qnn_cfg[section].end()) { + return; + } + value = atol(_qnn_cfg[section][key].c_str()); } - output += ')'; -} -static void ggmlqnn_get_graphkey_from_cgraph(const ggml_cgraph * cgraph, std::string & output) { - if (nullptr == cgraph || 0 == cgraph->n_nodes) { - GGMLQNN_LOG_WARN("empty ggml computational graph"); - return; +private: + void ltrim(std::string & str) { + if (str.empty()) return; + size_t len = 0; + char* temp = (char*)str.c_str(); + while (*temp && isblank(*temp)) { + ++len; + ++temp; + } + if (len > 0) str.erase(0, len); } - bool is_start = true; - for (int i = 0; i < cgraph->n_nodes; ++i) { - auto * op = cgraph->nodes[i]; - if (ggml_is_empty(op)) { - GGMLQNN_LOG_WARN("empty op in graph, skipping"); - continue; + void rtrim(std::string & str) { + if (str.empty()) return; + size_t len = str.length(); + size_t pos = len; + while (pos > 0) { + if (not isblank(str[pos - 1])) { + break; + } + --pos; } + if (pos != len) str.erase(pos); + } - if (op->op == GGML_OP_NONE) { - GGMLQNN_LOG_WARN("GGML_OP_NONE in graph, skipping"); - continue; - } + void trim(std::string& str) { + ltrim(str); + rtrim(str); + } - if (is_start) { - ggmlqnn_get_graphkey_from_op(cgraph->nodes[0], output); - is_start = false; - } else { - output += '#'; - ggmlqnn_get_opkey_with_srcop_desc(op, output); + void set_section_keyvalue(std::string & section, std::string & key, std::string & value) { + if (_qnn_cfg.find(section) == _qnn_cfg.end()) { + std::unordered_map kv_map; + _qnn_cfg[section] = kv_map; } + if (key != "" && value != "") _qnn_cfg[section][key] = value; } - if (cgraph->n_nodes > 1) { - auto * last_op = cgraph->nodes[cgraph->n_nodes - 1]; - output += ggmlqnn_get_ggml_type_name(last_op->type); - output += '_'; - ggmlqnn_append_tensor_dimensions(last_op, output); + bool parse_line(std::string & line, std::string & section, std::string & key, std::string & value) { + static std::string cur_section = ""; + std::string nodes[2] = {"#", ";"}; + for (int i = 0; i < 2; ++i) { + std::string::size_type pos = line.find(nodes[i]); + if (pos != std::string::npos) line.erase(pos); + } + trim(line); + if (line == "") return false; + if (line[0] == '[' && line[line.size() - 1] == ']') { + section = line.substr(1, line.size() - 2); + trim(section); + cur_section = section; + return false; + } + if (cur_section == "") return false; + bool is_key = true; + for (size_t i = 0; i < line.size(); ++i) { + if (line[i] == '=') { + is_key = false; + continue; + } + if (is_key) { + key += line[i]; + } else { + value += line[i]; + } + } + section = cur_section; + trim(key); + trim(value); + return true; } -} - -template -Fn ggmlqnn_load_qnn_functionpointers(void * handle, const char * function_name) { - return reinterpret_cast(dlsym(handle, function_name)); -} +private: + std::unordered_map> _qnn_cfg; + bool _load_success = false; + std::string _cfg_filename; +}; class qnn_interface { #define DEFINE_SHIM_FUNCTION_INTERFACE(F, pointer_name) \ @@ -1830,11 +1969,11 @@ class qnn_instance { bool is_valid_graph() const { return _qnn_graph_handle != nullptr; } - int init_htp_perfinfra(); + int htp_init_perfinfra(); - int set_rpc_polling(); + int htp_set_rpc_polling(); - int set_high_performance_mode(); + int htp_set_high_performance_mode(); std::string & get_qnn_graph_name() { return _graph_name; } @@ -1877,9 +2016,6 @@ class qnn_instance { return _device_id; } -public: - std::map>> _qnn_graph_map; - private: int load_system(); @@ -1901,7 +2037,7 @@ class qnn_instance { void htp_print_info(); - void htp_probe_device_meminfo(); + void htp_probe_rpc_meminfo(); void print_backend_info(); @@ -1924,7 +2060,7 @@ class qnn_instance { bool _do_node_validations = true; // flag to indicate whether all add_node calls need to be validated QnnLog_Level_t _qnn_log_level = QNN_LOG_LEVEL_DEBUG; - ggml_qnn_profile_level _profile_level = ggml_qnn_profile_level::profile_detail; + qnn_profile_level _profile_level = qnn_profile_level::profile_off; void * _system_lib_handle = nullptr; void * _loaded_lib_handle = nullptr; @@ -2314,7 +2450,7 @@ int qnn_instance::load_system() { if (nullptr == _system_lib_handle) { GGMLQNN_LOG_WARN("can not open QNN library %s, error: %s\n", system_lib_path.c_str(), dlerror()); //re-try with default path of QNN binary runtime lib - _lib_path = std::string(g_qnn_runtimelib_path); + _lib_path = std::string(g_qnn_params.qnn_runtimelib_path); #if !defined(__ANDROID__) && !defined(__linux__) system_lib_path = _lib_path + "QnnSystem.dll"; #else @@ -2411,16 +2547,16 @@ int qnn_instance::unload_system() { return result; } -static void ggml_qnn_logcallback(const char * fmt, +static void ggmlqnn_compute_logcallback(const char * fmt, QnnLog_Level_t level, uint64_t timestamp, va_list argp) { - if (0 == g_print_qnn_internal_log) + if (0 == g_qnn_params.print_qnn_internal_log) return; static std::mutex log_mutex; - static unsigned char s_ggml_qnn_logbuf[GGML_QNN_LOGBUF_LEN]; + static unsigned char s_ggmlqnn_compute_logbuf[GGML_QNN_LOGBUF_LEN]; const char * log_level_desc = ""; switch (level) { @@ -2447,9 +2583,9 @@ static void ggml_qnn_logcallback(const char * fmt, double ms = (double) timestamp / 1000000.0; { std::lock_guard lock(log_mutex); - memset(s_ggml_qnn_logbuf, 0, GGML_QNN_LOGBUF_LEN); - vsnprintf(reinterpret_cast(s_ggml_qnn_logbuf), GGML_QNN_LOGBUF_LEN, fmt, argp); - GGMLQNN_LOG_DEBUG("%8.1fms [%-7s] %s\n", ms, log_level_desc, s_ggml_qnn_logbuf); + memset(s_ggmlqnn_compute_logbuf, 0, GGML_QNN_LOGBUF_LEN); + vsnprintf(reinterpret_cast(s_ggmlqnn_compute_logbuf), GGML_QNN_LOGBUF_LEN, fmt, argp); + GGMLQNN_LOG_DEBUG("%8.1fms [%-7s] %s\n", ms, log_level_desc, s_ggmlqnn_compute_logbuf); } } @@ -2489,9 +2625,9 @@ int qnn_instance::qnn_init(const QnnSaver_Config_t ** saver_config) { _qnn_interface.set_qnn_interface(_loaded_backend); #if 1 - _qnn_interface.qnn_log_create(ggml_qnn_logcallback, _qnn_log_level, &_qnn_log_handle); + _qnn_interface.qnn_log_create(ggmlqnn_compute_logcallback, _qnn_log_level, &_qnn_log_handle); #else - _qnn_raw_interface.logCreate(ggml_qnn_logcallback, _qnn_log_level, &_qnn_log_handle); + _qnn_raw_interface.logCreate(ggmlqnn_compute_logcallback, _qnn_log_level, &_qnn_log_handle); #endif if (nullptr == _qnn_log_handle) { GGMLQNN_LOG_WARN("why failed to initialize qnn log\n"); //NPU backend not work on Qualcomm SoC based low-end phone @@ -2521,17 +2657,62 @@ int qnn_instance::qnn_init(const QnnSaver_Config_t ** saver_config) { } } - auto qnnstatus = _qnn_raw_interface.deviceCreate( - _qnn_log_handle, nullptr, &_qnn_device_handle); + auto qnnstatus = QNN_SUCCESS; + if (_device_id == QNN_BACKEND_NPU) { + //TODO: remove duplicated code between here and function htp_print_info + const QnnDevice_PlatformInfo_t * p_info = nullptr; + qcom_socinfo soc_info = {}; + qnnstatus = _qnn_raw_interface.deviceGetPlatformInfo(nullptr, &p_info); + if (qnnstatus == QNN_SUCCESS) { + GGMLQNN_LOG_INFO("device counts %d\n", p_info->v1.numHwDevices); + QnnDevice_HardwareDeviceInfo_t * infos = p_info->v1.hwDevices; + QnnHtpDevice_OnChipDeviceInfoExtension_t chipinfo = {}; + for (uint32_t i = 0; i < p_info->v1.numHwDevices; i++) { + GGMLQNN_LOG_INFO("deviceID:%d, deviceType:%d, numCores %d\n", (int) infos[i].v1.deviceId, + (int) infos[i].v1.deviceType, (int) infos[i].v1.numCores); + QnnDevice_DeviceInfoExtension_t devinfo = infos[i].v1.deviceInfoExtension; + chipinfo = devinfo->onChipDevice; + size_t htp_arch = (size_t) chipinfo.arch; + GGMLQNN_LOG_INFO("htp_type:%d(%s)\n", devinfo->devType, + (devinfo->devType == QNN_HTP_DEVICE_TYPE_ON_CHIP) ? "ON_CHIP" : ""); + soc_info = { chipinfo.socModel, htp_arch, chipinfo.vtcmSize }; + } + _qnn_raw_interface.deviceFreePlatformInfo(nullptr, p_info); + } else { + GGMLQNN_LOG_WARN("failed to get platform info, are we in emulator?\n"); + soc_info = { NONE, UNKNOWN_SM, 0 }; + } + + QnnHtpDevice_CustomConfig_t soc_customconfig; + soc_customconfig.option = QNN_HTP_DEVICE_CONFIG_OPTION_SOC; + soc_customconfig.socModel = soc_info.soc_model; + QnnDevice_Config_t soc_devconfig; + soc_devconfig.option = QNN_DEVICE_CONFIG_OPTION_CUSTOM; + soc_devconfig.customConfig = &soc_customconfig; + + /* + QnnHtpDevice_CustomConfig_t arch_customconfig; + arch_customconfig.option = QNN_HTP_DEVICE_CONFIG_OPTION_ARCH; + arch_customconfig.arch.arch = (QnnHtpDevice_Arch_t)soc_info.htp_arch; + arch_customconfig.arch.deviceId = 0; + QnnDevice_Config_t arch_devconfig; + arch_devconfig.option = QNN_DEVICE_CONFIG_OPTION_CUSTOM; + arch_devconfig.customConfig = &arch_customconfig; + */ + const QnnDevice_Config_t * p_deviceconfig[] = { &soc_devconfig, nullptr }; + qnnstatus = _qnn_raw_interface.deviceCreate(_qnn_log_handle, p_deviceconfig, &_qnn_device_handle); + } else { + qnnstatus = _qnn_interface.qnn_device_create(_qnn_log_handle, nullptr, &_qnn_device_handle); + } if (QNN_SUCCESS != qnnstatus && QNN_DEVICE_ERROR_UNSUPPORTED_FEATURE != qnnstatus) { GGMLQNN_LOG_WARN("failed to create QNN device\n"); } else { GGMLQNN_LOG_INFO("create device successfully\n"); } - if (ggml_qnn_profile_level::profile_off != _profile_level) { + if (qnn_profile_level::profile_off != _profile_level) { GGMLQNN_LOG_INFO("profiling turned on; level = %d", _profile_level); - if (ggml_qnn_profile_level::profile_basic == _profile_level) { + if (qnn_profile_level::profile_basic == _profile_level) { GGMLQNN_LOG_INFO("basic profiling requested. creating Qnn Profile object\n"); if (QNN_PROFILE_NO_ERROR != _qnn_raw_interface.profileCreate( _qnn_backend_handle, QNN_PROFILE_LEVEL_BASIC, &_qnn_profile_handle)) { @@ -2540,7 +2721,7 @@ int qnn_instance::qnn_init(const QnnSaver_Config_t ** saver_config) { } else { GGMLQNN_LOG_DEBUG("initialize qnn profile successfully\n"); } - } else if (ggml_qnn_profile_level::profile_detail == _profile_level) { + } else if (qnn_profile_level::profile_detail == _profile_level) { GGMLQNN_LOG_INFO("detailed profiling requested. Creating Qnn Profile object\n"); if (QNN_PROFILE_NO_ERROR != _qnn_raw_interface.profileCreate( _qnn_backend_handle, QNN_PROFILE_LEVEL_DETAILED, &_qnn_profile_handle)) { @@ -2553,7 +2734,14 @@ int qnn_instance::qnn_init(const QnnSaver_Config_t ** saver_config) { } #if defined(__ANDROID__) || defined(__linux__) - _rpc_lib_handle = dlopen("libcdsprpc.so", RTLD_NOW | RTLD_LOCAL); + //_rpc_lib_handle = dlopen("libcdsprpc.so", RTLD_NOW | RTLD_LOCAL); + std::filesystem::path full_path(std::string(g_qnn_params.qnn_runtimelib_path) + "libcdsprpc.so"); + full_path /= std::filesystem::path("libcdsprpc.so").filename(); + _rpc_lib_handle = dlopen(full_path.string().c_str(), RTLD_NOW | RTLD_LOCAL); + if (nullptr == _rpc_lib_handle) { + GGMLQNN_LOG_WARN("failed to load %s\n", full_path.c_str()); + _rpc_lib_handle = dlopen("libcdsprpc.so", RTLD_NOW | RTLD_LOCAL); + } #else _rpc_lib_handle = dlopen("libcdsprpc.dll", RTLD_NOW | RTLD_LOCAL); #endif @@ -2593,16 +2781,16 @@ int qnn_instance::qnn_init(const QnnSaver_Config_t ** saver_config) { if (_backend_name.find("Htp") != std::string::npos) { htp_print_info(); - htp_probe_device_meminfo(); + htp_probe_rpc_meminfo(); - if (0 != init_htp_perfinfra()) { + if (0 != htp_init_perfinfra()) { GGMLQNN_LOG_WARN("initialize HTP performance failure"); } -#if 0 - if (0 != set_rpc_polling()) { +#if 1 + if (0 != htp_set_rpc_polling()) { GGMLQNN_LOG_WARN("set RPC polling failure"); } - if (0 != set_high_performance_mode()) { + if (0 != htp_set_high_performance_mode()) { GGMLQNN_LOG_WARN("set HTP high performance mode failure"); } #else @@ -2628,7 +2816,8 @@ int qnn_instance::qnn_finalize() { Qnn_ErrorHandle_t error = QNN_SUCCESS; GGMLQNN_LOG_DEBUG("enter %s\n", __func__); - ggmqnn_reset_tensoridx(); + ggmlqnn_reset_tensoridx(); + ggmlqnn_reset_opcfgidx(); free_rpcmem(); unregister_rpcmem(); @@ -2706,7 +2895,6 @@ int qnn_instance::init_qnn_graph(const std::string & graph_name, QNNBackend devi GGMLQNN_LOG_DEBUG("[%s][%s]created", ggml_backend_qnn_get_devname(device), graph_name.c_str()); Qnn_ErrorHandle_t error = QNN_SUCCESS; - Qnn_GraphHandle_t graph_handle = nullptr; if (device == QNN_BACKEND_NPU) { QnnHtpGraph_CustomConfig_t hvx_config; hvx_config.option = QNN_HTP_GRAPH_CONFIG_OPTION_NUM_HVX_THREADS; @@ -2715,42 +2903,52 @@ int qnn_instance::init_qnn_graph(const std::string & graph_name, QNNBackend devi graph_hvx_config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM; graph_hvx_config.customConfig = &hvx_config; - QnnHtpGraph_CustomConfig_t dlbc_config; + QnnHtpGraph_CustomConfig_t dlbc_config = QNN_HTP_GRAPH_CUSTOM_CONFIG_INIT; dlbc_config.option = QNN_HTP_GRAPH_CONFIG_OPTION_OPTIMIZATION; dlbc_config.optimizationOption.type = QNN_HTP_GRAPH_OPTIMIZATION_TYPE_ENABLE_DLBC; - dlbc_config.optimizationOption.floatValue = 1.0; // set to 0.0 to turn off DLBC + if (0 == g_qnn_params.enable_dlbc) + dlbc_config.optimizationOption.floatValue = 0.0; // set to 0.0 to turn off DLBC + else + dlbc_config.optimizationOption.floatValue = 1.0; // set to 1.0 to turn on DLBC QnnGraph_Config_t graph_dlbc_config; graph_dlbc_config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM; graph_dlbc_config.customConfig = &dlbc_config; - QnnHtpGraph_CustomConfig_t opt_config; + QnnHtpGraph_CustomConfig_t opt_config = QNN_HTP_GRAPH_CUSTOM_CONFIG_INIT; + opt_config.option = QNN_HTP_GRAPH_CONFIG_OPTION_OPTIMIZATION; opt_config.optimizationOption.type = QNN_HTP_GRAPH_OPTIMIZATION_TYPE_FINALIZE_OPTIMIZATION_FLAG; opt_config.optimizationOption.floatValue = 1; // 1 / 3 QnnGraph_Config_t graph_opt_config; graph_opt_config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM; graph_opt_config.customConfig = &opt_config; - QnnHtpGraph_CustomConfig_t vtcm_config; + QnnHtpGraph_CustomConfig_t vtcm_config = QNN_HTP_GRAPH_CUSTOM_CONFIG_INIT; vtcm_config.option = QNN_HTP_GRAPH_CONFIG_OPTION_VTCM_SIZE; vtcm_config.vtcmSizeInMB = vtcm_size_in_mb; QnnGraph_Config_t graph_vtcm_config; graph_vtcm_config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM; graph_vtcm_config.customConfig = &vtcm_config; - QnnHtpGraph_CustomConfig_t fp16_config; - fp16_config.option = QNN_HTP_GRAPH_CONFIG_OPTION_PRECISION; - fp16_config.precision = QNN_PRECISION_FLOAT16; - QnnGraph_Config_t graph_fp16_config; - graph_fp16_config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM; - graph_fp16_config.customConfig = &fp16_config; - - const QnnGraph_Config_t * graph_configs[] = {&graph_hvx_config, &graph_dlbc_config, &graph_vtcm_config, - &graph_opt_config, &graph_fp16_config, nullptr}; - error = _qnn_interface.qnn_graph_create(_qnn_context_handle, graph_name.c_str(), graph_configs, &graph_handle); + std::vector graph_configs; + graph_configs.push_back(&graph_hvx_config); + graph_configs.push_back(&graph_dlbc_config); + graph_configs.push_back(&graph_vtcm_config); + graph_configs.push_back(&graph_opt_config); + if (1 == g_qnn_params.precision_mode) { + QnnHtpGraph_CustomConfig_t fp16_config = QNN_HTP_GRAPH_CUSTOM_CONFIG_INIT; + fp16_config.option = QNN_HTP_GRAPH_CONFIG_OPTION_PRECISION; + fp16_config.precision = QNN_PRECISION_FLOAT16; + QnnGraph_Config_t graph_fp16_config; + graph_fp16_config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM; + graph_fp16_config.customConfig = &fp16_config; + graph_configs.push_back(&graph_fp16_config); + } + graph_configs.push_back(nullptr); + error = _qnn_interface.qnn_graph_create(_qnn_context_handle, graph_name.c_str(), graph_configs.data(), &_qnn_graph_handle); + GGMLQNN_LOG_DEBUG("[%s][%s]created graph %p", ggml_backend_qnn_get_devname(device), graph_name.c_str(), _qnn_graph_handle); } else { - error = _qnn_interface.qnn_graph_create(_qnn_context_handle, graph_name.c_str(), nullptr, &graph_handle); + error = _qnn_interface.qnn_graph_create(_qnn_context_handle, graph_name.c_str(), nullptr, &_qnn_graph_handle); } - if (error != QNN_SUCCESS) { GGMLQNN_LOG_ERROR("[%s][%s]failed to create qnn graph, error: %s", ggml_backend_qnn_get_devname(device), graph_name.c_str(), @@ -2759,7 +2957,6 @@ int qnn_instance::init_qnn_graph(const std::string & graph_name, QNNBackend devi } GGMLQNN_LOG_DEBUG("[%s]create graph %s succeed", ggml_backend_qnn_get_devname(device), graph_name.c_str()); - _qnn_graph_handle = graph_handle; if (device == QNN_BACKEND_NPU) { htp_set_n_hvx_threads(hvx_threads); } @@ -2817,7 +3014,7 @@ int qnn_instance::finalize_qnn_graph() { return 0; } -int qnn_instance::init_htp_perfinfra() { +int qnn_instance::htp_init_perfinfra() { QnnDevice_Infrastructure_t device_infra = nullptr; int error = _qnn_raw_interface.deviceGetInfrastructure(&device_infra); if (error != QNN_SUCCESS) { @@ -2840,57 +3037,6 @@ int qnn_instance::init_htp_perfinfra() { return 0; } -int qnn_instance::set_rpc_polling() { - if (_qnn_rpc_pollingtime > 0) { - QnnHtpPerfInfrastructure_PowerConfig_t rpc_pollingtime; - memset(&rpc_pollingtime, 0, sizeof(rpc_pollingtime)); - rpc_pollingtime.option = QNN_HTP_PERF_INFRASTRUCTURE_POWER_CONFIGOPTION_RPC_POLLING_TIME; - rpc_pollingtime.rpcPollingTimeConfig = _qnn_rpc_pollingtime; - const QnnHtpPerfInfrastructure_PowerConfig_t * power_configs[] = {&rpc_pollingtime, nullptr}; - if (_qnn_htp_perfinfra) { - _qnn_htp_perfinfra->setPowerConfig(_qnn_htp_powerconfig_id, power_configs); - } - } - return 0; -} - -int qnn_instance::set_high_performance_mode() { - if (nullptr == _qnn_htp_perfinfra) { - GGMLQNN_LOG_DEBUG("perf intra is null\n"); - return 1; - } - - QnnHtpPerfInfrastructure_PowerConfig_t power_config; - memset(&power_config, 0, sizeof(power_config)); - power_config.option = QNN_HTP_PERF_INFRASTRUCTURE_POWER_CONFIGOPTION_DCVS_V3; - power_config.dcvsV3Config.dcvsEnable = 0; - power_config.dcvsV3Config.setDcvsEnable = 1; - power_config.dcvsV3Config.contextId = _qnn_htp_powerconfig_id; - power_config.dcvsV3Config.powerMode = QNN_HTP_PERF_INFRASTRUCTURE_POWERMODE_PERFORMANCE_MODE; - power_config.dcvsV3Config.setSleepLatency = 1; // True to consider Latency parameter otherwise False - power_config.dcvsV3Config.setBusParams = 1; // True to consider Bus parameter otherwise False - power_config.dcvsV3Config.setCoreParams = 1; // True to consider Core parameter otherwise False - power_config.dcvsV3Config.sleepDisable = 0; // True to consider sleep/LPM modes, False to enable - power_config.dcvsV3Config.setSleepDisable = 0; // True to consider sleep disable/enable parameter otherwise False - // set Sleep latency parameter - uint32_t latencyValue = 40; - power_config.dcvsV3Config.sleepLatency = latencyValue; // range 40-2000 micro sec - // set Bus Clock Parameters (refer QnnHtpPerfInfrastructure_VoltageCorner_t enum) - power_config.dcvsV3Config.busVoltageCornerMin = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; - power_config.dcvsV3Config.busVoltageCornerTarget = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; - power_config.dcvsV3Config.busVoltageCornerMax = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; - // set Core Clock Parameters (refer QnnHtpPerfInfrastructure_VoltageCorner_t enum) - power_config.dcvsV3Config.coreVoltageCornerMin = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; - power_config.dcvsV3Config.coreVoltageCornerTarget = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; - power_config.dcvsV3Config.coreVoltageCornerMax = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; - // set power config with different performance parameters - const QnnHtpPerfInfrastructure_PowerConfig_t * power_configs[] = {&power_config, nullptr}; - - _qnn_htp_perfinfra->setPowerConfig(_qnn_htp_powerconfig_id, power_configs); - - return 0; -} - void qnn_instance::htp_print_info() { const QnnDevice_PlatformInfo_t * p_info = nullptr; _qnn_raw_interface.deviceGetPlatformInfo(nullptr, &p_info); @@ -2922,7 +3068,7 @@ void qnn_instance::htp_print_info() { _qnn_raw_interface.deviceFreePlatformInfo(nullptr, p_info); } -void qnn_instance::htp_probe_device_meminfo() { +void qnn_instance::htp_probe_rpc_meminfo() { size_t candidate_size = 0; uint8_t * rpc_buffer = nullptr; const int SIZE_IN_MB = (1 << 20); @@ -2976,24 +3122,96 @@ void qnn_instance::print_backend_info() { print_property("Qnn group device", QNN_PROPERTY_GROUP_DEVICE); } -void qnn_instance::htp_set_memory_grow_size(size_t size) { - QnnHtpPerfInfrastructure_MemoryConfig_t grow_size_config = { - .option = QNN_HTP_PERF_INFRASTRUCTURE_MEMORY_CONFIGOPTION_GROW_SIZE, - .memGrowSizeConfig = (uint32_t)size, - }; +void qnn_instance::htp_set_memory_grow_size(size_t size) { + QnnHtpPerfInfrastructure_MemoryConfig_t grow_size_config = { + .option = QNN_HTP_PERF_INFRASTRUCTURE_MEMORY_CONFIGOPTION_GROW_SIZE, + .memGrowSizeConfig = (uint32_t)size, + }; + + const QnnHtpPerfInfrastructure_MemoryConfig_t *memory_config[] = { + &grow_size_config, + nullptr, + }; + Qnn_ErrorHandle_t ret = _qnn_htp_perfinfra->setMemoryConfig(_qnn_htp_device_id, _qnn_htp_core_id, memory_config); + if (ret != QNN_SUCCESS) { + GGMLQNN_LOG_WARN("failed to set HTP memory config"); + } else { + GGMLQNN_LOG_INFO("succeed to set HTP memory config"); + } +} + +void qnn_instance::htp_set_n_hvx_threads(size_t n_threads) { + QnnHtpGraph_CustomConfig_t htp_hvx_thread_config = { + .option = QNN_HTP_GRAPH_CONFIG_OPTION_NUM_HVX_THREADS, + .numHvxThreads = n_threads, + }; + + QnnGraph_Config_t hvx_thread_config = { + .option = QNN_GRAPH_CONFIG_OPTION_CUSTOM, + .customConfig = &htp_hvx_thread_config, + }; + + const QnnGraph_Config_t * graph_configs[] = {&hvx_thread_config, nullptr}; + Qnn_ErrorHandle_t ret = _qnn_raw_interface.graphSetConfig(_qnn_graph_handle, graph_configs); + if (ret != QNN_SUCCESS) { + GGMLQNN_LOG_WARN("failed to set QNN graph config: set hvx threads %d", n_threads); + } else { + GGMLQNN_LOG_INFO("succeed to set QNN graph config: set hvx threads %d", n_threads); + } +} + +int qnn_instance::htp_set_rpc_polling() { + if (_qnn_rpc_pollingtime > 0) { + QnnHtpPerfInfrastructure_PowerConfig_t rpc_pollingtime; + memset(&rpc_pollingtime, 0, sizeof(rpc_pollingtime)); + rpc_pollingtime.option = QNN_HTP_PERF_INFRASTRUCTURE_POWER_CONFIGOPTION_RPC_POLLING_TIME; + rpc_pollingtime.rpcPollingTimeConfig = _qnn_rpc_pollingtime; + const QnnHtpPerfInfrastructure_PowerConfig_t * power_configs[] = {&rpc_pollingtime, nullptr}; + if (_qnn_htp_perfinfra) { + _qnn_htp_perfinfra->setPowerConfig(_qnn_htp_powerconfig_id, power_configs); + } + } + return 0; +} + +int qnn_instance::htp_set_high_performance_mode() { + if (nullptr == _qnn_htp_perfinfra) { + GGMLQNN_LOG_DEBUG("perf intra is null\n"); + return 1; + } + + QnnHtpPerfInfrastructure_PowerConfig_t power_config; + memset(&power_config, 0, sizeof(power_config)); + power_config.option = QNN_HTP_PERF_INFRASTRUCTURE_POWER_CONFIGOPTION_DCVS_V3; + power_config.dcvsV3Config.dcvsEnable = 0; + power_config.dcvsV3Config.setDcvsEnable = 1; + power_config.dcvsV3Config.contextId = _qnn_htp_powerconfig_id; + power_config.dcvsV3Config.powerMode = QNN_HTP_PERF_INFRASTRUCTURE_POWERMODE_PERFORMANCE_MODE; + power_config.dcvsV3Config.setSleepLatency = 1; // True to consider Latency parameter otherwise False + power_config.dcvsV3Config.setBusParams = 1; // True to consider Bus parameter otherwise False + power_config.dcvsV3Config.setCoreParams = 1; // True to consider Core parameter otherwise False + power_config.dcvsV3Config.sleepDisable = 0; // True to consider sleep/LPM modes, False to enable + power_config.dcvsV3Config.setSleepDisable = 0; // True to consider sleep disable/enable parameter otherwise False + // set Sleep latency parameter + uint32_t latencyValue = 40; + power_config.dcvsV3Config.sleepLatency = latencyValue; // range 40-2000 micro sec + // set Bus Clock Parameters (refer QnnHtpPerfInfrastructure_VoltageCorner_t enum) + power_config.dcvsV3Config.busVoltageCornerMin = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; + power_config.dcvsV3Config.busVoltageCornerTarget = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; + power_config.dcvsV3Config.busVoltageCornerMax = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; + // set Core Clock Parameters (refer QnnHtpPerfInfrastructure_VoltageCorner_t enum) + power_config.dcvsV3Config.coreVoltageCornerMin = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; + power_config.dcvsV3Config.coreVoltageCornerTarget = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; + power_config.dcvsV3Config.coreVoltageCornerMax = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; + // set power config with different performance parameters + const QnnHtpPerfInfrastructure_PowerConfig_t * power_configs[] = {&power_config, nullptr}; + + _qnn_htp_perfinfra->setPowerConfig(_qnn_htp_powerconfig_id, power_configs); - const QnnHtpPerfInfrastructure_MemoryConfig_t *memory_config[] = { - &grow_size_config, - nullptr, - }; - Qnn_ErrorHandle_t ret = _qnn_htp_perfinfra->setMemoryConfig(_qnn_htp_device_id, _qnn_htp_core_id, memory_config); - if (ret != QNN_SUCCESS) { - GGMLQNN_LOG_WARN("failed to set HTP memory config"); - } else { - GGMLQNN_LOG_INFO("succeed to set HTP memory config"); - } + return 0; } +//TODO: merge code between this function and htp_set_rpc_polling,htp_set_high_performance_mode void qnn_instance::htp_enter_performance_mode() { QnnHtpPerfInfrastructure_PowerConfig_t dcvs_v3_config = { .option = QNN_HTP_PERF_INFRASTRUCTURE_POWER_CONFIGOPTION_DCVS_V3, @@ -3046,7 +3264,7 @@ void qnn_instance::htp_enter_performance_mode() { .rpcPollingTimeConfig = 9999, }; - const QnnHtpPerfInfrastructure_PowerConfig_t *power_configs[] = { + const QnnHtpPerfInfrastructure_PowerConfig_t * power_configs[] = { &dcvs_v3_config, &hmx_config, &rpc_ctrl_config, @@ -3061,26 +3279,6 @@ void qnn_instance::htp_enter_performance_mode() { } } -void qnn_instance::htp_set_n_hvx_threads(size_t n_threads) { - QnnHtpGraph_CustomConfig_t htp_hvx_thread_config = { - .option = QNN_HTP_GRAPH_CONFIG_OPTION_NUM_HVX_THREADS, - .numHvxThreads = n_threads, - }; - - QnnGraph_Config_t hvx_thread_config = { - .option = QNN_GRAPH_CONFIG_OPTION_CUSTOM, - .customConfig = &htp_hvx_thread_config, - }; - - const QnnGraph_Config_t * graph_configs[] = {&hvx_thread_config, nullptr}; - Qnn_ErrorHandle_t ret = _qnn_raw_interface.graphSetConfig(_qnn_graph_handle, graph_configs); - if (ret != QNN_SUCCESS) { - GGMLQNN_LOG_WARN("failed to set QNN graph config: set hvx threads %d", n_threads); - } else { - GGMLQNN_LOG_INFO("succeed to set QNN graph config: set hvx threads %d", n_threads); - } -} - static uint8_t * ggmlqnn_create_rpc_buffer(qnn_instance * instance, const ggml_tensor * ggml_tensor, Qnn_Tensor_t * qnn_tensor, bool b_copydata) { if (nullptr == instance || nullptr == ggml_tensor || nullptr == qnn_tensor) { GGMLQNN_LOG_WARN("invalid params\n"); @@ -3100,36 +3298,45 @@ static uint8_t * ggmlqnn_create_rpc_buffer(qnn_instance * instance, const ggml_t return qnn_rpcbuffer; } -static void ggmlqnn_print_tensors_info(const char * func_name, ggml_backend_qnn_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - //skip sanity check of params because of performance concern - if (nullptr != func_name && nullptr != ctx) { - GGMLQNN_LOG_DEBUG("call %s in dev %s\n", func_name, ctx->name); +static void ggmlqnn_load_cfg() { + char time_string[GGML_QNN_TMPBUF_LEN]; + memset(time_string, 0, GGML_QNN_TMPBUF_LEN); + ggmlqnn_get_timestring(time_string); + GGMLQNN_LOG_DEBUG("program running start time:%s", time_string); + std::string cfg_filename = std::string(g_qnn_params.qnn_runtimelib_path) + std::string(g_qnn_params.qnn_cfgfilename); + GGMLQNN_LOG_INFO("load ggml-qnn config from %s", cfg_filename.c_str()); + qnn_cfg qnncfg_instance; + qnncfg_instance.load(cfg_filename); + qnncfg_instance.dump([](const std::string & section, const std::string & key, const std::string value) { + std::ostringstream tmposs; + tmposs << "section[" << std::setw(10) << std::left << section << "],[" << std::setw(25) << std::left << key << "] = [" << value << "]" << std::endl; + GGMLQNN_LOG_INFO("%s", tmposs.str().c_str()); + }); + std::string precision_mode; + qnncfg_instance.get_intvalue("general", "print_qnn_internal_log", g_qnn_params.print_qnn_internal_log, 0); + qnncfg_instance.get_intvalue("general", "enable_perf", g_qnn_params.enable_perf, 0); + qnncfg_instance.get_intvalue("general", "print_tensors_info", g_qnn_params.print_tensors_info, 0); + qnncfg_instance.get_intvalue("general", "dump_op_info", g_qnn_params.dump_op_info, 0); + qnncfg_instance.get_intvalue("general", "inference_approach", g_qnn_params.inference_approach, 0); + qnncfg_instance.get_intvalue("general", "qnn_backend", g_qnn_params.qnn_backend, 2); + qnncfg_instance.get_intvalue("npu", "hvx_threads", g_qnn_params.hvx_threads, 4); + qnncfg_instance.get_intvalue("npu", "vtcm_size_in_mb", g_qnn_params.vtcm_size_in_mb, 8); + qnncfg_instance.get_intvalue("npu", "enable_dlbc", g_qnn_params.enable_dlbc, 0); + qnncfg_instance.get_stringvalue("npu", "precision_mode", precision_mode, "fp32"); + GGMLQNN_LOG_INFO("print_qnn_internal_log=%d", g_qnn_params.print_qnn_internal_log); + GGMLQNN_LOG_INFO("inference_approach=%d", g_qnn_params.inference_approach); + GGMLQNN_LOG_INFO("qnn_backend=%d", g_qnn_params.qnn_backend); + GGMLQNN_LOG_INFO("npu inference precision mode=%s", precision_mode.c_str()); + GGMLQNN_LOG_INFO("qnn runtime lib path=%s", g_qnn_params.qnn_runtimelib_path); + if (precision_mode.find("fp16") != std::string::npos) { + g_qnn_params.precision_mode = 1; + } else { + g_qnn_params.precision_mode = 0; } - GGMLQNN_LOG_DEBUG("%-6s: type = %i (%s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi, %5zi)", - src0->name, - src0->type, ggml_type_name(src0->type), src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], - src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3]); - GGMLQNN_LOG_DEBUG("%-6s: type = %i (%s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi, %5zi)", - src1->name, - src1->type, ggml_type_name(src1->type), src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3], - src1->nb[0], src1->nb[1], src1->nb[2], src1->nb[3]); - GGMLQNN_LOG_DEBUG("%-6s: type = %i (%s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi, %5zi)", - dst->name, - dst->type, ggml_type_name(dst->type), dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], - dst->nb[0], dst->nb[1], dst->nb[2], dst->nb[3]); - GGMLQNN_LOG_DEBUG("\n"); -} - -static void ggmlqnn_dump_op_info(const struct ggml_tensor * tensor) { - //skip sanity check of params because of performance concern - const struct ggml_tensor * src0 = tensor->src[0]; - struct ggml_tensor * src1 = tensor->src[1]; - struct ggml_tensor * dst = const_cast(tensor); - GGMLQNN_LOG_DEBUG("op name:%s, tensor type:%s", ggml_op_name(tensor->op), ggml_type_name(tensor->type)); - ggmlqnn_print_tensors_info(nullptr, nullptr, src0, src1, dst); } -static Qnn_Tensor_t * ggmlqnn_create_general_tensor(const ggml_tensor * tensor, const char * name, +static Qnn_Tensor_t * ggmlqnn_create_general_tensor(qnn_instance * instance, Qnn_GraphHandle_t graph_handle, + const ggml_tensor * tensor, const char * name, Qnn_TensorType_t qnn_tensor_type, Qnn_DataType_t qnn_data_type, uint32_t rank, uint32_t * dims, @@ -3140,12 +3347,12 @@ static Qnn_Tensor_t * ggmlqnn_create_general_tensor(const ggml_tensor * tensor, //ensure the tensor name is unique if (nullptr == name) { - snprintf(tensor_name, GGML_MAX_NAME, "tensor_%-8d", ggmqnn_get_tensoridx()); + snprintf(tensor_name, GGML_MAX_NAME, "tensor_%-8d", ggmlqnn_get_tensoridx()); } else { - snprintf(tensor_name, GGML_MAX_NAME, "tensor_%s%-8d", name, ggmqnn_get_tensoridx()); + snprintf(tensor_name, GGML_MAX_NAME, "tensor_%s%-8d", name, ggmlqnn_get_tensoridx()); } - GGMLQNN_LOG_DEBUG("init_tensor %d", ggmqnn_get_tensoridx()); - ggmqnn_inc_tensoridx(); + GGMLQNN_LOG_DEBUG("init_tensor %s", tensor_name); + ggmlqnn_inc_tensoridx(); uint32_t reverse_dims[GGML_MAX_DIMS] = {}; uint32_t transpose_dims[GGML_MAX_DIMS] = {}; @@ -3196,9 +3403,7 @@ static Qnn_Tensor_t * ggmlqnn_create_general_tensor(const ggml_tensor * tensor, } } }; - if (nullptr != name) { - QNN_VER_PTR(qnn_tensor)->name = name; - } + Qnn_Tensor_t * p_qnn_tensor = (Qnn_Tensor_t *)calloc(1, sizeof(Qnn_Tensor_t)); if (nullptr == p_qnn_tensor) { GGMLQNN_LOG_WARN("calloc failed"); @@ -3210,12 +3415,22 @@ static Qnn_Tensor_t * ggmlqnn_create_general_tensor(const ggml_tensor * tensor, GGMLQNN_LOG_WARN("init tensor failed"); return nullptr; } - QNN_VER_PTR(*p_qnn_tensor)->clientBuf = {data, data_size}; + + bool enable_npu_rpc = (instance->enable_qnn_rpc() && instance->get_device_id() == QNN_BACKEND_NPU); + if (enable_npu_rpc) { + QNN_VER_PTR(*p_qnn_tensor)->memType = QNN_TENSORMEMTYPE_MEMHANDLE; + QNN_VER_PTR(*p_qnn_tensor)->clientBuf = {.data=nullptr, .dataSize=0}; + } else { + QNN_VER_PTR(*p_qnn_tensor)->clientBuf = {data, data_size}; + } + QNN_INTERFACE_VER_TYPE qnn_raw_interface = instance->get_qnn_raw_interface(); + CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_qnn_tensor)); return p_qnn_tensor; } -static Qnn_Tensor_t * ggmlqnn_create_compute_tensor(qnn_instance * instance, Qnn_GraphHandle_t graph_handle, const ggml_tensor * tensor, Qnn_TensorType_t tensor_type) { +static Qnn_Tensor_t * ggmlqnn_create_compute_tensor(qnn_instance * instance, Qnn_GraphHandle_t graph_handle, + const ggml_tensor * tensor, Qnn_TensorType_t tensor_type) { Qnn_ErrorHandle_t error = QNN_SUCCESS; uint32_t dimensions[] = {(uint32_t) tensor->ne[0], (uint32_t) tensor->ne[1], (uint32_t) tensor->ne[2], (uint32_t) tensor->ne[3]}; @@ -3233,140 +3448,149 @@ static Qnn_Tensor_t * ggmlqnn_create_compute_tensor(qnn_instance * instance, Qnn } qnn_data_type = ggmlqnn_datatype_from_ggml_datatype(tensor->type); - Qnn_Tensor_t * p_qnn_tensor = ggmlqnn_create_general_tensor(tensor, nullptr, - qnn_tensor_type, qnn_data_type, - ggml_n_dims(tensor), dimensions, - nullptr, 0); - - bool enable_npu_rpc = (instance->enable_qnn_rpc() && instance->get_device_id() == QNN_BACKEND_NPU); - if (enable_npu_rpc) { - QNN_VER_PTR(*p_qnn_tensor)->memType = QNN_TENSORMEMTYPE_MEMHANDLE; - QNN_VER_PTR(*p_qnn_tensor)->clientBuf = {.data=nullptr, .dataSize=0}; - } - QNN_INTERFACE_VER_TYPE qnn_raw_interface = instance->get_qnn_raw_interface(); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_qnn_tensor)); - + Qnn_Tensor_t * p_qnn_tensor = ggmlqnn_create_general_tensor(instance, graph_handle, tensor, nullptr, + qnn_tensor_type, qnn_data_type, + ggml_n_dims(tensor), dimensions, + nullptr, 0); return p_qnn_tensor; } // ================================================================================================= // section-8: implementation of ggml-qnn backend // ================================================================================================= -//TODO: refine this function as it is a performance hotspot/bottleneck function -static bool ggml_qnn_can_handle_op(const ggml_backend_qnn_context * ctx, const struct ggml_tensor * tensor) { - if (tensor->op == GGML_OP_NONE) { - return true; +static bool ggmlqnn_same_types(const ggml_backend_qnn_context * ctx, const ggml_tensor * op_tensor) { + GGML_UNUSED(ctx); + ggml_tensor * src0 = op_tensor->src[0]; + ggml_tensor * src1 = op_tensor->src[1]; + if (nullptr != src1) { + if (src0->type != op_tensor->type || src1->type != op_tensor->type) { + return false; + } + } else { + if (src0->type != op_tensor->type) { + return false; + } } - if (ggml_is_empty(tensor) || tensor->op == GGML_OP_RESHAPE - || tensor->op == GGML_OP_TRANSPOSE - || tensor->op == GGML_OP_VIEW - || tensor->op == GGML_OP_PERMUTE - ) { + if (src0->type != GGML_TYPE_F32) return false; + return true; +} + +static bool ggmlqnn_compute_can_handle_op(const ggml_backend_qnn_context * ctx, const struct ggml_tensor * op_tensor) { + if (op_tensor->op == GGML_OP_NONE) { + return true; } - //TODO: add other op here - bool supported_op = ((tensor->op == GGML_OP_ADD) - || (tensor->op == GGML_OP_MUL_MAT) - || (tensor->op == GGML_OP_MUL) - ); - if (!supported_op) { + if (!ggmlqnn_k_op_caps[ggmlqnn_get_op_index(op_tensor)].supported) { return false; } - struct ggml_tensor * src0 = tensor->src[0]; - struct ggml_tensor * src1 = tensor->src[1]; + struct ggml_tensor * src0 = op_tensor->src[0]; + struct ggml_tensor * src1 = op_tensor->src[1]; - const int64_t ne00 = tensor->src[0]->ne[0]; - const int64_t ne01 = tensor->src[0]->ne[1]; + const int64_t ne00 = op_tensor->src[0]->ne[0]; + const int64_t ne01 = op_tensor->src[0]->ne[1]; + const int64_t ne0 = op_tensor->ne[0]; + const int64_t ne1 = op_tensor->ne[1]; - const int64_t ne10 = tensor->src[1]->ne[0]; - const int64_t ne11 = tensor->src[1]->ne[1]; - - const int64_t ne0 = tensor->ne[0]; - const int64_t ne1 = tensor->ne[1]; - - const uint32_t src0_rank = ggml_n_dims(src0); - const uint32_t src1_rank = ggml_n_dims(src1); + uint32_t src0_rank = ggml_n_dims(src0); + uint32_t src1_rank = 0; + if (nullptr != src1) { + src1_rank = ggml_n_dims(src1); + } GGML_UNUSED(ne01); - GGML_UNUSED(ne10); - GGML_UNUSED(ne11); GGML_UNUSED(ne0); GGML_UNUSED(ne1); + switch (op_tensor->op) { + case GGML_OP_ADD: + case GGML_OP_SUB: + { + //ggmlqnn_dump_op_info(op_tensor); + if (!ggml_are_same_shape(src0, src1)) { + return false; + } - if (tensor->op == GGML_OP_ADD) { - //ggmlqnn_dump_op_info(tensor); - if (!ggml_are_same_shape(src0, src1)) { - return false; + if (ne00 < 32) + return false; + + return ggmlqnn_same_types(ctx, op_tensor); } - if (ne00 < 32) - return false; - return (src0->type == GGML_TYPE_F32) && (src1->type == GGML_TYPE_F32); - } - if (tensor->op == GGML_OP_MUL_MAT) { - //ggmlqnn_dump_op_info(tensor); - if (src0_rank != src1_rank) // make QNN SDK happy - return false; - if (src0_rank < 2) // QNN's limitation, make QNN SDK happy - return false; - if (4 == src0_rank) //TODO: 4D matrix mulmat in CT - return false; - if ((src1->ne[2] != src0->ne[2]) || (src1->ne[3] != src0->ne[3])) // make QNN SDK happy - return false; + case GGML_OP_DIV: + case GGML_OP_MUL: { + //ggmlqnn_dump_op_info(op_tensor); + if (ctx->device == QNN_BACKEND_NPU) + return false; - if (ctx->device == QNN_BACKEND_NPU) - if (2 == src0_rank) - return (src0->type == GGML_TYPE_F32 - || src0->type == GGML_TYPE_Q4_0 || src0->type == GGML_TYPE_Q8_0 - || src0->type == GGML_TYPE_Q6_K || src0->type == GGML_TYPE_Q8_K - ) && (src1->type == GGML_TYPE_F32) && (tensor->type == GGML_TYPE_F32); - else - return (src0->type == GGML_TYPE_F32) && (src1->type == GGML_TYPE_F32) && (tensor->type == GGML_TYPE_F32); - else - return (src0->type == GGML_TYPE_F32 || ggml_is_quantized(src0->type)) - && (src1->type == GGML_TYPE_F32) && (tensor->type == GGML_TYPE_F32); - } + if (!ggml_are_same_shape(src0, src1)) { + return false; + } - if (tensor->op == GGML_OP_MUL) { - //ggmlqnn_dump_op_info(tensor); - if (ctx->device == QNN_BACKEND_NPU) - return false; - if ((src0_rank != 2) || (src1_rank != 2)) //TODO: 3D and 4D matrix - return false; - return (src0->type == GGML_TYPE_F32) - && (src1->type == GGML_TYPE_F32) - && (tensor->type == src1->type); - } + if ((src0_rank != 2) || (src1_rank != 2)) //TODO: 3D and 4D matrix + return false; - return false; + return ggmlqnn_same_types(ctx, op_tensor); + } + case GGML_OP_MUL_MAT: + { + ggmlqnn_dump_op_info(op_tensor); + if (src0_rank != src1_rank) // make QNN SDK happy + return false; + + if (src0_rank < 2) // QNN's limitation, make QNN SDK happy + return false; + + if (4 == src0_rank) //TODO: 4D matrix mulmat in CT + return false; + + if ((src1->ne[2] != src0->ne[2]) || (src1->ne[3] != src0->ne[3])) // make QNN SDK happy + return false; + + if (ctx->device == QNN_BACKEND_NPU) { + return (src0->type == GGML_TYPE_F32 + || src0->type == GGML_TYPE_Q4_0 || src0->type == GGML_TYPE_Q8_0 + || src0->type == GGML_TYPE_Q6_K || src0->type == GGML_TYPE_Q8_K + ) && (src1->type == GGML_TYPE_F32) && (op_tensor->type == GGML_TYPE_F32); + } else { + return (src0->type == GGML_TYPE_F32 || ggml_is_quantized(src0->type)) + && (src1->type == GGML_TYPE_F32) && (op_tensor->type == GGML_TYPE_F32); + } + } + case GGML_OP_LOG: + { + if (ctx->device == QNN_BACKEND_NPU) + return false; + } + case GGML_OP_SQRT: + default: + return ggmlqnn_same_types(ctx, op_tensor); + } } -static bool ggml_qnn_compute_forward(ggml_backend_t backend, struct ggml_tensor * dst) { - ggmlqnn_op_func_t func = nullptr; - ggml_backend_qnn_context * ctx = (ggml_backend_qnn_context *)backend->context; +static bool ggmlqnn_compute_compute_forward(ggml_backend_t backend, struct ggml_tensor * dst) { + ggmlqnn_op_func_t func = nullptr; + ggml_backend_qnn_context * ctx = (ggml_backend_qnn_context *)backend->context; switch (dst->op) { case GGML_OP_REPEAT: - ggml_qnn_repeat(ctx, dst); + ggmlqnn_compute_repeat(ctx, dst); break; case GGML_OP_GET_ROWS: - ggml_qnn_get_rows(ctx, dst); + ggmlqnn_compute_get_rows(ctx, dst); break; case GGML_OP_DUP: - ggml_qnn_dup(ctx, dst); + ggmlqnn_compute_dup(ctx, dst); break; case GGML_OP_ADD: - func = ggml_qnn_general_node; - break; - case GGML_OP_ACC: - ggml_qnn_acc(ctx, dst); - break; + case GGML_OP_SUB: case GGML_OP_MUL: - func = ggml_qnn_general_node; - break; case GGML_OP_DIV: - ggml_qnn_div(ctx, dst); + case GGML_OP_SQRT: + case GGML_OP_LOG: + func = ggmlqnn_compute_elementwise; + break; + case GGML_OP_ACC: + ggmlqnn_compute_acc(ctx, dst); break; case GGML_OP_UNARY: switch (ggml_get_unary_op(dst)) { @@ -3389,51 +3613,51 @@ static bool ggml_qnn_compute_forward(ggml_backend_t backend, struct ggml_tensor } break; case GGML_OP_NORM: - ggml_qnn_norm(ctx, dst); + ggmlqnn_compute_norm(ctx, dst); break; case GGML_OP_GROUP_NORM: - ggml_qnn_group_norm(ctx, dst); + ggmlqnn_compute_group_norm(ctx, dst); break; case GGML_OP_CONCAT: - ggml_qnn_concat(ctx, dst); + ggmlqnn_compute_concat(ctx, dst); break; case GGML_OP_UPSCALE: - ggml_qnn_upsample_nearest2d(ctx, dst); + ggmlqnn_compute_upsample_nearest2d(ctx, dst); break; case GGML_OP_PAD: - ggml_qnn_pad(ctx, dst); + ggmlqnn_compute_pad(ctx, dst); break; case GGML_OP_ARANGE: - ggml_qnn_arange(ctx, dst); + ggmlqnn_compute_arange(ctx, dst); break; case GGML_OP_TIMESTEP_EMBEDDING: - ggml_qnn_timestep_embedding(ctx, dst); + ggmlqnn_compute_timestep_embedding(ctx, dst); break; case GGML_OP_LEAKY_RELU: - ggml_qnn_leaky_relu(ctx, dst); + ggmlqnn_compute_leaky_relu(ctx, dst); break; case GGML_OP_RMS_NORM: - ggml_qnn_rms_norm(ctx, dst); + ggmlqnn_compute_rms_norm(ctx, dst); break; case GGML_OP_MUL_MAT: - ggml_qnn_mul_mat(ctx, dst); + ggmlqnn_compute_mul_mat(ctx, dst); break; case GGML_OP_MUL_MAT_ID: return false; case GGML_OP_SCALE: - ggml_qnn_scale(ctx, dst); + ggmlqnn_compute_scale(ctx, dst); break; case GGML_OP_SQR: - ggml_qnn_sqr(ctx, dst); + ggmlqnn_compute_sqr(ctx, dst); break; case GGML_OP_CLAMP: - ggml_qnn_clamp(ctx, dst); + ggmlqnn_compute_clamp(ctx, dst); break; case GGML_OP_CPY: - ggml_qnn_cpy(ctx, dst); + ggmlqnn_compute_cpy(ctx, dst); break; case GGML_OP_CONT: - ggml_qnn_dup(ctx, dst); + ggmlqnn_compute_dup(ctx, dst); break; case GGML_OP_NONE: case GGML_OP_RESHAPE: @@ -3442,25 +3666,25 @@ static bool ggml_qnn_compute_forward(ggml_backend_t backend, struct ggml_tensor case GGML_OP_TRANSPOSE: break; case GGML_OP_DIAG_MASK_INF: - ggml_qnn_diag_mask(ctx, dst, -INFINITY); + ggmlqnn_compute_diag_mask(ctx, dst, -INFINITY); break; case GGML_OP_SOFT_MAX: - ggml_qnn_softmax(ctx, dst); + ggmlqnn_compute_softmax(ctx, dst); break; case GGML_OP_ROPE: - ggml_qnn_rope(ctx, dst); + ggmlqnn_compute_rope(ctx, dst); break; case GGML_OP_IM2COL: - ggml_qnn_im2col(ctx, dst); + ggmlqnn_compute_im2col(ctx, dst); break; case GGML_OP_POOL_2D: - ggml_qnn_pool2d(ctx, dst); + ggmlqnn_compute_pool2d(ctx, dst); break; case GGML_OP_SUM_ROWS: - ggml_qnn_sum_rows(ctx, dst); + ggmlqnn_compute_sum_rows(ctx, dst); break; case GGML_OP_ARGSORT: - ggml_qnn_argsort(ctx, dst); + ggmlqnn_compute_argsort(ctx, dst); break; default: return false; @@ -3472,6 +3696,7 @@ static bool ggml_qnn_compute_forward(ggml_backend_t backend, struct ggml_tensor return true; } +//TODO: refine this data structure struct ggml_backend_qnn_buffer_context { ~ggml_backend_qnn_buffer_context() { if (buffer) { @@ -3572,7 +3797,7 @@ static const char * ggml_backend_qnn_buffer_type_name(ggml_backend_buffer_type_t } static ggml_backend_buffer_t ggml_backend_qnn_buffer_type_alloc_buffer( - ggml_backend_buffer_type_t buft, size_t size) { + ggml_backend_buffer_type_t buft, size_t size) { ggml_backend_qnn_buffer_context * ctx = new ggml_backend_qnn_buffer_context; size_t size_page = 0; @@ -3623,24 +3848,37 @@ static const char * ggml_backend_qnn_name(ggml_backend_t backend) { static void ggml_backend_qnn_free(ggml_backend_t backend) { GGMLQNN_LOG_DEBUG("enter %s", __func__ ); ggml_backend_qnn_context * ctx = (ggml_backend_qnn_context *) backend->context; - GGMLQNN_LOG_DEBUG("idx %d, name:%s", ctx->device, g_qnn_mgr[ctx->device].name); + GGMLQNN_LOG_DEBUG("device idx %d, name:%s", ctx->device, g_qnn_mgr[ctx->device].name); qnn_instance * instance = (qnn_instance*)g_qnn_mgr[ctx->device].instance; if (instance != nullptr) { - std::map>>::iterator graph_it; - - for (graph_it = instance->_qnn_graph_map.begin(); - graph_it != instance->_qnn_graph_map.end(); graph_it++) { - auto & graph_item = graph_it->second; - Qnn_GraphHandle_t & graph_handle = std::get<0>(graph_item); - qnn_tensors_t & tensors = std::get<1>(graph_item); - for (auto tensor_it = tensors.begin(); tensor_it != tensors.end(); ++tensor_it) { + std::map::iterator singlenode_graph_it; + for (singlenode_graph_it = ctx->qnn_singlenode_graph_map.begin(); + singlenode_graph_it != ctx->qnn_singlenode_graph_map.end(); singlenode_graph_it++) { + auto & graph_res = singlenode_graph_it->second; + Qnn_GraphHandle_t & graph_handle = std::get<0>(graph_res); + qnn_ptensors_t & ptensors = std::get<1>(graph_res); + for (auto tensor_it = ptensors.begin(); tensor_it != ptensors.end(); ++tensor_it) { free_qnn_tensor(*tensor_it); } GGML_UNUSED(graph_handle); - GGMLQNN_LOG_DEBUG("graph type:%s", graph_it->first.c_str()); + GGMLQNN_LOG_DEBUG("clean up graph:%s", singlenode_graph_it->first.c_str()); } - instance->_qnn_graph_map.clear(); + ctx->qnn_singlenode_graph_map.clear(); + + std::map::iterator multinode_graph_it; + for (multinode_graph_it = ctx->qnn_multinode_graph_map.begin(); + multinode_graph_it != ctx->qnn_multinode_graph_map.end(); multinode_graph_it++) { + auto & graph_res = multinode_graph_it->second; + Qnn_GraphHandle_t & graph_handle = std::get<0>(graph_res); + qnn_ptensors_t & ptensors = std::get<2>(graph_res); + for (auto tensor_it = ptensors.begin(); tensor_it != ptensors.end(); ++tensor_it) { + free_qnn_tensor(*tensor_it); + } + GGML_UNUSED(graph_handle); + GGMLQNN_LOG_DEBUG("clean up graph:%s", multinode_graph_it->first.c_str()); + } + ctx->qnn_multinode_graph_map.clear(); instance->qnn_finalize(); delete instance; @@ -3654,29 +3892,31 @@ static void ggml_backend_qnn_free(ggml_backend_t backend) { GGMLQNN_LOG_DEBUG("leave %s", __func__ ); } -static enum ggml_status ggml_backend_qnn_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph) { +//this is the first tech approach(or general approach in other ggml backends, such as ggml-sycl or ggml-cann) +static enum ggml_status ggmlqnn_backend_graph_compute_general(ggml_backend_t backend, struct ggml_cgraph * cgraph) { enum ggml_status result = GGML_STATUS_SUCCESS; - ggml_backend_qnn_context * ctx = (ggml_backend_qnn_context *) backend->context; + ggml_backend_qnn_context * ctx = (ggml_backend_qnn_context *)backend->context; GGML_UNUSED(ctx); - //GGMLQNN_LOG_DEBUG("device %d", ctx->device); - //GGMLQNN_LOG_DEBUG("cgraph->n_nodes %d", cgraph->n_nodes); - - if (0 == g_inference_approach) { - for (int i = 0; i < cgraph->n_nodes; i++) { - ggml_tensor * node = cgraph->nodes[i]; - if (ggml_is_empty(node) || node->op == GGML_OP_RESHAPE - || node->op == GGML_OP_TRANSPOSE || node->op == GGML_OP_VIEW - || node->op == GGML_OP_PERMUTE || node->op == GGML_OP_NONE) { - continue; - } - bool ok = ggml_qnn_compute_forward(backend, node); - if (!ok) { - GGMLQNN_LOG_DEBUG("%s: error: op not supported %s (%s)\n", __func__, node->name, ggml_op_name(node->op)); - } +#if 0 + GGMLQNN_LOG_DEBUG("device %d", ctx->device); + GGMLQNN_LOG_DEBUG("cgraph->n_nodes %d", cgraph->n_nodes); + int num_nodes = std::min(5, cgraph->n_nodes); + for (int i = 0; i < num_nodes; i++) { + ggml_tensor * node = cgraph->nodes[i]; + GGMLQNN_LOG_DEBUG("%s: op %s (%s)\n", __func__, node->name, ggml_op_name(node->op)); + } +#endif + for (int i = 0; i < cgraph->n_nodes; i++) { + ggml_tensor * node = cgraph->nodes[i]; + if (ggml_is_empty(node) || node->op == GGML_OP_RESHAPE + || node->op == GGML_OP_TRANSPOSE || node->op == GGML_OP_VIEW + || node->op == GGML_OP_PERMUTE || node->op == GGML_OP_NONE) { + continue; + } + bool ok = ggmlqnn_compute_compute_forward(backend, node); + if (!ok) { + GGMLQNN_LOG_DEBUG("%s: error: op not supported %s (%s)\n", __func__, node->name, ggml_op_name(node->op)); } - } else { - //offload entire cgraph to QNN CPU & GPU & NPU - return ggmlqnn_graph_compute(backend, cgraph); } return result; @@ -3765,10 +4005,32 @@ static void ggml_backend_qnn_device_get_props(ggml_backend_dev_t dev, static ggml_backend_t ggml_backend_qnn_device_init_backend(ggml_backend_dev_t dev, const char * params) { GGML_UNUSED(dev); + GGMLQNN_LOG_INFO("enter %s\n", __func__); + size_t dev_index = 0; + + //case-1: special scenario, such as test-backend-ops or other similar scenairo: calling ggml_backend_qnn_device_init_backend directly in user's applicaton + //call ggmlqnn_load_cfg accordingly in this place + ggmlqnn_load_cfg(); + GGMLQNN_LOG_INFO("user's specified qnn_backend in cfgfile = %d", g_qnn_params.qnn_backend); + GGMLQNN_LOG_INFO("user's sepcified qnn runtime lib path in cfgfile = %s", g_qnn_params.qnn_runtimelib_path); + if (nullptr == params) { - params = 0; + GGMLQNN_LOG_INFO("program specified param is nullptr\n"); + dev_index = (g_qnn_params.qnn_backend > 0) ? g_qnn_params.qnn_backend : 0; + if (dev_index >= GGML_QNN_MAX_DEVICES) { + GGMLQNN_LOG_INFO("assume the default ggml backend\n"); + return nullptr; + } + } else { + GGMLQNN_LOG_INFO("program specified param is not nullptr\n"); + //user's program calling ggml_backend_qnn_device_init_backend directly + dev_index = (int)(intptr_t)params; + g_qnn_params.qnn_backend = dev_index; + GGMLQNN_LOG_INFO("program specified dev_index %d\n", dev_index); } - ggml_backend_t qnn_backend = ggml_backend_qnn_init((int)(intptr_t)params, g_qnn_runtimelib_path); + GGMLQNN_LOG_INFO("qnn_backend=%d", dev_index); + ggml_backend_t qnn_backend = ggml_backend_qnn_init(dev_index, g_qnn_params.qnn_runtimelib_path); + GGMLQNN_LOG_INFO("leave %s\n", __func__); return qnn_backend; @@ -3812,7 +4074,7 @@ static ggml_backend_buffer_t ggml_backend_qnn_device_buffer_from_host_ptr(ggml_b static bool ggml_backend_qnn_device_supports_op(ggml_backend_dev_t dev, const struct ggml_tensor * op) { ggml_backend_qnn_context * ctx = (ggml_backend_qnn_context *) dev->context; - return (ggml_qnn_can_handle_op(ctx,op)); + return (ggmlqnn_compute_can_handle_op(ctx,op)); } static bool ggml_backend_qnn_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) { @@ -3849,7 +4111,7 @@ static ggml_backend_i ggml_backend_qnn_interface = { /* .graph_plan_free = */ nullptr, /* .graph_plan_update = */ nullptr, /* .graph_plan_compute = */ nullptr, - /* .graph_compute = */ ggml_backend_qnn_graph_compute, + /* .graph_compute = */ nullptr, /* .event_record = */ nullptr, /* .event_wait = */ nullptr, }; @@ -3909,7 +4171,6 @@ static void * ggml_backend_qnn_reg_get_proc_address(ggml_backend_reg_t reg, cons return nullptr; const char * slot_name = "ggml_backend_set_n_threads"; - //avoid buffer attack rather than strcmp if (0 == memcmp(name, slot_name, strlen(slot_name))) { return (void *)ggml_backend_qnn_set_n_threads; } @@ -3927,6 +4188,17 @@ ggml_backend_reg_t ggml_backend_qnn_reg() { static ggml_backend_reg reg; static bool initialized = false; GGMLQNN_LOG_DEBUG("enter ggml_backend_qnn_reg"); + //case-2: normal scenario, such as llama-cli or UI applicaton + //call ggmlqnn_load_cfg accordingly in this place + ggmlqnn_load_cfg(); + GGMLQNN_LOG_INFO("user's specified qnn_backend=%d", g_qnn_params.qnn_backend); + GGMLQNN_LOG_INFO("user's sepcified qnn runtime lib path=%s", g_qnn_params.qnn_runtimelib_path); + if (g_qnn_params.qnn_backend >= GGML_QNN_MAX_DEVICES) { + GGMLQNN_LOG_INFO("assume default ggml backend\n"); + GGMLQNN_LOG_DEBUG("leave ggml_backend_qnn_reg"); + return nullptr; + } + { static std::mutex mutex; std::lock_guard lock(mutex); @@ -3965,6 +4237,8 @@ ggml_backend_reg_t ggml_backend_qnn_reg() { ggml_backend_t ggml_backend_qnn_init(size_t device, const char * qnn_lib_path) { int result = 0; + GGMLQNN_LOG_INFO("enter %s\n", __func__); + if (nullptr == qnn_lib_path) return nullptr; @@ -3976,14 +4250,14 @@ ggml_backend_t ggml_backend_qnn_init(size_t device, const char * qnn_lib_path) { } if (nullptr != g_qnn_mgr[device].backend) { - GGMLQNN_LOG_WARN("qnn backend %d(%s) already loaded", device, ggml_backend_qnn_get_devname(device)); + GGMLQNN_LOG_INFO("qnn backend %d(%s) already loaded", device, ggml_backend_qnn_get_devname(device)); + GGMLQNN_LOG_INFO("leave %s\n", __func__); return g_qnn_mgr[device].backend; } - ggmlqnn_load_cfg(); - #if defined(__ANDROID__) std::string path = qnn_lib_path; + GGMLQNN_LOG_INFO("lib_path %s", path.c_str()); if (QNN_BACKEND_NPU == device) { if (0 == setenv("LD_LIBRARY_PATH", (path + @@ -4034,13 +4308,21 @@ ggml_backend_t ggml_backend_qnn_init(size_t device, const char * qnn_lib_path) { g_qnn_mgr[device].raw_interface = instance->get_qnn_raw_interface(); g_qnn_mgr[device].raw_system_interface = instance->get_qnn_raw_system_interface(); + if (0 == g_qnn_params.inference_approach) { + ggml_backend_qnn_interface.graph_compute = ggmlqnn_backend_graph_compute_general; + } else { + ggml_backend_qnn_interface.graph_compute = ggmlqnn_backend_graph_compute_special; + } + ggml_backend_t qnn_backend = new ggml_backend{ /* .guid = */ ggml_backend_qnn_guid(), /* .iface = */ ggml_backend_qnn_interface, /* .device = */ ggml_backend_reg_dev_get(ggml_backend_qnn_reg(), device), /* .context = */ &g_qnn_mgr[device] }; + g_qnn_mgr[device].backend = qnn_backend; + GGMLQNN_LOG_INFO("leave %s\n", __func__); return qnn_backend; } @@ -4065,7 +4347,7 @@ static inline uint32_t ggmlqnn_get_tensor_data_size(const ggml_tensor * tensor) static inline bool ggmlqnn_is_valid_params(ggml_backend_qnn_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - if ((nullptr == ctx) || (nullptr == src0) || (nullptr == src1) || (nullptr == dst)) { + if ((nullptr == ctx) || (nullptr == src0) || (nullptr == dst)) { GGMLQNN_LOG_WARN("invalid params\n"); return false; } @@ -4080,10 +4362,10 @@ static inline bool ggmlqnn_is_valid_params(ggml_backend_qnn_context * ctx, const } /* - * provide a general skeleton to offload ggml op to QNN backend: a single node contains 2 input - * tensor and 1 output tensor + * provide a general skeleton to offload ggml op to QNN backend: peform element-wise operation on 1/2 + * input tensors and 1 output tensors */ -void ggml_qnn_general_node(ggml_backend_qnn_context * ctx, ggml_tensor * op) { +static void ggmlqnn_compute_elementwise(ggml_backend_qnn_context * ctx, ggml_tensor * op) { Qnn_ErrorHandle_t error = QNN_SUCCESS; qnn_instance * instance = nullptr; Qnn_GraphHandle_t graph_handle = nullptr; @@ -4098,51 +4380,62 @@ void ggml_qnn_general_node(ggml_backend_qnn_context * ctx, ggml_tensor * op) { instance = ctx->instance; QNN_INTERFACE_VER_TYPE qnn_raw_interface = ctx->raw_interface; size_t qnn_op_index = ggmlqnn_get_op_index(op); - GGML_ASSERT(qnn_op_index < ggmlqnn_get_opcaps_size()); const char * qnn_op_name = ggmlqnn_k_op_caps[qnn_op_index].qnn_op_name; + size_t input_param_count = ggmlqnn_k_op_caps[qnn_op_index].input_param_count; std::string ggml_op_name_string = std::string("ggml_") + ggml_op_name(op->op); const char * ggml_op_name = ggml_op_name_string.c_str(); - qnn_perf op_perf = qnn_perf(ggml_op_name); - op_perf.start(); - - //ggmlqnn_print_tensors_info(__func__, ctx, src0, src1, dst); bool enable_npu_rpc = instance->enable_qnn_rpc() && ctx->device == QNN_BACKEND_NPU; std::string graph_name; ggmlqnn_get_graphkey_from_op(op, graph_name); - if (instance->_qnn_graph_map.find(graph_name) != instance->_qnn_graph_map.end()) { + + qnn_perf op_perf = qnn_perf(graph_name); + op_perf.start(); + + if (ctx->qnn_singlenode_graph_map.find(graph_name) != ctx->qnn_singlenode_graph_map.end()) { //retrieve computational resource from cached QNN graph - qnn_res_t & graph_item = instance->_qnn_graph_map[graph_name]; - graph_handle = std::get<0>(graph_item); - qnn_tensors_t & tensor = std::get<1>(graph_item); - p_tensor0 = tensor[0]; - p_tensor1 = tensor[1]; - p_tensor2 = tensor[2]; + qnn_singlenode_res_t & graph_item = ctx->qnn_singlenode_graph_map[graph_name]; + graph_handle = std::get<0>(graph_item); + qnn_ptensors_t & ptensors = std::get<1>(graph_item); + p_tensor0 = ptensors[0]; + if (2 == input_param_count) { + p_tensor1 = ptensors[1]; + p_tensor2 = ptensors[2]; + } else { + //now p_tensor1 is nullptr + p_tensor2 = ptensors[1]; + } } else { GGMLQNN_LOG_INFO("graph name %s", graph_name.c_str()); GGML_ASSERT(instance->get_device_id() == ctx->device); //create QNN graph - error = instance->init_qnn_graph(graph_name, static_cast(ctx->device), 8, 4); + error = instance->init_qnn_graph(graph_name, static_cast(ctx->device), g_qnn_params.vtcm_size_in_mb, g_qnn_params.hvx_threads); if (QNN_SUCCESS != error) { GGMLQNN_LOG_WARN("can't create qnn graph handle with graph name %s, error = %d\n", graph_name.c_str(), error); return; } graph_handle = instance->get_qnn_graph_handle(); + GGMLQNN_LOG_DEBUG("graph_handle %p", graph_handle); //create computational tensor p_tensor0 = ggmlqnn_create_compute_tensor(instance, graph_handle, src0, QNN_TENSOR_TYPE_APP_WRITE); - p_tensor1 = ggmlqnn_create_compute_tensor(instance, graph_handle, src1, QNN_TENSOR_TYPE_APP_WRITE); + if (2 == input_param_count) { + p_tensor1 = ggmlqnn_create_compute_tensor(instance, graph_handle, src1, QNN_TENSOR_TYPE_APP_WRITE); + } p_tensor2 = ggmlqnn_create_compute_tensor(instance, graph_handle, dst, QNN_TENSOR_TYPE_APP_READ); //compose QNN graph - Qnn_Tensor_t tensor_inputs[] = { - *p_tensor0, - *p_tensor1 - }; - Qnn_Tensor_t tensor_outputs[] = { + qnn_tensors_t input_tensors; + input_tensors.reserve(input_param_count); + input_tensors.push_back(*p_tensor0); + if (2 == input_param_count) { + input_tensors.push_back(*p_tensor1); + } + Qnn_Tensor_t output_tensors[] = { *p_tensor2 }; +#if 0 // keep them for understand code easily Qnn_OpConfig_t op_config = { QNN_OPCONFIG_VERSION_1, { ggml_op_name, @@ -4150,86 +4443,98 @@ void ggml_qnn_general_node(ggml_backend_qnn_context * ctx, ggml_tensor * op) { qnn_op_name, 0, nullptr, - 2, + input_param_count, tensor_inputs, 1, tensor_outputs } }; +#else + Qnn_OpConfig_t op_config = ggmlqnn_create_op_config(ggml_op_name, QNN_OP_PACKAGE_NAME_QTI_AISW, + qnn_op_name, nullptr, 0, + input_tensors.data(), input_param_count, output_tensors, 1); +#endif CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, op_config)); //finalize QNN graph CHECK_QNN_API(error, qnn_raw_interface.graphFinalize(graph_handle, nullptr, nullptr)); //cache QNN graph - qnn_tensors_t ggml_op_add_tensors; - ggml_op_add_tensors.reserve(3); - ggml_op_add_tensors.push_back(p_tensor0); - ggml_op_add_tensors.push_back(p_tensor1); - ggml_op_add_tensors.push_back(p_tensor2); - auto graph_item = std::make_tuple(graph_handle, ggml_op_add_tensors); - instance->_qnn_graph_map[graph_name] = graph_item; + qnn_ptensors_t qnn_elementwise_tensors; + qnn_elementwise_tensors.reserve(input_param_count + 1); + + qnn_elementwise_tensors.push_back(p_tensor0); + if (2 == input_param_count) { + qnn_elementwise_tensors.push_back(p_tensor1); + } + qnn_elementwise_tensors.push_back(p_tensor2); + auto graph_item = std::make_tuple(graph_handle, qnn_elementwise_tensors); + ctx->qnn_singlenode_graph_map[graph_name] = graph_item; } if (enable_npu_rpc) { uint8_t * qnn_buffer_0 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*p_tensor0)->memHandle)); - GGMLQNN_LOG_INFO("qnn_rpcbuffer_0 = %p\n", qnn_buffer_0); + GGMLQNN_LOG_DEBUG("qnn_rpcbuffer_0 = %p\n", qnn_buffer_0); if (nullptr != qnn_buffer_0) { memcpy(qnn_buffer_0, src0->data, ggml_nbytes(src0)); } - uint8_t * qnn_buffer_1 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*p_tensor1)->memHandle)); - GGMLQNN_LOG_INFO("qnn_rpcbuffer_1 = %p\n", qnn_buffer_1); - if (nullptr != qnn_buffer_1) { - memcpy(qnn_buffer_1, src1->data, ggml_nbytes(src1)); + if (2 == input_param_count) { + uint8_t *qnn_buffer_1 = static_cast(instance->get_rpcmem_from_memhandle( + QNN_VER_PTR(*p_tensor1)->memHandle)); + GGMLQNN_LOG_DEBUG("qnn_rpcbuffer_1 = %p\n", qnn_buffer_1); + if (nullptr != qnn_buffer_1) { + memcpy(qnn_buffer_1, src1->data, ggml_nbytes(src1)); + } } } else { QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggmlqnn_get_tensor_data_size(src0)}; - QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggmlqnn_get_tensor_data_size(src1)}; + if (2 == input_param_count) { + QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggmlqnn_get_tensor_data_size(src1)}; + } QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggmlqnn_get_tensor_data_size(dst)}; } - Qnn_Tensor_t tensor_inputs[] = { - *p_tensor0, - *p_tensor1 - }; - Qnn_Tensor_t tensor_outputs[] = { + qnn_tensors_t input_tensors; + input_tensors.reserve(input_param_count); + input_tensors.push_back(*p_tensor0); + if (2 == input_param_count) { + input_tensors.push_back(*p_tensor1); + } + Qnn_Tensor_t output_tensors[] = { *p_tensor2 }; CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, - tensor_inputs, 2, - tensor_outputs, 1, + input_tensors.data(), input_param_count, + output_tensors, 1, nullptr, nullptr)); if (enable_npu_rpc) { - //TODO:NPU RPC feature will failed with test-backend-ops uint8_t * qnn_buffer_2 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*p_tensor2)->memHandle)); if (nullptr != qnn_buffer_2) { memcpy(dst->data, qnn_buffer_2, ggml_nbytes(dst)); } } -#if GGMLQNN_PRINT_OP_ADD_LOG op_perf.info(); -#endif } /* * this function is AI-assisted code from Grok 3 for purpose of offload 4d matrix mulmat to QNN backend * various UT has verified and succeed but failed in CT of test-backend-ops * - * the logic of ggml_qnn_mul_mat_4d is similar to ggml_qnn_mul_mat but much more complicated - * than ggml_qnn_mul_mat, so it's a standalone function. - * it will be combined with ggml_qnn_mul_mat in the future + * the logic of ggmlqnn_compute_mul_mat_4d is similar to ggmlqnn_compute_mul_mat but much more complicated + * than ggmlqnn_compute_mul_mat, so it's a standalone function. + * it will be combined with ggmlqnn_compute_mul_mat in the future */ -static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) { - Qnn_ErrorHandle_t error = QNN_SUCCESS; - bool graph_initialized = false; - qnn_perf op_perf = qnn_perf("ggml_qnn_mul_mat_4d"); - qnn_instance *instance = ctx->instance; +static void ggmlqnn_compute_mul_mat_4d(ggml_backend_qnn_context * ctx, ggml_tensor * op) { + Qnn_ErrorHandle_t error = QNN_SUCCESS; + bool graph_initialized = false; + qnn_perf op_perf = qnn_perf("ggmlqnn_compute_mul_mat_4d"); + qnn_instance * instance = ctx->instance; QNN_INTERFACE_VER_TYPE qnn_raw_interface = ctx->raw_interface; - const ggml_tensor *src0 = op->src[0]; - const ggml_tensor *src1 = op->src[1]; - ggml_tensor *dst = op; + const ggml_tensor * src0 = op->src[0]; + const ggml_tensor * src1 = op->src[1]; + ggml_tensor * dst = op; GGMLQNN_CHECK_PARAMS(ctx, src0, src1, dst); GGML_ASSERT(ggml_n_dims(src0) == 4 && ggml_n_dims(src1) == 4); @@ -4241,32 +4546,31 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) ggmlqnn_print_tensors_info(__func__, ctx, src0, src1, dst); - Qnn_GraphHandle_t graph_handle = nullptr; - Qnn_Tensor_t *p_tensor0 = nullptr; - Qnn_Tensor_t *p_reshape0_out = nullptr; - Qnn_Tensor_t *p_tile0_out = nullptr; - Qnn_Tensor_t *p_tensor1 = nullptr; - Qnn_Tensor_t *p_permute1_out = nullptr; - Qnn_Tensor_t *p_reshape1_out = nullptr; - Qnn_Tensor_t *p_matmul_out = nullptr; - Qnn_Tensor_t *p_reshape2_out = nullptr; - - if (instance->_qnn_graph_map.find(graph_name) != instance->_qnn_graph_map.end()) { + Qnn_GraphHandle_t graph_handle = nullptr; + Qnn_Tensor_t * p_tensor0 = nullptr; + Qnn_Tensor_t * p_reshape0_out = nullptr; + Qnn_Tensor_t * p_tile0_out = nullptr; + Qnn_Tensor_t * p_tensor1 = nullptr; + Qnn_Tensor_t * p_permute1_out = nullptr; + Qnn_Tensor_t * p_reshape1_out = nullptr; + Qnn_Tensor_t * p_matmul_out = nullptr; + Qnn_Tensor_t * p_reshape2_out = nullptr; + + if (ctx->qnn_singlenode_graph_map.find(graph_name) != ctx->qnn_singlenode_graph_map.end()) { graph_initialized = true; - qnn_res_t &graph_item = instance->_qnn_graph_map[graph_name]; - graph_handle = std::get<0>(graph_item); - qnn_tensors_t &tensors = std::get<1>(graph_item); - p_tensor0 = tensors[0]; - p_reshape0_out = tensors[1]; - p_tile0_out = tensors[2]; - p_tensor1 = tensors[3]; - p_permute1_out = tensors[4]; - p_reshape1_out = tensors[5]; - p_matmul_out = tensors[6]; - p_reshape2_out = tensors[7]; + qnn_singlenode_res_t & graph_item = ctx->qnn_singlenode_graph_map[graph_name]; + graph_handle = std::get<0>(graph_item); + qnn_ptensors_t & tensors = std::get<1>(graph_item); + p_tensor0 = tensors[0]; + p_reshape0_out = tensors[1]; + p_tile0_out = tensors[2]; + p_tensor1 = tensors[3]; + p_permute1_out = tensors[4]; + p_reshape1_out = tensors[5]; + p_matmul_out = tensors[6]; + p_reshape2_out = tensors[7]; } else { - CHECK_QNN_API(error, qnn_raw_interface.graphCreate(instance->get_qnn_context_handle(), - graph_name.c_str(), NULL, &graph_handle)); + CHECK_QNN_API(error, qnn_raw_interface.graphCreate(instance->get_qnn_context_handle(), graph_name.c_str(), NULL, &graph_handle)); // Define dimensions uint32_t K = src0->ne[0]; // Inner dimension @@ -4279,127 +4583,136 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) GGML_ASSERT(src0->ne[0] == src1->ne[0]); // K must match // src0: [K, M, H0, B0] -> QNN: [B0, H0, M, K] - uint32_t src0_dims[] = {static_cast(src0->ne[3]), static_cast(src0->ne[2]), static_cast(src0->ne[1]), static_cast(src0->ne[0])}; - p_tensor0 = GQCGT(src0, "input0", QNN_TENSOR_TYPE_APP_WRITE, QNN_DATATYPE_FLOAT_32, 4, - src0_dims, nullptr, 0); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor0)); + uint32_t src0_dims[] = {static_cast(src0->ne[3]), static_cast(src0->ne[2]), + static_cast(src0->ne[1]), static_cast(src0->ne[0]) + }; + p_tensor0 = ggmlqnn_create_general_tensor(instance, graph_handle, src0, "input0", + QNN_TENSOR_TYPE_APP_WRITE, QNN_DATATYPE_FLOAT_32, 4, + src0_dims, nullptr, 0); // Reshape src0 to [B0, M, K] uint32_t reshape0_out_dims[] = {B0, M, K}; - p_reshape0_out = GQCGT(nullptr, "reshape0_out", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, 3, - reshape0_out_dims, nullptr, 0); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_reshape0_out)); - Qnn_Tensor_t reshape0_inputs[] = {*p_tensor0}; + p_reshape0_out = ggmlqnn_create_general_tensor(instance, graph_handle, nullptr, "reshape0_out", + QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, 3, + reshape0_out_dims, nullptr, 0); + + Qnn_Tensor_t reshape0_inputs[] = {*p_tensor0}; Qnn_Tensor_t reshape0_outputs[] = {*p_reshape0_out}; - Qnn_OpConfig_t reshape0_op = ggmlqnn_create_op_config("reshape0", QNN_OP_PACKAGE_NAME_QTI_AISW, - QNN_OP_RESHAPE, nullptr, 0, - reshape0_inputs, 1, reshape0_outputs, 1); + Qnn_OpConfig_t reshape0_op = ggmlqnn_create_op_config("reshape0", QNN_OP_PACKAGE_NAME_QTI_AISW, + QNN_OP_RESHAPE, nullptr, 0, + reshape0_inputs, 1, reshape0_outputs, 1); CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, reshape0_op)); // Tile src0 to match B1: [B0, M, K] -> [B1, M, K] uint32_t tile0_out_dims[] = {B1, M, K}; - p_tile0_out = GQCGT(nullptr, "tile0_out", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, 3, - tile0_out_dims, nullptr, 0); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tile0_out)); + p_tile0_out = ggmlqnn_create_general_tensor(instance, graph_handle, nullptr, "tile0_out", + QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, 3, + tile0_out_dims, nullptr, 0); + uint32_t tile_multiples[] = {B1 / B0, 1, 1}; uint32_t tile_dims[] = {3}; - Qnn_Tensor_t *p_tile_multiples = GQCGT(nullptr, "tile_multiples", QNN_TENSOR_TYPE_STATIC, QNN_DATATYPE_UINT_32, 1, - tile_dims, tile_multiples, sizeof(tile_multiples)); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tile_multiples)); - Qnn_Param_t tile_params[] = {{QNN_PARAMTYPE_TENSOR, "multiples", .tensorParam = *p_tile_multiples}}; - Qnn_Tensor_t tile0_inputs[] = {*p_reshape0_out}; - Qnn_Tensor_t tile0_outputs[] = {*p_tile0_out}; - Qnn_OpConfig_t tile0_op = ggmlqnn_create_op_config("tile0", QNN_OP_PACKAGE_NAME_QTI_AISW, - QNN_OP_TILE, tile_params, 1, - tile0_inputs, 1, tile0_outputs, 1); + Qnn_Tensor_t * p_tile_multiples = ggmlqnn_create_general_tensor(instance, graph_handle, nullptr, "tile_multiples", + QNN_TENSOR_TYPE_STATIC, QNN_DATATYPE_UINT_32, 1, + tile_dims, tile_multiples, sizeof(tile_multiples)); + + Qnn_Param_t tile_params[] = {{QNN_PARAMTYPE_TENSOR, "multiples", .tensorParam = *p_tile_multiples}}; + Qnn_Tensor_t tile0_inputs[] = {*p_reshape0_out}; + Qnn_Tensor_t tile0_outputs[] = {*p_tile0_out}; + Qnn_OpConfig_t tile0_op = ggmlqnn_create_op_config("tile0", QNN_OP_PACKAGE_NAME_QTI_AISW, + QNN_OP_TILE, tile_params, 1, + tile0_inputs, 1, tile0_outputs, 1); CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, tile0_op)); // src1: [N, K, H1, B1] -> QNN: [B1, H1, N, K] - uint32_t src1_dims[] = {static_cast(src1->ne[3]), static_cast(src1->ne[2]), static_cast(src1->ne[1]), static_cast(src1->ne[0])}; - p_tensor1 = GQCGT(src1, "input1", QNN_TENSOR_TYPE_APP_WRITE, QNN_DATATYPE_FLOAT_32, 4, - src1_dims, nullptr, 0); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor1)); + uint32_t src1_dims[] = {static_cast(src1->ne[3]), static_cast(src1->ne[2]), + static_cast(src1->ne[1]), static_cast(src1->ne[0]) + }; + p_tensor1 = ggmlqnn_create_general_tensor(instance, graph_handle, src1, "input1", + QNN_TENSOR_TYPE_APP_WRITE, QNN_DATATYPE_FLOAT_32, 4, + src1_dims, nullptr, 0); + // Permute src1 to [B1, H1, K, N] uint32_t perm_data[] = {0, 1, 3, 2}; uint32_t perm_dims[] = {4}; - Qnn_Tensor_t *p_perm = GQCGT(nullptr, "perm", QNN_TENSOR_TYPE_STATIC, QNN_DATATYPE_UINT_32, 1, - perm_dims, perm_data, sizeof(perm_data)); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_perm)); - uint32_t permute1_out_dims[] = {static_cast(src1->ne[3]), static_cast(src1->ne[2]), static_cast(src1->ne[0]), static_cast(src1->ne[1])}; - p_permute1_out = GQCGT(nullptr, "permute1_out", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, 4, - permute1_out_dims, nullptr, 0); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_permute1_out)); - Qnn_Param_t permute1_params[] = {{QNN_PARAMTYPE_TENSOR, "perm", .tensorParam = *p_perm}}; - Qnn_Tensor_t permute1_inputs[] = {*p_tensor1}; + Qnn_Tensor_t * p_perm = ggmlqnn_create_general_tensor(instance, graph_handle, nullptr, "perm", + QNN_TENSOR_TYPE_STATIC, QNN_DATATYPE_UINT_32, 1, + perm_dims, perm_data, sizeof(perm_data)); + + uint32_t permute1_out_dims[] = {static_cast(src1->ne[3]), static_cast(src1->ne[2]), + static_cast(src1->ne[0]), static_cast(src1->ne[1]) + }; + p_permute1_out = ggmlqnn_create_general_tensor(instance, graph_handle, nullptr, "permute1_out", + QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, 4, + permute1_out_dims, nullptr, 0); + + Qnn_Param_t permute1_params[] = {{QNN_PARAMTYPE_TENSOR, "perm", .tensorParam = *p_perm}}; + Qnn_Tensor_t permute1_inputs[] = {*p_tensor1}; Qnn_Tensor_t permute1_outputs[] = {*p_permute1_out}; - Qnn_OpConfig_t permute1_op = ggmlqnn_create_op_config("permute1", QNN_OP_PACKAGE_NAME_QTI_AISW, - QNN_OP_TRANSPOSE, permute1_params, 1, - permute1_inputs, 1, permute1_outputs, 1); + Qnn_OpConfig_t permute1_op = ggmlqnn_create_op_config("permute1", QNN_OP_PACKAGE_NAME_QTI_AISW, + QNN_OP_TRANSPOSE, permute1_params, 1, + permute1_inputs, 1, permute1_outputs, 1); CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, permute1_op)); // Reshape src1 to [B1, K, N] uint32_t reshape1_out_dims[] = {B1, K, N}; - p_reshape1_out = GQCGT(nullptr, "reshape1_out", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, 3, - reshape1_out_dims, nullptr, 0); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_reshape1_out)); - Qnn_Tensor_t reshape1_inputs[] = {*p_permute1_out}; + p_reshape1_out = ggmlqnn_create_general_tensor(instance, graph_handle, nullptr, "reshape1_out", + QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, 3, + reshape1_out_dims, nullptr, 0); + + Qnn_Tensor_t reshape1_inputs[] = {*p_permute1_out}; Qnn_Tensor_t reshape1_outputs[] = {*p_reshape1_out}; - Qnn_OpConfig_t reshape1_op = ggmlqnn_create_op_config("reshape1", QNN_OP_PACKAGE_NAME_QTI_AISW, - QNN_OP_RESHAPE, nullptr, 0, - reshape1_inputs, 1, reshape1_outputs, 1); + Qnn_OpConfig_t reshape1_op = ggmlqnn_create_op_config("reshape1", QNN_OP_PACKAGE_NAME_QTI_AISW, + QNN_OP_RESHAPE, nullptr, 0, + reshape1_inputs, 1, reshape1_outputs, 1); CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, reshape1_op)); // MatMul: [B1, M, K] x [B1, K, N] -> [B1, M, N] uint32_t matmul_out_dims[] = {B1, M, N}; - p_matmul_out = GQCGT(nullptr, "matmul_out", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, 3, - matmul_out_dims, nullptr, 0); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_matmul_out)); - Qnn_Tensor_t matmul_inputs[] = {*p_tile0_out, *p_reshape1_out}; - Qnn_Tensor_t matmul_outputs[] = {*p_matmul_out}; - Qnn_OpConfig_t matmul_op = ggmlqnn_create_op_config("matmul", QNN_OP_PACKAGE_NAME_QTI_AISW, - QNN_OP_MAT_MUL, nullptr, 0, - matmul_inputs, 2, matmul_outputs, 1); + p_matmul_out = ggmlqnn_create_general_tensor(instance, graph_handle, nullptr, "matmul_out", + QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, 3, + matmul_out_dims, nullptr, 0); + + Qnn_Tensor_t matmul_inputs[] = {*p_tile0_out, *p_reshape1_out}; + Qnn_Tensor_t matmul_outputs[] = {*p_matmul_out}; + Qnn_OpConfig_t matmul_op = ggmlqnn_create_op_config("matmul", QNN_OP_PACKAGE_NAME_QTI_AISW, + QNN_OP_MAT_MUL, nullptr, 0, + matmul_inputs, 2, matmul_outputs, 1); CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, matmul_op)); // Output: [N, M, H1, B1] -> QNN: [B1, H1, M, N] - uint32_t reshape2_out_dims[] = {static_cast(dst->ne[3]), static_cast(dst->ne[2]), static_cast(dst->ne[1]), static_cast(dst->ne[0])}; - p_reshape2_out = GQCGT(dst, "output", QNN_TENSOR_TYPE_APP_READ, QNN_DATATYPE_FLOAT_32, 4, - reshape2_out_dims, nullptr, 0); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_reshape2_out)); - Qnn_Tensor_t reshape2_inputs[] = {*p_matmul_out}; + uint32_t reshape2_out_dims[] = {static_cast(dst->ne[3]), static_cast(dst->ne[2]), + static_cast(dst->ne[1]), static_cast(dst->ne[0]) + }; + p_reshape2_out = ggmlqnn_create_general_tensor(instance, graph_handle, dst, "output", + QNN_TENSOR_TYPE_APP_READ, QNN_DATATYPE_FLOAT_32, 4, + reshape2_out_dims, nullptr, 0); + + Qnn_Tensor_t reshape2_inputs[] = {*p_matmul_out}; Qnn_Tensor_t reshape2_outputs[] = {*p_reshape2_out}; - Qnn_OpConfig_t reshape2_op = ggmlqnn_create_op_config("reshape2", QNN_OP_PACKAGE_NAME_QTI_AISW, - QNN_OP_RESHAPE, nullptr, 0, - reshape2_inputs, 1, reshape2_outputs, 1); + Qnn_OpConfig_t reshape2_op = ggmlqnn_create_op_config("reshape2", QNN_OP_PACKAGE_NAME_QTI_AISW, + QNN_OP_RESHAPE, nullptr, 0, + reshape2_inputs, 1, reshape2_outputs, 1); CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, reshape2_op)); // Finalize CHECK_QNN_API(error, qnn_raw_interface.graphFinalize(graph_handle, NULL, NULL)); // Cache - qnn_tensors_t ggml_op_mulmat_tensors = {p_tensor0, p_reshape0_out, p_tile0_out, p_tensor1, p_permute1_out, p_reshape1_out, p_matmul_out, p_reshape2_out}; - instance->_qnn_graph_map[graph_name] = std::make_tuple(graph_handle, ggml_op_mulmat_tensors); + qnn_ptensors_t ggml_op_mulmat_tensors = {p_tensor0, p_reshape0_out, p_tile0_out, p_tensor1, + p_permute1_out, p_reshape1_out, p_matmul_out, p_reshape2_out + }; + ctx->qnn_singlenode_graph_map[graph_name] = std::make_tuple(graph_handle, ggml_op_mulmat_tensors); } // Execute - QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, static_cast(ggml_nbytes(src0))}; - QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, static_cast(ggml_nbytes(src1))}; + QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, static_cast(ggml_nbytes(src0))}; + QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, static_cast(ggml_nbytes(src1))}; QNN_VER_PTR(*p_reshape2_out)->clientBuf = {dst->data, static_cast(ggml_nbytes(dst))}; - Qnn_Tensor_t input_tensors[] = {*p_tensor0, *p_tensor1}; - Qnn_Tensor_t output_tensors[] = {*p_reshape2_out}; - CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, input_tensors, 2, - output_tensors, 1, NULL, NULL)); - -#if 0 - // Log dst for debugging - float *dst_data = (float *)dst->data; - GGMLQNN_LOG_DEBUG("dst shape: [%d, %d, %d, %d]\n", dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]); - for (int i = 0; i < dst->ne[0] * dst->ne[1] * dst->ne[2] * dst->ne[3]; i++) { - GGMLQNN_LOG_DEBUG("dst[%d] = %f\n", i, dst_data[i]); - } -#endif + Qnn_Tensor_t input_tensors[] = {*p_tensor0, *p_tensor1}; + Qnn_Tensor_t output_tensors[] = {*p_reshape2_out}; + CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, input_tensors, 2, output_tensors, 1, NULL, NULL)); op_perf.info(); } @@ -4431,18 +4744,13 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) in the all, there is gap between ggml mulmat and QNN mulmat,we need to perform a transpose operation when offloading mulmat to QNN backend. this implementation will handle transpose - in func ggml_qnn_create_general_tensor() - * - * this function is a good example to illustrated the second technical approach "mapping the - * entire ggml computational graph to QNN graph" without complex C++ encapsulation. or another - * pipeline of "how to utilize the Hexagon NPU maximally through QNN SDK", details could be found at - * https://github.com/ggml-org/llama.cpp/pull/12049#issuecomment-2678308360 - * + in func ggmlqnn_compute_create_general_tensor() + * @param ctx the context of ggml-qnn backend * @param op the destination tensor where the result of the matrix multiplication will be stored. * - * @note the logic of ggml_qnn_mul_mat is similar to ggml_qnn_general_node but much more complicated - * than ggml_qnn_general_node. so it's a standalone function. accordingly, this is another + * @note the logic of ggmlqnn_compute_mul_mat is similar to ggmlqnn_compute_op_two_tensors but much more complicated + * than ggmlqnn_compute_op_two_tensors. so it's a standalone function. accordingly, this is another * typical skeleton for offload other ggml ops to QNN backend. MUL_MAT take most of the compute * time (about 95%).so to speed up llama inference, should focus on this func. there are three kinds * of MUL_MAT to compute: @@ -4451,9 +4759,8 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op) * mul_mat_q_f32: src0 is quantized (Q4_0, Q4_1, Q6_K...) * and src1 is F32, src0 -> f32 in src0', then src0' * src1 */ -void ggml_qnn_mul_mat(ggml_backend_qnn_context * ctx, ggml_tensor * op) { +static void ggmlqnn_compute_mul_mat(ggml_backend_qnn_context * ctx, ggml_tensor * op) { Qnn_ErrorHandle_t error = QNN_SUCCESS; - qnn_perf op_perf = qnn_perf("ggml_qnn_mul_mat"); qnn_instance * instance = nullptr; Qnn_GraphHandle_t graph_handle = nullptr; Qnn_Tensor_t * p_tensor0 = nullptr; @@ -4468,7 +4775,7 @@ void ggml_qnn_mul_mat(ggml_backend_qnn_context * ctx, ggml_tensor * op) { GGMLQNN_CHECK_PARAMS(ctx, src0, src1, dst); instance = ctx->instance; QNN_INTERFACE_VER_TYPE qnn_raw_interface = ctx->raw_interface; - op_perf.start(); + const enum ggml_type src0_type = src0->type; const uint32_t src0_rank = ggml_n_dims(src0); @@ -4476,20 +4783,25 @@ void ggml_qnn_mul_mat(ggml_backend_qnn_context * ctx, ggml_tensor * op) { GGML_ASSERT(src0_rank == src1_rank); GGML_ASSERT(src0_rank >= 2); //QNN SDK's limitation, make QNN SDK happy if (4 == src0_rank) { - return ggml_qnn_mul_mat_4d(ctx, op); + return ggmlqnn_compute_mul_mat_4d(ctx, op); } - void * wdata = ggmlqnn_type_trait(ctx, op); - const size_t desired_size = ctx->desired_size; ggmlqnn_print_tensors_info(__func__, ctx, src0, src1, dst); std::string graph_name; ggmlqnn_get_graphkey_from_op(op, graph_name); - if (instance->_qnn_graph_map.find(graph_name) != instance->_qnn_graph_map.end()) { + + qnn_perf op_perf = qnn_perf(graph_name); + op_perf.start(); + + void * wdata = ggmlqnn_type_trait(ctx, op); + const size_t desired_size = ctx->desired_size; + + if (ctx->qnn_singlenode_graph_map.find(graph_name) != ctx->qnn_singlenode_graph_map.end()) { //retrieve computational resource from cached QNN graph - qnn_res_t & graph_item = instance->_qnn_graph_map[graph_name]; + qnn_singlenode_res_t & graph_item = ctx->qnn_singlenode_graph_map[graph_name]; graph_handle = std::get<0>(graph_item); - qnn_tensors_t & tensors = std::get<1>(graph_item); + qnn_ptensors_t & tensors = std::get<1>(graph_item); p_tensor0 = tensors[0]; p_tensor1 = tensors[1]; p_tensor2 = tensors[2]; @@ -4498,7 +4810,7 @@ void ggml_qnn_mul_mat(ggml_backend_qnn_context * ctx, ggml_tensor * op) { } else { //create QNN graph GGMLQNN_LOG_INFO("graph name %s", graph_name.c_str()); - error = instance->init_qnn_graph(graph_name, static_cast(ctx->device), 8, 4); + error = instance->init_qnn_graph(graph_name, static_cast(ctx->device), g_qnn_params.vtcm_size_in_mb, g_qnn_params.hvx_threads); if (QNN_SUCCESS != error) { GGMLQNN_LOG_WARN("can't create qnn graph handle with graph name %s, error = %d\n", graph_name.c_str(), error); return; @@ -4506,12 +4818,15 @@ void ggml_qnn_mul_mat(ggml_backend_qnn_context * ctx, ggml_tensor * op) { graph_handle = instance->get_qnn_graph_handle(); //create computational tensor - p_tensor0 = GQCGT(src0, nullptr, QNN_TENSOR_TYPE_APP_WRITE, QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0); - p_tensor1 = GQCGT(src1, nullptr, QNN_TENSOR_TYPE_APP_WRITE, QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0); - p_tensor2 = GQCGT(dst, nullptr, QNN_TENSOR_TYPE_APP_READ, QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor0)); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor1)); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor2)); + p_tensor0 = ggmlqnn_create_general_tensor(instance, graph_handle, src0, nullptr, + QNN_TENSOR_TYPE_APP_WRITE, QNN_DATATYPE_FLOAT_32, src0_rank, + nullptr, nullptr, 0); + p_tensor1 = ggmlqnn_create_general_tensor(instance, graph_handle, src1, nullptr, + QNN_TENSOR_TYPE_APP_WRITE, QNN_DATATYPE_FLOAT_32, src0_rank, + nullptr, nullptr, 0); + p_tensor2 = ggmlqnn_create_general_tensor(instance, graph_handle, dst, nullptr, + QNN_TENSOR_TYPE_APP_READ, QNN_DATATYPE_FLOAT_32, src0_rank, + nullptr, nullptr, 0); //create param tensor for offload 2d/3d/4d matrix multiplication const uint32_t param_tensor_data[GGML_MAX_DIMS][GGML_MAX_DIMS] = { @@ -4521,32 +4836,36 @@ void ggml_qnn_mul_mat(ggml_backend_qnn_context * ctx, ggml_tensor * op) { {0, 1, 3, 2}, }; uint32_t param_tensor_dims[1] = {src0_rank}; - p_param_tensor = GQCGT(nullptr, "param", QNN_TENSOR_TYPE_STATIC, QNN_DATATYPE_UINT_32, 1, param_tensor_dims, (void *)(param_tensor_data[src0_rank - 1]), src0_rank * sizeof(uint32_t)); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_param_tensor)); + p_param_tensor = ggmlqnn_create_general_tensor(instance, graph_handle, nullptr, "param", + QNN_TENSOR_TYPE_STATIC, QNN_DATATYPE_UINT_32, 1, param_tensor_dims, + (void *)(param_tensor_data[src0_rank - 1]), src0_rank * sizeof(uint32_t)); //create transpose tensor - p_tensor2_transpose = GQCGT(dst, "transpose", QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, src0_rank, nullptr, nullptr, 0, true); - CHECK_QNN_API(error, qnn_raw_interface.tensorCreateGraphTensor(graph_handle, p_tensor2_transpose)); + p_tensor2_transpose = ggmlqnn_create_general_tensor(instance, graph_handle, dst, "transpose", + QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, src0_rank, + nullptr, nullptr, 0, true); //compose QNN graph: add mulmat node Qnn_Param_t out_0_params[] = {{QNN_PARAMTYPE_SCALAR, QNN_OP_MAT_MUL_PARAM_TRANSPOSE_IN1, .scalarParam = {QNN_DATATYPE_BOOL_8, .bool8Value = 1}}}; Qnn_Tensor_t out_0_inputs[] = {*p_tensor0, *p_tensor1}; Qnn_Tensor_t out_0_outputs[] = {*p_tensor2_transpose}; - Qnn_OpConfig_t out_0 = ggmlqnn_create_op_config("mulmat_opconfig", QNN_OP_PACKAGE_NAME_QTI_AISW, QNN_OP_MAT_MUL, out_0_params, 1, out_0_inputs, 2, out_0_outputs, 1); + Qnn_OpConfig_t out_0 = ggmlqnn_create_op_config("mulmat_opconfig", QNN_OP_PACKAGE_NAME_QTI_AISW, + QNN_OP_MAT_MUL, out_0_params, 1, out_0_inputs, 2, out_0_outputs, 1); CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle,out_0)); //compose QNN graph: add transpose node Qnn_Param_t out_trans1_0_params[] = { {QNN_PARAMTYPE_TENSOR, "perm", .tensorParam = *p_param_tensor}}; Qnn_Tensor_t out_trans1_0_inputs[] = {*p_tensor2_transpose}; Qnn_Tensor_t out_trans1_0_outputs[] = {*p_tensor2}; - Qnn_OpConfig_t out_trans1_0 = ggmlqnn_create_op_config("mulmat_transpose_opconfig", QNN_OP_PACKAGE_NAME_QTI_AISW, QNN_OP_TRANSPOSE, out_trans1_0_params, 1, out_trans1_0_inputs, 1, out_trans1_0_outputs, 1); + Qnn_OpConfig_t out_trans1_0 = ggmlqnn_create_op_config("mulmat_transpose_opconfig", QNN_OP_PACKAGE_NAME_QTI_AISW, + QNN_OP_TRANSPOSE, out_trans1_0_params, 1, out_trans1_0_inputs, 1, out_trans1_0_outputs, 1); CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle,out_trans1_0)); //finalize QNN graph CHECK_QNN_API(error, qnn_raw_interface.graphFinalize(graph_handle, nullptr, nullptr)); //cache QNN graph - qnn_tensors_t ggml_op_mulmat_tensors; + qnn_ptensors_t ggml_op_mulmat_tensors; ggml_op_mulmat_tensors.reserve(5); ggml_op_mulmat_tensors.push_back(p_tensor0); ggml_op_mulmat_tensors.push_back(p_tensor1); @@ -4554,7 +4873,7 @@ void ggml_qnn_mul_mat(ggml_backend_qnn_context * ctx, ggml_tensor * op) { ggml_op_mulmat_tensors.push_back(p_param_tensor); ggml_op_mulmat_tensors.push_back(p_tensor2_transpose); auto graph_item = std::make_tuple(graph_handle, ggml_op_mulmat_tensors); - instance->_qnn_graph_map[graph_name] = graph_item; + ctx->qnn_singlenode_graph_map[graph_name] = graph_item; } if (src0_type != GGML_TYPE_F32) { @@ -4579,127 +4898,127 @@ void ggml_qnn_mul_mat(ggml_backend_qnn_context * ctx, ggml_tensor * op) { op_perf.info(); } -void ggml_qnn_repeat(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +static void ggmlqnn_compute_repeat(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { GGML_UNUSED(ctx); GGML_UNUSED(dst); } -void ggml_qnn_div(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +static void ggmlqnn_compute_div(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { GGML_UNUSED(ctx); GGML_UNUSED(dst); } -void ggml_qnn_leaky_relu(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +static void ggmlqnn_compute_leaky_relu(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { GGML_UNUSED(ctx); GGML_UNUSED(dst); } -void ggml_qnn_concat(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +static void ggmlqnn_compute_concat(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { GGML_UNUSED(ctx); GGML_UNUSED(dst); } -void ggml_qnn_arange(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +static void ggmlqnn_compute_arange(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { GGML_UNUSED(ctx); GGML_UNUSED(dst); } -void ggml_qnn_sqr(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +static void ggmlqnn_compute_sqr(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { GGML_UNUSED(ctx); GGML_UNUSED(dst); } -void ggml_qnn_clamp(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +static void ggmlqnn_compute_clamp(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { GGML_UNUSED(ctx); GGML_UNUSED(dst); } -void ggml_qnn_scale(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +static void ggmlqnn_compute_scale(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { GGML_UNUSED(ctx); GGML_UNUSED(dst); } -void ggml_qnn_argsort(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +static void ggmlqnn_compute_argsort(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { GGML_UNUSED(ctx); GGML_UNUSED(dst); } -void ggml_qnn_norm(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +static void ggmlqnn_compute_norm(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { GGML_UNUSED(ctx); GGML_UNUSED(dst); } -void ggml_qnn_group_norm(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +static void ggmlqnn_compute_group_norm(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { GGML_UNUSED(ctx); GGML_UNUSED(dst); } -void ggml_qnn_acc(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +static void ggmlqnn_compute_acc(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { GGML_UNUSED(ctx); GGML_UNUSED(dst); } -void ggml_qnn_sum_rows(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +static void ggmlqnn_compute_sum_rows(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { GGML_UNUSED(ctx); GGML_UNUSED(dst); } -void ggml_qnn_upsample_nearest2d(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +static void ggmlqnn_compute_upsample_nearest2d(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { GGML_UNUSED(ctx); GGML_UNUSED(dst); } -void ggml_qnn_pad(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +static void ggmlqnn_compute_pad(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { GGML_UNUSED(ctx); GGML_UNUSED(dst); } -void ggml_qnn_pool2d(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +static void ggmlqnn_compute_pool2d(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { GGML_UNUSED(ctx); GGML_UNUSED(dst); } -void ggml_qnn_dup(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +static void ggmlqnn_compute_dup(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { GGML_UNUSED(ctx); GGML_UNUSED(dst); } -void ggml_qnn_rms_norm(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +static void ggmlqnn_compute_rms_norm(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { GGML_UNUSED(ctx); GGML_UNUSED(dst); } -void ggml_qnn_diag_mask(ggml_backend_qnn_context * ctx, ggml_tensor * dst, float value) { +static void ggmlqnn_compute_diag_mask(ggml_backend_qnn_context * ctx, ggml_tensor * dst, float value) { GGML_UNUSED(ctx); GGML_UNUSED(dst); GGML_UNUSED(value); } -void ggml_qnn_im2col(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +static void ggmlqnn_compute_im2col(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { GGML_UNUSED(ctx); GGML_UNUSED(dst); } -void ggml_qnn_timestep_embedding(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +static void ggmlqnn_compute_timestep_embedding(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { GGML_UNUSED(ctx); GGML_UNUSED(dst); } -void ggml_qnn_cpy(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { - ggml_qnn_dup(ctx, dst); +static void ggmlqnn_compute_cpy(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { + ggmlqnn_compute_dup(ctx, dst); } -void ggml_qnn_softmax(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +static void ggmlqnn_compute_softmax(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { GGML_UNUSED(ctx); GGML_UNUSED(dst); } -void ggml_qnn_get_rows(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +static void ggmlqnn_compute_get_rows(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { GGML_UNUSED(ctx); GGML_UNUSED(dst); } -void ggml_qnn_rope(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { +static void ggmlqnn_compute_rope(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { GGML_UNUSED(ctx); GGML_UNUSED(dst); } @@ -4707,13 +5026,14 @@ void ggml_qnn_rope(ggml_backend_qnn_context * ctx, ggml_tensor * dst) { // ================================================================================================= // section-10: second approach: mapping ggml computational cgraph to QNN graph // ================================================================================================= +// TODO: remove duplicated codes between section-9 and section-10 +// TODO: the graph algorithm in this section is naive, should optimized by AI experts // details: https://github.com/ggml-org/llama.cpp/pull/12326#issuecomment-2712838649 // ref: https://github.com/kantv-ai/kantv/blob/kantv-poc-with-qnn/core/ggml/jni/Inception_v3.cpp#L20634 -// TODO: mapping entire ggml cgraph to a single QNN graph -static enum ggml_status ggmlqnn_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph) { +static enum ggml_status ggmlqnn_backend_graph_compute_special(ggml_backend_t backend, struct ggml_cgraph * cgraph) { enum ggml_status ggml_result = GGML_STATUS_SUCCESS; Qnn_ErrorHandle_t qnn_error = QNN_SUCCESS; - qnn_perf op_perf = qnn_perf("ggmlqnn_graph_compute"); + qnn_perf op_perf = qnn_perf("ggml_backend_qnn_graph_compute_special"); qnn_instance * instance = nullptr; Qnn_GraphHandle_t graph_handle = nullptr; ggml_backend_qnn_context * ctx = (ggml_backend_qnn_context *) backend->context; @@ -4721,7 +5041,7 @@ static enum ggml_status ggmlqnn_graph_compute(ggml_backend_t backend, struct ggm QNN_INTERFACE_VER_TYPE qnn_raw_interface = ctx->raw_interface; op_perf.start(); - //now we got the entire ggml cgraph + //now we got the entire ggml cgraph or a ggml cgraph which contains multiple nodes GGMLQNN_LOG_DEBUG("qnn device %d(%s)", ctx->device, ggml_backend_qnn_get_devname(ctx->device)); GGMLQNN_LOG_DEBUG("cgraph->n_nodes %d", cgraph->n_nodes); int num_nodes = std::min(5, cgraph->n_nodes); @@ -4731,13 +5051,16 @@ static enum ggml_status ggmlqnn_graph_compute(ggml_backend_t backend, struct ggm GGMLQNN_LOG_DEBUG("%s: op %s (%s)\n", __func__, node->name, ggml_op_name(node->op)); } - //now we'll offload the entire ggml cgraph to a single opcfg QNN graph + //now we'll offload the ggml cgraph to a single QNN graph std::string graph_name; ggmlqnn_get_graphkey_from_cgraph(cgraph, graph_name); - if (instance->_qnn_graph_map.find(graph_name) != instance->_qnn_graph_map.end()) { + if (graph_name == "") + return GGML_STATUS_SUCCESS; + if (ctx->qnn_multinode_graph_map.find(graph_name) != ctx->qnn_multinode_graph_map.end()) { + GGMLQNN_LOG_DEBUG("graph name %s already create", graph_name.c_str()); //retrieve computational resource from cached QNN graph - qnn_res_t & graph_item = instance->_qnn_graph_map[graph_name]; - graph_handle = std::get<0>(graph_item); + qnn_multinode_res_t &graph_res = ctx->qnn_multinode_graph_map[graph_name]; + graph_handle = std::get<0>(graph_res); } else { //create QNN graph GGMLQNN_LOG_INFO("graph name %s", graph_name.c_str()); @@ -4748,20 +5071,14 @@ static enum ggml_status ggmlqnn_graph_compute(ggml_backend_t backend, struct ggm return ggml_result; } graph_handle = instance->get_qnn_graph_handle(); - //TODO: compose a single opcfg QNN graph + //TBD: compose a single QNN graph - //TODO: finalize QNN graph - //CHECK_QNN_API(qnn_error, qnn_raw_interface.graphFinalize(graph_handle, nullptr, nullptr)); + //finalize QNN graph + CHECK_QNN_API(qnn_error, qnn_raw_interface.graphFinalize(graph_handle, nullptr, nullptr)); - //cache QNN graph - qnn_tensors_t ggml_op_mulmat_tensors; - ggml_op_mulmat_tensors.reserve(0); - auto graph_item = std::make_tuple(graph_handle, ggml_op_mulmat_tensors); - instance->_qnn_graph_map[graph_name] = graph_item; + //TBD: cache QNN graph } - //exec QNN graph - - GGMLQNN_LOG_DEBUG("the second inference approach \"mapping cgraph to QNN graph\" is actually not supported now"); + //TBD: exec QNN graph return ggml_result; } diff --git a/scripts/build-run-android.sh b/scripts/build-run-android.sh index 393f4d458f41b..6f117680071e5 100755 --- a/scripts/build-run-android.sh +++ b/scripts/build-run-android.sh @@ -7,7 +7,6 @@ PWD=`pwd` ANDROID_PLATFORM=android-34 ANDROID_NDK=${PWD}/android-ndk-r26c REMOTE_PATH=/data/local/tmp/ -GGUF_MODEL_NAME=/sdcard/deepseek-r1-distill-qwen-1.5b-q4_0.gguf GGUF_MODEL_NAME=/sdcard/qwen1_5-1_8b-chat-q4_0.gguf #QNN SDK could be found at: @@ -18,8 +17,7 @@ QNN_SDK_INSTALL_PATH=/opt/qcom/aistack/qairt/ QNN_SDK_VERSION=2.32.0.250228 QNN_SDK_PATH=${QNN_SDK_INSTALL_PATH}/${QNN_SDK_VERSION} -#default is QNN NPU -qnnbackend=2 +qnnparams=" -mg 2 -ngl 99 " function dump_vars() { @@ -188,7 +186,7 @@ function run_llamacli() adb shell "cd ${REMOTE_PATH} \ && export LD_LIBRARY_PATH=${REMOTE_PATH} \ - && ${REMOTE_PATH}/llama-cli -mg ${qnnbackend} -ngl 99 -no-cnv -m ${GGUF_MODEL_NAME} -p \"introduce the movie Once Upon a Time in America briefly.\n\"" + && ${REMOTE_PATH}/llama-cli ${qnnparams} -no-cnv -m ${GGUF_MODEL_NAME} -p \"introduce the movie Once Upon a Time in America briefly.\n\"" } @@ -199,12 +197,11 @@ function run_llamabench() adb shell "cd ${REMOTE_PATH} \ && export LD_LIBRARY_PATH=${REMOTE_PATH} \ - && ${REMOTE_PATH}/llama-bench -mg ${qnnbackend} -m ${GGUF_MODEL_NAME}" + && ${REMOTE_PATH}/llama-bench ${qnnparams} -m ${GGUF_MODEL_NAME}" } -#refer to:https://github.com/ggml-org/llama.cpp/pull/12155 function run_test-ops() { prepare_run_on_phone test-backend-ops @@ -215,37 +212,6 @@ function run_test-ops() } -function run_test-op() -{ - prepare_run_on_phone test-backend-ops - - qnnbackendname=qnn-cpu - case $qnnbackend in - 0) - qnnbackendname=qnn-cpu - ;; - 1) - qnnbackendname=qnn-gpu - ;; - 2) - qnnbackendname=qnn-npu - ;; - *) - qnnbackendname=qnn-cpu - ;; - esac - - #debug - echo "adb shell cd ${REMOTE_PATH} \ - && export LD_LIBRARY_PATH=${REMOTE_PATH} \ - && ${REMOTE_PATH}/test-backend-ops test -o $opname -b $qnnbackendname " - - echo "\n" - adb shell "cd ${REMOTE_PATH} \ - && export LD_LIBRARY_PATH=${REMOTE_PATH} \ - && ${REMOTE_PATH}/test-backend-ops test -o $opname -b $qnnbackendname " - -} function print_oplist() { @@ -335,9 +301,8 @@ function show_usage() echo " $0 build" echo " $0 updateqnnlib" echo " $0 run_testops" - echo " $0 run_testop [ADD/MUL/MUL_MAT......(op from print_oplist)] [0 (QNN_CPU) / 1 (QNN_GPU) / 2 (QNN_NPU)]" - echo " $0 run_llamacli 0 (QNN_CPU) / 1 (QNN_GPU) / 2 (QNN_NPU) / 3 (ggml)" - echo " $0 run_llamabench 0 (QNN_CPU) / 1 (QNN_GPU) / 2 (QNN_NPU) / 3 (ggml)" + echo " $0 run_llamacli" + echo " $0 run_llamabench" echo -e "\n\n\n" } @@ -367,40 +332,19 @@ elif [ $# == 1 ]; then elif [ "$1" == "run_testops" ]; then run_test-ops exit 0 - - elif [ "$1" == "updateqnnlib" ]; then - update_qnn_libs - exit 0 - else - show_usage - exit 1 - fi -elif [ $# == 2 ]; then - qnnbackend=$2 - if [ ${qnnbackend} -gt 3 ]; then - show_usage - exit 1 - fi - - if [ "$1" == "run_llamacli" ]; then + elif [ "$1" == "run_llamacli" ]; then run_llamacli exit 0 elif [ "$1" == "run_llamabench" ]; then run_llamabench exit 0 - fi -elif [ $# == 3 ]; then - opname=$2 -#TODO: check opname in oplist -#opname can be found via print_oplist: - - qnnbackend=$3 - if [ ${qnnbackend} -gt 3 ]; then + elif [ "$1" == "updateqnnlib" ]; then + update_qnn_libs + exit 0 + else show_usage exit 1 fi - run_test-op - exit 0 else show_usage exit 1 diff --git a/scripts/ggml-qnn.cfg b/scripts/ggml-qnn.cfg index 5796e613ff2af..b1a697ae12ed9 100644 --- a/scripts/ggml-qnn.cfg +++ b/scripts/ggml-qnn.cfg @@ -1,9 +1,28 @@ [general] +#0: QNN-CPU backend +#1: QNN-GPU backend +#2: QNN-NPU(htp) backend +#3: default ggml backend +qnn_backend = 2 + # enable/disable QNN's internal log print_qnn_internal_log = 0 + +# enable/disable perf of op function +enable_perf = 0 + +# enable/disable print tensors info in op function +print_tensors_info = 0 + +# enable/disable dump op info in handle_op +dump_op_info = 0 + # 0: general approach,similar to ggml-sycl or ggml-cann # 1: mapping entire ggml cgraph to QNN graph inference_approach = 0 [npu] -npu_inference_datatype = "fp16" +hvx_threads = 4 +vtcm_size_in_mb = 8 +enable_dlbc = 1 +precision_mode = "fp16" From 34bb250f157ed9e007d688255d8b2f16c3971e2c Mon Sep 17 00:00:00 2001 From: zhouwg Date: Tue, 18 Mar 2025 22:45:26 +0800 Subject: [PATCH 65/76] ggml-qnn: self code-review --- ggml/src/ggml-qnn/ggml-qnn.cpp | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/ggml/src/ggml-qnn/ggml-qnn.cpp b/ggml/src/ggml-qnn/ggml-qnn.cpp index 834af1e08e30f..2e6281379d1b4 100644 --- a/ggml/src/ggml-qnn/ggml-qnn.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn.cpp @@ -2547,7 +2547,7 @@ int qnn_instance::unload_system() { return result; } -static void ggmlqnn_compute_logcallback(const char * fmt, +static void ggmlqnn_sdk_logcallback(const char * fmt, QnnLog_Level_t level, uint64_t timestamp, va_list argp) { @@ -2556,7 +2556,7 @@ static void ggmlqnn_compute_logcallback(const char * fmt, return; static std::mutex log_mutex; - static unsigned char s_ggmlqnn_compute_logbuf[GGML_QNN_LOGBUF_LEN]; + static unsigned char s_ggmlqnn_sdk_logbuf[GGML_QNN_LOGBUF_LEN]; const char * log_level_desc = ""; switch (level) { @@ -2583,9 +2583,9 @@ static void ggmlqnn_compute_logcallback(const char * fmt, double ms = (double) timestamp / 1000000.0; { std::lock_guard lock(log_mutex); - memset(s_ggmlqnn_compute_logbuf, 0, GGML_QNN_LOGBUF_LEN); - vsnprintf(reinterpret_cast(s_ggmlqnn_compute_logbuf), GGML_QNN_LOGBUF_LEN, fmt, argp); - GGMLQNN_LOG_DEBUG("%8.1fms [%-7s] %s\n", ms, log_level_desc, s_ggmlqnn_compute_logbuf); + memset(s_ggmlqnn_sdk_logbuf, 0, GGML_QNN_LOGBUF_LEN); + vsnprintf(reinterpret_cast(s_ggmlqnn_sdk_logbuf), GGML_QNN_LOGBUF_LEN, fmt, argp); + GGMLQNN_LOG_DEBUG("%8.1fms [%-7s] %s\n", ms, log_level_desc, s_ggmlqnn_sdk_logbuf); } } @@ -2625,9 +2625,9 @@ int qnn_instance::qnn_init(const QnnSaver_Config_t ** saver_config) { _qnn_interface.set_qnn_interface(_loaded_backend); #if 1 - _qnn_interface.qnn_log_create(ggmlqnn_compute_logcallback, _qnn_log_level, &_qnn_log_handle); + _qnn_interface.qnn_log_create(ggmlqnn_sdk_logcallback, _qnn_log_level, &_qnn_log_handle); #else - _qnn_raw_interface.logCreate(ggmlqnn_compute_logcallback, _qnn_log_level, &_qnn_log_handle); + _qnn_raw_interface.logCreate(ggmlqnn_sdk_logcallback, _qnn_log_level, &_qnn_log_handle); #endif if (nullptr == _qnn_log_handle) { GGMLQNN_LOG_WARN("why failed to initialize qnn log\n"); //NPU backend not work on Qualcomm SoC based low-end phone @@ -3476,7 +3476,7 @@ static bool ggmlqnn_same_types(const ggml_backend_qnn_context * ctx, const ggml_ return true; } -static bool ggmlqnn_compute_can_handle_op(const ggml_backend_qnn_context * ctx, const struct ggml_tensor * op_tensor) { +static bool ggmlqnn_can_handle_op(const ggml_backend_qnn_context * ctx, const struct ggml_tensor * op_tensor) { if (op_tensor->op == GGML_OP_NONE) { return true; } @@ -3567,7 +3567,7 @@ static bool ggmlqnn_compute_can_handle_op(const ggml_backend_qnn_context * ctx, } } -static bool ggmlqnn_compute_compute_forward(ggml_backend_t backend, struct ggml_tensor * dst) { +static bool ggmlqnn_compute_forward(ggml_backend_t backend, struct ggml_tensor * dst) { ggmlqnn_op_func_t func = nullptr; ggml_backend_qnn_context * ctx = (ggml_backend_qnn_context *)backend->context; @@ -3913,7 +3913,7 @@ static enum ggml_status ggmlqnn_backend_graph_compute_general(ggml_backend_t bac || node->op == GGML_OP_PERMUTE || node->op == GGML_OP_NONE) { continue; } - bool ok = ggmlqnn_compute_compute_forward(backend, node); + bool ok = ggmlqnn_compute_forward(backend, node); if (!ok) { GGMLQNN_LOG_DEBUG("%s: error: op not supported %s (%s)\n", __func__, node->name, ggml_op_name(node->op)); } @@ -4074,7 +4074,7 @@ static ggml_backend_buffer_t ggml_backend_qnn_device_buffer_from_host_ptr(ggml_b static bool ggml_backend_qnn_device_supports_op(ggml_backend_dev_t dev, const struct ggml_tensor * op) { ggml_backend_qnn_context * ctx = (ggml_backend_qnn_context *) dev->context; - return (ggmlqnn_compute_can_handle_op(ctx,op)); + return (ggmlqnn_can_handle_op(ctx,op)); } static bool ggml_backend_qnn_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) { @@ -5033,7 +5033,7 @@ static void ggmlqnn_compute_rope(ggml_backend_qnn_context * ctx, ggml_tensor * d static enum ggml_status ggmlqnn_backend_graph_compute_special(ggml_backend_t backend, struct ggml_cgraph * cgraph) { enum ggml_status ggml_result = GGML_STATUS_SUCCESS; Qnn_ErrorHandle_t qnn_error = QNN_SUCCESS; - qnn_perf op_perf = qnn_perf("ggml_backend_qnn_graph_compute_special"); + qnn_perf op_perf = qnn_perf("ggmlqnn_backend_graph_compute_special"); qnn_instance * instance = nullptr; Qnn_GraphHandle_t graph_handle = nullptr; ggml_backend_qnn_context * ctx = (ggml_backend_qnn_context *) backend->context; From f79812b373a248d290caef0cd1210b879b1ddb8b Mon Sep 17 00:00:00 2001 From: zhouwg Date: Wed, 19 Mar 2025 20:35:16 +0800 Subject: [PATCH 66/76] ggml-qnn: rebase upstream --- ggml/src/ggml-qnn/ggml-qnn.cpp | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/ggml/src/ggml-qnn/ggml-qnn.cpp b/ggml/src/ggml-qnn/ggml-qnn.cpp index 2e6281379d1b4..8e98a042df93b 100644 --- a/ggml/src/ggml-qnn/ggml-qnn.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn.cpp @@ -3299,6 +3299,12 @@ static uint8_t * ggmlqnn_create_rpc_buffer(qnn_instance * instance, const ggml_t } static void ggmlqnn_load_cfg() { + //this function can be called in various scenarios + static bool initialized = false; + if (initialized) { + GGMLQNN_LOG_DEBUG("qnn cfg file already loadded\n"); + return; + } char time_string[GGML_QNN_TMPBUF_LEN]; memset(time_string, 0, GGML_QNN_TMPBUF_LEN); ggmlqnn_get_timestring(time_string); @@ -3333,6 +3339,7 @@ static void ggmlqnn_load_cfg() { } else { g_qnn_params.precision_mode = 0; } + initialized = true; } static Qnn_Tensor_t * ggmlqnn_create_general_tensor(qnn_instance * instance, Qnn_GraphHandle_t graph_handle, @@ -4008,8 +4015,7 @@ static ggml_backend_t ggml_backend_qnn_device_init_backend(ggml_backend_dev_t de GGMLQNN_LOG_INFO("enter %s\n", __func__); size_t dev_index = 0; - //case-1: special scenario, such as test-backend-ops or other similar scenairo: calling ggml_backend_qnn_device_init_backend directly in user's applicaton - //call ggmlqnn_load_cfg accordingly in this place + //case-1: test-backend-ops or other similar scenairo: calling ggml_backend_dev_init(dev, reinterpret_cast(i)) directly in user's code ggmlqnn_load_cfg(); GGMLQNN_LOG_INFO("user's specified qnn_backend in cfgfile = %d", g_qnn_params.qnn_backend); GGMLQNN_LOG_INFO("user's sepcified qnn runtime lib path in cfgfile = %s", g_qnn_params.qnn_runtimelib_path); @@ -4188,8 +4194,8 @@ ggml_backend_reg_t ggml_backend_qnn_reg() { static ggml_backend_reg reg; static bool initialized = false; GGMLQNN_LOG_DEBUG("enter ggml_backend_qnn_reg"); + //case-2: normal scenario, such as llama-cli or UI applicaton - //call ggmlqnn_load_cfg accordingly in this place ggmlqnn_load_cfg(); GGMLQNN_LOG_INFO("user's specified qnn_backend=%d", g_qnn_params.qnn_backend); GGMLQNN_LOG_INFO("user's sepcified qnn runtime lib path=%s", g_qnn_params.qnn_runtimelib_path); @@ -4238,6 +4244,8 @@ ggml_backend_t ggml_backend_qnn_init(size_t device, const char * qnn_lib_path) { int result = 0; GGMLQNN_LOG_INFO("enter %s\n", __func__); + //case-3: calling ggml_backend_qnn_init() directly in user's code + ggmlqnn_load_cfg(); if (nullptr == qnn_lib_path) return nullptr; From 192d474830235b0b3569b55af4c2aa2fe9727eea Mon Sep 17 00:00:00 2001 From: zhouwg Date: Sat, 22 Mar 2025 22:44:41 +0800 Subject: [PATCH 67/76] ggml-qnn: add approach through Hexagon cDSP --- ggml/include/ggml-qnn.h | 17 +- ggml/src/ggml-qnn/CMakeLists.txt | 60 +- ggml/src/ggml-qnn/ggml-qnn.cpp | 1324 ++++++++++++++++--- ggml/src/ggml-qnn/kernels/ggmlop.h | 289 ++++ ggml/src/ggml-qnn/kernels/ggmlop_cdsp.c | 237 ++++ ggml/src/ggml-qnn/kernels/ggmlop_stub.c | 437 ++++++ ggml/src/ggml-qnn/kernels/libggmlop_skel.so | Bin 0 -> 13896 bytes scripts/build-run-android.sh | 4 +- scripts/ggml-qnn.cfg | 11 +- 9 files changed, 2139 insertions(+), 240 deletions(-) create mode 100644 ggml/src/ggml-qnn/kernels/ggmlop.h create mode 100644 ggml/src/ggml-qnn/kernels/ggmlop_cdsp.c create mode 100644 ggml/src/ggml-qnn/kernels/ggmlop_stub.c create mode 100755 ggml/src/ggml-qnn/kernels/libggmlop_skel.so diff --git a/ggml/include/ggml-qnn.h b/ggml/include/ggml-qnn.h index 06f143546ad24..2ff2bef9dcf7d 100644 --- a/ggml/include/ggml-qnn.h +++ b/ggml/include/ggml-qnn.h @@ -42,26 +42,11 @@ GGML_BACKEND_API ggml_backend_t ggml_backend_qnn_init(size_t dev_num, const char GGML_BACKEND_API bool ggml_backend_is_qnn(ggml_backend_t backend); -GGML_BACKEND_API void ggml_backend_qnn_set_n_threads(ggml_backend_t backend, int thread_counts); - GGML_BACKEND_API int ggml_backend_qnn_get_device_count(void); GGML_BACKEND_API ggml_backend_reg_t ggml_backend_qnn_reg(void); -inline const char * ggml_backend_qnn_get_devname(size_t dev_num) { - switch (dev_num) { - case QNN_BACKEND_CPU: - return "QNN-CPU"; - case QNN_BACKEND_GPU: - return "QNN-GPU"; - case QNN_BACKEND_NPU: - return "QNN-NPU"; - case QNN_BACKEND_GGML: - return "ggml"; //"fake" QNN backend, used for compare performance between QNN backend and original GGML - default: - return "unknown"; - } -} +const char * ggml_backend_qnn_get_devname(size_t dev_num); #ifdef __cplusplus } diff --git a/ggml/src/ggml-qnn/CMakeLists.txt b/ggml/src/ggml-qnn/CMakeLists.txt index fcbbc33a9b136..c63faca10e842 100644 --- a/ggml/src/ggml-qnn/CMakeLists.txt +++ b/ggml/src/ggml-qnn/CMakeLists.txt @@ -1,37 +1,59 @@ message(STATUS "Using QNN backend") message("CMAKE_SYSTEM_NAME : ${CMAKE_SYSTEM_NAME}") +if(NOT DEFINED QNN_SDK_PATH) + message(FATAL_ERROR "QNN_SDK_PATH not defined") +endif() + +if(NOT DEFINED HEXAGON_SDK_PATH) + message(FATAL_ERROR "HEXAGON_SDK_PATH not defined") +endif() + +message("QNN_SDK_PATH: ${QNN_SDK_PATH}") +message("HEXAGON_SDK_PATH: ${HEXAGON_SDK_PATH}") + if(CMAKE_SYSTEM_NAME STREQUAL "Android") find_library(LOG_LIB log) - set(QNN_LINK_LIBRARIES ${LOG_LIB}) + + add_library(cdsprpc + SHARED + IMPORTED) + set_target_properties(cdsprpc + PROPERTIES + IMPORTED_LOCATION + ${HEXAGON_SDK_PATH}/ipc/fastrpc/remote/ship/android_aarch64/libcdsprpc.so) + + set(QNN_LINK_LIBRARIES ${LOG_LIB} cdsprpc) set(QNN_DEFAULT_LIB_SEARCH_PATH "/data/local/tmp/" CACHE STRING "customized library search path for QNN backend") + + include_directories(${HEXAGON_SDK_PATH}/incs) + include_directories(${HEXAGON_SDK_PATH}/incs/stddef) + include_directories(${HEXAGON_SDK_PATH}/ipc/fastrpc/incs) + include_directories(${HEXAGON_SDK_PATH}/ipc/fastrpc/rpcmem/inc) + include_directories(${HEXAGON_SDK_PATH}/ipc/fastrpc/remote/ship/android_Debug_aarch64) + include_directories(${HEXAGON_SDK_PATH}/incs/qnx) + include_directories(${HEXAGON_SDK_PATH}/libs/common/qnx/ship/android_Debug_aarch64) + include_directories(${HEXAGON_SDK_PATH}/utils/examples) + include_directories(${HEXAGON_SDK_PATH}/ipc/fastrpc/rtld/ship/android_aarch64) + include_directories(${HEXAGON_SDK_PATH}/libs/atomic/inc) + include_directories(${HEXAGON_SDK_PATH}/libs/atomic/android_Debug_aarch64/ship) + include_directories(${CMAKE_SOURCE_DIR}/ggml/src/ggml-qnn/) + include_directories(${CMAKE_SOURCE_DIR}/ggml/src/ggml-qnn/kernels/) + elseif(CMAKE_SYSTEM_NAME STREQUAL "Windows") set(QNN_DEFAULT_LIB_SEARCH_PATH "C:\\" CACHE STRING "customized library search path for QNN backend") else() message(FATAL_ERROR "QNN now only available on Android and Windows(Windows on ARM)") endif() -if(NOT DEFINED GGML_QNN_SDK_PATH) -# try read from environment variable - if(DEFINED ENV{QNN_SDK_PATH}) - set(GGML_QNN_SDK_PATH $ENV{QNN_SDK_PATH}) - else() - message(FATAL_ERROR "GGML_QNN_SDK_PATH not defined") - endif() -endif() - -message("QNN_SDK_PATH: ${GGML_QNN_SDK_PATH}") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DGGML_USE_QNN") set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3") -file(GLOB QNN_SOURCES "${CMAKE_CURRENT_LIST_DIR}/*.cpp") - ggml_add_backend_library(ggml-qnn - ${QNN_SOURCES} -) +file(GLOB QNN_SOURCES "${CMAKE_CURRENT_LIST_DIR}/*.cpp" "${CMAKE_CURRENT_LIST_DIR}/kernels/*.c") +ggml_add_backend_library(ggml-qnn ${QNN_SOURCES}) -target_include_directories(ggml-qnn PRIVATE ${GGML_QNN_SDK_PATH}/include/QNN ${CMAKE_CURRENT_LIST_DIR}) +target_include_directories(ggml-qnn PRIVATE ${QNN_SDK_PATH}/include/QNN ${HEXAGON_SDK_PATH} ${CMAKE_CURRENT_LIST_DIR}) target_link_libraries(ggml-qnn PRIVATE ${QNN_LINK_LIBRARIES}) -string(REGEX REPLACE "/$" "" GGML_QNN_DEFAULT_LIB_SEARCH_PATH "${QNN_DEFAULT_LIB_SEARCH_PATH}") -target_compile_definitions(ggml-qnn PRIVATE GGML_QNN_DEFAULT_LIB_SEARCH_PATH="${QNN_DEFAULT_LIB_SEARCH_PATH}/") +string(REGEX REPLACE "/$" "" QNN_DEFAULT_LIB_SEARCH_PATH "${QNN_DEFAULT_LIB_SEARCH_PATH}") +target_compile_definitions(ggml-qnn PRIVATE QNN_DEFAULT_LIB_SEARCH_PATH="${QNN_DEFAULT_LIB_SEARCH_PATH}/") diff --git a/ggml/src/ggml-qnn/ggml-qnn.cpp b/ggml/src/ggml-qnn/ggml-qnn.cpp index 8e98a042df93b..23fe675aa97f3 100644 --- a/ggml/src/ggml-qnn/ggml-qnn.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn.cpp @@ -6,24 +6,26 @@ * https://developer.qualcomm.com/software/hexagon-dsp-sdk/tools * * this single-source-file or self-contained implementation of ggml-qnn backend has 10 sections: - * section-1 forward/prototype declaration - * section-2 global vars, macros, data structures - * section-3 ggml-qnn internal troubleshooting function/class - * section-4 helper function for WoA(Windows on ARM) - * section-5 general helper function - * section-6 QNN helper function + * section-1 forward/prototype declaration, global vars, macros, data structures + * section-2 ggml-qnn internal troubleshooting function/class + * section-3 helper function for WoA(Windows on ARM) + * section-4 general helper function + * section-5 QNN helper function + * section-6 Hexagon DSP helper function * section-7 ggml-qnn backend helper function / class * section-8 implementation of ggml-qnn backend according to ggml's backend subsystem - * section-9 implementation of general approach or the first tech approach - * section-10 implementation of the second tech approach:mapping the entire ggml cgraph to a single QNN graph + * section-9 implementation of general approach through QNN and Hexagon DSP + * section-10 implementation of special approach through QNN:mapping the entire ggml cgraph to a single QNN graph * - * currently provide following ggml op' QNN backend implementation: - * - GGML_OP_ADD/GGML_OP_SUB/GGML_OP_MUL/GGML_OP_DIV: - * this is a simple skeleton, can expand other ggml ops according to expertise - * - GGML_OP_LOG/GGML_OP_SQRT: + * currently provide following ggml op' implementation through QNN: + * - GGML_OP_ADD/GGML_OP_SUB/GGML_OP_MUL/GGML_OP_DIV/GGML_OP_LOG/GGML_OP_SQRT: * this is a simple skeleton, can expand other ggml ops according to expertise * - GGML_OP_MUL_MAT: - * this is a complicated skeleton, can expand other complex ggml ops accordingly + * this is a complicated skeleton, can expand other ggml ops accordingly + * + * currently provide following ggml op' implementation through Hexagon DSP: + * - GGML_OP_ADD: + * this is a skeleton, can expand other ggml ops accordingly * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to @@ -51,15 +53,6 @@ #include #include #include -#if defined(__ANDROID__) || defined(__linux__) -#include -#include -#include -#include -#include -#include -#include -#endif #include #include @@ -83,8 +76,15 @@ #include #include #include -#if (defined __ANDROID__) || (defined ANDROID) -#include "android/log.h" + +#if defined(__ANDROID__) || defined(__linux__) +#include +#include +#include +#include +#include +#include +#include #endif #if !defined(__ANDROID__) && !defined(__linux__) @@ -93,6 +93,18 @@ #include #endif +#if defined(__ANDROID__) +#include "android/log.h" + +#include "rpcmem.h" +#include "remote.h" +#include "os_defines.h" +#include "domain.h" +#include "AEEStdErr.h" +#include "HAP_power.h" +#include "HAP_farf.h" +#endif + #include "QnnTypes.h" #include "QnnCommon.h" #include "QnnContext.h" @@ -110,16 +122,20 @@ #include "ggml-impl.h" #include "ggml-backend-impl.h" +#include "kernels/ggmlop.h" + // ================================================================================================= -// section-1: forward/prototype declaration, macro +// section-1: forward/prototype declaration, global vars, macros, data structures // ================================================================================================= class qnn_instance; struct qnn_parameter; struct ggml_backend_qnn_context; typedef void (* ggmlqnn_op_func_t)(ggml_backend_qnn_context * ctx, ggml_tensor * op); +typedef int (* notif_callback_fn)(void * context, int domain, int session, remote_rpc_status_flags_t status); +typedef int (* ggmlhexagon_op_func_t)(remote_handle64 handle, const dsptensor * src0, const dsptensor * src1, dsptensor * dst); -//general function prototypes for ggml-qnn backend +static void * ggmlqnn_type_trait(ggml_backend_qnn_context * ctx, ggml_tensor * op); static void ggmlqnn_dump_tensor(const ggml_tensor * tensor, const char * name); static enum ggml_status ggmlqnn_backend_graph_compute_special(ggml_backend_t backend, struct ggml_cgraph * cgraph); static void ggmlqnn_log_internal(ggml_log_level level, const char * file, const char * func, int line, const char * format, ...); @@ -132,7 +148,8 @@ static Qnn_Tensor_t * ggmlqnn_create_general_tensor(qnn_instance * instance, Q void * data, uint32_t data_size, bool b_transpose = false); -//function prototypes for all op functions in the first tech approach(general approach in other backends) + +//function prototypes for all op functions in the general approach //general op function for elment-wise operation on 1/2 input tensors and 1 output tensor static void ggmlqnn_compute_elementwise(ggml_backend_qnn_context * ctx, ggml_tensor * dst); static void ggmlqnn_compute_mul_mat(ggml_backend_qnn_context * ctx, ggml_tensor * dst); @@ -163,7 +180,7 @@ static void ggmlqnn_compute_upsample_nearest2d(ggml_backend_qnn_context * ctx, g static void ggmlqnn_compute_timestep_embedding(ggml_backend_qnn_context * ctx, ggml_tensor * dst); static void ggmlqnn_compute_diag_mask(ggml_backend_qnn_context * ctx, ggml_tensor * dst, float value); -//function prototypes for all op functions in the second tech approach("mapping the entire cgraph to a single QNN graph") +//function prototypes for all op functions in the special approach("mapping the entire cgraph to a single QNN graph") static void ggmlqnn_graph_addnode(ggml_backend_qnn_context * ctx, struct ggml_cgraph * cgraph, Qnn_GraphHandle_t graph_handle, std::string & graph_name, ggml_tensor * op, bool is_reuse_graph = false); @@ -192,6 +209,7 @@ static void ggmlqnn_graph_addnode(ggml_backend_qnn_context * ctx, struct ggml_cg #define QNN_VER_PTR(x) (&((x).v1)) #define RPCMEM_DEFAULT_FLAGS 1 #define RPCMEM_HEAP_ID_SYSTEM 25 +#define STATUS_CONTEXT 0x12345678 #define QNN_TENSOR_GET_ID(tensor) get_qnn_tensorid(tensor) #define QNN_TENSOR_GET_NAME(tensor) get_qnn_tensorname(tensor) @@ -239,13 +257,15 @@ static void ggmlqnn_graph_addnode(ggml_backend_qnn_context * ctx, struct ggml_cg #define GGMLQNN_CHECK_PARAMS(ctx, src0, src1, dst) \ do { \ - if (!ggmlqnn_is_valid_params((ctx), (src0), (src1), (dst))) { \ - return; \ + if (g_qnn_params.inference_approach != DIRECT_USE_CDSP) { \ + if (!ggmlqnn_is_valid_params((ctx), (src0), (src1), (dst))) { \ + return; \ + } \ } \ - } while (0) + } while (0) \ // ================================================================================================= -// section-2: data type, data structure, global vars +// section-1: data type, data structure, global vars // ================================================================================================= using pfn_rpc_mem_init = void (*)(void); using pfn_rpc_mem_deinit = void (*)(void); @@ -268,10 +288,27 @@ using qnn_cgraph_node_t = std::tuple; using qnn_multinode_res_t = std::tuple; -enum class qnn_profile_level { - profile_off = 0, - profile_basic = 1, - profile_detail = 2 +enum qnn_profile_level { + PROFILE_OFF = 0, + PROFILE_BASIC = 1, + PROFILE_DETAIL = 2 +}; + +//0: general approach through QNN +//1: general approach through Hexagon cDSP +//2: special approach through QNN:mapping entire ggml cgraph to QNN graph +enum inference_approach { + QNN_GENERAL = 0, + DIRECT_USE_CDSP = 1, + QNN_SINGLEGRAPH = 2, +}; + +enum hexagon_dsp_type { + HEXAGON_ADSP = 0, + HEXAGON_MDSP = 1, + HEXAGON_SDSP = 2, + HEXAGON_CDSP = 3, + HEXAGON_CDSP1 = 4, }; enum qcom_htp_arch { @@ -328,6 +365,10 @@ struct ggml_backend_qnn_context { size_t work_size; size_t desired_size; int n_threads; + + size_t rpc_mempool_len; + void * rpc_mempool; + remote_handle64 ggmlop_handle; }; struct qnn_op_caps { @@ -347,21 +388,14 @@ struct qnn_parameter { int hvx_threads; int vtcm_size_in_mb; int enable_dlbc; - int inference_approach; // 0: general approach,similar to ggml-sycl or ggml-cann 1: mapping entire ggml cgraph to QNN graph - int qnn_backend; // 0: QNN-CPU backend, 1: QNN-GPU backend, 2: QNN-NPU backend + int inference_approach; // 0: QNN_GENERAL 1: DIRECT_USE_CDSP 2: QNN_SINGELGRAPH + int qnn_backend; // 0: QNN-CPU backend 1: QNN-GPU backend 2: QNN-NPU backend const char * qnn_cfgfilename; const char * qnn_runtimelib_path; }; -//TODO:I don't think threadsafe is required at the moment -// so we can uniform them to avoid compiler/toolchain's complains -#if !defined(__ANDROID__) && !defined(__linux__) -static std::atomic g_qnntensor_idx(0); //ensure every QNN tensor name is unique -static std::atomic g_qnnopcfg_idx(0); //ensure every QNN opconfig name is unique -#else static int32_t g_qnntensor_idx = 0; //ensure every QNN tensor name is unique static int32_t g_qnnopcfg_idx = 0; //ensure every QNN opconfig name is unique -#endif static struct qnn_parameter g_qnn_params = { .print_qnn_internal_log = 0, @@ -514,6 +548,14 @@ static struct ggml_backend_qnn_context g_qnn_mgr[GGML_QNN_MAX_DEVICES] = { .socinfo = {}}, }; +static domain hexagon_supported_domains[] = { + {ADSP_DOMAIN_ID, ADSP_DOMAIN}, + {MDSP_DOMAIN_ID, MDSP_DOMAIN}, + {SDSP_DOMAIN_ID, SDSP_DOMAIN}, + {CDSP_DOMAIN_ID, CDSP_DOMAIN}, + {CDSP1_DOMAIN_ID, CDSP1_DOMAIN} +}; + static constexpr const qnn_op_caps ggmlqnn_k_op_caps[] = { {true, GGML_OP_NONE, nullptr, 0, nullptr}, {false, GGML_OP_DUP}, @@ -624,7 +666,7 @@ static_assert(std::size(ggmlqnn_k_op_caps) == (GGML_OP_COUNT + GGML_UNARY_OP_COU "pls check ggmlqnn_k_op_caps and ensure is corresponding to latest ggml.h"); // ================================================================================================= -// section-3: ggml-qnn internal troubleshooting function/class +// section-2: ggml-qnn internal troubleshooting function/class // ================================================================================================= static void ggmlqnn_log_internal(ggml_log_level level, const char * file, const char * func, int line, const char * format, ...) { static std::mutex ggmlqnn_log_internal_mutex; @@ -762,7 +804,7 @@ class qnn_perf { }; // ================================================================================================= -// section-4: helper function for WoA(Window on ARM) +// section-3: helper function for WoA(Window on ARM) // ================================================================================================= #if !defined(__ANDROID__) && !defined(__linux__) #define RTLD_GLOBAL 0x100 @@ -817,7 +859,7 @@ static const char * dlerror(void) { #endif // ================================================================================================= -// section-5: general helper function +// section-4: general helper function // ================================================================================================= //TODO: merge the following 6 helper functions which used to ensure every QNN tensor/opcfg name is unique static void ggmlqnn_reset_tensoridx() { @@ -829,11 +871,7 @@ static void ggmlqnn_inc_tensoridx() { } static int32_t ggmlqnn_get_tensoridx() { -#if !defined(__ANDROID__) && !defined(__linux__) - return g_qnntensor_idx.load(); -#else return g_qnntensor_idx; -#endif } static void ggmlqnn_reset_opcfgidx() { @@ -845,11 +883,7 @@ static void ggmlqnn_inc_opcfgidx() { } static int32_t ggmlqnn_get_opcfgidx() { -#if !defined(__ANDROID__) && !defined(__linux__) - return g_qnnopcfg_idx.load(); -#else return g_qnnopcfg_idx; -#endif } static void * ggmlqnn_mallocz_aligned(size_t size, size_t alignment) { @@ -994,7 +1028,7 @@ static void ggmlqnn_get_timestring(char * p_currenttime) { } // ================================================================================================= -// section-6: QNN helper function +// section-5: QNN helper function // ================================================================================================= static inline uint32_t get_qnn_tensorid(const Qnn_Tensor_t & tensor) { if (tensor.version == QNN_TENSOR_VERSION_1) { @@ -1342,6 +1376,810 @@ static Qnn_OpConfig_t ggmlqnn_create_op_config(const char * name, const char * p return opcfg; } +// ================================================================================================= +// section-6: Hexagon DSP helper function +// ================================================================================================= +static const char * ggmlhexagon_get_dsp_name(int domain_id) { + switch (domain_id) { + case HEXAGON_ADSP: + return "Hexagon-aDSP"; + case HEXAGON_MDSP: + return "Hexagon-mDSP"; + case HEXAGON_SDSP: + return "Hexagon-sDSP"; + case HEXAGON_CDSP: + return "Hexagon-cDSP"; + case HEXAGON_CDSP1: + return "Hexagon-cDSP1"; + default: + return "Hexagon-unknown"; + } +} + +static int ggmlhexagon_pd_status_notifier_callback(void * context, int domain, int session, remote_rpc_status_flags_t status){ + int error = AEE_SUCCESS; + switch (status){ + case FASTRPC_USER_PD_UP: + GGMLQNN_LOG_DEBUG("PD is up\n"); + break; + case FASTRPC_USER_PD_EXIT: + GGMLQNN_LOG_DEBUG("PD closed\n"); + break; + case FASTRPC_USER_PD_FORCE_KILL: + GGMLQNN_LOG_DEBUG("PD force kill\n"); + break; + case FASTRPC_USER_PD_EXCEPTION: + GGMLQNN_LOG_DEBUG("PD exception\n"); + break; + case FASTRPC_DSP_SSR: + GGMLQNN_LOG_DEBUG("DSP SSR\n"); + break; + default : + error = AEE_EBADITEM; + break; + } + return error; +} + +static domain * ggmlhexagon_get_domain(int domain_id) { + int size = sizeof(hexagon_supported_domains) / sizeof(domain); + + for (size_t i = 0; i < size; i++) { + if (hexagon_supported_domains[i].id == domain_id) + return &hexagon_supported_domains[i]; + } + + return nullptr; +} + +static bool ggmlhexagon_is_cdsp(int domain_id) { + return (domain_id == HEXAGON_CDSP) || (domain_id == HEXAGON_CDSP1); +} + +static bool ggmlhexagon_is_valid_domain_id(int domain_id, int compute_only) { + int size = sizeof(hexagon_supported_domains) / sizeof(domain); + + if (compute_only) { + return ggmlhexagon_is_cdsp(domain_id); + } + + for (size_t i = 0; i < size; i++) { + if (hexagon_supported_domains[i].id == domain_id) + return true; + } + + return false; +} + +static int ggmlhexagon_get_domains_info(const char * domain_type, int * num_domains, fastrpc_domain ** domains_info) { + int hexagon_err = AEE_SUCCESS; + int ss_info = 0; + ss_info = strcmp(domain_type, "NSP")? HPASS: NSP; + system_req_payload req; + memset(&req, 0, sizeof(system_req_payload)); + req.id = FASTRPC_GET_DOMAINS; + req.sys.domains = nullptr; + fastrpc_domain * domain = nullptr; + + if (ss_info != 0) { + req.sys.flags = DOMAINS_LIST_FLAGS_SET_TYPE(req.sys.flags, ss_info); + } else { + req.sys.flags =0; + } + +#ifdef _WIN32 + hexagon_err = AEE_EUNSUPPORTED; + goto bail; +#endif + + if (remote_system_request) { + hexagon_err = remote_system_request(&req); + if (hexagon_err != AEE_SUCCESS) { + GGMLQNN_LOG_DEBUG("failure in remote_system_request call: %d", hexagon_err); + goto bail; + } + //allocate memory for domain-info array + req.sys.max_domains = req.sys.num_domains; + void * buffer = calloc(req.sys.num_domains, sizeof(fastrpc_domain)); + if (nullptr == buffer) { + hexagon_err = AEE_ENOMEMORY; + GGMLQNN_LOG_DEBUG("unable to allocate memory for req.sys.domains"); + goto bail; + } + req.sys.domains = static_cast(buffer); + hexagon_err = remote_system_request(&req); + if (hexagon_err != AEE_SUCCESS) { + GGMLQNN_LOG_DEBUG("failure in remote_system_request call: %d.\n", hexagon_err); + goto bail; + } + + for (int i = 0; i < req.sys.num_domains; i++) { + //verify that only requested type domains were returned + domain = &req.sys.domains[i]; + if (domain->type != ss_info) { + hexagon_err = -1; + GGMLQNN_LOG_DEBUG("incorrect data received from remote_system_request.\n"); + goto bail; + } + } + *domains_info = req.sys.domains; + *num_domains = req.sys.num_domains; + } else { + hexagon_err = AEE_EUNSUPPORTED; + goto bail; + } + +bail: + if (hexagon_err && !req.sys.domains) { + free(req.sys.domains); + } + return hexagon_err; +} + +static int ggmlhexagon_get_dsp_support(int * domain) { + int hexagon_error = AEE_SUCCESS; + *domain = HEXAGON_CDSP; + + if (remote_handle_control) { + struct remote_dsp_capability dsp_capability_domain = {HEXAGON_CDSP, DOMAIN_SUPPORT, 0}; + hexagon_error = remote_handle_control(DSPRPC_GET_DSP_INFO, &dsp_capability_domain, sizeof(struct remote_dsp_capability)); + if ((hexagon_error & 0xFF) == (AEE_EUNSUPPORTEDAPI & 0xFF)) { + GGMLQNN_LOG_DEBUG("FastRPC Capability API is not supported on this device"); + goto bail; + } + + if (0 == dsp_capability_domain.capability) { + dsp_capability_domain.domain = HEXAGON_ADSP; + dsp_capability_domain.attribute_ID = DOMAIN_SUPPORT; + dsp_capability_domain.capability = 0; + hexagon_error = remote_handle_control(DSPRPC_GET_DSP_INFO, &dsp_capability_domain, sizeof(struct remote_dsp_capability)); + if(dsp_capability_domain.capability) { + *domain = HEXAGON_ADSP; + } + } + + if (hexagon_error != AEE_SUCCESS) { + GGMLQNN_LOG_DEBUG("get_dsp_support failed with error 0x%x", hexagon_error); + goto bail; + } + } else { + hexagon_error = AEE_EUNSUPPORTEDAPI; + GGMLQNN_LOG_DEBUG("remote_dsp_capability interface is not supported on this device"); + } + +bail: + return hexagon_error; +} + +static int ggmlhexagon_get_vtcm_info(int domain, uint32_t * capability, uint32_t attr) { + int hexagon_error = AEE_SUCCESS; + *capability = 0; + + if (attr == VTCM_PAGE || attr == VTCM_COUNT) { + } else { + hexagon_error = AEE_EBADPARM; + GGMLQNN_LOG_DEBUG("unsupported attr, only VTCM_PAGE and VTCM_COUNT supported"); + goto bail; + } + + if (remote_handle_control) { + if (domain == HEXAGON_ADSP || domain == HEXAGON_CDSP) { + /* + * query the DSP for VTCM information + * since the ADSP does not have a dedicated VTCM, we expect the output to be 0 + */ + struct remote_dsp_capability dsp_capability_vtcm_dsp; + dsp_capability_vtcm_dsp.domain = (uint32_t)domain; + dsp_capability_vtcm_dsp.attribute_ID = attr; + dsp_capability_vtcm_dsp.capability = (uint32_t)0; + hexagon_error = remote_handle_control(DSPRPC_GET_DSP_INFO, &dsp_capability_vtcm_dsp, sizeof(struct remote_dsp_capability)); + if ((hexagon_error & 0xFF) == (AEE_EUNSUPPORTEDAPI & 0xFF)) { + GGMLQNN_LOG_DEBUG("FastRPC Capability API is not supported on this device"); + GGMLQNN_LOG_DEBUG("running the use case without checking the capability"); + hexagon_error = AEE_SUCCESS; + goto bail; + } else if (hexagon_error == AEE_SUCCESS) { + *capability = dsp_capability_vtcm_dsp.capability; + } else { + GGMLQNN_LOG_DEBUG("get_vtcm_info failed with error 0x%x", hexagon_error); + goto bail; + } + } else { + hexagon_error = AEE_EUNSUPPORTED; + GGMLQNN_LOG_DEBUG("unsupported domain %d", domain); + goto bail; + } + } else { + hexagon_error = AEE_EUNSUPPORTEDAPI; + GGMLQNN_LOG_DEBUG("remote_dsp_capability interface is not supported on this device"); + } + +bail: + return hexagon_error; +} + +static bool ggmlhexagon_is_unsignedpd_supported(int domain_id) { + int hexagon_error = AEE_SUCCESS; + if (remote_handle_control) { + struct remote_dsp_capability dsp_capability_domain = {static_cast(domain_id), UNSIGNED_PD_SUPPORT, 0}; + hexagon_error = remote_handle_control(DSPRPC_GET_DSP_INFO, &dsp_capability_domain, sizeof(struct remote_dsp_capability)); + if ((hexagon_error & 0xFF) == (AEE_EUNSUPPORTEDAPI & 0xFF)) { + GGMLQNN_LOG_WARN("FastRPC Capability API is not supported on this device. Falling back to signed pd"); + return false; + } + + if (hexagon_error) { + GGMLQNN_LOG_WARN("error 0x%x: FastRPC Capability API failed. falling back to signed pd", hexagon_error); + return false; + } + + if (dsp_capability_domain.capability == 1) { + return true; + } + } else { + hexagon_error = AEE_EUNSUPPORTEDAPI; + GGMLQNN_LOG_WARN("remote_dsp_capability interface is not supported on this device.falling back to signed pd"); + return false; + } + + return false; +} + +static bool ggmlhexagon_get_unsignedpd_support(void) { + return ggmlhexagon_is_unsignedpd_supported(HEXAGON_CDSP); +} + +static bool ggmlhexagon_is_async_fastrpc_supported(int domain) { + int hexagon_error = AEE_SUCCESS; + if (remote_handle_control) { + if (domain == HEXAGON_CDSP) { + /* + * Query the DSP for ASYNC_FASTRPC_SUPPORT information + * Async fastrpc is supported only on CDSP + */ + struct remote_dsp_capability dsp_capability_async_support; + dsp_capability_async_support.domain = (uint32_t)domain; + dsp_capability_async_support.attribute_ID = ASYNC_FASTRPC_SUPPORT; + dsp_capability_async_support.capability = (uint32_t)0; + hexagon_error = remote_handle_control(DSPRPC_GET_DSP_INFO, &dsp_capability_async_support, sizeof(struct remote_dsp_capability)); + if ((hexagon_error & 0xFF) == (AEE_EUNSUPPORTEDAPI & 0xFF)) { + GGMLQNN_LOG_WARN("FastRPC Capability API is not supported on this device"); + hexagon_error = AEE_SUCCESS; + goto bail; + } else if (dsp_capability_async_support.capability == 1) { + return true; + } + + if (hexagon_error != AEE_SUCCESS){ + GGMLQNN_LOG_WARN("failed with error 0x%x", hexagon_error); + goto bail; + } + } else { + hexagon_error = AEE_EUNSUPPORTED; + GGMLQNN_LOG_WARN("async FastRPC is not supported on domain %d", domain); + goto bail; + } + } else { + hexagon_error = AEE_EUNSUPPORTEDAPI; + GGMLQNN_LOG_WARN("remote_dsp_capability interface is not supported on this device"); + } + +bail: + return false; +} + +static void ggmlhexagon_set_rpc_latency(int domain, int qos, int latency) { + int hexagon_error = AEE_SUCCESS; + + if (remote_handle_control) { + struct remote_rpc_control_latency data; +#if 1 + data.enable = RPC_PM_QOS; + data.latency = 300; +#else + data.enable = RPC_POLL_QOS; + data.latency = 1000; +#endif + data.enable = qos; + data.latency = latency; + hexagon_error = remote_handle64_control(DSPRPC_GET_DSP_INFO, DSPRPC_CONTROL_LATENCY, (void*)&data, sizeof(data)); + if (hexagon_error != AEE_SUCCESS){ + GGMLQNN_LOG_WARN("failed with error 0x%x", hexagon_error); + goto bail; + } else { + GGMLQNN_LOG_INFO("set rpc qos %d, latency %d\n", qos, latency); + } + } else { + hexagon_error = AEE_EUNSUPPORTEDAPI; + GGMLQNN_LOG_WARN("remote_dsp_capability interface is not supported on this device"); + } + +bail: + return; +} + +static bool ggmlhexagon_is_status_notification_supported(int domain) { + int hexagon_error = AEE_SUCCESS; + + if (remote_handle_control) { + /* + * Query the DSP for STATUS_NOTIFICATION_SUPPORT information + * DSP User PD status notification Support + */ + struct remote_dsp_capability dsp_capability_status_notification_support; + dsp_capability_status_notification_support.domain = (uint32_t)domain; + dsp_capability_status_notification_support.attribute_ID = STATUS_NOTIFICATION_SUPPORT; + dsp_capability_status_notification_support.capability = (uint32_t)0; + hexagon_error = remote_handle_control(DSPRPC_GET_DSP_INFO, &dsp_capability_status_notification_support, sizeof(struct remote_dsp_capability)); + if ((hexagon_error & 0xFF) == (AEE_EUNSUPPORTEDAPI & 0xFF)) { + GGMLQNN_LOG_WARN("FastRPC Capability API is not supported on this device"); + hexagon_error = AEE_SUCCESS; + goto bail; + } else if (1 == dsp_capability_status_notification_support.capability) { + return true; + } + + if (hexagon_error != AEE_SUCCESS){ + GGMLQNN_LOG_WARN("failed with error 0x%x", hexagon_error); + goto bail; + } + } else { + hexagon_error = AEE_EUNSUPPORTEDAPI; + GGMLQNN_LOG_WARN("remote_dsp_capability interface is not supported on this device"); + } + +bail: + return false; +} + +static int ggmlhexagon_get_hmx_support_info(int domain, uint32_t * capability, uint32_t attr) { + int hexagon_error = AEE_SUCCESS; + *capability = 0; + + if (attr != HMX_SUPPORT_SPATIAL && attr != HMX_SUPPORT_DEPTH) { + hexagon_error = AEE_EBADPARM; + GGMLQNN_LOG_WARN("unsupported attr, only HMX_SUPPORT_SPATIAL and HMX_SUPPORT_DEPTH supported"); + goto bail; + } + + if (remote_handle_control) { + if (domain == HEXAGON_CDSP) { + /* + * Query the DSP for HMX SUPPORT information + * HMX is supported on CDSP only + */ + struct remote_dsp_capability dsp_capability_hmx_dsp; + dsp_capability_hmx_dsp.domain = (uint32_t)domain; + dsp_capability_hmx_dsp.attribute_ID = attr; + dsp_capability_hmx_dsp.capability = (uint32_t)0; + hexagon_error = remote_handle_control(DSPRPC_GET_DSP_INFO, &dsp_capability_hmx_dsp, sizeof(struct remote_dsp_capability)); + if ((hexagon_error & 0xFF) == (AEE_EUNSUPPORTEDAPI & 0xFF)) { + GGMLQNN_LOG_DEBUG("FastRPC Capability API is not supported on this device"); + hexagon_error = AEE_SUCCESS; + goto bail; + } + else if (hexagon_error == AEE_SUCCESS) { + *capability = dsp_capability_hmx_dsp.capability; + } else { + GGMLQNN_LOG_DEBUG("get_hmx_support_info failed with Error 0x%x", hexagon_error); + goto bail; + } + } else { + hexagon_error = AEE_EUNSUPPORTED; + GGMLQNN_LOG_DEBUG("HMX support is not there for domain %d", domain); + goto bail; + } + } else { + hexagon_error = AEE_EUNSUPPORTEDAPI; + GGMLQNN_LOG_DEBUG("remote_dsp_capability interface is not supported on this device"); + } + +bail: + return hexagon_error; +} + +static int ggmlhexagon_get_hex_arch_ver(int domain, uint32_t * capability) { + int hexagon_error = AEE_SUCCESS; + *capability = 0; + if(remote_handle_control) { + /* + * Query the Hexagon processor architecture version information + */ + struct remote_dsp_capability dsp_capability_arch_ver; + dsp_capability_arch_ver.domain = (uint32_t)domain; + dsp_capability_arch_ver.attribute_ID = ARCH_VER; + dsp_capability_arch_ver.capability = (uint32_t)0; + hexagon_error = remote_handle_control(DSPRPC_GET_DSP_INFO, &dsp_capability_arch_ver, sizeof(struct remote_dsp_capability)); + if ((hexagon_error & 0xFF) == (AEE_EUNSUPPORTEDAPI & 0xFF)) { + GGMLQNN_LOG_DEBUG("FastRPC Capability API is not supported on this device"); + hexagon_error = AEE_SUCCESS; + goto bail; + } else if (hexagon_error == AEE_SUCCESS) { + *capability = dsp_capability_arch_ver.capability; + } else { + GGMLQNN_LOG_DEBUG("get_hex_arch_ver failed with error 0x%x", hexagon_error); + goto bail; + } + } else { + hexagon_error = AEE_EUNSUPPORTEDAPI; + GGMLQNN_LOG_DEBUG("remote_dsp_capability interface is not supported on this device"); + } + + bail: + return hexagon_error; +} + +static int ggmlhexagon_get_hvx_support_info(int domain, uint32_t * capability, uint32_t attr) +{ + int hexagon_error = AEE_SUCCESS; + *capability = 0; + if (attr == HVX_SUPPORT_64B) { + hexagon_error = AEE_EBADPARM; + GGMLQNN_LOG_DEBUG("latest targets have 128 byte HVX register, use HVX_SUPPORT_128B instead of HVX_SUPPORT_64B"); + goto bail; + } + + if (attr != HVX_SUPPORT_128B) { + hexagon_error = AEE_EBADPARM; + GGMLQNN_LOG_DEBUG("unsupported attr. only HVX_SUPPORT_128B supported"); + goto bail; + } + + if (remote_handle_control) { + if (domain == HEXAGON_CDSP) { + /* + * Query the DSP for HVX SUPPORT information + * HVX is supported on CDSP only + */ + struct remote_dsp_capability dsp_capability_hvx_dsp; + dsp_capability_hvx_dsp.domain = (uint32_t)domain; + dsp_capability_hvx_dsp.attribute_ID = attr; + dsp_capability_hvx_dsp.capability = (uint32_t)0; + hexagon_error = remote_handle_control(DSPRPC_GET_DSP_INFO, &dsp_capability_hvx_dsp, sizeof(struct remote_dsp_capability)); + if ((hexagon_error & 0xFF)==(AEE_EUNSUPPORTEDAPI & 0xFF)) { + GGMLQNN_LOG_DEBUG("FastRPC Capability API is not supported on this device"); + hexagon_error = AEE_SUCCESS; + goto bail; + } else if (hexagon_error == AEE_SUCCESS) { + *capability = dsp_capability_hvx_dsp.capability; + } else { + GGMLQNN_LOG_DEBUG("failed with error 0x%x", hexagon_error); + goto bail; + } + } else { + hexagon_error = AEE_EUNSUPPORTED; + GGMLQNN_LOG_DEBUG("HVX support is not available on domain %d", domain); + goto bail; + } + } else { + hexagon_error = AEE_EUNSUPPORTEDAPI; + GGMLQNN_LOG_DEBUG("remote_dsp_capability interface is not supported on this device"); + } + +bail: + return hexagon_error; +} + +static int ggmlhexagon_request_status_notifications(int domain_id, void * context, notif_callback_fn call_back_fn) { + int hexagon_error = AEE_SUCCESS; + struct remote_rpc_notif_register notif; + bool status_notification_support; + + notif.context = context; + notif.domain = domain_id; + notif.notifier_fn = call_back_fn; + + status_notification_support = ggmlhexagon_is_status_notification_supported(domain_id); + if (status_notification_support) { + hexagon_error = remote_session_control(FASTRPC_REGISTER_STATUS_NOTIFICATIONS, (void*)¬if, sizeof(notif)); + if (hexagon_error != AEE_SUCCESS) { + GGMLQNN_LOG_DEBUG("error 0x%x: remote_session_control failed to enable status notifications", hexagon_error); + } + } else { + hexagon_error = AEE_EUNSUPPORTEDAPI; + } + + return hexagon_error; +} + +//TODO:not work on cDSP currently +static AEEResult ggmlhexagon_set_clocks(remote_handle64 handle, int32 power_level, int32 latency, int32 dcvs_enabled) { +#if 0 + GGMLQNN_LOG_DEBUG("----------- entering power set clocks"); + + HAP_power_request_t request; + memset(&request, 0, sizeof(HAP_power_request_t)); + request.type = HAP_power_set_apptype; + request.apptype = HAP_POWER_COMPUTE_CLIENT_CLASS; + + void * benchmark_ctx = (void*)(handle); + int retval = HAP_power_set(benchmark_ctx, &request); + if (retval) { + GGMLQNN_LOG_WARN("failed first power vote"); + return AEE_EFAILED; + } + + //configure clocks & DCVS mode + memset(&request, 0, sizeof(HAP_power_request_t)); + request.type = HAP_power_set_DCVS_v2; + request.dcvs_v2.dcvs_enable = TRUE; + request.dcvs_v2.dcvs_params.target_corner = (HAP_dcvs_voltage_corner_t)power_level; + if (dcvs_enabled) { + request.dcvs_v2.dcvs_params.min_corner = HAP_DCVS_VCORNER_DISABLE; + request.dcvs_v2.dcvs_params.max_corner = HAP_DCVS_VCORNER_DISABLE; + } else { + request.dcvs_v2.dcvs_params.min_corner = request.dcvs_v2.dcvs_params.target_corner; + request.dcvs_v2.dcvs_params.max_corner = request.dcvs_v2.dcvs_params.target_corner; + } + request.dcvs_v2.dcvs_option = HAP_DCVS_V2_PERFORMANCE_MODE; + request.dcvs_v2.set_dcvs_params = TRUE; + request.dcvs_v2.set_latency = TRUE; + request.dcvs_v2.latency = latency; + retval = HAP_power_set(benchmark_ctx, &request); + if (retval) { + GGMLQNN_LOG_WARN("failed to vote for performance mode"); + return AEE_EFAILED; + } + + memset(&request, 0, sizeof(HAP_power_request_t)); + request.type = HAP_power_set_HVX; + request.hvx.power_up = TRUE; + retval = HAP_power_set(benchmark_ctx, &request); + if (retval) { + GGMLQNN_LOG_WARN("failed to vote for HVX power"); + return AEE_EFAILED; + } +#endif + return AEE_SUCCESS; +} + +static int ggmlhexagon_init_dsp(ggml_backend_qnn_context * ctx) { + int hexagon_error = AEE_SUCCESS; + + int domain_id = HEXAGON_CDSP; + const char * domain_type = "NSP"; + + int unsignedpd_flag = 1; + bool is_unsignedpd_enabled = false; + int use_logical_id = 0; + int core_id = -1; + fastrpc_domain * domains_info = NULL; + fastrpc_domain * domain_info = NULL; + int num_domains = -1; + + domain * my_domain = NULL; + char * uri = NULL; + + char * ggmlop_domain_uri = NULL; + int ggmlop_domain_uri_len = 0; + + if (nullptr == ctx) + return 1; + GGMLQNN_LOG_INFO("init Hexagon DSP with backend %d(%s)", ctx->device, ggml_backend_qnn_get_devname(ctx->device)); + //TODO: reasonable rpc memory pool size and use it practically + ctx->ggmlop_handle = -1; + ctx->rpc_mempool_len = (1 << 20) * 512; + ctx->rpc_mempool = rpcmem_alloc(RPCMEM_HEAP_ID_SYSTEM, RPCMEM_DEFAULT_FLAGS, ctx->rpc_mempool_len); + if (nullptr == ctx->rpc_mempool) { + hexagon_error = AEE_ENORPCMEMORY; + printf("rpc memory alloc failed", hexagon_error); + ctx->rpc_mempool_len = 0; + return 2; + } + + if (domain_id == -1) { + if (domain_type != NULL) { + if ((strcmp(domain_type, "NSP") != 0 && strcmp(domain_type, "HPASS") != 0)) { + GGMLQNN_LOG_WARN("invalid domain_type %s. possible values are NSP or HPASS", domain_type); + goto bail; + } else { + hexagon_error = ggmlhexagon_get_domains_info(domain_type, &num_domains, &domains_info); + if (hexagon_error == AEE_EUNSUPPORTED) { + GGMLQNN_LOG_DEBUG("API is not supported on this target so cannot get domains info from the device. falling back to legacy approach of using default domain id"); + hexagon_error = ggmlhexagon_get_dsp_support(&domain_id); + if (hexagon_error != AEE_SUCCESS) { + GGMLQNN_LOG_DEBUG("error: 0x%x, defaulting to CDSP domain", hexagon_error); + } + } else if (hexagon_error != AEE_SUCCESS) { + GGMLQNN_LOG_DEBUG("error in getting domains information"); + goto bail; + } else { + if (core_id != -1) { + if (core_id < 0 || core_id >= num_domains) { + GGMLQNN_LOG_DEBUG("invalid core_id = %d for %s. core_id should be between 0 to %d", core_id, domain_type, num_domains - 1); + hexagon_error = AEE_EBADPARM; + goto bail; + } + } else { + core_id = 0; + } + use_logical_id = 1; + domain_id = domains_info[core_id].id; + } + } + } else { + GGMLQNN_LOG_DEBUG("DSP domain is not provided, retrieving DSP information using Remote APIs"); + hexagon_error = ggmlhexagon_get_dsp_support(&domain_id); + if (hexagon_error != AEE_SUCCESS) { + GGMLQNN_LOG_DEBUG("error: 0x%x, defaulting to CDSP domain", hexagon_error); + } + } + } + + if (0 == use_logical_id) { + if (!ggmlhexagon_is_valid_domain_id(domain_id, 0)) { + hexagon_error = AEE_EBADPARM; + GGMLQNN_LOG_DEBUG("error 0x%x: invalid domain %d", hexagon_error, domain_id); + goto bail; + } + + my_domain = ggmlhexagon_get_domain(domain_id); + if (nullptr == my_domain) { + GGMLQNN_LOG_DEBUG("unable to get domain struct %d", domain_id); + goto bail; + } + uri = my_domain->uri; + } else { + domain_info = &domains_info[domain_id]; + uri = (char *)malloc(MAX_DOMAIN_NAMELEN); + if (nullptr == uri) { + hexagon_error = AEE_ENOMEMORY; + GGMLQNN_LOG_DEBUG("unable to allocated memory for uri of size: %d", MAX_DOMAIN_NAMELEN); + goto bail; + } + snprintf(uri, MAX_DOMAIN_NAMELEN, "%s%s", "&_dom=", domain_info->name); + } + GGMLQNN_LOG_INFO("\ndomain uri=%s\n", uri); + + if (1 == unsignedpd_flag) { + is_unsignedpd_enabled = ggmlhexagon_is_unsignedpd_supported(domain_id); + if (!is_unsignedpd_enabled) { + GGMLQNN_LOG_DEBUG("overriding user request for unsigned PD, only signed offload is allowed on domain %d", domain_id); + unsignedpd_flag = 0; + } + } + + GGMLQNN_LOG_INFO("using Hexagon domain %d(%s)", domain_id, ggmlhexagon_get_dsp_name(domain_id)); + GGMLQNN_LOG_INFO("unsignedpd_enabled %d", is_unsignedpd_enabled); + if (is_unsignedpd_enabled) { + if (remote_session_control) { + struct remote_rpc_control_unsigned_module data; + data.enable = 1; + data.domain = domain_id; + hexagon_error = remote_session_control(DSPRPC_CONTROL_UNSIGNED_MODULE, (void *)&data, sizeof(data)); + GGMLQNN_LOG_DEBUG("remote_session_control returned %d for configuring unsigned PD success", hexagon_error); + if (AEE_SUCCESS != hexagon_error) { + GGMLQNN_LOG_DEBUG("error 0x%x: remote_session_control failed", hexagon_error); + } + } else { + GGMLQNN_LOG_DEBUG("unsigned PD not supported on this device"); + hexagon_error = AEE_EUNSUPPORTED; + GGMLQNN_LOG_DEBUG("error 0x%x: remote_session_control interface is not supported on this device", hexagon_error); + } + } + + hexagon_error = ggmlhexagon_request_status_notifications(domain_id, (void *)STATUS_CONTEXT, ggmlhexagon_pd_status_notifier_callback); + if (AEE_SUCCESS != hexagon_error) { + if (AEE_EUNSUPPORTEDAPI != hexagon_error) { + GGMLQNN_LOG_WARN("error 0x%x: hexagon_request_status_notifications failed", hexagon_error); + } + GGMLQNN_LOG_WARN("error 0x%x: failed to compute on domain %d", hexagon_error, domain_id); + goto bail; + } + + ggmlop_domain_uri_len = strlen(ggmlop_URI) + MAX_DOMAIN_NAMELEN; + ggmlop_domain_uri = (char *)malloc(ggmlop_domain_uri_len); + snprintf(ggmlop_domain_uri, ggmlop_domain_uri_len, "%s%s", ggmlop_URI, uri); + GGMLQNN_LOG_INFO("ggmlop domain uri:%s\n", ggmlop_domain_uri); + hexagon_error = ggmlop_open(ggmlop_domain_uri, &ctx->ggmlop_handle); + if (AEE_SUCCESS == hexagon_error) { + GGMLQNN_LOG_INFO("succeed to open domain %d(%s)", domain_id, ggmlhexagon_get_dsp_name(domain_id)); + GGMLQNN_LOG_INFO("only support GGML_OP_ADD on cDSP currently\n"); + ggmlhexagon_set_clocks(ctx->ggmlop_handle, HAP_DCVS_V2_DUTY_CYCLE_MODE, 40, 1); + ggmlhexagon_set_rpc_latency(domain_id, RPC_POLL_QOS, 1000); + } else { + GGMLQNN_LOG_WARN("error 0x%x: failed to compute on domain %d(%s)", hexagon_error, domain_id, + ggmlhexagon_get_dsp_name(domain_id)); + goto bail; + } + + return 0; +bail: + if (ggmlop_domain_uri) { + free(ggmlop_domain_uri); + } + + if (uri) { + free(uri); + } + + if (ctx->rpc_mempool) { + rpcmem_free(ctx->rpc_mempool); + ctx->rpc_mempool = nullptr; + ctx->rpc_mempool_len = 0; + ctx->ggmlop_handle = -1; + } + + return -1; +} + +static void ggmlhexagon_close_cdsp(ggml_backend_qnn_context * ctx) { + int hexagon_error = AEE_SUCCESS; + GGMLQNN_LOG_DEBUG("enter %s", __func__); + if (-1 != ctx->ggmlop_handle) { + hexagon_error = ggmlop_close(ctx->ggmlop_handle); + if (AEE_SUCCESS != hexagon_error) { + GGMLQNN_LOG_WARN("error 0x%x: failed to close ggmlop handle", hexagon_error); + } else { + ctx->ggmlop_handle = -1; + } + } + + if (ctx->rpc_mempool) { + rpcmem_free(ctx->rpc_mempool); + ctx->rpc_mempool = nullptr; + ctx->rpc_mempool_len = 0; + } + GGMLQNN_LOG_DEBUG("leave %s", __func__); +} + +static void ggmlhexagon_compute(ggml_backend_qnn_context * ctx, struct ggml_tensor * op) { + //skip sanity check because already checked in other place + struct dsptensor dsptensor_0; + struct dsptensor dsptensor_1; + struct dsptensor dsptensor_2; + + int hexagon_error = AEE_SUCCESS; + ggmlhexagon_op_func_t op_func = nullptr; + void * wdata = nullptr; + + ggml_tensor * src0 = op->src[0]; + //TODO: src1 might-be nullptr + ggml_tensor * src1 = op->src[1]; + ggml_tensor * dst = op; + ggml_type src0_type = src0->type; + + switch (op->op) { + case GGML_OP_ADD: + op_func = ggmlop_add; + break; + case GGML_OP_MUL_MAT: { + wdata = ggmlqnn_type_trait(ctx, op); + op_func = ggmlop_mulmat; + break; + } + default: + return; + } + + if ((GGML_OP_MUL_MAT == op->op) && (src0_type != GGML_TYPE_F32)) { + dsptensor_0.data = static_cast(wdata); + dsptensor_0.dataLen = ctx->desired_size; + } else { + dsptensor_0.data = static_cast(src0->data); + dsptensor_0.dataLen = ggml_nbytes(src0); + } + dsptensor_1.data = static_cast(src1->data); + dsptensor_2.data = static_cast(dst->data); + dsptensor_0.type = GGML_TYPE_F32; + dsptensor_1.type = GGML_TYPE_F32; + dsptensor_2.type = GGML_TYPE_F32; + dsptensor_0.ne[0] = src0->ne[0]; + dsptensor_0.ne[1] = src0->ne[1]; + dsptensor_0.ne[2] = src0->ne[2]; + dsptensor_0.ne[3] = src0->ne[3]; + dsptensor_0.nb[0] = src0->nb[0]; + dsptensor_0.nb[1] = src0->nb[1]; + dsptensor_0.nb[2] = src0->nb[2]; + dsptensor_0.nb[3] = src0->nb[3]; + dsptensor_1.dataLen = ggml_nbytes(src1); + dsptensor_2.dataLen = ggml_nbytes(dst); + hexagon_error = op_func(ctx->ggmlop_handle, &dsptensor_0, &dsptensor_1, &dsptensor_2); + if (AEE_SUCCESS != hexagon_error) { + GGMLQNN_LOG_WARN("ggmlop computation fail on cdsp"); + } +} + // ================================================================================================= // section-7:ggml-qnn backend helper function / class // ================================================================================================= @@ -1383,6 +2221,19 @@ static const char * ggmlqnn_get_htparch_desc(size_t htp_arch) { } } +static const char * ggmlqnn_get_inference_approach_name(int inference_approach) { + switch (inference_approach) { + case 0: + return "QNN_GENERAL"; + case 1: + return "DIRECT_USE_CDSP"; + case 2: + return "QNN_SINGLEGRAPH"; + default: + return "unknown approach"; + } +} + static struct qcom_socinfo * ggmlqnn_get_socinfo_from_socmodel(uint32_t soc_model) { size_t items = sizeof(g_qnn_soc_info_table) / sizeof(g_qnn_soc_info_table[0]); for (size_t idx = 0; idx < items; idx++) { @@ -2060,7 +2911,7 @@ class qnn_instance { bool _do_node_validations = true; // flag to indicate whether all add_node calls need to be validated QnnLog_Level_t _qnn_log_level = QNN_LOG_LEVEL_DEBUG; - qnn_profile_level _profile_level = qnn_profile_level::profile_off; + qnn_profile_level _profile_level = PROFILE_OFF; void * _system_lib_handle = nullptr; void * _loaded_lib_handle = nullptr; @@ -2710,9 +3561,9 @@ int qnn_instance::qnn_init(const QnnSaver_Config_t ** saver_config) { GGMLQNN_LOG_INFO("create device successfully\n"); } - if (qnn_profile_level::profile_off != _profile_level) { + if (PROFILE_OFF != _profile_level) { GGMLQNN_LOG_INFO("profiling turned on; level = %d", _profile_level); - if (qnn_profile_level::profile_basic == _profile_level) { + if (PROFILE_BASIC == _profile_level) { GGMLQNN_LOG_INFO("basic profiling requested. creating Qnn Profile object\n"); if (QNN_PROFILE_NO_ERROR != _qnn_raw_interface.profileCreate( _qnn_backend_handle, QNN_PROFILE_LEVEL_BASIC, &_qnn_profile_handle)) { @@ -2721,7 +3572,7 @@ int qnn_instance::qnn_init(const QnnSaver_Config_t ** saver_config) { } else { GGMLQNN_LOG_DEBUG("initialize qnn profile successfully\n"); } - } else if (qnn_profile_level::profile_detail == _profile_level) { + } else if (PROFILE_DETAIL == _profile_level) { GGMLQNN_LOG_INFO("detailed profiling requested. Creating Qnn Profile object\n"); if (QNN_PROFILE_NO_ERROR != _qnn_raw_interface.profileCreate( _qnn_backend_handle, QNN_PROFILE_LEVEL_DETAILED, &_qnn_profile_handle)) { @@ -3279,6 +4130,38 @@ void qnn_instance::htp_enter_performance_mode() { } } +static void ggmlqnn_set_runtime_path(size_t device, const std::string & path) { + if ((QNN_BACKEND_NPU == device) || (DIRECT_USE_CDSP == g_qnn_params.inference_approach)) { + if (0 == setenv("LD_LIBRARY_PATH", + (path + + ":/vendor/dsp/cdsp:/vendor/lib64:/vendor/dsp/dsp:/vendor/dsp/images").c_str(), + 1)) { + GGMLQNN_LOG_INFO("QNN NPU backend setenv successfully"); + } else { + GGMLQNN_LOG_ERROR("QNN NPU backend setenv failure"); + } + if (0 == setenv("ADSP_LIBRARY_PATH", + (path + + ";/vendor/dsp/cdsp;/vendor/lib/rfsa/adsp;/system/lib/rfsa/adsp;/vendor/dsp/dsp;/vendor/dsp/images;/dsp").c_str(), + 1)) { + GGMLQNN_LOG_INFO("QNN NPU backend setenv successfully"); + } else { + GGMLQNN_LOG_ERROR("QNN NPU backend setenv failure"); + } + } else { + if (0 == setenv("LD_LIBRARY_PATH", + (path + + ":/vendor/dsp/cdsp:/vendor/lib64:/vendor/dsp/dsp:/vendor/dsp/images").c_str(), + 1)) { + GGMLQNN_LOG_INFO("%s backend setenv successfully\n", + ggml_backend_qnn_get_devname(device)); + } else { + GGMLQNN_LOG_ERROR("%s backend setenv failure\n", + ggml_backend_qnn_get_devname(device)); + } + } +} + static uint8_t * ggmlqnn_create_rpc_buffer(qnn_instance * instance, const ggml_tensor * ggml_tensor, Qnn_Tensor_t * qnn_tensor, bool b_copydata) { if (nullptr == instance || nullptr == ggml_tensor || nullptr == qnn_tensor) { GGMLQNN_LOG_WARN("invalid params\n"); @@ -3302,7 +4185,7 @@ static void ggmlqnn_load_cfg() { //this function can be called in various scenarios static bool initialized = false; if (initialized) { - GGMLQNN_LOG_DEBUG("qnn cfg file already loadded\n"); + GGMLQNN_LOG_INFO("qnn cfg file already loadded\n"); return; } char time_string[GGML_QNN_TMPBUF_LEN]; @@ -3330,7 +4213,8 @@ static void ggmlqnn_load_cfg() { qnncfg_instance.get_intvalue("npu", "enable_dlbc", g_qnn_params.enable_dlbc, 0); qnncfg_instance.get_stringvalue("npu", "precision_mode", precision_mode, "fp32"); GGMLQNN_LOG_INFO("print_qnn_internal_log=%d", g_qnn_params.print_qnn_internal_log); - GGMLQNN_LOG_INFO("inference_approach=%d", g_qnn_params.inference_approach); + GGMLQNN_LOG_INFO("inference_approach=%d(%s)", g_qnn_params.inference_approach, + ggmlqnn_get_inference_approach_name(g_qnn_params.inference_approach)); GGMLQNN_LOG_INFO("qnn_backend=%d", g_qnn_params.qnn_backend); GGMLQNN_LOG_INFO("npu inference precision mode=%s", precision_mode.c_str()); GGMLQNN_LOG_INFO("qnn runtime lib path=%s", g_qnn_params.qnn_runtimelib_path); @@ -3488,6 +4372,12 @@ static bool ggmlqnn_can_handle_op(const ggml_backend_qnn_context * ctx, const st return true; } + if (DIRECT_USE_CDSP == g_qnn_params.inference_approach) { + //FIXME: mulmat on cDSP doesn't work as expected + if (op_tensor->op != GGML_OP_ADD) + return false; + } + if (!ggmlqnn_k_op_caps[ggmlqnn_get_op_index(op_tensor)].supported) { return false; } @@ -3854,9 +4744,13 @@ static const char * ggml_backend_qnn_name(ggml_backend_t backend) { static void ggml_backend_qnn_free(ggml_backend_t backend) { GGMLQNN_LOG_DEBUG("enter %s", __func__ ); - ggml_backend_qnn_context * ctx = (ggml_backend_qnn_context *) backend->context; + ggml_backend_qnn_context * ctx = (ggml_backend_qnn_context *)backend->context; GGMLQNN_LOG_DEBUG("device idx %d, name:%s", ctx->device, g_qnn_mgr[ctx->device].name); + if (DIRECT_USE_CDSP == g_qnn_params.inference_approach) { + ggmlhexagon_close_cdsp(ctx); + } + qnn_instance * instance = (qnn_instance*)g_qnn_mgr[ctx->device].instance; if (instance != nullptr) { std::map::iterator singlenode_graph_it; @@ -3899,20 +4793,11 @@ static void ggml_backend_qnn_free(ggml_backend_t backend) { GGMLQNN_LOG_DEBUG("leave %s", __func__ ); } -//this is the first tech approach(or general approach in other ggml backends, such as ggml-sycl or ggml-cann) static enum ggml_status ggmlqnn_backend_graph_compute_general(ggml_backend_t backend, struct ggml_cgraph * cgraph) { enum ggml_status result = GGML_STATUS_SUCCESS; ggml_backend_qnn_context * ctx = (ggml_backend_qnn_context *)backend->context; GGML_UNUSED(ctx); -#if 0 - GGMLQNN_LOG_DEBUG("device %d", ctx->device); - GGMLQNN_LOG_DEBUG("cgraph->n_nodes %d", cgraph->n_nodes); - int num_nodes = std::min(5, cgraph->n_nodes); - for (int i = 0; i < num_nodes; i++) { - ggml_tensor * node = cgraph->nodes[i]; - GGMLQNN_LOG_DEBUG("%s: op %s (%s)\n", __func__, node->name, ggml_op_name(node->op)); - } -#endif + for (int i = 0; i < cgraph->n_nodes; i++) { ggml_tensor * node = cgraph->nodes[i]; if (ggml_is_empty(node) || node->op == GGML_OP_RESHAPE @@ -3989,7 +4874,7 @@ static enum ggml_backend_dev_type ggml_backend_qnn_device_get_type(ggml_backend_ if (QNN_BACKEND_CPU == ctx->device) return GGML_BACKEND_DEVICE_TYPE_ACCEL; else if (QNN_BACKEND_GPU == ctx->device) - return GGML_BACKEND_DEVICE_TYPE_GPU; + return GGML_BACKEND_DEVICE_TYPE_ACCEL; else if (QNN_BACKEND_NPU == ctx->device) return GGML_BACKEND_DEVICE_TYPE_ACCEL; else @@ -4197,8 +5082,10 @@ ggml_backend_reg_t ggml_backend_qnn_reg() { //case-2: normal scenario, such as llama-cli or UI applicaton ggmlqnn_load_cfg(); + GGMLQNN_LOG_INFO("inference approach=%d(%s)", g_qnn_params.inference_approach, + ggmlqnn_get_inference_approach_name(g_qnn_params.inference_approach)); GGMLQNN_LOG_INFO("user's specified qnn_backend=%d", g_qnn_params.qnn_backend); - GGMLQNN_LOG_INFO("user's sepcified qnn runtime lib path=%s", g_qnn_params.qnn_runtimelib_path); + GGMLQNN_LOG_INFO("user's specified qnn runtime lib path=%s", g_qnn_params.qnn_runtimelib_path); if (g_qnn_params.qnn_backend >= GGML_QNN_MAX_DEVICES) { GGMLQNN_LOG_INFO("assume default ggml backend\n"); GGMLQNN_LOG_DEBUG("leave ggml_backend_qnn_reg"); @@ -4234,6 +5121,58 @@ ggml_backend_reg_t ggml_backend_qnn_reg() { return ® } +const char * ggml_backend_qnn_get_devname(size_t dev_num) { + if (DIRECT_USE_CDSP == g_qnn_params.inference_approach) { + if (dev_num == QNN_BACKEND_GGML) + return "ggml"; + else + return "ggml-hexagon"; + } + + switch (dev_num) { + case QNN_BACKEND_CPU: + return "QNN-CPU"; + case QNN_BACKEND_GPU: + return "QNN-GPU"; + case QNN_BACKEND_NPU: + return "QNN-NPU"; + case QNN_BACKEND_GGML: + return "ggml"; //"fake" QNN backend, used for compare performance between QNN backend and original GGML + default: + return "unknown"; + } +} + +static qnn_instance * ggmlqnn_init_qnn_instance(size_t device, const char * qnn_lib_path) { + int result = 0; + GGMLQNN_LOG_INFO("inference approach=%d(%s)", g_qnn_params.inference_approach, + ggmlqnn_get_inference_approach_name(g_qnn_params.inference_approach)); + + qnn_instance * instance = nullptr; + instance = new qnn_instance(qnn_lib_path, g_qnn_mgr[device].lib, ""); + result = instance->qnn_init(nullptr); + if (0 != result) { + GGMLQNN_LOG_WARN("init qnn subsystem failed with qnn backend %s, pls check why\n", + ggml_backend_qnn_get_devname(device)); + delete instance; + return nullptr; + } + qnn_interface qnn_interface = instance->get_qnn_interface(); + if (!qnn_interface.is_loaded()) { + GGMLQNN_LOG_WARN("qnn subsystem failure\n"); + delete instance; + return nullptr; + } + + std::string device_name = ggml_backend_qnn_get_devname(device); + GGMLQNN_LOG_INFO("qnn device name %s", device_name.c_str()); + g_qnn_mgr[device].instance = instance; + g_qnn_mgr[device].raw_interface = instance->get_qnn_raw_interface(); + g_qnn_mgr[device].raw_system_interface = instance->get_qnn_raw_system_interface(); + + return instance; +} + /** * * @param device 0: QNN_BACKEND_CPU 1: QNN_BACKEND_GPU 2: QNN_BACKEND_NPU @@ -4257,69 +5196,27 @@ ggml_backend_t ggml_backend_qnn_init(size_t device, const char * qnn_lib_path) { return nullptr; } - if (nullptr != g_qnn_mgr[device].backend) { - GGMLQNN_LOG_INFO("qnn backend %d(%s) already loaded", device, ggml_backend_qnn_get_devname(device)); - GGMLQNN_LOG_INFO("leave %s\n", __func__); - return g_qnn_mgr[device].backend; - } - #if defined(__ANDROID__) std::string path = qnn_lib_path; GGMLQNN_LOG_INFO("lib_path %s", path.c_str()); - if (QNN_BACKEND_NPU == device) { - if (0 == setenv("LD_LIBRARY_PATH", - (path + - ":/vendor/dsp/cdsp:/vendor/lib64:/vendor/dsp/dsp:/vendor/dsp/images").c_str(), - 1)) { - GGMLQNN_LOG_INFO("QNN NPU backend setenv successfully"); - } else { - GGMLQNN_LOG_ERROR("QNN NPU backend setenv failure"); - } - if (0 == setenv("ADSP_LIBRARY_PATH", - (path + - ";/vendor/dsp/cdsp;/vendor/lib/rfsa/adsp;/system/lib/rfsa/adsp;/vendor/dsp/dsp;/vendor/dsp/images;/dsp").c_str(), - 1)) { - GGMLQNN_LOG_INFO("QNN NPU backend setenv successfully"); - } else { - GGMLQNN_LOG_ERROR("QNN NPU backend setenv failure"); - } - } else { - if (0 == setenv("LD_LIBRARY_PATH", - (path + - ":/vendor/dsp/cdsp:/vendor/lib64:/vendor/dsp/dsp:/vendor/dsp/images").c_str(), - 1)) { - GGMLQNN_LOG_INFO("%s backend setenv successfully\n", ggml_backend_qnn_get_devname(device)); - } else { - GGMLQNN_LOG_ERROR("%s backend setenv failure\n", ggml_backend_qnn_get_devname(device)); - } - } + ggmlqnn_set_runtime_path(device, path); #endif - qnn_instance * instance = nullptr; - instance = new qnn_instance(qnn_lib_path, g_qnn_mgr[device].lib, ""); - result = instance->qnn_init(nullptr); - if (0 != result) { - GGMLQNN_LOG_WARN("init qnn subsystem failed with qnn backend %s, pls check why\n", ggml_backend_qnn_get_devname(device)); - delete instance; - return nullptr; - } - qnn_interface qnn_interface = instance->get_qnn_interface(); - if (!qnn_interface.is_loaded()) { - GGMLQNN_LOG_WARN("qnn subsystem failure\n"); - delete instance; - return nullptr; + if (nullptr != g_qnn_mgr[device].backend) { + GGMLQNN_LOG_INFO("backend %d(%s) already loaded", device, + ggml_backend_qnn_get_devname(device)); + GGMLQNN_LOG_INFO("leave %s\n", __func__); + return g_qnn_mgr[device].backend; } - std::string device_name = ggml_backend_qnn_get_devname(device); - GGMLQNN_LOG_INFO("qnn device name %s", device_name.c_str()); - g_qnn_mgr[device].instance = instance; - g_qnn_mgr[device].raw_interface = instance->get_qnn_raw_interface(); - g_qnn_mgr[device].raw_system_interface = instance->get_qnn_raw_system_interface(); + qnn_instance * instance = ggmlqnn_init_qnn_instance(device, qnn_lib_path); + if (nullptr == instance) + return nullptr; - if (0 == g_qnn_params.inference_approach) { - ggml_backend_qnn_interface.graph_compute = ggmlqnn_backend_graph_compute_general; - } else { + if (QNN_SINGLEGRAPH == g_qnn_params.inference_approach) { ggml_backend_qnn_interface.graph_compute = ggmlqnn_backend_graph_compute_special; + } else { + ggml_backend_qnn_interface.graph_compute = ggmlqnn_backend_graph_compute_general; } ggml_backend_t qnn_backend = new ggml_backend{ @@ -4329,7 +5226,16 @@ ggml_backend_t ggml_backend_qnn_init(size_t device, const char * qnn_lib_path) { /* .context = */ &g_qnn_mgr[device] }; - g_qnn_mgr[device].backend = qnn_backend; + g_qnn_mgr[device].backend = qnn_backend; + if (DIRECT_USE_CDSP == g_qnn_params.inference_approach) { + int result = ggmlhexagon_init_dsp(&g_qnn_mgr[device]); + if (0 != result) { + GGMLQNN_LOG_INFO("init hexagon dsp failure"); + ggml_backend_qnn_free(qnn_backend); + return nullptr; + } + } + GGMLQNN_LOG_INFO("leave %s\n", __func__); return qnn_backend; @@ -4338,7 +5244,7 @@ ggml_backend_t ggml_backend_qnn_init(size_t device, const char * qnn_lib_path) { GGML_BACKEND_DL_IMPL(ggml_backend_qnn_reg) // ================================================================================================= -// section-9: general approach: offload GGML op to QNN backend +// section-9: general approach: offload GGML op to QNN backend or offload GGML op to Hexagon DSP directly // ================================================================================================= static inline uint32_t ggmlqnn_get_tensor_data_size(const ggml_tensor * tensor) { /* @@ -4370,7 +5276,7 @@ static inline bool ggmlqnn_is_valid_params(ggml_backend_qnn_context * ctx, const } /* - * provide a general skeleton to offload ggml op to QNN backend: peform element-wise operation on 1/2 + * provide a general skeleton to offload ggml op to QNN backend or Hexagon cDSP: peform element-wise operation on 1/2 * input tensors and 1 output tensors */ static void ggmlqnn_compute_elementwise(ggml_backend_qnn_context * ctx, ggml_tensor * op) { @@ -4393,20 +5299,25 @@ static void ggmlqnn_compute_elementwise(ggml_backend_qnn_context * ctx, ggml_ten std::string ggml_op_name_string = std::string("ggml_") + ggml_op_name(op->op); const char * ggml_op_name = ggml_op_name_string.c_str(); - bool enable_npu_rpc = instance->enable_qnn_rpc() && ctx->device == QNN_BACKEND_NPU; - std::string graph_name; ggmlqnn_get_graphkey_from_op(op, graph_name); qnn_perf op_perf = qnn_perf(graph_name); op_perf.start(); + if (DIRECT_USE_CDSP == g_qnn_params.inference_approach) { + ggmlhexagon_compute(ctx, op); + op_perf.info(); + return; + } + + bool enable_npu_rpc = instance->enable_qnn_rpc() && ctx->device == QNN_BACKEND_NPU; if (ctx->qnn_singlenode_graph_map.find(graph_name) != ctx->qnn_singlenode_graph_map.end()) { //retrieve computational resource from cached QNN graph - qnn_singlenode_res_t & graph_item = ctx->qnn_singlenode_graph_map[graph_name]; - graph_handle = std::get<0>(graph_item); - qnn_ptensors_t & ptensors = std::get<1>(graph_item); - p_tensor0 = ptensors[0]; + qnn_singlenode_res_t & graph_item = ctx->qnn_singlenode_graph_map[graph_name]; + graph_handle = std::get<0>(graph_item); + qnn_ptensors_t & ptensors = std::get<1>(graph_item); + p_tensor0 = ptensors[0]; if (2 == input_param_count) { p_tensor1 = ptensors[1]; p_tensor2 = ptensors[2]; @@ -4415,10 +5326,12 @@ static void ggmlqnn_compute_elementwise(ggml_backend_qnn_context * ctx, ggml_ten p_tensor2 = ptensors[1]; } } else { - GGMLQNN_LOG_INFO("graph name %s", graph_name.c_str()); GGML_ASSERT(instance->get_device_id() == ctx->device); + GGMLQNN_LOG_INFO("graph name %s", graph_name.c_str()); //create QNN graph - error = instance->init_qnn_graph(graph_name, static_cast(ctx->device), g_qnn_params.vtcm_size_in_mb, g_qnn_params.hvx_threads); + error = instance->init_qnn_graph(graph_name, static_cast(ctx->device), + g_qnn_params.vtcm_size_in_mb, + g_qnn_params.hvx_threads); if (QNN_SUCCESS != error) { GGMLQNN_LOG_WARN("can't create qnn graph handle with graph name %s, error = %d\n", graph_name.c_str(), error); return; @@ -4431,7 +5344,7 @@ static void ggmlqnn_compute_elementwise(ggml_backend_qnn_context * ctx, ggml_ten if (2 == input_param_count) { p_tensor1 = ggmlqnn_create_compute_tensor(instance, graph_handle, src1, QNN_TENSOR_TYPE_APP_WRITE); } - p_tensor2 = ggmlqnn_create_compute_tensor(instance, graph_handle, dst, QNN_TENSOR_TYPE_APP_READ); + p_tensor2 = ggmlqnn_create_compute_tensor(instance, graph_handle, dst, QNN_TENSOR_TYPE_APP_READ); //compose QNN graph qnn_tensors_t input_tensors; @@ -4443,25 +5356,12 @@ static void ggmlqnn_compute_elementwise(ggml_backend_qnn_context * ctx, ggml_ten Qnn_Tensor_t output_tensors[] = { *p_tensor2 }; -#if 0 // keep them for understand code easily - Qnn_OpConfig_t op_config = { - QNN_OPCONFIG_VERSION_1, { - ggml_op_name, - QNN_OP_PACKAGE_NAME_QTI_AISW, - qnn_op_name, - 0, - nullptr, - input_param_count, - tensor_inputs, - 1, - tensor_outputs - } - }; -#else - Qnn_OpConfig_t op_config = ggmlqnn_create_op_config(ggml_op_name, QNN_OP_PACKAGE_NAME_QTI_AISW, - qnn_op_name, nullptr, 0, - input_tensors.data(), input_param_count, output_tensors, 1); -#endif + Qnn_OpConfig_t op_config = ggmlqnn_create_op_config(ggml_op_name, + QNN_OP_PACKAGE_NAME_QTI_AISW, + qnn_op_name, nullptr, 0, + input_tensors.data(), + input_param_count, output_tensors, + 1); CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, op_config)); //finalize QNN graph CHECK_QNN_API(error, qnn_raw_interface.graphFinalize(graph_handle, nullptr, nullptr)); @@ -4475,20 +5375,21 @@ static void ggmlqnn_compute_elementwise(ggml_backend_qnn_context * ctx, ggml_ten qnn_elementwise_tensors.push_back(p_tensor1); } qnn_elementwise_tensors.push_back(p_tensor2); - auto graph_item = std::make_tuple(graph_handle, qnn_elementwise_tensors); + auto graph_item = std::make_tuple(graph_handle, qnn_elementwise_tensors); ctx->qnn_singlenode_graph_map[graph_name] = graph_item; } if (enable_npu_rpc) { - uint8_t * qnn_buffer_0 = static_cast(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*p_tensor0)->memHandle)); + uint8_t * qnn_buffer_0 = static_cast(instance->get_rpcmem_from_memhandle( + QNN_VER_PTR(*p_tensor0)->memHandle)); GGMLQNN_LOG_DEBUG("qnn_rpcbuffer_0 = %p\n", qnn_buffer_0); if (nullptr != qnn_buffer_0) { memcpy(qnn_buffer_0, src0->data, ggml_nbytes(src0)); } if (2 == input_param_count) { - uint8_t *qnn_buffer_1 = static_cast(instance->get_rpcmem_from_memhandle( - QNN_VER_PTR(*p_tensor1)->memHandle)); + uint8_t * qnn_buffer_1 = static_cast(instance->get_rpcmem_from_memhandle( + QNN_VER_PTR(*p_tensor1)->memHandle)); GGMLQNN_LOG_DEBUG("qnn_rpcbuffer_1 = %p\n", qnn_buffer_1); if (nullptr != qnn_buffer_1) { memcpy(qnn_buffer_1, src1->data, ggml_nbytes(src1)); @@ -4784,7 +5685,6 @@ static void ggmlqnn_compute_mul_mat(ggml_backend_qnn_context * ctx, ggml_tensor instance = ctx->instance; QNN_INTERFACE_VER_TYPE qnn_raw_interface = ctx->raw_interface; - const enum ggml_type src0_type = src0->type; const uint32_t src0_rank = ggml_n_dims(src0); const uint32_t src1_rank = ggml_n_dims(src1); @@ -4802,39 +5702,51 @@ static void ggmlqnn_compute_mul_mat(ggml_backend_qnn_context * ctx, ggml_tensor qnn_perf op_perf = qnn_perf(graph_name); op_perf.start(); + if (DIRECT_USE_CDSP == g_qnn_params.inference_approach) { + ggmlhexagon_compute(ctx, op); + op_perf.info(); + return; + } + void * wdata = ggmlqnn_type_trait(ctx, op); const size_t desired_size = ctx->desired_size; if (ctx->qnn_singlenode_graph_map.find(graph_name) != ctx->qnn_singlenode_graph_map.end()) { //retrieve computational resource from cached QNN graph - qnn_singlenode_res_t & graph_item = ctx->qnn_singlenode_graph_map[graph_name]; - graph_handle = std::get<0>(graph_item); - qnn_ptensors_t & tensors = std::get<1>(graph_item); - p_tensor0 = tensors[0]; - p_tensor1 = tensors[1]; - p_tensor2 = tensors[2]; - p_param_tensor = tensors[3]; - p_tensor2_transpose = tensors[4]; + qnn_singlenode_res_t & graph_item = ctx->qnn_singlenode_graph_map[graph_name]; + graph_handle = std::get<0>(graph_item); + qnn_ptensors_t &tensors = std::get<1>(graph_item); + p_tensor0 = tensors[0]; + p_tensor1 = tensors[1]; + p_tensor2 = tensors[2]; + p_param_tensor = tensors[3]; + p_tensor2_transpose = tensors[4]; } else { //create QNN graph GGMLQNN_LOG_INFO("graph name %s", graph_name.c_str()); - error = instance->init_qnn_graph(graph_name, static_cast(ctx->device), g_qnn_params.vtcm_size_in_mb, g_qnn_params.hvx_threads); + error = instance->init_qnn_graph(graph_name, static_cast(ctx->device), + g_qnn_params.vtcm_size_in_mb, + g_qnn_params.hvx_threads); if (QNN_SUCCESS != error) { - GGMLQNN_LOG_WARN("can't create qnn graph handle with graph name %s, error = %d\n", graph_name.c_str(), error); + GGMLQNN_LOG_WARN("can't create qnn graph handle with graph name %s, error = %d\n", + graph_name.c_str(), error); return; } graph_handle = instance->get_qnn_graph_handle(); //create computational tensor p_tensor0 = ggmlqnn_create_general_tensor(instance, graph_handle, src0, nullptr, - QNN_TENSOR_TYPE_APP_WRITE, QNN_DATATYPE_FLOAT_32, src0_rank, - nullptr, nullptr, 0); + QNN_TENSOR_TYPE_APP_WRITE, + QNN_DATATYPE_FLOAT_32, src0_rank, + nullptr, nullptr, 0); p_tensor1 = ggmlqnn_create_general_tensor(instance, graph_handle, src1, nullptr, - QNN_TENSOR_TYPE_APP_WRITE, QNN_DATATYPE_FLOAT_32, src0_rank, - nullptr, nullptr, 0); + QNN_TENSOR_TYPE_APP_WRITE, + QNN_DATATYPE_FLOAT_32, src0_rank, + nullptr, nullptr, 0); p_tensor2 = ggmlqnn_create_general_tensor(instance, graph_handle, dst, nullptr, - QNN_TENSOR_TYPE_APP_READ, QNN_DATATYPE_FLOAT_32, src0_rank, - nullptr, nullptr, 0); + QNN_TENSOR_TYPE_APP_READ, + QNN_DATATYPE_FLOAT_32, src0_rank, + nullptr, nullptr, 0); //create param tensor for offload 2d/3d/4d matrix multiplication const uint32_t param_tensor_data[GGML_MAX_DIMS][GGML_MAX_DIMS] = { @@ -4845,29 +5757,43 @@ static void ggmlqnn_compute_mul_mat(ggml_backend_qnn_context * ctx, ggml_tensor }; uint32_t param_tensor_dims[1] = {src0_rank}; p_param_tensor = ggmlqnn_create_general_tensor(instance, graph_handle, nullptr, "param", - QNN_TENSOR_TYPE_STATIC, QNN_DATATYPE_UINT_32, 1, param_tensor_dims, - (void *)(param_tensor_data[src0_rank - 1]), src0_rank * sizeof(uint32_t)); + QNN_TENSOR_TYPE_STATIC, + QNN_DATATYPE_UINT_32, 1, + param_tensor_dims, + (void *) (param_tensor_data[src0_rank - 1]), + src0_rank * sizeof(uint32_t)); //create transpose tensor - p_tensor2_transpose = ggmlqnn_create_general_tensor(instance, graph_handle, dst, "transpose", - QNN_TENSOR_TYPE_NATIVE, QNN_DATATYPE_FLOAT_32, src0_rank, - nullptr, nullptr, 0, true); + p_tensor2_transpose = ggmlqnn_create_general_tensor(instance, graph_handle, dst, + "transpose", + QNN_TENSOR_TYPE_NATIVE, + QNN_DATATYPE_FLOAT_32, src0_rank, + nullptr, nullptr, 0, true); //compose QNN graph: add mulmat node - Qnn_Param_t out_0_params[] = {{QNN_PARAMTYPE_SCALAR, QNN_OP_MAT_MUL_PARAM_TRANSPOSE_IN1, .scalarParam = {QNN_DATATYPE_BOOL_8, .bool8Value = 1}}}; - Qnn_Tensor_t out_0_inputs[] = {*p_tensor0, *p_tensor1}; + Qnn_Param_t out_0_params[] = { + {QNN_PARAMTYPE_SCALAR, QNN_OP_MAT_MUL_PARAM_TRANSPOSE_IN1, .scalarParam = { + QNN_DATATYPE_BOOL_8, .bool8Value = 1}}}; + Qnn_Tensor_t out_0_inputs[] = {*p_tensor0, *p_tensor1}; Qnn_Tensor_t out_0_outputs[] = {*p_tensor2_transpose}; - Qnn_OpConfig_t out_0 = ggmlqnn_create_op_config("mulmat_opconfig", QNN_OP_PACKAGE_NAME_QTI_AISW, - QNN_OP_MAT_MUL, out_0_params, 1, out_0_inputs, 2, out_0_outputs, 1); - CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle,out_0)); + Qnn_OpConfig_t out_0 = ggmlqnn_create_op_config("mulmat_opconfig", + QNN_OP_PACKAGE_NAME_QTI_AISW, + QNN_OP_MAT_MUL, out_0_params, 1, + out_0_inputs, 2, out_0_outputs, 1); + CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, out_0)); //compose QNN graph: add transpose node - Qnn_Param_t out_trans1_0_params[] = { {QNN_PARAMTYPE_TENSOR, "perm", .tensorParam = *p_param_tensor}}; - Qnn_Tensor_t out_trans1_0_inputs[] = {*p_tensor2_transpose}; + Qnn_Param_t out_trans1_0_params[] = { + {QNN_PARAMTYPE_TENSOR, "perm", .tensorParam = *p_param_tensor}}; + Qnn_Tensor_t out_trans1_0_inputs[] = {*p_tensor2_transpose}; Qnn_Tensor_t out_trans1_0_outputs[] = {*p_tensor2}; - Qnn_OpConfig_t out_trans1_0 = ggmlqnn_create_op_config("mulmat_transpose_opconfig", QNN_OP_PACKAGE_NAME_QTI_AISW, - QNN_OP_TRANSPOSE, out_trans1_0_params, 1, out_trans1_0_inputs, 1, out_trans1_0_outputs, 1); - CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle,out_trans1_0)); + Qnn_OpConfig_t out_trans1_0 = ggmlqnn_create_op_config("mulmat_transpose_opconfig", + QNN_OP_PACKAGE_NAME_QTI_AISW, + QNN_OP_TRANSPOSE, + out_trans1_0_params, 1, + out_trans1_0_inputs, 1, + out_trans1_0_outputs, 1); + CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle, out_trans1_0)); //finalize QNN graph CHECK_QNN_API(error, qnn_raw_interface.graphFinalize(graph_handle, nullptr, nullptr)); @@ -4880,7 +5806,7 @@ static void ggmlqnn_compute_mul_mat(ggml_backend_qnn_context * ctx, ggml_tensor ggml_op_mulmat_tensors.push_back(p_tensor2); ggml_op_mulmat_tensors.push_back(p_param_tensor); ggml_op_mulmat_tensors.push_back(p_tensor2_transpose); - auto graph_item = std::make_tuple(graph_handle, ggml_op_mulmat_tensors); + auto graph_item = std::make_tuple(graph_handle, ggml_op_mulmat_tensors); ctx->qnn_singlenode_graph_map[graph_name] = graph_item; } @@ -5032,7 +5958,7 @@ static void ggmlqnn_compute_rope(ggml_backend_qnn_context * ctx, ggml_tensor * d } // ================================================================================================= -// section-10: second approach: mapping ggml computational cgraph to QNN graph +// section-10: special approach: mapping ggml computational cgraph to QNN graph // ================================================================================================= // TODO: remove duplicated codes between section-9 and section-10 // TODO: the graph algorithm in this section is naive, should optimized by AI experts diff --git a/ggml/src/ggml-qnn/kernels/ggmlop.h b/ggml/src/ggml-qnn/kernels/ggmlop.h new file mode 100644 index 0000000000000..b45070c20001b --- /dev/null +++ b/ggml/src/ggml-qnn/kernels/ggmlop.h @@ -0,0 +1,289 @@ +#ifndef _GGMLOP_H +#define _GGMLOP_H +/// @file ggmlop.idl +/// +//qidl copyright +//qidl nested=false +#include +#include +#include +#include + +#ifndef __QAIC_HEADER +#define __QAIC_HEADER(ff) ff +#endif //__QAIC_HEADER + +#ifndef __QAIC_HEADER_EXPORT +#define __QAIC_HEADER_EXPORT +#endif // __QAIC_HEADER_EXPORT + +#ifndef __QAIC_HEADER_ATTRIBUTE +#define __QAIC_HEADER_ATTRIBUTE +#endif // __QAIC_HEADER_ATTRIBUTE + +#ifndef __QAIC_IMPL +#define __QAIC_IMPL(ff) ff +#endif //__QAIC_IMPL + +#ifndef __QAIC_IMPL_EXPORT +#define __QAIC_IMPL_EXPORT +#endif // __QAIC_IMPL_EXPORT + +#ifndef __QAIC_IMPL_ATTRIBUTE +#define __QAIC_IMPL_ATTRIBUTE +#endif // __QAIC_IMPL_ATTRIBUTE +#ifndef _QAIC_ENV_H +#define _QAIC_ENV_H + +#include +#ifdef _WIN32 +#include "qtest_stdlib.h" +#else +#define MALLOC malloc +#define FREE free +#endif + +#ifdef __GNUC__ +#ifdef __clang__ +#pragma GCC diagnostic ignored "-Wunknown-pragmas" +#else +#pragma GCC diagnostic ignored "-Wpragmas" +#endif +#pragma GCC diagnostic ignored "-Wuninitialized" +#pragma GCC diagnostic ignored "-Wunused-parameter" +#pragma GCC diagnostic ignored "-Wunused-function" +#endif + +#ifndef _ATTRIBUTE_UNUSED + +#ifdef _WIN32 +#define _ATTRIBUTE_UNUSED +#else +#define _ATTRIBUTE_UNUSED __attribute__ ((unused)) +#endif + +#endif // _ATTRIBUTE_UNUSED + +#ifndef _ATTRIBUTE_VISIBILITY + +#ifdef _WIN32 +#define _ATTRIBUTE_VISIBILITY +#else +#define _ATTRIBUTE_VISIBILITY __attribute__ ((visibility("default"))) +#endif + +#endif // _ATTRIBUTE_VISIBILITY + +#ifndef __QAIC_REMOTE +#define __QAIC_REMOTE(ff) ff +#endif //__QAIC_REMOTE + +#ifndef __QAIC_HEADER +#define __QAIC_HEADER(ff) ff +#endif //__QAIC_HEADER + +#ifndef __QAIC_HEADER_EXPORT +#define __QAIC_HEADER_EXPORT +#endif // __QAIC_HEADER_EXPORT + +#ifndef __QAIC_HEADER_ATTRIBUTE +#define __QAIC_HEADER_ATTRIBUTE +#endif // __QAIC_HEADER_ATTRIBUTE + +#ifndef __QAIC_IMPL +#define __QAIC_IMPL(ff) ff +#endif //__QAIC_IMPL + +#ifndef __QAIC_IMPL_EXPORT +#define __QAIC_IMPL_EXPORT +#endif // __QAIC_IMPL_EXPORT + +#ifndef __QAIC_IMPL_ATTRIBUTE +#define __QAIC_IMPL_ATTRIBUTE +#endif // __QAIC_IMPL_ATTRIBUTE + +#ifndef __QAIC_STUB +#define __QAIC_STUB(ff) ff +#endif //__QAIC_STUB + +#ifndef __QAIC_STUB_EXPORT +#define __QAIC_STUB_EXPORT +#endif // __QAIC_STUB_EXPORT + +#ifndef __QAIC_STUB_ATTRIBUTE +#define __QAIC_STUB_ATTRIBUTE +#endif // __QAIC_STUB_ATTRIBUTE + +#ifndef __QAIC_SKEL +#define __QAIC_SKEL(ff) ff +#endif //__QAIC_SKEL__ + +#ifndef __QAIC_SKEL_EXPORT +#define __QAIC_SKEL_EXPORT +#endif // __QAIC_SKEL_EXPORT + +#ifndef __QAIC_SKEL_ATTRIBUTE +#define __QAIC_SKEL_ATTRIBUTE +#endif // __QAIC_SKEL_ATTRIBUTE + +#ifdef __QAIC_DEBUG__ + #ifndef __QAIC_DBG_PRINTF__ + #include + #define __QAIC_DBG_PRINTF__( ee ) do { printf ee ; } while(0) + #endif +#else + #define __QAIC_DBG_PRINTF__( ee ) (void)0 +#endif + + +#define _OFFSET(src, sof) ((void*)(((char*)(src)) + (sof))) + +#define _COPY(dst, dof, src, sof, sz) \ + do {\ + struct __copy { \ + char ar[sz]; \ + };\ + *(struct __copy*)_OFFSET(dst, dof) = *(struct __copy*)_OFFSET(src, sof);\ + } while (0) + +#define _COPYIF(dst, dof, src, sof, sz) \ + do {\ + if(_OFFSET(dst, dof) != _OFFSET(src, sof)) {\ + _COPY(dst, dof, src, sof, sz); \ + } \ + } while (0) + +_ATTRIBUTE_UNUSED +static __inline void _qaic_memmove(void* dst, void* src, int size) { + int i = 0; + for(i = 0; i < size; ++i) { + ((char*)dst)[i] = ((char*)src)[i]; + } +} + +#define _MEMMOVEIF(dst, src, sz) \ + do {\ + if(dst != src) {\ + _qaic_memmove(dst, src, sz);\ + } \ + } while (0) + + +#define _ASSIGN(dst, src, sof) \ + do {\ + dst = OFFSET(src, sof); \ + } while (0) + +#define _STD_STRLEN_IF(str) (str == 0 ? 0 : strlen(str)) + +#include "AEEStdErr.h" + +#ifdef _WIN32 +#define _QAIC_FARF(level, msg, ...) (void)0 +#else +#define _QAIC_FARF(level, msg, ...) \ + do {\ + if(0 == (HAP_debug_v2) ) {\ + (void)0; \ + } else { \ + FARF(level, msg , ##__VA_ARGS__); \ + } \ + }while(0) +#endif //_WIN32 for _QAIC_FARF + +#define _TRY(ee, func) \ + do { \ + if (AEE_SUCCESS != ((ee) = func)) {\ + __QAIC_DBG_PRINTF__((__FILE__ ":%d:error:%d:%s\n", __LINE__, (int)(ee),#func));\ + goto ee##bail;\ + } \ + } while (0) + +#define _TRY_FARF(ee, func) \ + do { \ + if (AEE_SUCCESS != ((ee) = func)) {\ + goto ee##farf##bail;\ + } \ + } while (0) + +#define _QAIC_CATCH(exception) exception##bail: if (exception != AEE_SUCCESS) + +#define _CATCH_FARF(exception) exception##farf##bail: if (exception != AEE_SUCCESS) + +#define _QAIC_ASSERT(nErr, ff) _TRY(nErr, 0 == (ff) ? AEE_EBADPARM : AEE_SUCCESS) + +#ifdef __QAIC_DEBUG__ +#define _QAIC_ALLOCATE(nErr, pal, size, alignment, pv) _TRY(nErr, _allocator_alloc(pal, __FILE_LINE__, size, alignment, (void**)&pv));\ + _QAIC_ASSERT(nErr,pv || !(size)) +#else +#define _QAIC_ALLOCATE(nErr, pal, size, alignment, pv) _TRY(nErr, _allocator_alloc(pal, 0, size, alignment, (void**)&pv));\ + _QAIC_ASSERT(nErr,pv || !(size)) +#endif + + +#endif // _QAIC_ENV_H + +#ifdef __cplusplus +extern "C" { +#endif +#if !defined(__QAIC_STRING1_OBJECT_DEFINED__) && !defined(__STRING1_OBJECT__) +#define __QAIC_STRING1_OBJECT_DEFINED__ +#define __STRING1_OBJECT__ +typedef struct _cstring1_s { + char* data; + int dataLen; +} _cstring1_t; + +#endif /* __QAIC_STRING1_OBJECT_DEFINED__ */ +/// Enabling stub-skel mismatch check feature in the auto-gen files. +/// Please refer to the IDL documentation for more details on the feature. +/// It is fully supported only on Kailua and later targets. +#define IDL_VERSION "0.0.1" +typedef struct dsptensor dsptensor; +struct dsptensor { + int64_t ne[4]; + int64_t nb[4]; + int32_t flags; + int32_t type; + float* data; + int dataLen; +}; +/** + * Opens the handle in the specified domain. If this is the first + * handle, this creates the session. Typically this means opening + * the device, aka open("/dev/adsprpc-smd"), then calling ioctl + * device APIs to create a PD on the DSP to execute our code in, + * then asking that PD to dlopen the .so and dlsym the skel function. + * + * @param uri, _URI"&_dom=aDSP" + * _URI is a QAIC generated uri, or + * "file:///?_skel_handle_invoke&_modver=1.0" + * If the _dom parameter is not present, _dom=DEFAULT is assumed + * but not forwarded. + * Reserved uri keys: + * [0]: first unamed argument is the skel invoke function + * _dom: execution domain name, _dom=mDSP/aDSP/DEFAULT + * _modver: module version, _modver=1.0 + * _*: any other key name starting with an _ is reserved + * Unknown uri keys/values are forwarded as is. + * @param h, resulting handle + * @retval, 0 on success + */ +__QAIC_HEADER_EXPORT int __QAIC_HEADER(ggmlop_open)(const char* uri, remote_handle64* h) __QAIC_HEADER_ATTRIBUTE; +/** + * Closes a handle. If this is the last handle to close, the session + * is closed as well, releasing all the allocated resources. + + * @param h, the handle to close + * @retval, 0 on success, should always succeed + */ +__QAIC_HEADER_EXPORT int __QAIC_HEADER(ggmlop_close)(remote_handle64 h) __QAIC_HEADER_ATTRIBUTE; +__QAIC_HEADER_EXPORT int __QAIC_HEADER(ggmlop_add)(remote_handle64 _h, const dsptensor* src0, const dsptensor* src1, dsptensor* dst) __QAIC_HEADER_ATTRIBUTE; +__QAIC_HEADER_EXPORT int __QAIC_HEADER(ggmlop_mulmat)(remote_handle64 _h, const dsptensor* src0, const dsptensor* src1, dsptensor* dst) __QAIC_HEADER_ATTRIBUTE; +#ifndef ggmlop_URI +#define ggmlop_URI "file:///libggmlop_skel.so?ggmlop_skel_handle_invoke&_modver=1.0&_idlver=0.0.1" +#endif /*ggmlop_URI*/ +#ifdef __cplusplus +} +#endif +#endif //_GGMLOP_H diff --git a/ggml/src/ggml-qnn/kernels/ggmlop_cdsp.c b/ggml/src/ggml-qnn/kernels/ggmlop_cdsp.c new file mode 100644 index 0000000000000..0350942648e2d --- /dev/null +++ b/ggml/src/ggml-qnn/kernels/ggmlop_cdsp.c @@ -0,0 +1,237 @@ +/* ggml op functions, running on Hexagon cDSP as libggmlop_skel.so + * + * currently I didn't find a general approach to compile/build this hexagon-kernel file, a manual build approach can works fine in my local dev envs. I'm working on this build issue. + * + */ + +#if 0 +#include +#include +#include +#include "HAP_farf.h" +#include "ggmlop.h" + +#define GGML_ASSERT(x) do { } while(0) +#define MIN(a, b) ((a) < (b) ? (a) : (b)) +#define GGML_RESTRICT + +int ggmlop_open(const char * uri, remote_handle64 * handle) { + void * tptr = NULL; + FARF(HIGH, "uri %s", uri); + tptr = (void *)malloc(1); + *handle = (remote_handle64)tptr; + assert(*handle); + return 0; +} + +int ggmlop_close(remote_handle64 handle) { + if (handle) + free((void*)handle); + return 0; +} + +int ggmlop_add(remote_handle64 h, const dsptensor * src0, const dsptensor * src1, dsptensor * dst) { + FARF(HIGH, "=============== DSP: ggmlop_add "); + for (size_t idx = 0; idx < src0->dataLen; idx++) { + dst->data[idx] = src0->data[idx] + src1->data[idx]; + } + + return 0; +} + +static void ggmldsp_dump_tensor(struct dsptensor * src0) { + FARF(HIGH, "ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi, %5zi)\n", + src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], + src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3]); +} + +static int ggmldsp_is_contiguous(const struct dsptensor * tensor) { + int n = 0; + size_t next_nb = sizeof(float); + if (tensor->ne[0] != 1 && tensor->nb[0] != next_nb) { + return 0; + } + next_nb *= tensor->ne[0]; + for (int i = 1; i < 4; i++) { + if (tensor->ne[i] != 1) { + if (i > n) { + if (tensor->nb[i] != next_nb) { + return 0; + } + next_nb *= tensor->ne[i]; + } else { + next_nb = tensor->ne[i] * tensor->nb[i]; + } + } + } + return 1; +} + +//FIXME: unknown issue on cDSP +int ggmlop_mulmat(remote_handle64 h, const struct dsptensor * src00, const struct dsptensor * src10, dsptensor * dst) { + FARF(HIGH, "=============== DSP: ggmlop_mulmat "); + + dsptensor * src0 = (dsptensor*)src00; + dsptensor * src1 = (dsptensor*)src10; + const int64_t ne00 = src0->ne[0]; + (void) (ne00); + const int64_t ne01 = (src0)->ne[1]; + (void) (ne01); + const int64_t ne02 = (src0)->ne[2]; + (void) (ne02); + const int64_t ne03 = (src0)->ne[3]; + (void) (ne03); + const size_t nb00 = (src0)->nb[0]; + (void) (nb00); + const size_t nb01 = (src0)->nb[1]; + (void) (nb01); + const size_t nb02 = (src0)->nb[2]; + (void) (nb02); + const size_t nb03 = (src0)->nb[3]; + (void) (nb03); + const int64_t ne10 = (src1)->ne[0]; + (void) (ne10); + const int64_t ne11 = (src1)->ne[1]; + (void) (ne11); + const int64_t ne12 = (src1)->ne[2]; + (void) (ne12); + const int64_t ne13 = (src1)->ne[3]; + (void) (ne13); + const size_t nb10 = (src1)->nb[0]; + (void) (nb10); + const size_t nb11 = (src1)->nb[1]; + (void) (nb11); + const size_t nb12 = (src1)->nb[2]; + (void) (nb12); + const size_t nb13 = (src1)->nb[3]; + (void) (nb13); + const int64_t ne0 = (dst)->ne[0]; + (void) (ne0); + const int64_t ne1 = (dst)->ne[1]; + (void) (ne1); + const int64_t ne2 = (dst)->ne[2]; + (void) (ne2); + const int64_t ne3 = (dst)->ne[3]; + (void) (ne3); + const size_t nb0 = (dst)->nb[0]; + (void) (nb0); + const size_t nb1 = (dst)->nb[1]; + (void) (nb1); + const size_t nb2 = (dst)->nb[2]; + (void) (nb2); + const size_t nb3 = (dst)->nb[3]; + (void) (nb3); + + ggmldsp_dump_tensor(src0); + ggmldsp_dump_tensor(src1); + + const int vec_dot_type = 0; + int64_t const vec_dot_num_rows = 1; + + GGML_ASSERT(ne0 == ne01); + GGML_ASSERT(ne1 == ne11); + GGML_ASSERT(ne2 == ne12); + GGML_ASSERT(ne3 == ne13); + + GGML_ASSERT(nb00 == sizeof(float)); + GGML_ASSERT(nb10 == sizeof(float)); + + GGML_ASSERT(nb0 == sizeof(float)); + GGML_ASSERT(nb0 <= nb1); + GGML_ASSERT(nb1 <= nb2); + GGML_ASSERT(nb2 <= nb3); + + const int64_t nr0 = ne0; + const int64_t nr1 = ne1 * ne2 * ne3; + + int chunk_size = 16; + if (nr0 == 1 || nr1 == 1) { + chunk_size = 64; + } + int64_t nchunk0 = (nr0 + chunk_size - 1) / chunk_size; + int64_t nchunk1 = (nr1 + chunk_size - 1) / chunk_size; + + if (nchunk0 * nchunk1 < nth * 4) { + nchunk0 = nr0 > nr1 ? nth : 1; + nchunk1 = nr0 > nr1 ? 1 : nth; + } + const int64_t dr0 = (nr0 + nchunk0 - 1) / nchunk0; + const int64_t dr1 = (nr1 + nchunk1 - 1) / nchunk1; + + int current_chunk = 0; + + const int64_t ith0 = current_chunk % nchunk0; + const int64_t ith1 = current_chunk / nchunk0; + + const int64_t ir0_start = dr0 * ith0; + const int64_t ir0_end = MIN(ir0_start + dr0, nr0); + + const int64_t ir1_start = dr1 * ith1; + const int64_t ir1_end = MIN(ir1_start + dr1, nr1); + + int64_t num_rows_per_vec_dot = vec_dot_num_rows; + + const int src1_cont = ggmldsp_is_contiguous(src1); + const int64_t r2 = ne12 / ne02; + const int64_t r3 = ne13 / ne03; + + const void * wdata = src1->data; + const size_t row_size = sizeof(float) * ne10; + assert(ne12 % ne02 == 0); + assert(ne13 % ne03 == 0); + + const int64_t blck_0 = 16; + const int64_t blck_1 = 16; + + float tmp[32]; + + for (int64_t iir1 = ir1_start; iir1 < ir1_end; iir1 += blck_1) { + for (int64_t iir0 = ir0_start; iir0 < ir0_end; iir0 += blck_0) { + for (int64_t ir1 = iir1; + ir1 < iir1 + blck_1 && ir1 < ir1_end; ir1 += num_rows_per_vec_dot) { + const int64_t i13 = (ir1 / (ne12 * ne1)); + const int64_t i12 = (ir1 - i13 * ne12 * ne1) / ne1; + const int64_t i11 = (ir1 - i13 * ne12 * ne1 - i12 * ne1); + + const int64_t i03 = i13 / r3; + const int64_t i02 = i12 / r2; + + const int64_t i1 = i11; + const int64_t i2 = i12; + const int64_t i3 = i13; + + const char * src0_row = (const char *)src0->data + (0 + i02 * nb02 + i03 * nb03); + + const char * src1_col = (const char *)wdata + + (src1_cont || src1->type != vec_dot_type + ? (i11 + i12 * ne11 + i13 * ne12 * ne11) * row_size + : (i11 * nb11 + i12 * nb12 + i13 * nb13)); + float * dst_col = (float *)((char *) dst->data + + (i1 * nb1 + i2 * nb2 + i3 * nb3)); + + + for (int64_t ir0 = iir0; + ir0 < iir0 + blck_0 && ir0 < ir0_end; ir0 += num_rows_per_vec_dot) { + + float sumf = 0.0; + const float * GGML_RESTRICT x = (float*)src0_row + ir0 * nb01; + const float * GGML_RESTRICT y = (float*)src1_col; + float * GGML_RESTRICT s = &tmp[ir0 - iir0]; + for (int i = 0; i < ne00; i++) { + sumf += x[i] * y[i]; + } + *s = sumf; + + } + + for (int cn = 0; cn < num_rows_per_vec_dot; ++cn) { + memcpy(&dst_col[iir0 + cn * nb1 / nb0], tmp + (cn * 16), + (MIN(iir0 + blck_0, ir0_end) - iir0) * sizeof(float)); + } + } + } + } + + return 0; +} +#endif diff --git a/ggml/src/ggml-qnn/kernels/ggmlop_stub.c b/ggml/src/ggml-qnn/kernels/ggmlop_stub.c new file mode 100644 index 0000000000000..6313348d7ea2d --- /dev/null +++ b/ggml/src/ggml-qnn/kernels/ggmlop_stub.c @@ -0,0 +1,437 @@ +#ifndef _GGMLOP_STUB_H +#define _GGMLOP_STUB_H +/// @file ggmlop.idl +/// +//qidl copyright +//qidl nested=false +#include "ggmlop.h" +#include +#ifndef _WIN32 +#include "HAP_farf.h" +#include +#endif //_WIN32 for HAP_farf +#ifndef _ALLOCATOR_H +#define _ALLOCATOR_H + +#include +#include + +typedef struct _heap _heap; +struct _heap { + _heap* pPrev; + const char* loc; + uint64_t buf; +}; + +typedef struct _allocator { + _heap* pheap; + uint8_t* stack; + uint8_t* stackEnd; + int nSize; +} _allocator; + +_ATTRIBUTE_UNUSED +static __inline int _heap_alloc(_heap** ppa, const char* loc, int size, void** ppbuf) { + _heap* pn = 0; + pn = MALLOC((size_t)size + sizeof(_heap) - sizeof(uint64_t)); + if(pn != 0) { + pn->pPrev = *ppa; + pn->loc = loc; + *ppa = pn; + *ppbuf = (void*)&(pn->buf); + return 0; + } else { + return -1; + } +} +#define _ALIGN_SIZE(x, y) (((x) + (y-1)) & ~(y-1)) + +_ATTRIBUTE_UNUSED +static __inline int _allocator_alloc(_allocator* me, + const char* loc, + int size, + unsigned int al, + void** ppbuf) { + if(size < 0) { + return -1; + } else if (size == 0) { + *ppbuf = 0; + return 0; + } + if((_ALIGN_SIZE((uintptr_t)me->stackEnd, al) + (size_t)size) < (uintptr_t)me->stack + (size_t)me->nSize) { + *ppbuf = (uint8_t*)_ALIGN_SIZE((uintptr_t)me->stackEnd, al); + me->stackEnd = (uint8_t*)_ALIGN_SIZE((uintptr_t)me->stackEnd, al) + size; + return 0; + } else { + return _heap_alloc(&me->pheap, loc, size, ppbuf); + } +} + +_ATTRIBUTE_UNUSED +static __inline void _allocator_deinit(_allocator* me) { + _heap* pa = me->pheap; + while(pa != 0) { + _heap* pn = pa; + const char* loc = pn->loc; + (void)loc; + pa = pn->pPrev; + FREE(pn); + } +} + +_ATTRIBUTE_UNUSED +static __inline void _allocator_init(_allocator* me, uint8_t* stack, int stackSize) { + me->stack = stack; + me->stackEnd = stack + stackSize; + me->nSize = stackSize; + me->pheap = 0; +} + + +#endif // _ALLOCATOR_H + +#ifndef SLIM_H +#define SLIM_H + +#include + +//a C data structure for the idl types that can be used to implement +//static and dynamic language bindings fairly efficiently. +// +//the goal is to have a minimal ROM and RAM footprint and without +//doing too many allocations. A good way to package these things seemed +//like the module boundary, so all the idls within one module can share +//all the type references. + + +#define PARAMETER_IN 0x0 +#define PARAMETER_OUT 0x1 +#define PARAMETER_INOUT 0x2 +#define PARAMETER_ROUT 0x3 +#define PARAMETER_INROUT 0x4 + +//the types that we get from idl +#define TYPE_OBJECT 0x0 +#define TYPE_INTERFACE 0x1 +#define TYPE_PRIMITIVE 0x2 +#define TYPE_ENUM 0x3 +#define TYPE_STRING 0x4 +#define TYPE_WSTRING 0x5 +#define TYPE_STRUCTURE 0x6 +#define TYPE_UNION 0x7 +#define TYPE_ARRAY 0x8 +#define TYPE_SEQUENCE 0x9 + +//these require the pack/unpack to recurse +//so it's a hint to those languages that can optimize in cases where +//recursion isn't necessary. +#define TYPE_COMPLEX_STRUCTURE (0x10 | TYPE_STRUCTURE) +#define TYPE_COMPLEX_UNION (0x10 | TYPE_UNION) +#define TYPE_COMPLEX_ARRAY (0x10 | TYPE_ARRAY) +#define TYPE_COMPLEX_SEQUENCE (0x10 | TYPE_SEQUENCE) + + +typedef struct Type Type; + +#define INHERIT_TYPE\ + int32_t nativeSize; /*in the simple case its the same as wire size and alignment*/\ + union {\ + struct {\ + const uintptr_t p1;\ + const uintptr_t p2;\ + } _cast;\ + struct {\ + uint32_t iid;\ + uint32_t bNotNil;\ + } object;\ + struct {\ + const Type *arrayType;\ + int32_t nItems;\ + } array;\ + struct {\ + const Type *seqType;\ + int32_t nMaxLen;\ + } seqSimple; \ + struct {\ + uint32_t bFloating;\ + uint32_t bSigned;\ + } prim; \ + const SequenceType* seqComplex;\ + const UnionType *unionType;\ + const StructType *structType;\ + int32_t stringMaxLen;\ + uint8_t bInterfaceNotNil;\ + } param;\ + uint8_t type;\ + uint8_t nativeAlignment\ + +typedef struct UnionType UnionType; +typedef struct StructType StructType; +typedef struct SequenceType SequenceType; +struct Type { + INHERIT_TYPE; +}; + +struct SequenceType { + const Type * seqType; + uint32_t nMaxLen; + uint32_t inSize; + uint32_t routSizePrimIn; + uint32_t routSizePrimROut; +}; + +//byte offset from the start of the case values for +//this unions case value array. it MUST be aligned +//at the alignment requrements for the descriptor +// +//if negative it means that the unions cases are +//simple enumerators, so the value read from the descriptor +//can be used directly to find the correct case +typedef union CaseValuePtr CaseValuePtr; +union CaseValuePtr { + const uint8_t* value8s; + const uint16_t* value16s; + const uint32_t* value32s; + const uint64_t* value64s; +}; + +//these are only used in complex cases +//so I pulled them out of the type definition as references to make +//the type smaller +struct UnionType { + const Type *descriptor; + uint32_t nCases; + const CaseValuePtr caseValues; + const Type * const *cases; + int32_t inSize; + int32_t routSizePrimIn; + int32_t routSizePrimROut; + uint8_t inAlignment; + uint8_t routAlignmentPrimIn; + uint8_t routAlignmentPrimROut; + uint8_t inCaseAlignment; + uint8_t routCaseAlignmentPrimIn; + uint8_t routCaseAlignmentPrimROut; + uint8_t nativeCaseAlignment; + uint8_t bDefaultCase; +}; + +struct StructType { + uint32_t nMembers; + const Type * const *members; + int32_t inSize; + int32_t routSizePrimIn; + int32_t routSizePrimROut; + uint8_t inAlignment; + uint8_t routAlignmentPrimIn; + uint8_t routAlignmentPrimROut; +}; + +typedef struct Parameter Parameter; +struct Parameter { + INHERIT_TYPE; + uint8_t mode; + uint8_t bNotNil; +}; + +#define SLIM_IFPTR32(is32,is64) (sizeof(uintptr_t) == 4 ? (is32) : (is64)) +#define SLIM_SCALARS_IS_DYNAMIC(u) (((u) & 0x00ffffff) == 0x00ffffff) + +typedef struct Method Method; +struct Method { + uint32_t uScalars; //no method index + int32_t primInSize; + int32_t primROutSize; + int maxArgs; + int numParams; + const Parameter * const *params; + uint8_t primInAlignment; + uint8_t primROutAlignment; +}; + +typedef struct Interface Interface; + +struct Interface { + int nMethods; + const Method * const *methodArray; + int nIIds; + const uint32_t *iids; + const uint16_t* methodStringArray; + const uint16_t* methodStrings; + const char* strings; +}; + + +#endif //SLIM_H + + +#ifndef _GGMLOP_SLIM_H +#define _GGMLOP_SLIM_H +#include + +#ifndef __QAIC_SLIM +#define __QAIC_SLIM(ff) ff +#endif +#ifndef __QAIC_SLIM_EXPORT +#define __QAIC_SLIM_EXPORT +#endif + +static const Type types[5]; +static const Type* const typeArrays[5] = {&(types[0]),&(types[0]),&(types[2]),&(types[2]),&(types[3])}; +static const StructType structTypes[1] = {{0x5,&(typeArrays[0]),0x50,0x4,0x48,0x8,0x4,0x8}}; +static const Type types[5] = {{0x20,{{(const uintptr_t)&(types[1]),(const uintptr_t)0x4}}, 8,0x8},{0x8,{{(const uintptr_t)0,(const uintptr_t)1}}, 2,0x8},{0x4,{{(const uintptr_t)0,(const uintptr_t)1}}, 2,0x4},{SLIM_IFPTR32(0x8,0x10),{{(const uintptr_t)&(types[4]),(const uintptr_t)0x0}}, 9,SLIM_IFPTR32(0x4,0x8)},{0x4,{{(const uintptr_t)0,(const uintptr_t)1}}, 2,0x4}}; +static const Parameter parameters[5] = {{SLIM_IFPTR32(0x8,0x10),{{(const uintptr_t)0x0,0}}, 4,SLIM_IFPTR32(0x4,0x8),0,0},{SLIM_IFPTR32(0x4,0x8),{{(const uintptr_t)0xdeadc0de,(const uintptr_t)0}}, 0,SLIM_IFPTR32(0x4,0x8),3,0},{SLIM_IFPTR32(0x4,0x8),{{(const uintptr_t)0xdeadc0de,(const uintptr_t)0}}, 0,SLIM_IFPTR32(0x4,0x8),0,0},{SLIM_IFPTR32(0x50,0x58),{{(const uintptr_t)&(structTypes[0]),0}}, 22,0x8,0,0},{SLIM_IFPTR32(0x50,0x58),{{(const uintptr_t)&(structTypes[0]),0}}, 22,0x8,3,0}}; +static const Parameter* const parameterArrays[6] = {(&(parameters[3])),(&(parameters[3])),(&(parameters[4])),(&(parameters[0])),(&(parameters[1])),(&(parameters[2]))}; +static const Method methods[3] = {{REMOTE_SCALARS_MAKEX(0,0,0x2,0x0,0x0,0x1),0x4,0x0,2,2,(&(parameterArrays[3])),0x4,0x1},{REMOTE_SCALARS_MAKEX(0,0,0x0,0x0,0x1,0x0),0x0,0x0,1,1,(&(parameterArrays[5])),0x1,0x0},{REMOTE_SCALARS_MAKEX(0,0,0x3,0x2,0x0,0x0),0xa4,0x48,3,3,(&(parameterArrays[0])),0x8,0x8}}; +static const Method* const methodArrays[4] = {&(methods[0]),&(methods[1]),&(methods[2]),&(methods[2])}; +static const char strings[65] = "mulmat\0flags\0close\0src1\0data\0type\0src0\0open\0dst\0add\0uri\0nb\0ne\0h\0"; +static const uint16_t methodStrings[43] = {0,34,59,56,7,29,24,19,59,56,7,29,24,44,59,56,7,29,24,48,34,59,56,7,29,24,19,59,56,7,29,24,44,59,56,7,29,24,39,52,62,13,62}; +static const uint16_t methodStringsArrays[4] = {38,41,19,0}; +__QAIC_SLIM_EXPORT const Interface __QAIC_SLIM(ggmlop_slim) = {4,&(methodArrays[0]),0,0,&(methodStringsArrays [0]),methodStrings,strings}; +#endif //_GGMLOP_SLIM_H + + +#ifdef __cplusplus +extern "C" { +#endif +__QAIC_STUB_EXPORT int __QAIC_STUB(ggmlop_open)(const char* uri, remote_handle64* h) __QAIC_STUB_ATTRIBUTE { + return __QAIC_REMOTE(remote_handle64_open)(uri, h); +} +__QAIC_STUB_EXPORT int __QAIC_STUB(ggmlop_close)(remote_handle64 h) __QAIC_STUB_ATTRIBUTE { + return __QAIC_REMOTE(remote_handle64_close)(h); +} +static __inline int _stub_unpack(_ATTRIBUTE_UNUSED remote_arg* _praROutPost, _ATTRIBUTE_UNUSED remote_arg* _ppraROutPost[1], _ATTRIBUTE_UNUSED void* _primROut, _ATTRIBUTE_UNUSED uint64_t _rout0[4], _ATTRIBUTE_UNUSED uint64_t _rout1[4], _ATTRIBUTE_UNUSED uint32_t _rout2[1], _ATTRIBUTE_UNUSED uint32_t _rout3[1], _ATTRIBUTE_UNUSED char* _rout4[1], _ATTRIBUTE_UNUSED uint32_t _rout4Len[1]) { + int _nErr = 0; + remote_arg* _praROutPostStart = _praROutPost; + remote_arg** _ppraROutPostStart = _ppraROutPost; + _ppraROutPost = &_praROutPost; + _COPY(_rout0, 0, _primROut, 0, 32); + _COPY(_rout1, 0, _primROut, 32, 32); + _COPY(_rout2, 0, _primROut, 64, 4); + _COPY(_rout3, 0, _primROut, 68, 4); + _ppraROutPostStart[0] += (_praROutPost - _praROutPostStart) +1; + return _nErr; +} +static __inline int _stub_pack(_ATTRIBUTE_UNUSED _allocator* _al, _ATTRIBUTE_UNUSED remote_arg* _praIn, _ATTRIBUTE_UNUSED remote_arg* _ppraIn[1], _ATTRIBUTE_UNUSED remote_arg* _praROut, _ATTRIBUTE_UNUSED remote_arg* _ppraROut[1], _ATTRIBUTE_UNUSED remote_arg* _praHIn, _ATTRIBUTE_UNUSED remote_arg* _ppraHIn[1], _ATTRIBUTE_UNUSED remote_arg* _praHROut, _ATTRIBUTE_UNUSED remote_arg* _ppraHROut[1], _ATTRIBUTE_UNUSED void* _primIn, _ATTRIBUTE_UNUSED void* _primROut, _ATTRIBUTE_UNUSED uint64_t _rout0[4], _ATTRIBUTE_UNUSED uint64_t _rout1[4], _ATTRIBUTE_UNUSED uint32_t _rout2[1], _ATTRIBUTE_UNUSED uint32_t _rout3[1], _ATTRIBUTE_UNUSED char* _rout4[1], _ATTRIBUTE_UNUSED uint32_t _rout4Len[1]) { + int _nErr = 0; + remote_arg* _praInStart = _praIn; + remote_arg** _ppraInStart = _ppraIn; + remote_arg* _praROutStart = _praROut; + remote_arg** _ppraROutStart = _ppraROut; + _ppraIn = &_praIn; + _ppraROut = &_praROut; + _COPY(_primIn, 0, _rout4Len, 0, 4); + _praROut[0].buf.pv = _rout4[0]; + _praROut[0].buf.nLen = (4 * _rout4Len[0]); + _ppraInStart[0] += (_praIn - _praInStart) + 0; + _ppraROutStart[0] += (_praROut - _praROutStart) +1; + return _nErr; +} +static __inline int _stub_pack_1(_ATTRIBUTE_UNUSED _allocator* _al, _ATTRIBUTE_UNUSED remote_arg* _praIn, _ATTRIBUTE_UNUSED remote_arg* _ppraIn[1], _ATTRIBUTE_UNUSED remote_arg* _praROut, _ATTRIBUTE_UNUSED remote_arg* _ppraROut[1], _ATTRIBUTE_UNUSED remote_arg* _praHIn, _ATTRIBUTE_UNUSED remote_arg* _ppraHIn[1], _ATTRIBUTE_UNUSED remote_arg* _praHROut, _ATTRIBUTE_UNUSED remote_arg* _ppraHROut[1], _ATTRIBUTE_UNUSED void* _primIn, _ATTRIBUTE_UNUSED void* _primROut, _ATTRIBUTE_UNUSED uint64_t _in0[4], _ATTRIBUTE_UNUSED uint64_t _in1[4], _ATTRIBUTE_UNUSED uint32_t _in2[1], _ATTRIBUTE_UNUSED uint32_t _in3[1], _ATTRIBUTE_UNUSED char* _in4[1], _ATTRIBUTE_UNUSED uint32_t _in4Len[1]) { + int _nErr = 0; + remote_arg* _praInStart = _praIn; + remote_arg** _ppraInStart = _ppraIn; + remote_arg* _praROutStart = _praROut; + remote_arg** _ppraROutStart = _ppraROut; + _ppraIn = &_praIn; + _ppraROut = &_praROut; + _COPY(_primIn, 0, _in0, 0, 32); + _COPY(_primIn, 32, _in1, 0, 32); + _COPY(_primIn, 64, _in2, 0, 4); + _COPY(_primIn, 68, _in3, 0, 4); + _COPY(_primIn, 72, _in4Len, 0, 4); + _praIn[0].buf.pv = (void*) _in4[0]; + _praIn[0].buf.nLen = (4 * _in4Len[0]); + _ppraInStart[0] += (_praIn - _praInStart) + 1; + _ppraROutStart[0] += (_praROut - _praROutStart) +0; + return _nErr; +} +static __inline void _count(int _numIn[1], int _numROut[1], int _numInH[1], int _numROutH[1], _ATTRIBUTE_UNUSED uint64_t _rout0[4], _ATTRIBUTE_UNUSED uint64_t _rout1[4], _ATTRIBUTE_UNUSED uint32_t _rout2[1], _ATTRIBUTE_UNUSED uint32_t _rout3[1], _ATTRIBUTE_UNUSED char* _rout4[1], _ATTRIBUTE_UNUSED uint32_t _rout4Len[1]) { + _numIn[0] += 0; + _numROut[0] += 1; + _numInH[0] += 0; + _numROutH[0] += 0; +} +static __inline void _count_1(int _numIn[1], int _numROut[1], int _numInH[1], int _numROutH[1], _ATTRIBUTE_UNUSED uint64_t _in0[4], _ATTRIBUTE_UNUSED uint64_t _in1[4], _ATTRIBUTE_UNUSED uint32_t _in2[1], _ATTRIBUTE_UNUSED uint32_t _in3[1], _ATTRIBUTE_UNUSED char* _in4[1], _ATTRIBUTE_UNUSED uint32_t _in4Len[1]) { + _numIn[0] += 1; + _numROut[0] += 0; + _numInH[0] += 0; + _numROutH[0] += 0; +} +static __inline int _stub_method(remote_handle64 _handle, uint32_t _mid, uint64_t _in0[SLIM_IFPTR32(10, 11)], uint64_t _in1[SLIM_IFPTR32(10, 11)], uint64_t _rout2[SLIM_IFPTR32(10, 11)]) { + remote_arg* _pra = 0; + int _numIn[1] = {0}; + int _numROut[1] = {0}; + int _numInH[1] = {0}; + int _numROutH[1] = {0}; + _allocator _al[1] = {{0}}; + uint64_t _primIn[21]= {0}; + uint64_t _primROut[9]= {0}; + remote_arg* _praIn = 0; + remote_arg* _praROut = 0; + remote_arg* _praROutPost = 0; + remote_arg** _ppraROutPost = &_praROutPost; + remote_arg** _ppraIn = &_praIn; + remote_arg** _ppraROut = &_praROut; + remote_arg* _praHIn = 0; + remote_arg** _ppraHIn = &_praHIn; + remote_arg* _praHROut = 0; + remote_arg** _ppraHROut = &_praHROut; + int _nErr = 0; + _numIn[0] = 0; + _numROut[0] = 0; + _numInH[0] = 0; + _numROutH[0] = 0; + _count_1(_numIn, _numROut, _numInH, _numROutH, (uint64_t*)&(((uint64_t*)_in0)[0]), (uint64_t*)&(((uint64_t*)_in0)[4]), (uint32_t*)&(((uint32_t*)_in0)[16]), (uint32_t*)&(((uint32_t*)_in0)[17]), SLIM_IFPTR32((char**)&(((uint32_t*)_in0)[18]), (char**)&(((uint64_t*)_in0)[9])), SLIM_IFPTR32((uint32_t*)&(((uint32_t*)_in0)[19]), (uint32_t*)&(((uint32_t*)_in0)[20]))); + _count_1(_numIn, _numROut, _numInH, _numROutH, (uint64_t*)&(((uint64_t*)_in1)[0]), (uint64_t*)&(((uint64_t*)_in1)[4]), (uint32_t*)&(((uint32_t*)_in1)[16]), (uint32_t*)&(((uint32_t*)_in1)[17]), SLIM_IFPTR32((char**)&(((uint32_t*)_in1)[18]), (char**)&(((uint64_t*)_in1)[9])), SLIM_IFPTR32((uint32_t*)&(((uint32_t*)_in1)[19]), (uint32_t*)&(((uint32_t*)_in1)[20]))); + _count(_numIn, _numROut, _numInH, _numROutH, (uint64_t*)&(((uint64_t*)_rout2)[0]), (uint64_t*)&(((uint64_t*)_rout2)[4]), (uint32_t*)&(((uint32_t*)_rout2)[16]), (uint32_t*)&(((uint32_t*)_rout2)[17]), SLIM_IFPTR32((char**)&(((uint32_t*)_rout2)[18]), (char**)&(((uint64_t*)_rout2)[9])), SLIM_IFPTR32((uint32_t*)&(((uint32_t*)_rout2)[19]), (uint32_t*)&(((uint32_t*)_rout2)[20]))); + if(_numIn[0]>=255){ + _QAIC_FARF(RUNTIME_ERROR, "ERROR: Unsupported number of input buffers\n"); + return AEE_EUNSUPPORTED; + } + if(_numROut[0]>=255){ + _QAIC_FARF(RUNTIME_ERROR, "ERROR: Unsupported number of output buffers\n"); + return AEE_EUNSUPPORTED; + } + _allocator_init(_al, 0, 0); + _QAIC_ALLOCATE(_nErr, _al, ((((((((_numIn[0] + _numROut[0]) + _numInH[0]) + _numROutH[0]) + 1) + 1) + 0) + 0) * sizeof(_pra[0])), 4, _pra); + _QAIC_ASSERT(_nErr, _pra); + _pra[0].buf.pv = (void*)_primIn; + _pra[0].buf.nLen = sizeof(_primIn); + _pra[(_numIn[0] + 1)].buf.pv = (void*)_primROut; + _pra[(_numIn[0] + 1)].buf.nLen = sizeof(_primROut); + _praIn = (_pra + 1); + _praROut = (_praIn + _numIn[0] + 1); + _praROutPost = _praROut; + if(_praHIn == 0) + { + _praHIn = ((_praROut + _numROut[0]) + 1); + } + if(_praHROut == 0) + (_praHROut = _praHIn + _numInH[0] + 0); + _TRY(_nErr, _stub_pack_1(_al, (_praIn + 0), _ppraIn, (_praROut + 0), _ppraROut, _praHIn, _ppraHIn, _praHROut, _ppraHROut, ((char*)_primIn + 0), 0, (uint64_t*)&(((uint64_t*)_in0)[0]), (uint64_t*)&(((uint64_t*)_in0)[4]), (uint32_t*)&(((uint32_t*)_in0)[16]), (uint32_t*)&(((uint32_t*)_in0)[17]), SLIM_IFPTR32((char**)&(((uint32_t*)_in0)[18]), (char**)&(((uint64_t*)_in0)[9])), SLIM_IFPTR32((uint32_t*)&(((uint32_t*)_in0)[19]), (uint32_t*)&(((uint32_t*)_in0)[20])))); + _TRY(_nErr, _stub_pack_1(_al, (_praIn + 0), _ppraIn, (_praROut + 0), _ppraROut, _praHIn, _ppraHIn, _praHROut, _ppraHROut, ((char*)_primIn + 80), 0, (uint64_t*)&(((uint64_t*)_in1)[0]), (uint64_t*)&(((uint64_t*)_in1)[4]), (uint32_t*)&(((uint32_t*)_in1)[16]), (uint32_t*)&(((uint32_t*)_in1)[17]), SLIM_IFPTR32((char**)&(((uint32_t*)_in1)[18]), (char**)&(((uint64_t*)_in1)[9])), SLIM_IFPTR32((uint32_t*)&(((uint32_t*)_in1)[19]), (uint32_t*)&(((uint32_t*)_in1)[20])))); + _TRY(_nErr, _stub_pack(_al, (_praIn + 0), _ppraIn, (_praROut + 0), _ppraROut, _praHIn, _ppraHIn, _praHROut, _ppraHROut, ((char*)_primIn + 160), ((char*)_primROut + 0), (uint64_t*)&(((uint64_t*)_rout2)[0]), (uint64_t*)&(((uint64_t*)_rout2)[4]), (uint32_t*)&(((uint32_t*)_rout2)[16]), (uint32_t*)&(((uint32_t*)_rout2)[17]), SLIM_IFPTR32((char**)&(((uint32_t*)_rout2)[18]), (char**)&(((uint64_t*)_rout2)[9])), SLIM_IFPTR32((uint32_t*)&(((uint32_t*)_rout2)[19]), (uint32_t*)&(((uint32_t*)_rout2)[20])))); + _QAIC_ASSERT(_nErr, (_numInH[0] + 0) <= 15); + _QAIC_ASSERT(_nErr, (_numROutH[0] + 0) <= 15); + _TRY_FARF(_nErr, __QAIC_REMOTE(remote_handle64_invoke)(_handle, REMOTE_SCALARS_MAKEX(0, _mid, (_numIn[0] + 1), (_numROut[0] + 1), (_numInH[0] + 0), (_numROutH[0] + 0)), _pra)); + _TRY(_nErr, _stub_unpack((_praROutPost + 0), _ppraROutPost, ((char*)_primROut + 0), (uint64_t*)&(((uint64_t*)_rout2)[0]), (uint64_t*)&(((uint64_t*)_rout2)[4]), (uint32_t*)&(((uint32_t*)_rout2)[16]), (uint32_t*)&(((uint32_t*)_rout2)[17]), SLIM_IFPTR32((char**)&(((uint32_t*)_rout2)[18]), (char**)&(((uint64_t*)_rout2)[9])), SLIM_IFPTR32((uint32_t*)&(((uint32_t*)_rout2)[19]), (uint32_t*)&(((uint32_t*)_rout2)[20])))); + _QAIC_CATCH(_nErr) {} + _CATCH_FARF(_nErr) { + _QAIC_FARF(RUNTIME_ERROR, "ERROR 0x%x: handle=0x%"PRIx64", scalar=0x%x, method ID=%d: %s failed\n", _nErr , _handle, REMOTE_SCALARS_MAKEX(0, _mid, (_numIn[0] + 1), (_numROut[0] + 1), (_numInH[0] + 0), (_numROutH[0] + 0)), _mid, __func__); + } + _allocator_deinit(_al); + return _nErr; +} +__QAIC_STUB_EXPORT int __QAIC_STUB(ggmlop_add)(remote_handle64 _handle, const dsptensor* src0, const dsptensor* src1, dsptensor* dst) __QAIC_STUB_ATTRIBUTE { + uint32_t _mid = 2; + return _stub_method(_handle, _mid, (uint64_t*)src0, (uint64_t*)src1, (uint64_t*)dst); +} +__QAIC_STUB_EXPORT int __QAIC_STUB(ggmlop_mulmat)(remote_handle64 _handle, const dsptensor* src0, const dsptensor* src1, dsptensor* dst) __QAIC_STUB_ATTRIBUTE { + uint32_t _mid = 3; + return _stub_method(_handle, _mid, (uint64_t*)src0, (uint64_t*)src1, (uint64_t*)dst); +} +#ifdef __cplusplus +} +#endif +#endif //_GGMLOP_STUB_H diff --git a/ggml/src/ggml-qnn/kernels/libggmlop_skel.so b/ggml/src/ggml-qnn/kernels/libggmlop_skel.so new file mode 100755 index 0000000000000000000000000000000000000000..9d4be24f3263907fa6c2bf83eacbf6fe17941dd9 GIT binary patch literal 13896 zcmeHOdsJJ;nIBy}5RQWd78t+9!wzX6~KgP(@9pLZM)CP_ieOj7pK%Si)EjXel!=6)Rv1 zS=NkP&U17*V-(etpkr~l0lXB)ImRezkVJB@GcwW{2Bb)Gl@w2d%CR3jMOoGiUJf1Y zO~zh#x3So0!atza*3J%(*Wo+h?lAg1n@WxAj3!z66QI;qf`PFmpy@1)-JVcJ26%lE zUI9$?qWYwxrPhEhSrE^k!BW{q;DS{AAn1TAK~F<&K;w|SPL#xd$o{@093(8|C;jgN zZa{u2k1(}!G>fq<5`GS}AeXTnpy_NLdlgg;WdAK0fD-;4q5=BU7o?w04Il-QeGZQscc;(oPcVZ7m-nEfqaB0ppu^eG-rD7Ec67FPtu1Lf=y&@Z$mri7vdHD?b2?hu zyP$TD+u`%~G|A?j&CN3F@xo8pby$@gakO{!dJec9J>BihvD@$I>43d%cWb-P@9uWA zbUQoUP-*cvnw&nj?++OGg0tO4b@sJ;x~N(3AC|t@?exNm+f7M+&jAM#;c5Fv8)7J~ z9Zg~O`P|)N%b+1-OP$V+4v#CT#thG>3r{A|R@V$S>#0SzEtVpND#P%d2{eY}K=X(+ z)WxJl_+2Oo=YwWTJdKNViJu3WFhs;TXO1NMI6`Ay4oaB;otwZDE6#(U7{=^2#56>j z2egQph%`qG8yL$2je?$*_}_v)EAiPmj6IjYhb5l!U6lBOWtdkb-UR-##NQA8hQ!-o zM~6sd)qt*%_zj>GDSZHRO9IbJ{6nCWpLA+L?GoPzO13E73wl)IPlA3Ufqz2cp9Q5z z`WHc;OW*~GzXlpk;Lk}s!$UkS@lj9}Fxk<89+h|+1Of0lGyo7zg3qOnMtFjFDg7zp zsS^U^+VJ1zWO5T%f7t&`gQQ^EbC)J_H~6Y5Y!3^9mlgi zMS-%QRA76jAwTf~>s$O<_Vp9f|MNT(P6RaqmJQ)#?ak@wyX%F);37T}TqZ1ghzWnc ziwfP(W39PZy+6(J*u+l$vEUZrv0$1dEx?4wL^Ii!l7HZjqJUigfAsw_$!w$>c5nP!?$oN*?S`iU%rJ|QsB_$-$pws(9T>a=Q_3N z0NS4k+;xUFX9BiAf@3!mQuiN5o2K(F@9({le^`Nj<0_gS)bmZ=+`7n6&ZWqZ&cd|M zm|0&AZ|TV!jSOAm7d-oUGixx^M~0T8j~UPUGvS|B;B@TX>3OEc_5O9h?2ILY`@Ful z6nx`FOMY&xKA;r*UX{?~ed1i*@vCpDdas(Tuaev#Sn`$j(g!CTTyM86608eHf}EAR zUIzaCkx0<>SuT9Isd&>vNY6%GmtH!buU7>s?CL<-g7q1~#j4}X@>VhBb-;j`w>_ z!h&Z{h3k&9ou}^GbWYj8+3OxGraS^C`gql>?FMKk%9Ph`*JTRO3f+Ca<) zUCNh5`9kr0R!dnhAH1BG1HJ+*=MMot0W90#Y^?7HFxgoEJB=e{!OD6Tuv*K4IpEKV zzAFL@_zJLw567zD{~6$uz%SOC;R_2WY+1ucfK#mD7z?Y1>+YT2mwh%eWU-e8-RH}K zE>JV56?yZ)uLM6L)q??Zz)IM*)vE)W#(y*vhF#vS3A_baix3Q|EJDm$uLuahL+2F73;Ej--dR0q4uuh)%M#0j{~|7EA+H zSB=EZ)^0oA3cUJd_*W z9~*)E?||>BpBLC>PYcWqM}}U8EM@yku_MT5w`&8p!3wMz z&`-P2Ph;pOP&265PvBRApXn!H4p{D|m5KcX*^`O=1iT9EFZPo~6Bq&>7%dA@e<{#k z9G;=33uRT*Uw$*!&|?2)#U0SMpkH#JZ>_^P9xn@q;6IONL=B#+5Ml%HyMX-ep8vaC z*l2WenVBK<`}U>7`D;!YU$U{in70m9;Ca+Tk92g6e1_JH!`Invrfmi&jq`Ull}@_$8` z$9!29R>n9>)^);$kYzP)=$ie<$&lwW7-GD_xCJFV4~rH5)Oj`Qa<0nRGKmQntz zTn^C!p406Tf>qouS{LH}OdR6AMKTt(>hqHhb+6`a%ZhD6 zPQ!+Q?BQIV^XCc5;cULqTNz^Y<*@rv^K#cO6ym(R7C>H;2YAI_=jA_QUOs)#r#~-0uspx~f9GSHnbp%gOmmi2z*f(uifp-QXjp>s3_|ws4a{q|nFb9U zuMe6q-z>RPQgg zur1#|!?p~YnhH&3{g&?w`Yk42zhxk#-|}PREz)}4=8zZl=--6-`3~gEtgYU!z}j~f z@>2zhf_X@L%Qnz+_B?A~{KB3Ejm)oy-`Z+JLmJQObC8xQ3|e^5Y{+O`IElQMw>A~# z@Zav)*!XY$4CK8W_3Gw8pX5@ReG07gD_~!-r+U9&%3X65`ACkrwGVT1AFUZA&#bK8 zzfeFOk;bt8_9gk&C;GOyXCcbZkbIjf`nI@7A^NtsXHk^->qNhD*W~agdNw!yyFVNG zUX6M)Fdk;sSJ}UXD2x1H*gvqpIryIGvZ28y`sefVBK3Rgc((VXX~@|0*{rY*ZNDYT zXFOG|wXf`B19`QIz`(E5H?M4C1Nqqd=IB>GlT*tA2JCBBX68Melac>Jtl#wtfg4iX zz2H6l-RR?UgH|O!82s@>Nlq@_TZz|(EC#l>@4fKN=?0ZlK9kC4yx02>-g&z5j#CF( z2AT~@??8CpeV9oBW$e=L|Id?lFr<+l}h0=gdWpV}IJG7Ws$cw{K`E*85{n>Loghk~@f%dWIfrkdx*MhXo53#+s- z1%G^W?cT^xHukOyt2Hr=9pi9EWT*)50E~ZRG>c#9=Y>#EZPEA|VPiS=T|WoS0nOsS z>(}rj!4&L;SAs4BtpuG5+5)-|^hrD;&&(Tu1#Nwhud`2fkUGJ+=t%s-(LV?Z6<9?#Uh#b?AB9%TH=}u0LPE z9P}ECDbFm5@u6d*++NiU<-P?sbiC?D`k4hcu6_~uLi({NQP|{a)29}}2W=zkkFNgJ z8w&g3(Mk&!(BI7(4xb;a;$U+?2;n_&V6?K{(6GT4sj5F4sT?pzf*UXv750Hq#J=?#VD=%^0*wsVmxh54-m7z3kgpr1by zv9d6GS7CW{G^bo)XTsr8CI|q+<C9#{Ocw7j0!vozf$6 zp$NXKDW+arcD~$zJlUbWlM6={#n_(u$%Xs47}lks=S45I zZ>}X0%)$2{)tQtq#8N$gijOP1pv5^!-S|Tke~K zxf0{gnrk^6d=B%^1jhOtjQwM0B0<$z%cS~6E~XyQ#3pQu#+s4Fg;eiiZR)>LF&KOl zW$myW!FPwjAmgp~)!7XVk6Q+%O zu@uZ92CuE9KWl`kDL;Tfy6YQlGz)CKbMXRiI)sEOAN7x2eU ztP;L*;=XEa?B-i$m}X+>@Bshl=rZ`BzItr4nSXfn_20bNpvPF1^RuRlVr}A@`rI*h|nQ3S*Dz`;IJ6pC*&J$Wtj(t*dcJHKe``noNd|E7J-@MpF zm>Zj@DO+RDwgfzj{0@+S5eSgWcc!S~Kag5#Bu z;M1UwfZhX2YnqzYDSUg?&>FR^djB!_Ogy_FrD5IbM;cOQ0`>qW*=vJC^YH!hU3}Y| zGs61j{nu*oJLh72^IZJeyz3`~#ly+pju&J7rL|7}ZoHDMV)I3-xok7rz*5*!oUhDJ z;MXMZCicf@-GNi(a-4>g!>)9+*3#i@^|AN~oUhwe%$l8kC-Wcl;z+97Wn$vFShLU1 z=&X!R!Md7Qmz%ZC)YsA8+<|kgt#SgL3Ks(yr36?hlFrrIJH1BNrjioFYUz}ZR3w>L z4&e{#o7Ne++{Gn^6(FV(!`7_^)2)0`H9j<78Rc6W&$m|2M`e7PL4m@*h99Dj;)jkk zo&eQJPSM%cKuY|~iq36Sz>nZZ18RaEoo|sYoq0t$h7(=JoipG=7mW9Q``WVmK;PF08jBD#9G8gL@%Nw z7Ijo8n96=b_?I{03Cbjlybzp3J_=dLJR2*;cBQj3N+(<7k5;7>&(SU-{hAcE^aoJt#fTKnTpqy ztX)@Hy0K-|$D1X`$S9d@Ds>@SQ_1?`^=nI7)|EOpmacP`G;Jt#7q>V!wrp&6HLWdP zzrnT2z^tCmPFzRWROIpci@xCUbQVdqbL_TlFDf&Z&{S9C_jo#dMfQq%bJflrjye&= z>su5DP3>Jp(*1|R4%E<4SlH(DwH5mO2RqzbeFuHLhQdx~x2tWdqy^~m_QEb3 zU?fbU7^r{Hi-g_F$7o8^9gX}#cUQCI$84_(J8%hSmi#a9j1n8%DU4 zMK_n6z0URyXH$neFE7uOmuJmO*o@^Bw685$l~>^Pcds$vXH{Ntf`+(ZlvUcxN@q*4 zS4@zJ*>C;KR*mo9K1Q1vyF7k3T{fhvhs@Y~unPk+PH~UQNYg1ZcDp;AB$7C9ho2e6 z%Th*>^}G9!Al|Mrc6-EauF>5Vzr!V(a(1@6z+nGNrl1oC%;@rax_tm~(&~X3{KTzd zRp@kgAtml%M{lRm)!Nxicfy#_*GAX6oK4^kcE)KOf7rsCkfxk*YP-(b|uofuM~HC*tcMR8JE*866aOY9!@FlGGTAIm<;>z z8&~qaPbuzNnRqXeG?ciPNTE;K*C?f(-oOk_7v+H&oFQV`uToIrz2>WNR{T}5Urj%f zguk1F{}q`0rJ5<=?+7UEL-$DdC!n+^HB0yv(2Ky-1f(AY{RJ>5;kQAr0n?tD1blknmsoC_=`=`cRlCgBQTdOrFQ z>7H*5@Bkiz4&V%y!gf(+Y5dc@*B0Q!d%r&5#Cx$10w>>`TVt7BXR%$Ud;Al<##*ca%Tm;`N7J@}|EWn0?(^?)1b<+3`=0Xgrv0(SLlet;$3N{(x|~b8 z5I_5pI;?*59rHia<4Myq?GiBU%=)_z*}VK+f>w6==0tk*{fXi*B26D;gY;;9pli64 zMtV{FP^9lta*$apk>WQN(xdNM6r2X+xUOJr8K=cp3I>U8 z1C{k?J&@M~Nz7!u_&Oc;oAxynX^$WWX$OZ7^>KStKiXeVq&mvMATdp#R9A7Dr$oeI z8G}+DvUvcJiXlDPk5Ke}NV$EG%X+kKQKY?=9Ay9QMS`qH`wfb+jC5sR?@Q35b&VqR zlN@Bd01{+-w11>1%Scz|g9&=HkD+KI1u4irJPJ(qOj7zd^!^EBOAaKX_&6dxA9A?2 zfU&1#07^O(o Date: Sun, 23 Mar 2025 22:45:22 +0800 Subject: [PATCH 68/76] ggml-qnn: refine general approach through Hexagon cDSP --- ggml/src/ggml-qnn/CMakeLists.txt | 2 +- ggml/src/ggml-qnn/ggml-qnn.cpp | 268 ++++---- .../{ggmlop_stub.c => ggmlop_ap_skel.c} | 88 ++- .../kernels/{ggmlop.h => ggmlop_ap_skel.h} | 23 +- ggml/src/ggml-qnn/kernels/ggmlop_cdsp.c | 413 +++++++++--- ggml/src/ggml-qnn/kernels/ggmlop_cdsp_skel.c | 596 ++++++++++++++++++ ggml/src/ggml-qnn/kernels/libggmlop_skel.so | Bin 13896 -> 13704 bytes 7 files changed, 1116 insertions(+), 274 deletions(-) rename ggml/src/ggml-qnn/kernels/{ggmlop_stub.c => ggmlop_ap_skel.c} (70%) rename ggml/src/ggml-qnn/kernels/{ggmlop.h => ggmlop_ap_skel.h} (95%) create mode 100644 ggml/src/ggml-qnn/kernels/ggmlop_cdsp_skel.c diff --git a/ggml/src/ggml-qnn/CMakeLists.txt b/ggml/src/ggml-qnn/CMakeLists.txt index c63faca10e842..d5a16ffd4e1a1 100644 --- a/ggml/src/ggml-qnn/CMakeLists.txt +++ b/ggml/src/ggml-qnn/CMakeLists.txt @@ -49,7 +49,7 @@ endif() set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DGGML_USE_QNN") set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3") -file(GLOB QNN_SOURCES "${CMAKE_CURRENT_LIST_DIR}/*.cpp" "${CMAKE_CURRENT_LIST_DIR}/kernels/*.c") +file(GLOB QNN_SOURCES "${CMAKE_CURRENT_LIST_DIR}/*.cpp" "${CMAKE_CURRENT_LIST_DIR}/kernels/ggmlop_ap_skel.c") ggml_add_backend_library(ggml-qnn ${QNN_SOURCES}) target_include_directories(ggml-qnn PRIVATE ${QNN_SDK_PATH}/include/QNN ${HEXAGON_SDK_PATH} ${CMAKE_CURRENT_LIST_DIR}) diff --git a/ggml/src/ggml-qnn/ggml-qnn.cpp b/ggml/src/ggml-qnn/ggml-qnn.cpp index 23fe675aa97f3..70bdc625fe37b 100644 --- a/ggml/src/ggml-qnn/ggml-qnn.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn.cpp @@ -122,7 +122,7 @@ #include "ggml-impl.h" #include "ggml-backend-impl.h" -#include "kernels/ggmlop.h" +#include "kernels/ggmlop_ap_skel.h" // ================================================================================================= // section-1: forward/prototype declaration, global vars, macros, data structures @@ -132,7 +132,7 @@ struct qnn_parameter; struct ggml_backend_qnn_context; typedef void (* ggmlqnn_op_func_t)(ggml_backend_qnn_context * ctx, ggml_tensor * op); -typedef int (* notif_callback_fn)(void * context, int domain, int session, remote_rpc_status_flags_t status); +typedef int (* notify_callback_fn)(void * context, int domain, int session, remote_rpc_status_flags_t status); typedef int (* ggmlhexagon_op_func_t)(remote_handle64 handle, const dsptensor * src0, const dsptensor * src1, dsptensor * dst); static void * ggmlqnn_type_trait(ggml_backend_qnn_context * ctx, ggml_tensor * op); @@ -288,10 +288,15 @@ using qnn_cgraph_node_t = std::tuple; using qnn_multinode_res_t = std::tuple; +enum qnn_index_type { + QNN_TENSOR_INDEX = 0, + QNN_OPCFG_INDEX = 1, +}; + enum qnn_profile_level { PROFILE_OFF = 0, PROFILE_BASIC = 1, - PROFILE_DETAIL = 2 + PROFILE_DETAIL = 2, }; //0: general approach through QNN @@ -861,63 +866,34 @@ static const char * dlerror(void) { // ================================================================================================= // section-4: general helper function // ================================================================================================= -//TODO: merge the following 6 helper functions which used to ensure every QNN tensor/opcfg name is unique -static void ggmlqnn_reset_tensoridx() { +//ensure every QNN tensor/opcfg name is unique +static void ggmlqnn_reset_idx() { g_qnntensor_idx = 0; -} - -static void ggmlqnn_inc_tensoridx() { - g_qnntensor_idx++; -} - -static int32_t ggmlqnn_get_tensoridx() { - return g_qnntensor_idx; -} - -static void ggmlqnn_reset_opcfgidx() { g_qnnopcfg_idx = 0; } -static void ggmlqnn_inc_opcfgidx() { - g_qnnopcfg_idx++; -} - -static int32_t ggmlqnn_get_opcfgidx() { - return g_qnnopcfg_idx; -} - -static void * ggmlqnn_mallocz_aligned(size_t size, size_t alignment) { - uint8_t * buffer = NULL; - size_t * sp = NULL; - buffer = static_cast(calloc(1, size + GGMLQNN_MEM_ADD(alignment))); - if (!buffer) - return NULL; - sp = (size_t *)buffer; - *sp = size; - buffer = (uint8_t *)(((uintptr_t) buffer + GGMLQNN_MEM_ADD(alignment)) & ~GGMLQNN_MEM_MASK(alignment)); - buffer[-1] = buffer - (uint8_t *)sp; - return buffer; -} - -static void * ggmlqnn_malloc_aligned(size_t size, size_t alignment) { - uint8_t * buffer = NULL; - size_t * sp = NULL; - buffer = static_cast(malloc(size + GGMLQNN_MEM_ADD(alignment))); - if (!buffer) - return NULL; - sp = (size_t *)buffer; - *sp = size; - buffer = (uint8_t *)(((uintptr_t) buffer + GGMLQNN_MEM_ADD(alignment)) & ~GGMLQNN_MEM_MASK(alignment)); - buffer[-1] = buffer - (uint8_t *)sp; - return buffer; +static void ggmlqnn_inc_idx(int idx_type) { + switch (idx_type) { + case QNN_TENSOR_INDEX: + g_qnntensor_idx++; + break; + case QNN_OPCFG_INDEX: + g_qnnopcfg_idx++; + break; + default: + break; + } } -static void ggmqnn_free_aligned(void * ptr) { - uint8_t * old = (uint8_t *)ptr; - if (!old) - return; - old -= old[-1]; - free(old); +static int32_t ggmlqnn_get_idx(int idx_type) { + switch (idx_type) { + case QNN_TENSOR_INDEX: + return g_qnntensor_idx; + case QNN_OPCFG_INDEX: + return g_qnnopcfg_idx; + default: + break; + } } static intptr_t ggmlqnn_align_to(size_t alignment, intptr_t offset) { @@ -994,25 +970,6 @@ static char * ggmlqnn_strndup(const char * source, size_t maxlen) { #endif } -static void * ggmlqnn_host_malloc(size_t buffer_size, size_t page_size) { - void * data = nullptr; -#if defined(__ANDROID__) || defined(__linux__) - int result = posix_memalign((void **)&data, page_size, buffer_size); - if (result != 0) { - GGMLQNN_LOG_WARN("%s: error: posix_memalign failed\n", __func__); - return nullptr; - } -#else - //GGMLQNN_LOG_DEBUG("buffer_size %d, page_size %d\n", buffer_size, page_size); - data = ggmlqnn_malloc_aligned(buffer_size, page_size); - if (nullptr == data) { - GGMLQNN_LOG_WARN("%s: error: host_malloc failed\n", __func__); - } -#endif - - return data; -} - static void ggmlqnn_get_timestring(char * p_currenttime) { time_t n_seconds = 0; struct tm * p_tm = nullptr; @@ -1027,6 +984,37 @@ static void ggmlqnn_get_timestring(char * p_currenttime) { p_tm->tm_hour, p_tm->tm_min, p_tm->tm_sec); } +//fix some tricky memory issue +typedef int (*pfn_mallopt)(int, int); +typedef int (*pfn_android_mallopt)(int, void *, size_t); +static void ggmlqnn_disable_android_tags(int disable) { + if (0 == disable) + return; + + void * lib_handle = dlopen("libc.so", RTLD_LAZY); + if (nullptr != lib_handle) { + int api_level = android_get_device_api_level(); + GGMLQNN_LOG_INFO("device_api_level=%d", api_level); + if (api_level >= 31) { //ANDROID 12 + pfn_mallopt mallopt = reinterpret_cast(dlsym(lib_handle, "mallopt")); + if (mallopt) { + mallopt(M_BIONIC_SET_HEAP_TAGGING_LEVEL, M_HEAP_TAGGING_LEVEL_NONE); + } + return; + } else if (api_level >= 30) { //ANDROID 11 + /* android_get_device_api_level() < 31 */ + pfn_android_mallopt android_mallopt = reinterpret_cast(dlsym( + lib_handle, "android_mallopt")); + if (android_mallopt) { + int android_malloc_tag_level = 0; + int tmp = 0; + android_mallopt(8, &tmp, sizeof(tmp)); + } + } + dlclose(lib_handle); + } +} + // ================================================================================================= // section-5: QNN helper function // ================================================================================================= @@ -1359,12 +1347,12 @@ static Qnn_OpConfig_t ggmlqnn_create_op_config(const char * name, const char * p //ensure the opcfg name is unique if (nullptr == name) { - snprintf(opcfg_name, GGML_MAX_NAME, "opcfg_%-8d", ggmlqnn_get_opcfgidx()); + snprintf(opcfg_name, GGML_MAX_NAME, "opcfg_%-8d", ggmlqnn_get_idx(QNN_OPCFG_INDEX)); } else { - snprintf(opcfg_name, GGML_MAX_NAME, "opcfg_%s_%-8d", name, ggmlqnn_get_opcfgidx()); + snprintf(opcfg_name, GGML_MAX_NAME, "opcfg_%s_%-8d", name, ggmlqnn_get_idx(QNN_OPCFG_INDEX)); } GGMLQNN_LOG_DEBUG("create qnn opconfig %s", opcfg_name); - ggmlqnn_inc_opcfgidx(); + ggmlqnn_inc_idx(QNN_OPCFG_INDEX); Qnn_OpConfigV1_t v1 = {opcfg_name, package, type, num_params, params, @@ -1860,7 +1848,7 @@ static int ggmlhexagon_get_hvx_support_info(int domain, uint32_t * capability, u return hexagon_error; } -static int ggmlhexagon_request_status_notifications(int domain_id, void * context, notif_callback_fn call_back_fn) { +static int ggmlhexagon_request_status_notifications(int domain_id, void * context, notify_callback_fn call_back_fn) { int hexagon_error = AEE_SUCCESS; struct remote_rpc_notif_register notif; bool status_notification_support; @@ -2019,17 +2007,8 @@ static int ggmlhexagon_init_dsp(ggml_backend_qnn_context * ctx) { goto bail; } uri = my_domain->uri; - } else { - domain_info = &domains_info[domain_id]; - uri = (char *)malloc(MAX_DOMAIN_NAMELEN); - if (nullptr == uri) { - hexagon_error = AEE_ENOMEMORY; - GGMLQNN_LOG_DEBUG("unable to allocated memory for uri of size: %d", MAX_DOMAIN_NAMELEN); - goto bail; - } - snprintf(uri, MAX_DOMAIN_NAMELEN, "%s%s", "&_dom=", domain_info->name); } - GGMLQNN_LOG_INFO("\ndomain uri=%s\n", uri); + GGMLQNN_LOG_INFO("domain uri=%s\n", uri); if (1 == unsignedpd_flag) { is_unsignedpd_enabled = ggmlhexagon_is_unsignedpd_supported(domain_id); @@ -2078,7 +2057,7 @@ static int ggmlhexagon_init_dsp(ggml_backend_qnn_context * ctx) { ggmlhexagon_set_clocks(ctx->ggmlop_handle, HAP_DCVS_V2_DUTY_CYCLE_MODE, 40, 1); ggmlhexagon_set_rpc_latency(domain_id, RPC_POLL_QOS, 1000); } else { - GGMLQNN_LOG_WARN("error 0x%x: failed to compute on domain %d(%s)", hexagon_error, domain_id, + GGMLQNN_LOG_INFO("error 0x%x: failed to open domain %d(%s)", hexagon_error, domain_id, ggmlhexagon_get_dsp_name(domain_id)); goto bail; } @@ -2089,10 +2068,6 @@ static int ggmlhexagon_init_dsp(ggml_backend_qnn_context * ctx) { free(ggmlop_domain_uri); } - if (uri) { - free(uri); - } - if (ctx->rpc_mempool) { rpcmem_free(ctx->rpc_mempool); ctx->rpc_mempool = nullptr; @@ -2153,27 +2128,54 @@ static void ggmlhexagon_compute(ggml_backend_qnn_context * ctx, struct ggml_tens } if ((GGML_OP_MUL_MAT == op->op) && (src0_type != GGML_TYPE_F32)) { - dsptensor_0.data = static_cast(wdata); - dsptensor_0.dataLen = ctx->desired_size; + dsptensor_0.data = wdata; + dsptensor_0.data_len = ctx->desired_size; } else { - dsptensor_0.data = static_cast(src0->data); - dsptensor_0.dataLen = ggml_nbytes(src0); - } - dsptensor_1.data = static_cast(src1->data); - dsptensor_2.data = static_cast(dst->data); - dsptensor_0.type = GGML_TYPE_F32; - dsptensor_1.type = GGML_TYPE_F32; - dsptensor_2.type = GGML_TYPE_F32; + dsptensor_0.data = src0->data; + dsptensor_0.data_len= ggml_nbytes(src0); + } + + dsptensor_1.data = src1->data; + dsptensor_2.data = dst->data; + dsptensor_0.ne[0] = src0->ne[0]; dsptensor_0.ne[1] = src0->ne[1]; dsptensor_0.ne[2] = src0->ne[2]; dsptensor_0.ne[3] = src0->ne[3]; + dsptensor_0.nb[0] = src0->nb[0]; dsptensor_0.nb[1] = src0->nb[1]; dsptensor_0.nb[2] = src0->nb[2]; dsptensor_0.nb[3] = src0->nb[3]; - dsptensor_1.dataLen = ggml_nbytes(src1); - dsptensor_2.dataLen = ggml_nbytes(dst); + + dsptensor_1.ne[0] = src1->ne[0]; + dsptensor_1.ne[1] = src1->ne[1]; + dsptensor_1.ne[2] = src1->ne[2]; + dsptensor_1.ne[3] = src1->ne[3]; + + dsptensor_1.nb[0] = src1->nb[0]; + dsptensor_1.nb[1] = src1->nb[1]; + dsptensor_1.nb[2] = src1->nb[2]; + dsptensor_1.nb[3] = src1->nb[3]; + + dsptensor_2.ne[0] = dst->ne[0]; + dsptensor_2.ne[1] = dst->ne[1]; + dsptensor_2.ne[2] = dst->ne[2]; + dsptensor_2.ne[3] = dst->ne[3]; + + dsptensor_2.nb[0] = dst->nb[0]; + dsptensor_2.nb[1] = dst->nb[1]; + dsptensor_2.nb[2] = dst->nb[2]; + dsptensor_2.nb[3] = dst->nb[3]; + + dsptensor_0.data_len = ggml_nbytes(src0); + dsptensor_1.data_len = ggml_nbytes(src1); + dsptensor_2.data_len = ggml_nbytes(dst); + + dsptensor_0.type = src0->type; + dsptensor_1.type = src1->type; + dsptensor_2.type = dst->type; + hexagon_error = op_func(ctx->ggmlop_handle, &dsptensor_0, &dsptensor_1, &dsptensor_2); if (AEE_SUCCESS != hexagon_error) { GGMLQNN_LOG_WARN("ggmlop computation fail on cdsp"); @@ -3667,8 +3669,7 @@ int qnn_instance::qnn_finalize() { Qnn_ErrorHandle_t error = QNN_SUCCESS; GGMLQNN_LOG_DEBUG("enter %s\n", __func__); - ggmlqnn_reset_tensoridx(); - ggmlqnn_reset_opcfgidx(); + ggmlqnn_reset_idx(); free_rpcmem(); unregister_rpcmem(); @@ -4192,6 +4193,8 @@ static void ggmlqnn_load_cfg() { memset(time_string, 0, GGML_QNN_TMPBUF_LEN); ggmlqnn_get_timestring(time_string); GGMLQNN_LOG_DEBUG("program running start time:%s", time_string); + ggmlqnn_disable_android_tags(1); + std::string cfg_filename = std::string(g_qnn_params.qnn_runtimelib_path) + std::string(g_qnn_params.qnn_cfgfilename); GGMLQNN_LOG_INFO("load ggml-qnn config from %s", cfg_filename.c_str()); qnn_cfg qnncfg_instance; @@ -4238,12 +4241,12 @@ static Qnn_Tensor_t * ggmlqnn_create_general_tensor(qnn_instance * instance, Qnn //ensure the tensor name is unique if (nullptr == name) { - snprintf(tensor_name, GGML_MAX_NAME, "tensor_%-8d", ggmlqnn_get_tensoridx()); + snprintf(tensor_name, GGML_MAX_NAME, "tensor_%-8d", ggmlqnn_get_idx(QNN_TENSOR_INDEX)); } else { - snprintf(tensor_name, GGML_MAX_NAME, "tensor_%s%-8d", name, ggmlqnn_get_tensoridx()); + snprintf(tensor_name, GGML_MAX_NAME, "tensor_%s%-8d", name, ggmlqnn_get_idx(QNN_TENSOR_INDEX)); } GGMLQNN_LOG_DEBUG("init_tensor %s", tensor_name); - ggmlqnn_inc_tensoridx(); + ggmlqnn_inc_idx(QNN_TENSOR_INDEX); uint32_t reverse_dims[GGML_MAX_DIMS] = {}; uint32_t transpose_dims[GGML_MAX_DIMS] = {}; @@ -4362,20 +4365,46 @@ static bool ggmlqnn_same_types(const ggml_backend_qnn_context * ctx, const ggml_ return false; } } + if (src0->type != GGML_TYPE_F32) return false; + return true; } +static bool ggmlhexagon_can_handle_op(const ggml_backend_qnn_context * ctx, const struct ggml_tensor * op_tensor) { + struct ggml_tensor * src0 = op_tensor->src[0]; + struct ggml_tensor * src1 = op_tensor->src[1]; + + const int64_t ne00 = op_tensor->src[0]->ne[0]; + uint32_t src0_rank = ggml_n_dims(src0); + uint32_t src1_rank = 0; + if (nullptr != src1) { + src1_rank = ggml_n_dims(src1); + } + + //FIXME: mulmat on cDSP doesn't work as expected + if (op_tensor->op != GGML_OP_ADD) + return false; + + //ggmlqnn_dump_op_info(op_tensor); + if (!ggml_are_same_shape(src0, src1)) { + return false; + } + + if (ne00 < 32) + return false; + + return ggmlqnn_same_types(ctx, op_tensor); +} + static bool ggmlqnn_can_handle_op(const ggml_backend_qnn_context * ctx, const struct ggml_tensor * op_tensor) { if (op_tensor->op == GGML_OP_NONE) { return true; } if (DIRECT_USE_CDSP == g_qnn_params.inference_approach) { - //FIXME: mulmat on cDSP doesn't work as expected - if (op_tensor->op != GGML_OP_ADD) - return false; + return ggmlhexagon_can_handle_op(ctx, op_tensor); } if (!ggmlqnn_k_op_caps[ggmlqnn_get_op_index(op_tensor)].supported) { @@ -4384,25 +4413,17 @@ static bool ggmlqnn_can_handle_op(const ggml_backend_qnn_context * ctx, const st struct ggml_tensor * src0 = op_tensor->src[0]; struct ggml_tensor * src1 = op_tensor->src[1]; - const int64_t ne00 = op_tensor->src[0]->ne[0]; - const int64_t ne01 = op_tensor->src[0]->ne[1]; - const int64_t ne0 = op_tensor->ne[0]; - const int64_t ne1 = op_tensor->ne[1]; - uint32_t src0_rank = ggml_n_dims(src0); uint32_t src1_rank = 0; if (nullptr != src1) { src1_rank = ggml_n_dims(src1); } - GGML_UNUSED(ne01); - GGML_UNUSED(ne0); - GGML_UNUSED(ne1); + switch (op_tensor->op) { case GGML_OP_ADD: case GGML_OP_SUB: { - //ggmlqnn_dump_op_info(op_tensor); if (!ggml_are_same_shape(src0, src1)) { return false; } @@ -4415,7 +4436,6 @@ static bool ggmlqnn_can_handle_op(const ggml_backend_qnn_context * ctx, const st case GGML_OP_DIV: case GGML_OP_MUL: { - //ggmlqnn_dump_op_info(op_tensor); if (ctx->device == QNN_BACKEND_NPU) return false; @@ -4597,7 +4617,7 @@ static bool ggmlqnn_compute_forward(ggml_backend_t backend, struct ggml_tensor * struct ggml_backend_qnn_buffer_context { ~ggml_backend_qnn_buffer_context() { if (buffer) { - free(buffer); + ggml_aligned_free(buffer, 0); } for (auto * sub_buffer : sub_buffers) { @@ -4709,7 +4729,7 @@ static ggml_backend_buffer_t ggml_backend_qnn_buffer_type_alloc_buffer( if ((size_aligned % size_page) != 0) { size_aligned += (size_page - (size_aligned % size_page)); } - ctx->buffer = ggmlqnn_host_malloc(size_aligned, size_page); + ctx->buffer = ggml_aligned_malloc(size_aligned); ctx->buffer_size = size_aligned; if (nullptr == ctx->buffer) { GGMLQNN_LOG_WARN("%s: failed to allocate %d MiB\n", __func__, size / (1 << 20)); diff --git a/ggml/src/ggml-qnn/kernels/ggmlop_stub.c b/ggml/src/ggml-qnn/kernels/ggmlop_ap_skel.c similarity index 70% rename from ggml/src/ggml-qnn/kernels/ggmlop_stub.c rename to ggml/src/ggml-qnn/kernels/ggmlop_ap_skel.c index 6313348d7ea2d..1e1ce6488d25e 100644 --- a/ggml/src/ggml-qnn/kernels/ggmlop_stub.c +++ b/ggml/src/ggml-qnn/kernels/ggmlop_ap_skel.c @@ -1,10 +1,3 @@ -#ifndef _GGMLOP_STUB_H -#define _GGMLOP_STUB_H -/// @file ggmlop.idl -/// -//qidl copyright -//qidl nested=false -#include "ggmlop.h" #include #ifndef _WIN32 #include "HAP_farf.h" @@ -15,6 +8,7 @@ #include #include +#include "ggmlop_ap_skel.h" typedef struct _heap _heap; struct _heap { @@ -277,16 +271,16 @@ struct Interface { #endif static const Type types[5]; -static const Type* const typeArrays[5] = {&(types[0]),&(types[0]),&(types[2]),&(types[2]),&(types[3])}; -static const StructType structTypes[1] = {{0x5,&(typeArrays[0]),0x50,0x4,0x48,0x8,0x4,0x8}}; -static const Type types[5] = {{0x20,{{(const uintptr_t)&(types[1]),(const uintptr_t)0x4}}, 8,0x8},{0x8,{{(const uintptr_t)0,(const uintptr_t)1}}, 2,0x8},{0x4,{{(const uintptr_t)0,(const uintptr_t)1}}, 2,0x4},{SLIM_IFPTR32(0x8,0x10),{{(const uintptr_t)&(types[4]),(const uintptr_t)0x0}}, 9,SLIM_IFPTR32(0x4,0x8)},{0x4,{{(const uintptr_t)0,(const uintptr_t)1}}, 2,0x4}}; -static const Parameter parameters[5] = {{SLIM_IFPTR32(0x8,0x10),{{(const uintptr_t)0x0,0}}, 4,SLIM_IFPTR32(0x4,0x8),0,0},{SLIM_IFPTR32(0x4,0x8),{{(const uintptr_t)0xdeadc0de,(const uintptr_t)0}}, 0,SLIM_IFPTR32(0x4,0x8),3,0},{SLIM_IFPTR32(0x4,0x8),{{(const uintptr_t)0xdeadc0de,(const uintptr_t)0}}, 0,SLIM_IFPTR32(0x4,0x8),0,0},{SLIM_IFPTR32(0x50,0x58),{{(const uintptr_t)&(structTypes[0]),0}}, 22,0x8,0,0},{SLIM_IFPTR32(0x50,0x58),{{(const uintptr_t)&(structTypes[0]),0}}, 22,0x8,3,0}}; +static const Type* const typeArrays[6] = {&(types[0]),&(types[1]),&(types[1]),&(types[0]),&(types[0]),&(types[3])}; +static const StructType structTypes[1] = {{0x6,&(typeArrays[0]),0x58,0x4,0x50,0x8,0x4,0x8}}; +static const Type types[5] = {{0x4,{{(const uintptr_t)0,(const uintptr_t)1}}, 2,0x4},{0x20,{{(const uintptr_t)&(types[2]),(const uintptr_t)0x4}}, 8,0x8},{0x8,{{(const uintptr_t)0,(const uintptr_t)1}}, 2,0x8},{SLIM_IFPTR32(0x8,0x10),{{(const uintptr_t)&(types[4]),(const uintptr_t)0x0}}, 9,SLIM_IFPTR32(0x4,0x8)},{0x4,{{(const uintptr_t)0,(const uintptr_t)1}}, 2,0x4}}; +static const Parameter parameters[5] = {{SLIM_IFPTR32(0x8,0x10),{{(const uintptr_t)0x0,0}}, 4,SLIM_IFPTR32(0x4,0x8),0,0},{SLIM_IFPTR32(0x4,0x8),{{(const uintptr_t)0xdeadc0de,(const uintptr_t)0}}, 0,SLIM_IFPTR32(0x4,0x8),3,0},{SLIM_IFPTR32(0x4,0x8),{{(const uintptr_t)0xdeadc0de,(const uintptr_t)0}}, 0,SLIM_IFPTR32(0x4,0x8),0,0},{SLIM_IFPTR32(0x58,0x60),{{(const uintptr_t)&(structTypes[0]),0}}, 22,0x8,0,0},{SLIM_IFPTR32(0x58,0x60),{{(const uintptr_t)&(structTypes[0]),0}}, 22,0x8,3,0}}; static const Parameter* const parameterArrays[6] = {(&(parameters[3])),(&(parameters[3])),(&(parameters[4])),(&(parameters[0])),(&(parameters[1])),(&(parameters[2]))}; -static const Method methods[3] = {{REMOTE_SCALARS_MAKEX(0,0,0x2,0x0,0x0,0x1),0x4,0x0,2,2,(&(parameterArrays[3])),0x4,0x1},{REMOTE_SCALARS_MAKEX(0,0,0x0,0x0,0x1,0x0),0x0,0x0,1,1,(&(parameterArrays[5])),0x1,0x0},{REMOTE_SCALARS_MAKEX(0,0,0x3,0x2,0x0,0x0),0xa4,0x48,3,3,(&(parameterArrays[0])),0x8,0x8}}; +static const Method methods[3] = {{REMOTE_SCALARS_MAKEX(0,0,0x2,0x0,0x0,0x1),0x4,0x0,2,2,(&(parameterArrays[3])),0x4,0x1},{REMOTE_SCALARS_MAKEX(0,0,0x0,0x0,0x1,0x0),0x0,0x0,1,1,(&(parameterArrays[5])),0x1,0x0},{REMOTE_SCALARS_MAKEX(0,0,0x3,0x2,0x0,0x0),0xb4,0x50,3,3,(&(parameterArrays[0])),0x8,0x8}}; static const Method* const methodArrays[4] = {&(methods[0]),&(methods[1]),&(methods[2]),&(methods[2])}; -static const char strings[65] = "mulmat\0flags\0close\0src1\0data\0type\0src0\0open\0dst\0add\0uri\0nb\0ne\0h\0"; -static const uint16_t methodStrings[43] = {0,34,59,56,7,29,24,19,59,56,7,29,24,44,59,56,7,29,24,48,34,59,56,7,29,24,19,59,56,7,29,24,44,59,56,7,29,24,39,52,62,13,62}; -static const uint16_t methodStringsArrays[4] = {38,41,19,0}; +static const char strings[68] = "mulmat\0flags\0close\0src1\0data\0type\0src0\0open\0dst\0add\0uri\0op\0nb\0ne\0h\0"; +static const uint16_t methodStrings[49] = {0,34,29,62,59,56,7,24,19,29,62,59,56,7,24,44,29,62,59,56,7,24,48,34,29,62,59,56,7,24,19,29,62,59,56,7,24,44,29,62,59,56,7,24,39,52,65,13,65}; +static const uint16_t methodStringsArrays[4] = {44,47,22,0}; __QAIC_SLIM_EXPORT const Interface __QAIC_SLIM(ggmlop_slim) = {4,&(methodArrays[0]),0,0,&(methodStringsArrays [0]),methodStrings,strings}; #endif //_GGMLOP_SLIM_H @@ -300,19 +294,20 @@ __QAIC_STUB_EXPORT int __QAIC_STUB(ggmlop_open)(const char* uri, remote_handle64 __QAIC_STUB_EXPORT int __QAIC_STUB(ggmlop_close)(remote_handle64 h) __QAIC_STUB_ATTRIBUTE { return __QAIC_REMOTE(remote_handle64_close)(h); } -static __inline int _stub_unpack(_ATTRIBUTE_UNUSED remote_arg* _praROutPost, _ATTRIBUTE_UNUSED remote_arg* _ppraROutPost[1], _ATTRIBUTE_UNUSED void* _primROut, _ATTRIBUTE_UNUSED uint64_t _rout0[4], _ATTRIBUTE_UNUSED uint64_t _rout1[4], _ATTRIBUTE_UNUSED uint32_t _rout2[1], _ATTRIBUTE_UNUSED uint32_t _rout3[1], _ATTRIBUTE_UNUSED char* _rout4[1], _ATTRIBUTE_UNUSED uint32_t _rout4Len[1]) { +static __inline int _stub_unpack(_ATTRIBUTE_UNUSED remote_arg* _praROutPost, _ATTRIBUTE_UNUSED remote_arg* _ppraROutPost[1], _ATTRIBUTE_UNUSED void* _primROut, _ATTRIBUTE_UNUSED uint32_t _rout0[1], _ATTRIBUTE_UNUSED uint64_t _rout1[4], _ATTRIBUTE_UNUSED uint64_t _rout2[4], _ATTRIBUTE_UNUSED uint32_t _rout3[1], _ATTRIBUTE_UNUSED uint32_t _rout4[1], _ATTRIBUTE_UNUSED char* _rout5[1], _ATTRIBUTE_UNUSED uint32_t _rout5Len[1]) { int _nErr = 0; remote_arg* _praROutPostStart = _praROutPost; remote_arg** _ppraROutPostStart = _ppraROutPost; _ppraROutPost = &_praROutPost; - _COPY(_rout0, 0, _primROut, 0, 32); - _COPY(_rout1, 0, _primROut, 32, 32); - _COPY(_rout2, 0, _primROut, 64, 4); - _COPY(_rout3, 0, _primROut, 68, 4); + _COPY(_rout0, 0, _primROut, 0, 4); + _COPY(_rout1, 0, _primROut, 8, 32); + _COPY(_rout2, 0, _primROut, 40, 32); + _COPY(_rout3, 0, _primROut, 72, 4); + _COPY(_rout4, 0, _primROut, 76, 4); _ppraROutPostStart[0] += (_praROutPost - _praROutPostStart) +1; return _nErr; } -static __inline int _stub_pack(_ATTRIBUTE_UNUSED _allocator* _al, _ATTRIBUTE_UNUSED remote_arg* _praIn, _ATTRIBUTE_UNUSED remote_arg* _ppraIn[1], _ATTRIBUTE_UNUSED remote_arg* _praROut, _ATTRIBUTE_UNUSED remote_arg* _ppraROut[1], _ATTRIBUTE_UNUSED remote_arg* _praHIn, _ATTRIBUTE_UNUSED remote_arg* _ppraHIn[1], _ATTRIBUTE_UNUSED remote_arg* _praHROut, _ATTRIBUTE_UNUSED remote_arg* _ppraHROut[1], _ATTRIBUTE_UNUSED void* _primIn, _ATTRIBUTE_UNUSED void* _primROut, _ATTRIBUTE_UNUSED uint64_t _rout0[4], _ATTRIBUTE_UNUSED uint64_t _rout1[4], _ATTRIBUTE_UNUSED uint32_t _rout2[1], _ATTRIBUTE_UNUSED uint32_t _rout3[1], _ATTRIBUTE_UNUSED char* _rout4[1], _ATTRIBUTE_UNUSED uint32_t _rout4Len[1]) { +static __inline int _stub_pack(_ATTRIBUTE_UNUSED _allocator* _al, _ATTRIBUTE_UNUSED remote_arg* _praIn, _ATTRIBUTE_UNUSED remote_arg* _ppraIn[1], _ATTRIBUTE_UNUSED remote_arg* _praROut, _ATTRIBUTE_UNUSED remote_arg* _ppraROut[1], _ATTRIBUTE_UNUSED remote_arg* _praHIn, _ATTRIBUTE_UNUSED remote_arg* _ppraHIn[1], _ATTRIBUTE_UNUSED remote_arg* _praHROut, _ATTRIBUTE_UNUSED remote_arg* _ppraHROut[1], _ATTRIBUTE_UNUSED void* _primIn, _ATTRIBUTE_UNUSED void* _primROut, _ATTRIBUTE_UNUSED uint32_t _rout0[1], _ATTRIBUTE_UNUSED uint64_t _rout1[4], _ATTRIBUTE_UNUSED uint64_t _rout2[4], _ATTRIBUTE_UNUSED uint32_t _rout3[1], _ATTRIBUTE_UNUSED uint32_t _rout4[1], _ATTRIBUTE_UNUSED char* _rout5[1], _ATTRIBUTE_UNUSED uint32_t _rout5Len[1]) { int _nErr = 0; remote_arg* _praInStart = _praIn; remote_arg** _ppraInStart = _ppraIn; @@ -320,14 +315,14 @@ static __inline int _stub_pack(_ATTRIBUTE_UNUSED _allocator* _al, _ATTRIBUTE_UNU remote_arg** _ppraROutStart = _ppraROut; _ppraIn = &_praIn; _ppraROut = &_praROut; - _COPY(_primIn, 0, _rout4Len, 0, 4); - _praROut[0].buf.pv = _rout4[0]; - _praROut[0].buf.nLen = (4 * _rout4Len[0]); + _COPY(_primIn, 0, _rout5Len, 0, 4); + _praROut[0].buf.pv = _rout5[0]; + _praROut[0].buf.nLen = (4 * _rout5Len[0]); _ppraInStart[0] += (_praIn - _praInStart) + 0; _ppraROutStart[0] += (_praROut - _praROutStart) +1; return _nErr; } -static __inline int _stub_pack_1(_ATTRIBUTE_UNUSED _allocator* _al, _ATTRIBUTE_UNUSED remote_arg* _praIn, _ATTRIBUTE_UNUSED remote_arg* _ppraIn[1], _ATTRIBUTE_UNUSED remote_arg* _praROut, _ATTRIBUTE_UNUSED remote_arg* _ppraROut[1], _ATTRIBUTE_UNUSED remote_arg* _praHIn, _ATTRIBUTE_UNUSED remote_arg* _ppraHIn[1], _ATTRIBUTE_UNUSED remote_arg* _praHROut, _ATTRIBUTE_UNUSED remote_arg* _ppraHROut[1], _ATTRIBUTE_UNUSED void* _primIn, _ATTRIBUTE_UNUSED void* _primROut, _ATTRIBUTE_UNUSED uint64_t _in0[4], _ATTRIBUTE_UNUSED uint64_t _in1[4], _ATTRIBUTE_UNUSED uint32_t _in2[1], _ATTRIBUTE_UNUSED uint32_t _in3[1], _ATTRIBUTE_UNUSED char* _in4[1], _ATTRIBUTE_UNUSED uint32_t _in4Len[1]) { +static __inline int _stub_pack_1(_ATTRIBUTE_UNUSED _allocator* _al, _ATTRIBUTE_UNUSED remote_arg* _praIn, _ATTRIBUTE_UNUSED remote_arg* _ppraIn[1], _ATTRIBUTE_UNUSED remote_arg* _praROut, _ATTRIBUTE_UNUSED remote_arg* _ppraROut[1], _ATTRIBUTE_UNUSED remote_arg* _praHIn, _ATTRIBUTE_UNUSED remote_arg* _ppraHIn[1], _ATTRIBUTE_UNUSED remote_arg* _praHROut, _ATTRIBUTE_UNUSED remote_arg* _ppraHROut[1], _ATTRIBUTE_UNUSED void* _primIn, _ATTRIBUTE_UNUSED void* _primROut, _ATTRIBUTE_UNUSED uint32_t _in0[1], _ATTRIBUTE_UNUSED uint64_t _in1[4], _ATTRIBUTE_UNUSED uint64_t _in2[4], _ATTRIBUTE_UNUSED uint32_t _in3[1], _ATTRIBUTE_UNUSED uint32_t _in4[1], _ATTRIBUTE_UNUSED char* _in5[1], _ATTRIBUTE_UNUSED uint32_t _in5Len[1]) { int _nErr = 0; remote_arg* _praInStart = _praIn; remote_arg** _ppraInStart = _ppraIn; @@ -335,38 +330,39 @@ static __inline int _stub_pack_1(_ATTRIBUTE_UNUSED _allocator* _al, _ATTRIBUTE_U remote_arg** _ppraROutStart = _ppraROut; _ppraIn = &_praIn; _ppraROut = &_praROut; - _COPY(_primIn, 0, _in0, 0, 32); - _COPY(_primIn, 32, _in1, 0, 32); - _COPY(_primIn, 64, _in2, 0, 4); - _COPY(_primIn, 68, _in3, 0, 4); - _COPY(_primIn, 72, _in4Len, 0, 4); - _praIn[0].buf.pv = (void*) _in4[0]; - _praIn[0].buf.nLen = (4 * _in4Len[0]); + _COPY(_primIn, 0, _in0, 0, 4); + _COPY(_primIn, 8, _in1, 0, 32); + _COPY(_primIn, 40, _in2, 0, 32); + _COPY(_primIn, 72, _in3, 0, 4); + _COPY(_primIn, 76, _in4, 0, 4); + _COPY(_primIn, 80, _in5Len, 0, 4); + _praIn[0].buf.pv = (void*) _in5[0]; + _praIn[0].buf.nLen = (4 * _in5Len[0]); _ppraInStart[0] += (_praIn - _praInStart) + 1; _ppraROutStart[0] += (_praROut - _praROutStart) +0; return _nErr; } -static __inline void _count(int _numIn[1], int _numROut[1], int _numInH[1], int _numROutH[1], _ATTRIBUTE_UNUSED uint64_t _rout0[4], _ATTRIBUTE_UNUSED uint64_t _rout1[4], _ATTRIBUTE_UNUSED uint32_t _rout2[1], _ATTRIBUTE_UNUSED uint32_t _rout3[1], _ATTRIBUTE_UNUSED char* _rout4[1], _ATTRIBUTE_UNUSED uint32_t _rout4Len[1]) { +static __inline void _count(int _numIn[1], int _numROut[1], int _numInH[1], int _numROutH[1], _ATTRIBUTE_UNUSED uint32_t _rout0[1], _ATTRIBUTE_UNUSED uint64_t _rout1[4], _ATTRIBUTE_UNUSED uint64_t _rout2[4], _ATTRIBUTE_UNUSED uint32_t _rout3[1], _ATTRIBUTE_UNUSED uint32_t _rout4[1], _ATTRIBUTE_UNUSED char* _rout5[1], _ATTRIBUTE_UNUSED uint32_t _rout5Len[1]) { _numIn[0] += 0; _numROut[0] += 1; _numInH[0] += 0; _numROutH[0] += 0; } -static __inline void _count_1(int _numIn[1], int _numROut[1], int _numInH[1], int _numROutH[1], _ATTRIBUTE_UNUSED uint64_t _in0[4], _ATTRIBUTE_UNUSED uint64_t _in1[4], _ATTRIBUTE_UNUSED uint32_t _in2[1], _ATTRIBUTE_UNUSED uint32_t _in3[1], _ATTRIBUTE_UNUSED char* _in4[1], _ATTRIBUTE_UNUSED uint32_t _in4Len[1]) { +static __inline void _count_1(int _numIn[1], int _numROut[1], int _numInH[1], int _numROutH[1], _ATTRIBUTE_UNUSED uint32_t _in0[1], _ATTRIBUTE_UNUSED uint64_t _in1[4], _ATTRIBUTE_UNUSED uint64_t _in2[4], _ATTRIBUTE_UNUSED uint32_t _in3[1], _ATTRIBUTE_UNUSED uint32_t _in4[1], _ATTRIBUTE_UNUSED char* _in5[1], _ATTRIBUTE_UNUSED uint32_t _in5Len[1]) { _numIn[0] += 1; _numROut[0] += 0; _numInH[0] += 0; _numROutH[0] += 0; } -static __inline int _stub_method(remote_handle64 _handle, uint32_t _mid, uint64_t _in0[SLIM_IFPTR32(10, 11)], uint64_t _in1[SLIM_IFPTR32(10, 11)], uint64_t _rout2[SLIM_IFPTR32(10, 11)]) { +static __inline int _stub_method(remote_handle64 _handle, uint32_t _mid, uint64_t _in0[SLIM_IFPTR32(11, 12)], uint64_t _in1[SLIM_IFPTR32(11, 12)], uint64_t _rout2[SLIM_IFPTR32(11, 12)]) { remote_arg* _pra = 0; int _numIn[1] = {0}; int _numROut[1] = {0}; int _numInH[1] = {0}; int _numROutH[1] = {0}; _allocator _al[1] = {{0}}; - uint64_t _primIn[21]= {0}; - uint64_t _primROut[9]= {0}; + uint64_t _primIn[23]= {0}; + uint64_t _primROut[10]= {0}; remote_arg* _praIn = 0; remote_arg* _praROut = 0; remote_arg* _praROutPost = 0; @@ -382,9 +378,9 @@ static __inline int _stub_method(remote_handle64 _handle, uint32_t _mid, uint64_ _numROut[0] = 0; _numInH[0] = 0; _numROutH[0] = 0; - _count_1(_numIn, _numROut, _numInH, _numROutH, (uint64_t*)&(((uint64_t*)_in0)[0]), (uint64_t*)&(((uint64_t*)_in0)[4]), (uint32_t*)&(((uint32_t*)_in0)[16]), (uint32_t*)&(((uint32_t*)_in0)[17]), SLIM_IFPTR32((char**)&(((uint32_t*)_in0)[18]), (char**)&(((uint64_t*)_in0)[9])), SLIM_IFPTR32((uint32_t*)&(((uint32_t*)_in0)[19]), (uint32_t*)&(((uint32_t*)_in0)[20]))); - _count_1(_numIn, _numROut, _numInH, _numROutH, (uint64_t*)&(((uint64_t*)_in1)[0]), (uint64_t*)&(((uint64_t*)_in1)[4]), (uint32_t*)&(((uint32_t*)_in1)[16]), (uint32_t*)&(((uint32_t*)_in1)[17]), SLIM_IFPTR32((char**)&(((uint32_t*)_in1)[18]), (char**)&(((uint64_t*)_in1)[9])), SLIM_IFPTR32((uint32_t*)&(((uint32_t*)_in1)[19]), (uint32_t*)&(((uint32_t*)_in1)[20]))); - _count(_numIn, _numROut, _numInH, _numROutH, (uint64_t*)&(((uint64_t*)_rout2)[0]), (uint64_t*)&(((uint64_t*)_rout2)[4]), (uint32_t*)&(((uint32_t*)_rout2)[16]), (uint32_t*)&(((uint32_t*)_rout2)[17]), SLIM_IFPTR32((char**)&(((uint32_t*)_rout2)[18]), (char**)&(((uint64_t*)_rout2)[9])), SLIM_IFPTR32((uint32_t*)&(((uint32_t*)_rout2)[19]), (uint32_t*)&(((uint32_t*)_rout2)[20]))); + _count_1(_numIn, _numROut, _numInH, _numROutH, (uint32_t*)&(((uint32_t*)_in0)[0]), (uint64_t*)&(((uint64_t*)_in0)[1]), (uint64_t*)&(((uint64_t*)_in0)[5]), (uint32_t*)&(((uint32_t*)_in0)[18]), (uint32_t*)&(((uint32_t*)_in0)[19]), SLIM_IFPTR32((char**)&(((uint32_t*)_in0)[20]), (char**)&(((uint64_t*)_in0)[10])), SLIM_IFPTR32((uint32_t*)&(((uint32_t*)_in0)[21]), (uint32_t*)&(((uint32_t*)_in0)[22]))); + _count_1(_numIn, _numROut, _numInH, _numROutH, (uint32_t*)&(((uint32_t*)_in1)[0]), (uint64_t*)&(((uint64_t*)_in1)[1]), (uint64_t*)&(((uint64_t*)_in1)[5]), (uint32_t*)&(((uint32_t*)_in1)[18]), (uint32_t*)&(((uint32_t*)_in1)[19]), SLIM_IFPTR32((char**)&(((uint32_t*)_in1)[20]), (char**)&(((uint64_t*)_in1)[10])), SLIM_IFPTR32((uint32_t*)&(((uint32_t*)_in1)[21]), (uint32_t*)&(((uint32_t*)_in1)[22]))); + _count(_numIn, _numROut, _numInH, _numROutH, (uint32_t*)&(((uint32_t*)_rout2)[0]), (uint64_t*)&(((uint64_t*)_rout2)[1]), (uint64_t*)&(((uint64_t*)_rout2)[5]), (uint32_t*)&(((uint32_t*)_rout2)[18]), (uint32_t*)&(((uint32_t*)_rout2)[19]), SLIM_IFPTR32((char**)&(((uint32_t*)_rout2)[20]), (char**)&(((uint64_t*)_rout2)[10])), SLIM_IFPTR32((uint32_t*)&(((uint32_t*)_rout2)[21]), (uint32_t*)&(((uint32_t*)_rout2)[22]))); if(_numIn[0]>=255){ _QAIC_FARF(RUNTIME_ERROR, "ERROR: Unsupported number of input buffers\n"); return AEE_EUNSUPPORTED; @@ -409,13 +405,13 @@ static __inline int _stub_method(remote_handle64 _handle, uint32_t _mid, uint64_ } if(_praHROut == 0) (_praHROut = _praHIn + _numInH[0] + 0); - _TRY(_nErr, _stub_pack_1(_al, (_praIn + 0), _ppraIn, (_praROut + 0), _ppraROut, _praHIn, _ppraHIn, _praHROut, _ppraHROut, ((char*)_primIn + 0), 0, (uint64_t*)&(((uint64_t*)_in0)[0]), (uint64_t*)&(((uint64_t*)_in0)[4]), (uint32_t*)&(((uint32_t*)_in0)[16]), (uint32_t*)&(((uint32_t*)_in0)[17]), SLIM_IFPTR32((char**)&(((uint32_t*)_in0)[18]), (char**)&(((uint64_t*)_in0)[9])), SLIM_IFPTR32((uint32_t*)&(((uint32_t*)_in0)[19]), (uint32_t*)&(((uint32_t*)_in0)[20])))); - _TRY(_nErr, _stub_pack_1(_al, (_praIn + 0), _ppraIn, (_praROut + 0), _ppraROut, _praHIn, _ppraHIn, _praHROut, _ppraHROut, ((char*)_primIn + 80), 0, (uint64_t*)&(((uint64_t*)_in1)[0]), (uint64_t*)&(((uint64_t*)_in1)[4]), (uint32_t*)&(((uint32_t*)_in1)[16]), (uint32_t*)&(((uint32_t*)_in1)[17]), SLIM_IFPTR32((char**)&(((uint32_t*)_in1)[18]), (char**)&(((uint64_t*)_in1)[9])), SLIM_IFPTR32((uint32_t*)&(((uint32_t*)_in1)[19]), (uint32_t*)&(((uint32_t*)_in1)[20])))); - _TRY(_nErr, _stub_pack(_al, (_praIn + 0), _ppraIn, (_praROut + 0), _ppraROut, _praHIn, _ppraHIn, _praHROut, _ppraHROut, ((char*)_primIn + 160), ((char*)_primROut + 0), (uint64_t*)&(((uint64_t*)_rout2)[0]), (uint64_t*)&(((uint64_t*)_rout2)[4]), (uint32_t*)&(((uint32_t*)_rout2)[16]), (uint32_t*)&(((uint32_t*)_rout2)[17]), SLIM_IFPTR32((char**)&(((uint32_t*)_rout2)[18]), (char**)&(((uint64_t*)_rout2)[9])), SLIM_IFPTR32((uint32_t*)&(((uint32_t*)_rout2)[19]), (uint32_t*)&(((uint32_t*)_rout2)[20])))); + _TRY(_nErr, _stub_pack_1(_al, (_praIn + 0), _ppraIn, (_praROut + 0), _ppraROut, _praHIn, _ppraHIn, _praHROut, _ppraHROut, ((char*)_primIn + 0), 0, (uint32_t*)&(((uint32_t*)_in0)[0]), (uint64_t*)&(((uint64_t*)_in0)[1]), (uint64_t*)&(((uint64_t*)_in0)[5]), (uint32_t*)&(((uint32_t*)_in0)[18]), (uint32_t*)&(((uint32_t*)_in0)[19]), SLIM_IFPTR32((char**)&(((uint32_t*)_in0)[20]), (char**)&(((uint64_t*)_in0)[10])), SLIM_IFPTR32((uint32_t*)&(((uint32_t*)_in0)[21]), (uint32_t*)&(((uint32_t*)_in0)[22])))); + _TRY(_nErr, _stub_pack_1(_al, (_praIn + 0), _ppraIn, (_praROut + 0), _ppraROut, _praHIn, _ppraHIn, _praHROut, _ppraHROut, ((char*)_primIn + 88), 0, (uint32_t*)&(((uint32_t*)_in1)[0]), (uint64_t*)&(((uint64_t*)_in1)[1]), (uint64_t*)&(((uint64_t*)_in1)[5]), (uint32_t*)&(((uint32_t*)_in1)[18]), (uint32_t*)&(((uint32_t*)_in1)[19]), SLIM_IFPTR32((char**)&(((uint32_t*)_in1)[20]), (char**)&(((uint64_t*)_in1)[10])), SLIM_IFPTR32((uint32_t*)&(((uint32_t*)_in1)[21]), (uint32_t*)&(((uint32_t*)_in1)[22])))); + _TRY(_nErr, _stub_pack(_al, (_praIn + 0), _ppraIn, (_praROut + 0), _ppraROut, _praHIn, _ppraHIn, _praHROut, _ppraHROut, ((char*)_primIn + 176), ((char*)_primROut + 0), (uint32_t*)&(((uint32_t*)_rout2)[0]), (uint64_t*)&(((uint64_t*)_rout2)[1]), (uint64_t*)&(((uint64_t*)_rout2)[5]), (uint32_t*)&(((uint32_t*)_rout2)[18]), (uint32_t*)&(((uint32_t*)_rout2)[19]), SLIM_IFPTR32((char**)&(((uint32_t*)_rout2)[20]), (char**)&(((uint64_t*)_rout2)[10])), SLIM_IFPTR32((uint32_t*)&(((uint32_t*)_rout2)[21]), (uint32_t*)&(((uint32_t*)_rout2)[22])))); _QAIC_ASSERT(_nErr, (_numInH[0] + 0) <= 15); _QAIC_ASSERT(_nErr, (_numROutH[0] + 0) <= 15); _TRY_FARF(_nErr, __QAIC_REMOTE(remote_handle64_invoke)(_handle, REMOTE_SCALARS_MAKEX(0, _mid, (_numIn[0] + 1), (_numROut[0] + 1), (_numInH[0] + 0), (_numROutH[0] + 0)), _pra)); - _TRY(_nErr, _stub_unpack((_praROutPost + 0), _ppraROutPost, ((char*)_primROut + 0), (uint64_t*)&(((uint64_t*)_rout2)[0]), (uint64_t*)&(((uint64_t*)_rout2)[4]), (uint32_t*)&(((uint32_t*)_rout2)[16]), (uint32_t*)&(((uint32_t*)_rout2)[17]), SLIM_IFPTR32((char**)&(((uint32_t*)_rout2)[18]), (char**)&(((uint64_t*)_rout2)[9])), SLIM_IFPTR32((uint32_t*)&(((uint32_t*)_rout2)[19]), (uint32_t*)&(((uint32_t*)_rout2)[20])))); + _TRY(_nErr, _stub_unpack((_praROutPost + 0), _ppraROutPost, ((char*)_primROut + 0), (uint32_t*)&(((uint32_t*)_rout2)[0]), (uint64_t*)&(((uint64_t*)_rout2)[1]), (uint64_t*)&(((uint64_t*)_rout2)[5]), (uint32_t*)&(((uint32_t*)_rout2)[18]), (uint32_t*)&(((uint32_t*)_rout2)[19]), SLIM_IFPTR32((char**)&(((uint32_t*)_rout2)[20]), (char**)&(((uint64_t*)_rout2)[10])), SLIM_IFPTR32((uint32_t*)&(((uint32_t*)_rout2)[21]), (uint32_t*)&(((uint32_t*)_rout2)[22])))); _QAIC_CATCH(_nErr) {} _CATCH_FARF(_nErr) { _QAIC_FARF(RUNTIME_ERROR, "ERROR 0x%x: handle=0x%"PRIx64", scalar=0x%x, method ID=%d: %s failed\n", _nErr , _handle, REMOTE_SCALARS_MAKEX(0, _mid, (_numIn[0] + 1), (_numROut[0] + 1), (_numInH[0] + 0), (_numROutH[0] + 0)), _mid, __func__); @@ -431,7 +427,3 @@ __QAIC_STUB_EXPORT int __QAIC_STUB(ggmlop_mulmat)(remote_handle64 _handle, const uint32_t _mid = 3; return _stub_method(_handle, _mid, (uint64_t*)src0, (uint64_t*)src1, (uint64_t*)dst); } -#ifdef __cplusplus -} -#endif -#endif //_GGMLOP_STUB_H diff --git a/ggml/src/ggml-qnn/kernels/ggmlop.h b/ggml/src/ggml-qnn/kernels/ggmlop_ap_skel.h similarity index 95% rename from ggml/src/ggml-qnn/kernels/ggmlop.h rename to ggml/src/ggml-qnn/kernels/ggmlop_ap_skel.h index b45070c20001b..0301f8f78f8d2 100644 --- a/ggml/src/ggml-qnn/kernels/ggmlop.h +++ b/ggml/src/ggml-qnn/kernels/ggmlop_ap_skel.h @@ -1,14 +1,12 @@ -#ifndef _GGMLOP_H -#define _GGMLOP_H -/// @file ggmlop.idl -/// -//qidl copyright -//qidl nested=false +#ifndef _GGMLOP_AP_SKEL_H +#define _GGMLOP_AP_SKEL_H + #include #include #include #include + #ifndef __QAIC_HEADER #define __QAIC_HEADER(ff) ff #endif //__QAIC_HEADER @@ -241,12 +239,13 @@ typedef struct _cstring1_s { #define IDL_VERSION "0.0.1" typedef struct dsptensor dsptensor; struct dsptensor { + int32_t type; int64_t ne[4]; int64_t nb[4]; + int32_t op; int32_t flags; - int32_t type; - float* data; - int dataLen; + void * data; + int data_len; }; /** * Opens the handle in the specified domain. If this is the first @@ -278,12 +277,12 @@ __QAIC_HEADER_EXPORT int __QAIC_HEADER(ggmlop_open)(const char* uri, remote_hand * @retval, 0 on success, should always succeed */ __QAIC_HEADER_EXPORT int __QAIC_HEADER(ggmlop_close)(remote_handle64 h) __QAIC_HEADER_ATTRIBUTE; -__QAIC_HEADER_EXPORT int __QAIC_HEADER(ggmlop_add)(remote_handle64 _h, const dsptensor* src0, const dsptensor* src1, dsptensor* dst) __QAIC_HEADER_ATTRIBUTE; -__QAIC_HEADER_EXPORT int __QAIC_HEADER(ggmlop_mulmat)(remote_handle64 _h, const dsptensor* src0, const dsptensor* src1, dsptensor* dst) __QAIC_HEADER_ATTRIBUTE; +__QAIC_HEADER_EXPORT int __QAIC_HEADER(ggmlop_add)(remote_handle64 _h, const dsptensor * src0, const dsptensor * src1, dsptensor * dst) __QAIC_HEADER_ATTRIBUTE; +__QAIC_HEADER_EXPORT int __QAIC_HEADER(ggmlop_mulmat)(remote_handle64 _h, const dsptensor * src0, const dsptensor * src1, dsptensor * dst) __QAIC_HEADER_ATTRIBUTE; #ifndef ggmlop_URI #define ggmlop_URI "file:///libggmlop_skel.so?ggmlop_skel_handle_invoke&_modver=1.0&_idlver=0.0.1" #endif /*ggmlop_URI*/ #ifdef __cplusplus } #endif -#endif //_GGMLOP_H +#endif //_GGMLOP_AP_SKEL_H diff --git a/ggml/src/ggml-qnn/kernels/ggmlop_cdsp.c b/ggml/src/ggml-qnn/kernels/ggmlop_cdsp.c index 0350942648e2d..bddafa29ea81e 100644 --- a/ggml/src/ggml-qnn/kernels/ggmlop_cdsp.c +++ b/ggml/src/ggml-qnn/kernels/ggmlop_cdsp.c @@ -1,22 +1,16 @@ -/* ggml op functions, running on Hexagon cDSP as libggmlop_skel.so - * - * currently I didn't find a general approach to compile/build this hexagon-kernel file, a manual build approach can works fine in my local dev envs. I'm working on this build issue. - * - */ - -#if 0 #include #include +#include +#include +#include #include #include "HAP_farf.h" -#include "ggmlop.h" +#include "ggmlop_ap_skel.h" -#define GGML_ASSERT(x) do { } while(0) -#define MIN(a, b) ((a) < (b) ? (a) : (b)) -#define GGML_RESTRICT +#define ggml_tensor dsptensor -int ggmlop_open(const char * uri, remote_handle64 * handle) { - void * tptr = NULL; +int ggmlop_open(const char*uri, remote_handle64* handle) { + void *tptr = NULL; FARF(HIGH, "uri %s", uri); tptr = (void *)malloc(1); *handle = (remote_handle64)tptr; @@ -30,100 +24,338 @@ int ggmlop_close(remote_handle64 handle) { return 0; } -int ggmlop_add(remote_handle64 h, const dsptensor * src0, const dsptensor * src1, dsptensor * dst) { - FARF(HIGH, "=============== DSP: ggmlop_add "); - for (size_t idx = 0; idx < src0->dataLen; idx++) { - dst->data[idx] = src0->data[idx] + src1->data[idx]; +static void ggml_dump_tensor(struct ggml_tensor * tensor) { + FARF(HIGH, "ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi, %5zi)\n", + tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3], + tensor->nb[0], tensor->nb[1], tensor->nb[2], tensor->nb[3]); +} + +static void ggml_abort(const char * file, int line, const char * fmt, ...) { + //abort(); + return; +} + +#define GGML_MAX_DIMS 4 +#define GGML_UNUSED(x) (void)(x) +#define GGML_PAD(x, n) (((x) + (n) - 1) & ~((n) - 1)) +#define GGML_ABORT(...) ggml_abort(__FILE__, __LINE__, __VA_ARGS__) +#define GGML_ASSERT(x) if (!(x)) GGML_ABORT("GGML_ASSERT(%s) failed", #x) +#define MIN(a, b) ((a) < (b) ? (a) : (b)) +#define MAX(a, b) ((a) > (b) ? (a) : (b)) +#define GGML_RESTRICT + +#define static_assert(a, b) do { } while (0) + +#define GGML_TENSOR_LOCALS_1(type, prefix, pointer, array) \ + const type prefix##0 = (pointer)->array[0]; \ + GGML_UNUSED(prefix##0); +#define GGML_TENSOR_LOCALS_2(type, prefix, pointer, array) \ + GGML_TENSOR_LOCALS_1 (type, prefix, pointer, array) \ + const type prefix##1 = (pointer)->array[1]; \ + GGML_UNUSED(prefix##1); +#define GGML_TENSOR_LOCALS_3(type, prefix, pointer, array) \ + GGML_TENSOR_LOCALS_2 (type, prefix, pointer, array) \ + const type prefix##2 = (pointer)->array[2]; \ + GGML_UNUSED(prefix##2); +#define GGML_TENSOR_LOCALS(type, prefix, pointer, array) \ + GGML_TENSOR_LOCALS_3 (type, prefix, pointer, array) \ + const type prefix##3 = (pointer)->array[3]; \ + GGML_UNUSED(prefix##3); + +#define GGML_TENSOR_UNARY_OP_LOCALS \ + GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \ + GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \ + GGML_TENSOR_LOCALS(int64_t, ne, dst, ne) \ + GGML_TENSOR_LOCALS(size_t, nb, dst, nb) + +#define GGML_TENSOR_BINARY_OP_LOCALS \ + GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \ + GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \ + GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne) \ + GGML_TENSOR_LOCALS(size_t, nb1, src1, nb) \ + GGML_TENSOR_LOCALS(int64_t, ne, dst, ne) \ + GGML_TENSOR_LOCALS(size_t, nb, dst, nb) + +#define GGML_TENSOR_BINARY_OP_LOCALS01 \ + GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \ + GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \ + GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne) \ + GGML_TENSOR_LOCALS(size_t, nb1, src1, nb) + +enum ggml_type { + GGML_TYPE_F32 = 0, + GGML_TYPE_F16 = 1, + GGML_TYPE_Q4_0 = 2, + GGML_TYPE_Q4_1 = 3, + // GGML_TYPE_Q4_2 = 4, support has been removed + // GGML_TYPE_Q4_3 = 5, support has been removed + GGML_TYPE_Q5_0 = 6, + GGML_TYPE_Q5_1 = 7, + GGML_TYPE_Q8_0 = 8, + GGML_TYPE_Q8_1 = 9, + GGML_TYPE_Q2_K = 10, + GGML_TYPE_Q3_K = 11, + GGML_TYPE_Q4_K = 12, + GGML_TYPE_Q5_K = 13, + GGML_TYPE_Q6_K = 14, + GGML_TYPE_Q8_K = 15, + GGML_TYPE_IQ2_XXS = 16, + GGML_TYPE_IQ2_XS = 17, + GGML_TYPE_IQ3_XXS = 18, + GGML_TYPE_IQ1_S = 19, + GGML_TYPE_IQ4_NL = 20, + GGML_TYPE_IQ3_S = 21, + GGML_TYPE_IQ2_S = 22, + GGML_TYPE_IQ4_XS = 23, + GGML_TYPE_I8 = 24, + GGML_TYPE_I16 = 25, + GGML_TYPE_I32 = 26, + GGML_TYPE_I64 = 27, + GGML_TYPE_F64 = 28, + GGML_TYPE_IQ1_M = 29, + GGML_TYPE_BF16 = 30, + // GGML_TYPE_Q4_0_4_4 = 31, support has been removed from gguf files + // GGML_TYPE_Q4_0_4_8 = 32, + // GGML_TYPE_Q4_0_8_8 = 33, + GGML_TYPE_TQ1_0 = 34, + GGML_TYPE_TQ2_0 = 35, + // GGML_TYPE_IQ4_NL_4_4 = 36, + // GGML_TYPE_IQ4_NL_4_8 = 37, + // GGML_TYPE_IQ4_NL_8_8 = 38, + GGML_TYPE_COUNT = 39, +}; + +static bool ggml_is_empty(const struct ggml_tensor * tensor) { + for (int i = 0; i < GGML_MAX_DIMS; ++i) { + if (tensor->ne[i] == 0) { + // empty if any dimension has no elements + return true; + } } + return false; +} - return 0; +static bool ggml_can_repeat(const struct ggml_tensor * t0, const struct ggml_tensor * t1) { + static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function"); + + return ggml_is_empty(t0) ? ggml_is_empty(t1) : + (t1->ne[0]%t0->ne[0] == 0) && + (t1->ne[1]%t0->ne[1] == 0) && + (t1->ne[2]%t0->ne[2] == 0) && + (t1->ne[3]%t0->ne[3] == 0); } -static void ggmldsp_dump_tensor(struct dsptensor * src0) { - FARF(HIGH, "ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi, %5zi)\n", - src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], - src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3]); +static bool ggml_are_same_shape(const struct ggml_tensor * t0, const struct ggml_tensor * t1) { + static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function"); + + return + (t0->ne[0] == t1->ne[0]) && + (t0->ne[1] == t1->ne[1]) && + (t0->ne[2] == t1->ne[2]) && + (t0->ne[3] == t1->ne[3]); +} + +static int64_t ggml_nrows(const struct ggml_tensor * tensor) { + static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function"); + + return tensor->ne[1]*tensor->ne[2]*tensor->ne[3]; } -static int ggmldsp_is_contiguous(const struct dsptensor * tensor) { - int n = 0; - size_t next_nb = sizeof(float); - if (tensor->ne[0] != 1 && tensor->nb[0] != next_nb) { - return 0; +static bool ggml_is_contiguous_n(const struct ggml_tensor * tensor, int n) { + size_t next_nb = sizeof(float);//ggml_type_size(tensor->type); + if (tensor->ne[0] != 1/*ggml_blck_size(tensor->type)*/ && tensor->nb[0] != next_nb) { + return false; } - next_nb *= tensor->ne[0]; - for (int i = 1; i < 4; i++) { + next_nb *= tensor->ne[0]/1/*ggml_blck_size(tensor->type)*/; + for (int i = 1; i < GGML_MAX_DIMS; i++) { if (tensor->ne[i] != 1) { if (i > n) { if (tensor->nb[i] != next_nb) { - return 0; + return false; } next_nb *= tensor->ne[i]; } else { - next_nb = tensor->ne[i] * tensor->nb[i]; + // this dimension does not need to be contiguous + next_nb = tensor->ne[i]*tensor->nb[i]; + } + } + } + return true; +} + +static bool ggml_is_contiguous_0(const struct ggml_tensor * tensor) { + return ggml_is_contiguous_n(tensor, 0); +} + +static bool ggml_is_contiguous(const struct ggml_tensor * tensor) { + return ggml_is_contiguous_0(tensor); +} + +inline static void ggml_vec_add_f32 (const int n, float * z, const float * x, const float * y) { for (int i = 0; i < n; ++i) z[i] = x[i] + y[i]; } + +static void ggml_compute_forward_add_f32( + const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + + ggml_dump_tensor(src0); + ggml_dump_tensor(src1); +#if 1 + float * a = (float*)src0->data; + float * b = (float*)src1->data; + float * c = (float*)dst->data; + //TODO: Hexagon SIMD + for (size_t idx = 0; idx < src0->data_len; idx++) { + *c = *a + *b; + a++; + b++; + c++; + } + return; +#endif + GGML_ASSERT(ggml_can_repeat(src1, src0) && ggml_are_same_shape(src0, dst)); + + const int ith = 0; + const int nth = 1; + + const int nr = ggml_nrows(src0); + + GGML_TENSOR_BINARY_OP_LOCALS + + GGML_ASSERT( nb0 == sizeof(float)); + GGML_ASSERT(nb00 == sizeof(float)); + + // rows per thread + const int dr = (nr + nth - 1)/nth; + + // row range for this thread + const int ir0 = dr*ith; + const int ir1 = MIN(ir0 + dr, nr); + + if (nb10 == sizeof(float)) { + for (int ir = ir0; ir < ir1; ++ir) { + // src1 is broadcastable across src0 and dst in i1, i2, i3 + const int64_t i03 = ir/(ne02*ne01); + const int64_t i02 = (ir - i03*ne02*ne01)/ne01; + const int64_t i01 = (ir - i03*ne02*ne01 - i02*ne01); + + const int64_t i13 = i03 % ne13; + const int64_t i12 = i02 % ne12; + const int64_t i11 = i01 % ne11; + const int64_t nr0 = ne00 / ne10; + + float * dst_ptr = (float *) ((char *) dst->data + i03*nb3 + i02*nb2 + i01*nb1 ); + float * src0_ptr = (float *) ((char *) src0->data + i03*nb03 + i02*nb02 + i01*nb01); + float * src1_ptr = (float *) ((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11); + + for (int64_t r = 0; r < nr0; ++r) { +#ifdef GGML_USE_ACCELERATE + vDSP_vadd(src0_ptr + r*ne10, 1, src1_ptr, 1, dst_ptr + r*ne10, 1, ne10); +#else + ggml_vec_add_f32(ne10, dst_ptr + r*ne10, src0_ptr + r*ne10, src1_ptr); +#endif + } + } + } else { + // src1 is not contiguous + for (int ir = ir0; ir < ir1; ++ir) { + // src1 is broadcastable across src0 and dst in i1, i2, i3 + const int64_t i03 = ir/(ne02*ne01); + const int64_t i02 = (ir - i03*ne02*ne01)/ne01; + const int64_t i01 = (ir - i03*ne02*ne01 - i02*ne01); + + const int64_t i13 = i03 % ne13; + const int64_t i12 = i02 % ne12; + const int64_t i11 = i01 % ne11; + + float * dst_ptr = (float *) ((char *) dst->data + i03*nb3 + i02*nb2 + i01*nb1 ); + float * src0_ptr = (float *) ((char *) src0->data + i03*nb03 + i02*nb02 + i01*nb01); + + for (int64_t i0 = 0; i0 < ne0; ++i0) { + const int64_t i10 = i0 % ne10; + float * src1_ptr = (float *) ((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11 + i10*nb10); + + dst_ptr[i0] = src0_ptr[i0] + *src1_ptr; + } + } + } +} + +int ggmlop_add(remote_handle64 h, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) +{ + FARF(HIGH, "=============== DSP: ggmlop_add "); + switch (src0->type) { + case GGML_TYPE_F32: + { + if (src1->type == GGML_TYPE_F32) { + ggml_compute_forward_add_f32(src0, src1, dst); + } else { + GGML_ABORT("fatal error"); + } + } break; + case GGML_TYPE_F16: + { + if (src1->type == GGML_TYPE_F16) { + //ggml_compute_forward_add_f16_f16(dst); + } + else if (src1->type == GGML_TYPE_F32) { + //ggml_compute_forward_add_f16_f32(dst); + } + else { + GGML_ABORT("fatal error"); } + } break; + case GGML_TYPE_BF16: + { + if (src1->type == GGML_TYPE_BF16) { + //ggml_compute_forward_add_bf16_bf16(dst); + } + else if (src1->type == GGML_TYPE_F32) { + //ggml_compute_forward_add_bf16_f32(dst); + } + else { + GGML_ABORT("fatal error"); + } + } break; + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q4_1: + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q5_1: + case GGML_TYPE_Q8_0: + case GGML_TYPE_Q2_K: + case GGML_TYPE_Q3_K: + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + case GGML_TYPE_Q6_K: + case GGML_TYPE_TQ1_0: + case GGML_TYPE_TQ2_0: + case GGML_TYPE_IQ2_XXS: + case GGML_TYPE_IQ2_XS: + case GGML_TYPE_IQ3_XXS: + case GGML_TYPE_IQ1_S: + case GGML_TYPE_IQ1_M: + case GGML_TYPE_IQ4_NL: + case GGML_TYPE_IQ4_XS: + case GGML_TYPE_IQ3_S: + case GGML_TYPE_IQ2_S: + { + //ggml_compute_forward_add_q_f32(dst); + } break; + default: + { + GGML_ABORT("fatal error"); } } - return 1; + + return 0; } -//FIXME: unknown issue on cDSP -int ggmlop_mulmat(remote_handle64 h, const struct dsptensor * src00, const struct dsptensor * src10, dsptensor * dst) { + +int ggmlop_mulmat(remote_handle64 h, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { FARF(HIGH, "=============== DSP: ggmlop_mulmat "); - dsptensor * src0 = (dsptensor*)src00; - dsptensor * src1 = (dsptensor*)src10; - const int64_t ne00 = src0->ne[0]; - (void) (ne00); - const int64_t ne01 = (src0)->ne[1]; - (void) (ne01); - const int64_t ne02 = (src0)->ne[2]; - (void) (ne02); - const int64_t ne03 = (src0)->ne[3]; - (void) (ne03); - const size_t nb00 = (src0)->nb[0]; - (void) (nb00); - const size_t nb01 = (src0)->nb[1]; - (void) (nb01); - const size_t nb02 = (src0)->nb[2]; - (void) (nb02); - const size_t nb03 = (src0)->nb[3]; - (void) (nb03); - const int64_t ne10 = (src1)->ne[0]; - (void) (ne10); - const int64_t ne11 = (src1)->ne[1]; - (void) (ne11); - const int64_t ne12 = (src1)->ne[2]; - (void) (ne12); - const int64_t ne13 = (src1)->ne[3]; - (void) (ne13); - const size_t nb10 = (src1)->nb[0]; - (void) (nb10); - const size_t nb11 = (src1)->nb[1]; - (void) (nb11); - const size_t nb12 = (src1)->nb[2]; - (void) (nb12); - const size_t nb13 = (src1)->nb[3]; - (void) (nb13); - const int64_t ne0 = (dst)->ne[0]; - (void) (ne0); - const int64_t ne1 = (dst)->ne[1]; - (void) (ne1); - const int64_t ne2 = (dst)->ne[2]; - (void) (ne2); - const int64_t ne3 = (dst)->ne[3]; - (void) (ne3); - const size_t nb0 = (dst)->nb[0]; - (void) (nb0); - const size_t nb1 = (dst)->nb[1]; - (void) (nb1); - const size_t nb2 = (dst)->nb[2]; - (void) (nb2); - const size_t nb3 = (dst)->nb[3]; - (void) (nb3); - - ggmldsp_dump_tensor(src0); - ggmldsp_dump_tensor(src1); + GGML_TENSOR_BINARY_OP_LOCALS + + ggml_dump_tensor(src0); + ggml_dump_tensor(src1); const int vec_dot_type = 0; int64_t const vec_dot_num_rows = 1; @@ -145,9 +377,12 @@ int ggmlop_mulmat(remote_handle64 h, const struct dsptensor * src00, const struc const int64_t nr1 = ne1 * ne2 * ne3; int chunk_size = 16; + int nth = 1; + if (nr0 == 1 || nr1 == 1) { chunk_size = 64; } + int64_t nchunk0 = (nr0 + chunk_size - 1) / chunk_size; int64_t nchunk1 = (nr1 + chunk_size - 1) / chunk_size; @@ -155,6 +390,7 @@ int ggmlop_mulmat(remote_handle64 h, const struct dsptensor * src00, const struc nchunk0 = nr0 > nr1 ? nth : 1; nchunk1 = nr0 > nr1 ? 1 : nth; } + const int64_t dr0 = (nr0 + nchunk0 - 1) / nchunk0; const int64_t dr1 = (nr1 + nchunk1 - 1) / nchunk1; @@ -171,7 +407,7 @@ int ggmlop_mulmat(remote_handle64 h, const struct dsptensor * src00, const struc int64_t num_rows_per_vec_dot = vec_dot_num_rows; - const int src1_cont = ggmldsp_is_contiguous(src1); + const int src1_cont = ggml_is_contiguous(src1); const int64_t r2 = ne12 / ne02; const int64_t r3 = ne13 / ne03; @@ -234,4 +470,3 @@ int ggmlop_mulmat(remote_handle64 h, const struct dsptensor * src00, const struc return 0; } -#endif diff --git a/ggml/src/ggml-qnn/kernels/ggmlop_cdsp_skel.c b/ggml/src/ggml-qnn/kernels/ggmlop_cdsp_skel.c new file mode 100644 index 0000000000000..33d47174bf5ef --- /dev/null +++ b/ggml/src/ggml-qnn/kernels/ggmlop_cdsp_skel.c @@ -0,0 +1,596 @@ +#include +#ifndef _WIN32 +#include "HAP_farf.h" +#endif //_WIN32 for HAP_farf +#ifndef _ALLOCATOR_H +#define _ALLOCATOR_H + +#include +#include +#include "version_note.h" +#include "ggmlop_ap_skel.h" + +typedef struct _heap _heap; +struct _heap { + _heap* pPrev; + const char* loc; + uint64_t buf; +}; + +typedef struct _allocator { + _heap* pheap; + uint8_t* stack; + uint8_t* stackEnd; + int nSize; +} _allocator; + +_ATTRIBUTE_UNUSED +static __inline int _heap_alloc(_heap** ppa, const char* loc, int size, void** ppbuf) { + _heap* pn = 0; + pn = MALLOC((size_t)size + sizeof(_heap) - sizeof(uint64_t)); + if(pn != 0) { + pn->pPrev = *ppa; + pn->loc = loc; + *ppa = pn; + *ppbuf = (void*)&(pn->buf); + return 0; + } else { + return -1; + } +} +#define _ALIGN_SIZE(x, y) (((x) + (y-1)) & ~(y-1)) + +_ATTRIBUTE_UNUSED +static __inline int _allocator_alloc(_allocator* me, + const char* loc, + int size, + unsigned int al, + void** ppbuf) { + if(size < 0) { + return -1; + } else if (size == 0) { + *ppbuf = 0; + return 0; + } + if((_ALIGN_SIZE((uintptr_t)me->stackEnd, al) + (size_t)size) < (uintptr_t)me->stack + (size_t)me->nSize) { + *ppbuf = (uint8_t*)_ALIGN_SIZE((uintptr_t)me->stackEnd, al); + me->stackEnd = (uint8_t*)_ALIGN_SIZE((uintptr_t)me->stackEnd, al) + size; + return 0; + } else { + return _heap_alloc(&me->pheap, loc, size, ppbuf); + } +} + +_ATTRIBUTE_UNUSED +static __inline void _allocator_deinit(_allocator* me) { + _heap* pa = me->pheap; + while(pa != 0) { + _heap* pn = pa; + const char* loc = pn->loc; + (void)loc; + pa = pn->pPrev; + FREE(pn); + } +} + +_ATTRIBUTE_UNUSED +static __inline void _allocator_init(_allocator* me, uint8_t* stack, int stackSize) { + me->stack = stack; + me->stackEnd = stack + stackSize; + me->nSize = stackSize; + me->pheap = 0; +} + + +#endif // _ALLOCATOR_H + +#ifndef SLIM_H +#define SLIM_H + +#include + +//a C data structure for the idl types that can be used to implement +//static and dynamic language bindings fairly efficiently. +// +//the goal is to have a minimal ROM and RAM footprint and without +//doing too many allocations. A good way to package these things seemed +//like the module boundary, so all the idls within one module can share +//all the type references. + + +#define PARAMETER_IN 0x0 +#define PARAMETER_OUT 0x1 +#define PARAMETER_INOUT 0x2 +#define PARAMETER_ROUT 0x3 +#define PARAMETER_INROUT 0x4 + +//the types that we get from idl +#define TYPE_OBJECT 0x0 +#define TYPE_INTERFACE 0x1 +#define TYPE_PRIMITIVE 0x2 +#define TYPE_ENUM 0x3 +#define TYPE_STRING 0x4 +#define TYPE_WSTRING 0x5 +#define TYPE_STRUCTURE 0x6 +#define TYPE_UNION 0x7 +#define TYPE_ARRAY 0x8 +#define TYPE_SEQUENCE 0x9 + +//these require the pack/unpack to recurse +//so it's a hint to those languages that can optimize in cases where +//recursion isn't necessary. +#define TYPE_COMPLEX_STRUCTURE (0x10 | TYPE_STRUCTURE) +#define TYPE_COMPLEX_UNION (0x10 | TYPE_UNION) +#define TYPE_COMPLEX_ARRAY (0x10 | TYPE_ARRAY) +#define TYPE_COMPLEX_SEQUENCE (0x10 | TYPE_SEQUENCE) + + +typedef struct Type Type; + +#define INHERIT_TYPE\ + int32_t nativeSize; /*in the simple case its the same as wire size and alignment*/\ + union {\ + struct {\ + const uintptr_t p1;\ + const uintptr_t p2;\ + } _cast;\ + struct {\ + uint32_t iid;\ + uint32_t bNotNil;\ + } object;\ + struct {\ + const Type *arrayType;\ + int32_t nItems;\ + } array;\ + struct {\ + const Type *seqType;\ + int32_t nMaxLen;\ + } seqSimple; \ + struct {\ + uint32_t bFloating;\ + uint32_t bSigned;\ + } prim; \ + const SequenceType* seqComplex;\ + const UnionType *unionType;\ + const StructType *structType;\ + int32_t stringMaxLen;\ + uint8_t bInterfaceNotNil;\ + } param;\ + uint8_t type;\ + uint8_t nativeAlignment\ + +typedef struct UnionType UnionType; +typedef struct StructType StructType; +typedef struct SequenceType SequenceType; +struct Type { + INHERIT_TYPE; +}; + +struct SequenceType { + const Type * seqType; + uint32_t nMaxLen; + uint32_t inSize; + uint32_t routSizePrimIn; + uint32_t routSizePrimROut; +}; + +//byte offset from the start of the case values for +//this unions case value array. it MUST be aligned +//at the alignment requrements for the descriptor +// +//if negative it means that the unions cases are +//simple enumerators, so the value read from the descriptor +//can be used directly to find the correct case +typedef union CaseValuePtr CaseValuePtr; +union CaseValuePtr { + const uint8_t* value8s; + const uint16_t* value16s; + const uint32_t* value32s; + const uint64_t* value64s; +}; + +//these are only used in complex cases +//so I pulled them out of the type definition as references to make +//the type smaller +struct UnionType { + const Type *descriptor; + uint32_t nCases; + const CaseValuePtr caseValues; + const Type * const *cases; + int32_t inSize; + int32_t routSizePrimIn; + int32_t routSizePrimROut; + uint8_t inAlignment; + uint8_t routAlignmentPrimIn; + uint8_t routAlignmentPrimROut; + uint8_t inCaseAlignment; + uint8_t routCaseAlignmentPrimIn; + uint8_t routCaseAlignmentPrimROut; + uint8_t nativeCaseAlignment; + uint8_t bDefaultCase; +}; + +struct StructType { + uint32_t nMembers; + const Type * const *members; + int32_t inSize; + int32_t routSizePrimIn; + int32_t routSizePrimROut; + uint8_t inAlignment; + uint8_t routAlignmentPrimIn; + uint8_t routAlignmentPrimROut; +}; + +typedef struct Parameter Parameter; +struct Parameter { + INHERIT_TYPE; + uint8_t mode; + uint8_t bNotNil; +}; + +#define SLIM_IFPTR32(is32,is64) (sizeof(uintptr_t) == 4 ? (is32) : (is64)) +#define SLIM_SCALARS_IS_DYNAMIC(u) (((u) & 0x00ffffff) == 0x00ffffff) + +typedef struct Method Method; +struct Method { + uint32_t uScalars; //no method index + int32_t primInSize; + int32_t primROutSize; + int maxArgs; + int numParams; + const Parameter * const *params; + uint8_t primInAlignment; + uint8_t primROutAlignment; +}; + +typedef struct Interface Interface; + +struct Interface { + int nMethods; + const Method * const *methodArray; + int nIIds; + const uint32_t *iids; + const uint16_t* methodStringArray; + const uint16_t* methodStrings; + const char* strings; +}; + + +#endif //SLIM_H + + +#ifndef _GGMLOP_SLIM_H +#define _GGMLOP_SLIM_H +#include + +#ifndef __QAIC_SLIM +#define __QAIC_SLIM(ff) ff +#endif +#ifndef __QAIC_SLIM_EXPORT +#define __QAIC_SLIM_EXPORT +#endif + +static const Type types[5]; +static const Type* const typeArrays[6] = {&(types[0]),&(types[1]),&(types[1]),&(types[0]),&(types[0]),&(types[3])}; +static const StructType structTypes[1] = {{0x6,&(typeArrays[0]),0x58,0x4,0x50,0x8,0x4,0x8}}; +static const Type types[5] = {{0x4,{{(const uintptr_t)0,(const uintptr_t)1}}, 2,0x4},{0x20,{{(const uintptr_t)&(types[2]),(const uintptr_t)0x4}}, 8,0x8},{0x8,{{(const uintptr_t)0,(const uintptr_t)1}}, 2,0x8},{SLIM_IFPTR32(0x8,0x10),{{(const uintptr_t)&(types[4]),(const uintptr_t)0x0}}, 9,SLIM_IFPTR32(0x4,0x8)},{0x4,{{(const uintptr_t)0,(const uintptr_t)1}}, 2,0x4}}; +static const Parameter parameters[5] = {{SLIM_IFPTR32(0x8,0x10),{{(const uintptr_t)0x0,0}}, 4,SLIM_IFPTR32(0x4,0x8),0,0},{SLIM_IFPTR32(0x4,0x8),{{(const uintptr_t)0xdeadc0de,(const uintptr_t)0}}, 0,SLIM_IFPTR32(0x4,0x8),3,0},{SLIM_IFPTR32(0x4,0x8),{{(const uintptr_t)0xdeadc0de,(const uintptr_t)0}}, 0,SLIM_IFPTR32(0x4,0x8),0,0},{SLIM_IFPTR32(0x58,0x60),{{(const uintptr_t)&(structTypes[0]),0}}, 22,0x8,0,0},{SLIM_IFPTR32(0x58,0x60),{{(const uintptr_t)&(structTypes[0]),0}}, 22,0x8,3,0}}; +static const Parameter* const parameterArrays[6] = {(&(parameters[3])),(&(parameters[3])),(&(parameters[4])),(&(parameters[0])),(&(parameters[1])),(&(parameters[2]))}; +static const Method methods[3] = {{REMOTE_SCALARS_MAKEX(0,0,0x2,0x0,0x0,0x1),0x4,0x0,2,2,(&(parameterArrays[3])),0x4,0x1},{REMOTE_SCALARS_MAKEX(0,0,0x0,0x0,0x1,0x0),0x0,0x0,1,1,(&(parameterArrays[5])),0x1,0x0},{REMOTE_SCALARS_MAKEX(0,0,0x3,0x2,0x0,0x0),0xb4,0x50,3,3,(&(parameterArrays[0])),0x8,0x8}}; +static const Method* const methodArrays[4] = {&(methods[0]),&(methods[1]),&(methods[2]),&(methods[2])}; +static const char strings[68] = "mulmat\0flags\0close\0src1\0data\0type\0src0\0open\0dst\0add\0uri\0op\0nb\0ne\0h\0"; +static const uint16_t methodStrings[49] = {0,34,29,62,59,56,7,24,19,29,62,59,56,7,24,44,29,62,59,56,7,24,48,34,29,62,59,56,7,24,19,29,62,59,56,7,24,44,29,62,59,56,7,24,39,52,65,13,65}; +static const uint16_t methodStringsArrays[4] = {44,47,22,0}; +__QAIC_SLIM_EXPORT const Interface __QAIC_SLIM(ggmlop_slim) = {4,&(methodArrays[0]),0,0,&(methodStringsArrays [0]),methodStrings,strings}; +#endif //_GGMLOP_SLIM_H +extern int adsp_mmap_fd_getinfo(int, uint32_t *); +#ifdef __cplusplus +extern "C" { +#endif +_ATTRIBUTE_VISIBILITY uint32_t ggmlop_skel_handle_invoke_qaic_version = 10048; +_ATTRIBUTE_VISIBILITY char ggmlop_skel_handle_invoke_uri[77+1]="file:///libggmlop_skel.so?ggmlop_skel_handle_invoke&_modver=1.0&_idlver=0.0.1"; +static __inline int _skel_pack(_ATTRIBUTE_UNUSED remote_arg* _praROutPost, _ATTRIBUTE_UNUSED remote_arg* _ppraROutPost[1], _ATTRIBUTE_UNUSED void* _primROut, _ATTRIBUTE_UNUSED uint32_t _rout0[1], _ATTRIBUTE_UNUSED uint64_t _rout1[4], _ATTRIBUTE_UNUSED uint64_t _rout2[4], _ATTRIBUTE_UNUSED uint32_t _rout3[1], _ATTRIBUTE_UNUSED uint32_t _rout4[1], _ATTRIBUTE_UNUSED char* _rout5[1], _ATTRIBUTE_UNUSED uint32_t _rout5Len[1]) { + int _nErr = 0; + remote_arg* _praROutPostStart = _praROutPost; + remote_arg** _ppraROutPostStart = _ppraROutPost; + _ppraROutPost = &_praROutPost; + _COPY(_primROut, 0, _rout0, 0, 4); + _COPY(_primROut, 8, _rout1, 0, 32); + _COPY(_primROut, 40, _rout2, 0, 32); + _COPY(_primROut, 72, _rout3, 0, 4); + _COPY(_primROut, 76, _rout4, 0, 4); + _ppraROutPostStart[0] += (_praROutPost - _praROutPostStart) +1; + return _nErr; +} +static __inline int _skel_unpack(_ATTRIBUTE_UNUSED _allocator* _al, _ATTRIBUTE_UNUSED remote_arg* _praIn, _ATTRIBUTE_UNUSED remote_arg* _ppraIn[1], _ATTRIBUTE_UNUSED remote_arg* _praROut, _ATTRIBUTE_UNUSED remote_arg* _ppraROut[1], _ATTRIBUTE_UNUSED remote_arg* _praHIn, _ATTRIBUTE_UNUSED remote_arg* _ppraHIn[1], _ATTRIBUTE_UNUSED remote_arg* _praHROut, _ATTRIBUTE_UNUSED remote_arg* _ppraHROut[1], _ATTRIBUTE_UNUSED void* _primIn, _ATTRIBUTE_UNUSED void* _primROut, _ATTRIBUTE_UNUSED uint32_t _rout0[1], _ATTRIBUTE_UNUSED uint64_t _rout1[4], _ATTRIBUTE_UNUSED uint64_t _rout2[4], _ATTRIBUTE_UNUSED uint32_t _rout3[1], _ATTRIBUTE_UNUSED uint32_t _rout4[1], _ATTRIBUTE_UNUSED char* _rout5[1], _ATTRIBUTE_UNUSED uint32_t _rout5Len[1]) { + int _nErr = 0; + remote_arg* _praInStart = _praIn; + remote_arg** _ppraInStart = _ppraIn; + remote_arg* _praROutStart = _praROut; + remote_arg** _ppraROutStart = _ppraROut; + _ppraIn = &_praIn; + _ppraROut = &_praROut; + _COPY(_rout5Len, 0, _primIn, 0, 4); + _QAIC_ASSERT(_nErr, ((_praROut[0].buf.nLen / 4)) >= (size_t)(_rout5Len[0])); + _rout5[0] = _praROut[0].buf.pv; + _ppraInStart[0] += (_praIn - _praInStart) + 0; + _ppraROutStart[0] += (_praROut - _praROutStart) +1; + _QAIC_CATCH(_nErr) {} + return _nErr; +} +static __inline int _skel_unpack_1(_ATTRIBUTE_UNUSED _allocator* _al, _ATTRIBUTE_UNUSED remote_arg* _praIn, _ATTRIBUTE_UNUSED remote_arg* _ppraIn[1], _ATTRIBUTE_UNUSED remote_arg* _praROut, _ATTRIBUTE_UNUSED remote_arg* _ppraROut[1], _ATTRIBUTE_UNUSED remote_arg* _praHIn, _ATTRIBUTE_UNUSED remote_arg* _ppraHIn[1], _ATTRIBUTE_UNUSED remote_arg* _praHROut, _ATTRIBUTE_UNUSED remote_arg* _ppraHROut[1], _ATTRIBUTE_UNUSED void* _primIn, _ATTRIBUTE_UNUSED void* _primROut, _ATTRIBUTE_UNUSED uint32_t _in0[1], _ATTRIBUTE_UNUSED uint64_t _in1[4], _ATTRIBUTE_UNUSED uint64_t _in2[4], _ATTRIBUTE_UNUSED uint32_t _in3[1], _ATTRIBUTE_UNUSED uint32_t _in4[1], _ATTRIBUTE_UNUSED char* _in5[1], _ATTRIBUTE_UNUSED uint32_t _in5Len[1]) { + int _nErr = 0; + remote_arg* _praInStart = _praIn; + remote_arg** _ppraInStart = _ppraIn; + remote_arg* _praROutStart = _praROut; + remote_arg** _ppraROutStart = _ppraROut; + _ppraIn = &_praIn; + _ppraROut = &_praROut; + _COPY(_in0, 0, _primIn, 0, 4); + _COPY(_in1, 0, _primIn, 8, 32); + _COPY(_in2, 0, _primIn, 40, 32); + _COPY(_in3, 0, _primIn, 72, 4); + _COPY(_in4, 0, _primIn, 76, 4); + _COPY(_in5Len, 0, _primIn, 80, 4); + _QAIC_ASSERT(_nErr, ((_praIn[0].buf.nLen / 4)) >= (size_t)(_in5Len[0])); + _in5[0] = _praIn[0].buf.pv; + _ppraInStart[0] += (_praIn - _praInStart) + 1; + _ppraROutStart[0] += (_praROut - _praROutStart) +0; + _QAIC_CATCH(_nErr) {} + return _nErr; +} +static __inline int _skel_method(int (*_pfn)(remote_handle64, const dsptensor*, const dsptensor*, dsptensor*), remote_handle64 _h, uint32_t _sc, remote_arg* _pra) { + remote_arg* _praEnd = 0; + uint64_t _in0[SLIM_IFPTR32(11, 12)] = {0}; + uint64_t _in1[SLIM_IFPTR32(11, 12)] = {0}; + uint64_t _rout2[SLIM_IFPTR32(11, 12)] = {0}; + uint64_t* _primIn= 0; + int _numIn[1] = {0}; + uint64_t* _primROut= 0; + int _numInH[1] = {0}; + int _numROut[1] = {0}; + remote_arg* _praIn = 0; + remote_arg* _praROut = 0; + remote_arg* _praROutPost = 0; + remote_arg** _ppraROutPost = &_praROutPost; + _allocator _al[1] = {{0}}; + remote_arg** _ppraIn = &_praIn; + remote_arg** _ppraROut = &_praROut; + remote_arg* _praHIn = 0; + remote_arg** _ppraHIn = &_praHIn; + remote_arg* _praHROut = 0; + remote_arg** _ppraHROut = &_praHROut; + int _nErr = 0; + _praEnd = ((_pra + REMOTE_SCALARS_INBUFS(_sc)) + REMOTE_SCALARS_OUTBUFS(_sc) + REMOTE_SCALARS_INHANDLES(_sc) + REMOTE_SCALARS_OUTHANDLES(_sc)); + _QAIC_ASSERT(_nErr, REMOTE_SCALARS_INBUFS(_sc)>=1); + _QAIC_ASSERT(_nErr, REMOTE_SCALARS_OUTBUFS(_sc)>=1); + _QAIC_ASSERT(_nErr, REMOTE_SCALARS_INHANDLES(_sc)==0); + _QAIC_ASSERT(_nErr, REMOTE_SCALARS_OUTHANDLES(_sc)==0); + _QAIC_ASSERT(_nErr, (_pra + ((1 + 1) + (((0 + 0) + 0) + 0))) <= _praEnd); + _numIn[0] = (REMOTE_SCALARS_INBUFS(_sc) - 1); + _QAIC_ASSERT(_nErr, _pra[0].buf.nLen >= 180); + _primIn = _pra[0].buf.pv; + _QAIC_ASSERT(_nErr, _pra[(_numIn[0] + 1)].buf.nLen >= 80); + _primROut = _pra[(_numIn[0] + 1)].buf.pv; + _numInH[0] = REMOTE_SCALARS_INHANDLES(_sc); + _numROut[0] = REMOTE_SCALARS_OUTBUFS(_sc); + _praIn = (_pra + 1); + _praROut = (_praIn + _numIn[0] + 1); + _praROutPost = _praROut; + _allocator_init(_al, 0, 0); + if(_praHIn == 0) + { + _praHIn = ((_praROut + _numROut[0]) + 1); + } + if(_praHROut == 0) + (_praHROut = _praHIn + _numInH[0] + 0); + _TRY(_nErr, _skel_unpack_1(_al, (_praIn + 0), _ppraIn, (_praROut + 0), _ppraROut, _praHIn, _ppraHIn, _praHROut, _ppraHROut, ((char*)_primIn + 0), 0, (uint32_t*)&(((uint32_t*)_in0)[0]), (uint64_t*)&(((uint64_t*)_in0)[1]), (uint64_t*)&(((uint64_t*)_in0)[5]), (uint32_t*)&(((uint32_t*)_in0)[18]), (uint32_t*)&(((uint32_t*)_in0)[19]), SLIM_IFPTR32((char**)&(((uint32_t*)_in0)[20]), (char**)&(((uint64_t*)_in0)[10])), SLIM_IFPTR32((uint32_t*)&(((uint32_t*)_in0)[21]), (uint32_t*)&(((uint32_t*)_in0)[22])))); + _TRY(_nErr, _skel_unpack_1(_al, (_praIn + 0), _ppraIn, (_praROut + 0), _ppraROut, _praHIn, _ppraHIn, _praHROut, _ppraHROut, ((char*)_primIn + 88), 0, (uint32_t*)&(((uint32_t*)_in1)[0]), (uint64_t*)&(((uint64_t*)_in1)[1]), (uint64_t*)&(((uint64_t*)_in1)[5]), (uint32_t*)&(((uint32_t*)_in1)[18]), (uint32_t*)&(((uint32_t*)_in1)[19]), SLIM_IFPTR32((char**)&(((uint32_t*)_in1)[20]), (char**)&(((uint64_t*)_in1)[10])), SLIM_IFPTR32((uint32_t*)&(((uint32_t*)_in1)[21]), (uint32_t*)&(((uint32_t*)_in1)[22])))); + _TRY(_nErr, _skel_unpack(_al, (_praIn + 0), _ppraIn, (_praROut + 0), _ppraROut, _praHIn, _ppraHIn, _praHROut, _ppraHROut, ((char*)_primIn + 176), ((char*)_primROut + 0), (uint32_t*)&(((uint32_t*)_rout2)[0]), (uint64_t*)&(((uint64_t*)_rout2)[1]), (uint64_t*)&(((uint64_t*)_rout2)[5]), (uint32_t*)&(((uint32_t*)_rout2)[18]), (uint32_t*)&(((uint32_t*)_rout2)[19]), SLIM_IFPTR32((char**)&(((uint32_t*)_rout2)[20]), (char**)&(((uint64_t*)_rout2)[10])), SLIM_IFPTR32((uint32_t*)&(((uint32_t*)_rout2)[21]), (uint32_t*)&(((uint32_t*)_rout2)[22])))); + _TRY(_nErr, _pfn(_h, (const dsptensor*)_in0, (const dsptensor*)_in1, (dsptensor*)_rout2)); + _TRY(_nErr, _skel_pack((_praROutPost + 0), _ppraROutPost, ((char*)_primROut + 0), (uint32_t*)&(((uint32_t*)_rout2)[0]), (uint64_t*)&(((uint64_t*)_rout2)[1]), (uint64_t*)&(((uint64_t*)_rout2)[5]), (uint32_t*)&(((uint32_t*)_rout2)[18]), (uint32_t*)&(((uint32_t*)_rout2)[19]), SLIM_IFPTR32((char**)&(((uint32_t*)_rout2)[20]), (char**)&(((uint64_t*)_rout2)[10])), SLIM_IFPTR32((uint32_t*)&(((uint32_t*)_rout2)[21]), (uint32_t*)&(((uint32_t*)_rout2)[22])))); + _QAIC_CATCH(_nErr) {} + _allocator_deinit(_al); + return _nErr; +} +static __inline int _skel_method_1(int (*_pfn)(remote_handle64), uint32_t _sc, remote_arg* _pra) { + remote_arg* _praEnd = 0; + remote_handle64 _in0[1] = {0}; + remote_arg* _praRHandleIn = _pra + REMOTE_SCALARS_INBUFS(_sc) + REMOTE_SCALARS_OUTBUFS(_sc); + int _nErr = 0; + _praEnd = ((_pra + REMOTE_SCALARS_INBUFS(_sc)) + REMOTE_SCALARS_OUTBUFS(_sc) + REMOTE_SCALARS_INHANDLES(_sc) + REMOTE_SCALARS_OUTHANDLES(_sc)); + _QAIC_ASSERT(_nErr, REMOTE_SCALARS_INBUFS(_sc)==0); + _QAIC_ASSERT(_nErr, REMOTE_SCALARS_OUTBUFS(_sc)==0); + _QAIC_ASSERT(_nErr, REMOTE_SCALARS_INHANDLES(_sc)==1); + _QAIC_ASSERT(_nErr, REMOTE_SCALARS_OUTHANDLES(_sc)==0); + _QAIC_ASSERT(_nErr, (_pra + ((0 + 0) + (((1 + 0) + 0) + 0))) <= _praEnd); + _COPY(_in0, 0, &(_praRHandleIn[0].h64), 0, sizeof(remote_handle64)); + _TRY(_nErr, _pfn((remote_handle64)*_in0)); + _QAIC_CATCH(_nErr) {} + return _nErr; +} +static __inline int _compare_versions(char* stub_ver, char* skel_ver, int* result) { + unsigned long int major_stub = 0, minor_stub = 0, patch_stub = 0; + unsigned long int major_skel = 0, minor_skel = 0, patch_skel = 0; + char *saveptr1 = NULL; + char *token1 = NULL; + char *saveptr2 = NULL; + char *token2 = NULL; + int i=0; + for (i=0, token1 = strtok_r(stub_ver, ".", &saveptr1); i<3 && token1 != NULL; i++, token1 = strtok_r(NULL, ".", &saveptr1)) + { + unsigned long int tn = strtoul(token1, NULL,10); + if( tn > 999) + { + *result=-1; + return 0; + } + else + { + if(i==0) major_stub=tn; + if(i==1) minor_stub=tn; + if(i==2) patch_stub=tn; + } + } + for (i=0, token2 = strtok_r(skel_ver, ".", &saveptr2); i<3 && token2 != NULL; i++, token2 = strtok_r(NULL, ".", &saveptr2)) + { + unsigned long int tn = strtoul(token2, NULL,10); + if( tn > 999) + { + *result=-1; + return 0; + } + else + { + if(i==0) major_skel=tn; + if(i==1) minor_skel=tn; + if(i==2) patch_skel=tn; + } + } + if(major_stub=patch_stub)) + { + *result=1; + return 0; + } + } + *result=-1; + return 0; +} +static __inline int _stub_skel_version_check(char*_in0, int* resVal) { + int _nErr = 0; + char* p = strstr(_in0, "_idlver="); + if(!p) + { + *resVal = -1; + return 0; + } + p+=8; + int i=0,len=0, comVer=0,num_delimit=0, updtInxStub=0, updtInxSkel=0; + for(i=0;i2) + { + *resVal = -1; + return 0; + } + if ((p[i]>='0' && p[i]<='9') || (p[i]=='.')) + { + len++; + if(p[i]=='.') + { + num_delimit++; + } + } + else if(p[i]=='&') + { + break; + } + else + { + *resVal = -1; + return 0; + } + } + char* stubVer=(char*)MALLOC(len+1); + _QAIC_ASSERT(_nErr, stubVer!=NULL); + for(i=0;i='0' && p[i]<='9') || (p[i]=='.')) + { + stubVer[updtInxStub]=p[i]; + updtInxStub++; + } + else if(p[i]=='&') + { + break; + } + } + stubVer[len]='\0'; + char* skelVer=(char*)MALLOC(strlen(IDL_VERSION)+1); + _QAIC_ASSERT(_nErr, skelVer!=NULL); + for(i=0;i< strlen(IDL_VERSION);i++) + { + skelVer[updtInxSkel]=IDL_VERSION[i]; + updtInxSkel++; + } + skelVer[strlen(IDL_VERSION)]='\0'; + _TRY(_nErr, _compare_versions(stubVer, skelVer, &comVer)); + *resVal = 0; + if (comVer==-1) + { + *resVal = -1; + } + FREE(stubVer); + FREE(skelVer); + _QAIC_CATCH(_nErr) {} + return 0; +} +static __inline int _skel_method_2(int (*_pfn)(const char*, remote_handle64*), uint32_t _sc, remote_arg* _pra) { + remote_arg* _praEnd = 0; + char* _in0[1] = {0}; + uint32_t _in0Len[1] = {0}; + remote_handle64 _rout1[1] = {0}; + uint32_t* _primIn= 0; + remote_arg* _praRHandleROut = _pra + REMOTE_SCALARS_INBUFS(_sc) + REMOTE_SCALARS_OUTBUFS(_sc) + REMOTE_SCALARS_INHANDLES(_sc) ; + remote_arg* _praIn = 0; + int _nErr = 0; + _praEnd = ((_pra + REMOTE_SCALARS_INBUFS(_sc)) + REMOTE_SCALARS_OUTBUFS(_sc) + REMOTE_SCALARS_INHANDLES(_sc) + REMOTE_SCALARS_OUTHANDLES(_sc)); + _QAIC_ASSERT(_nErr, REMOTE_SCALARS_INBUFS(_sc)==2); + _QAIC_ASSERT(_nErr, REMOTE_SCALARS_OUTBUFS(_sc)==0); + _QAIC_ASSERT(_nErr, REMOTE_SCALARS_INHANDLES(_sc)==0); + _QAIC_ASSERT(_nErr, REMOTE_SCALARS_OUTHANDLES(_sc)==1); + _QAIC_ASSERT(_nErr, (_pra + ((2 + 0) + (((0 + 1) + 0) + 0))) <= _praEnd); + _QAIC_ASSERT(_nErr, _pra[0].buf.nLen >= 4); + _primIn = _pra[0].buf.pv; + _COPY(_in0Len, 0, _primIn, 0, 4); + _praIn = (_pra + 1); + _QAIC_ASSERT(_nErr, ((_praIn[0].buf.nLen / 1)) >= (size_t)(_in0Len[0])); + _in0[0] = _praIn[0].buf.pv; + _QAIC_ASSERT(_nErr, (_in0Len[0] > 0) && (_in0[0][(_in0Len[0] - 1)] == 0)); + int resVal; + _TRY(_nErr, _stub_skel_version_check(*_in0, &resVal)); + if(resVal==-1) + { + return AEE_ESTUBSKELVERMISMATCH; + } + _TRY(_nErr, _pfn((const char*)*_in0, (remote_handle64*)_rout1)); + _COPY(&(_praRHandleROut[0].h64), 0, _rout1, 0, sizeof(remote_handle64)); + _QAIC_CATCH(_nErr) {} + return _nErr; +} +__QAIC_SKEL_EXPORT int __QAIC_SKEL(ggmlop_skel_handle_invoke)(remote_handle64 _h, uint32_t _sc, remote_arg* _pra) __QAIC_SKEL_ATTRIBUTE { + switch(REMOTE_SCALARS_METHOD(_sc)){ + case 0: + return _skel_method_2(__QAIC_IMPL(ggmlop_open), _sc, _pra); + case 1: + return _skel_method_1(__QAIC_IMPL(ggmlop_close), _sc, _pra); + case 2: + return _skel_method(__QAIC_IMPL(ggmlop_add), _h, _sc, _pra); + case 3: + return _skel_method(__QAIC_IMPL(ggmlop_mulmat), _h, _sc, _pra); + } + return AEE_EUNSUPPORTED; +} + +/* Library version needs to be added in the name member of note_type structure in below format + * "lib.ver.1.0.0." + "" + ":" + "" + */ +const lib_ver_note_t so_ver __attribute__ ((section (".note.lib.ver"))) + __attribute__ ((visibility ("default"))) = { + 100, + 0, + 0, + "lib.ver.1.0.0.libggmlop_skel.so:4.5.0", + }; diff --git a/ggml/src/ggml-qnn/kernels/libggmlop_skel.so b/ggml/src/ggml-qnn/kernels/libggmlop_skel.so index 9d4be24f3263907fa6c2bf83eacbf6fe17941dd9..8dcdba3c2fd271b8a8ec61bc77f7b443bf6a9c1c 100755 GIT binary patch delta 5270 zcmcIo4RBM}l|JuEmSsag7P9;=o^9C%#~2xmVZnHxo^6n+Nt6sjoZ<<#fy6b0sKEg@ zH9U{`31kJ!%Nw#xsawQOlk6s&)yA2W>4f$nD;YyGX-1|Vl8sWu(99%h*CZ4slS$(J zt|SxTrqi@LduOiB`OdlLzI*OH=jh!xy5sQn<03mGii-L)L|idE_3)vw#1PC8FZIK zqEhGyQ(sQ<-G$nC+B2(T#Y8$fS93m`GJ=CHZUGf0&w-JU%he3s98g zB={ibtqJ~R&KnZ^Sgj{|4vhCHS{EKPSO!{pq6V zLuG0%g@(_h9}^r2@z=kiM12;i(`PCC)5nibY5!xlD;^1Ixuu8Z8yh$;w6^s4rl#&^ zI`xx05Xl^+_8rR9CenvaJ9Ni!{U6DbCJm}4ojDWye)X%^qt0jKLG^QeiF2Xb<9af9=C)A81!pCV+mruC#v%tlU-B~1&Lehc8G<|;~pTN*eACvUcdm-;L zkDqz|nxgc7<%>T14_Iug6jp05M*}&~b%0+s9t|7v0)>|%(sOhgq18YDXpbvu120a9t z2l_gw2>Jo20aV8vEG`eL8JHsmYzK~Qw2d+Mh-!w6ZOpmx0hVG6MFVi-PQ~O7tJxTI zdLK4D?k_1e;Lp6dWBHPBgBujVujBi)NN!zbFNVcJwlUv6&6-!4aT?3#$JV4Z=iMo+ z(7vCyFuij-_AFAV1@dy&pBDk7BZU&RRa`n35Zs;H^TSbS{A?F?fwlbGu& zBIg2MIo@daVlNiu&AP!ZW#72DSIf*V7eZQ1eu3~yZGC=D-pTJ#`yFs33ws`MyxlM! zJ~Jk59EkX}z4;Z+1M&g&4osBMXqq%4yey;9K)!@9lg$0B($F8%jbqxo`K`>iSF0>o zK5v6K#t+%cpPAeUt-HWkvjT$8Uw4kthU^=5^lO#sd(~e~ZJKBrQs=)D4G5BaF!O$D zAG|QMmLXI_+PMO!Q#SrFJ_}~d?V$cE%|q&sMxyF5dB1vPWJo2c@!&@7%Yss2zgApW zzk)vyL*3q|A}+}iTp%4fwNQWJ)Im?i*Yrz6ZUtKow-)7W$R@7g1d?gn?$ zK;)!$q^ML`#B*`HrKg(b;hL77IbTd|;yjJE^l+Wf-qK@)43b+d3n{}xej!4hHFq7g zAzjKhL<40AcZ*EspS!8ASdt&BK-gcCa?ZB9somKY4U}S|upj*&=vE8uo0{wDSZ?iI zC=r>@%U1S)ZTAB3-I|(FP{X+ey6iM~YL8BAX1hOaimI1*&@{5^uYi9X{FZxIc(AKx zrRBA#dQyvKELBSSr|#!FXv9OTh+7M0X$vt6xr>73e8v&kDj-hSMe6haaB5Rm<}W^C zQm@aY=bSwg-f)cWK6CJuQP$2p8;+J?yaxq`ui&cNwwq&@Zgd`OvK=*q4cE(L!z7o{ zdAV%MIQ5lEs-ij}=(r*mKZd2{+*cHwxIld+(uug-H6a(jOFtpOZ4X|tPRwg+WBync zTNm~=9&7fHpWHb?gI=c^8(*9BVsX{Tm35%hoD&q9?Lii%KJJ{{Qm1aaSf_3Qb%Qpx z8T?N0Z)p#^~uII!LI`c zE^u4vSb50gZ}td&TC?@j0mvJ}Lji;I+GLf)+TEBVr?n{9`JFmdhJ6!kg;sm8k>k)% zAgsU>)f`VTfu1{lF7Phrxp&@#ck%{!_FH%c>IRK_2EH8p zEysWv@U&x%^Q zqIw0+tr_QT#~_Q;;iJylUerHY5tFPI1Y@Iw><_&hD8bnkdkua=_~k&U$KV&-mf(3S z>+xb|gb`bymD6TRkeY3Y^T*Ed7(ih64a!5R2XmBSv7(pxzY|t4p@;d`>9#;8Ng+2A=&wK*mg8+ zqw^DvUC7D0uwjzP_Ujy5LULINb|V*jpZ4XP`3mP;S6MNqxe~M7tE+2Sgk7Id*uUU^ zk@{k?(*ep0%ey~70O*hUY;Fo(^W~i7dPyfsw%*~3#C5&CGm^zR;$jQna!0OCeRhoL zmRXijFI}rQ8>4BTU;WkZH>d43eJGb5kBLhAihzmq?3vb>5O-=ZCIYUnVUy^=C z&MSWXZ%F#Ho%DAl{f|4XJmGKU`$uprk?+da3Ga`2zg@D!IG*gBe(Pla3%R}smEg~k z9T~ur9Vyvo)~QWzGDhn^6Wg!u(bn14FeL6%w(SUdZ z&=s0mdTL~BWqVqB z1nkqBdo+9Y%JkI=*Rl3J+DF+9WyZMGE+55duWtwFY1jMJUH90-yy(`W*q+ta+c&wwyZQXP ze0rSr`bhHmEX`5%Q@ll7!rR0vpeI53TLms0PphBg-XY-ShW>jfFjvp$nb7`Nv`TsC zp@(-o;@iAs>n>mQmX$Tj#mer^om+Qoqhw3!AMtJ58JFBXzP6@L+_|l0iC6~WS|YAq zExHnh$#D-qw&UT>UWjJeE$4PiMI~;yG{vxr7Lz?bM+p^C1FfPJluo%c?>1C*8*tC(1zRf`Q- zMVP>bo{r9MsWEM@m?Ly+C&X2m{EE!4#<&u{Bg$zn19`RL;>}7MxYM}o@@B`q^>%C^ z6Xa*1=W-*u7u{?$6Tdw3%QWwt=N)dx$ABXNoxwl^y#^h(HDmtd)i|Ly)9_UBy=-`= zGr+W}iUvpOu{xPysv&%#zpz90N|Y+0-}#Z{YSK5k7XnehzhVJ@AGk{vPm(V!Q+- z@ehE#!2Fa1Qs&SB2>9_%4>%6Yk8>KI0_I1TFqlID;65By!Gu!2Q7d6#4ikfs@~` zl(^so1o6*XNF_etUx5=}uM+qiaO&5s5nx?ROqhi;{|(odBr+(A=F&~z1;BUVfgIcL z<*6T-Hw*4z#lQn`yp{{HXfXtDPY=KjECZfye2cfw!2KzhoBfk^r22GTq9msl&Uj8% zZ_x%8R|vJ*vx|#a%`$Dx;#I>J7kdP)s^(7Zcun5$#+pJwlLVvYUlK$yTD^3BS`0f8 z(>j;_keRL8drKWA?mmhbt1Zdk>B@VTwy1W#iRVVFaQNO@SrD?d6U$23hZWi{mX))g mF3={Hl}o;b@D-(nhcj5>RfdP*1@sgz=-MKpXHx{zLHED+M+R8{ delta 6105 zcmcgwdr(`~oj&*Kfp`m9h=+NIr-9m7U|V(#p;uP|3!69!!@5dfV;;eF@k?V9f)nhk z^(%HNdwhKx_u-*MPMf-En^gu+*E^H67oiaBb#_uMpRYcdOUTLGSGR6v*sL_^G?SycmPUwC z`Tr~H(=&wNh zWbjJNd8RFe>u~+AEBO5)xqpDIZ9k{y+FkigL2uqWL9do4nt!p7-aL+fs$eYWo#oQ5 zKVcy)MrY8Q`)<%nIMSEO5u%Y0G#nH=9%Tc^+bcF-fIVG`O<5 z%h}bQJMit2MN!GiA*++|$iUuFn#-LRiv4O&tsT9qd-9KNI35c0dllRpA?3FI?uJ&j zrRts6z07ul&$`ttHvAiA!KU=JLs#CUqhvjF^rOHT=2UuAz%Rlda5M-=P}Z5a-YIJTbO4P5pXI_!+`#6zEk@0R0{^Xlz4 zLLuO*@X;lxw+nHe;Rdt#3Ig6l?-RD&2n{3PtH2LClRaDQNuGp2(EApAG23p2P9V{4 z*LV`qnMCVo$co)uJ@&fSeRRG#=@yo%5lc0WrGm6ThL;L@G4#cy0%n23OC?}>ikGex zoG&e1E8tQr0bQ$NUhVM$yT{BTT`~qs#^NZo-!gBZOV({+TXxxBs9lA49!rylcxw|j z^MqOSV*ngZkP13mD~H_ZcnLZ#xDT!PpL{gZ+Qz1)`my%gvVDcCuaw`kk;lP3LM@J^ z9)UDQ+sf%QCbmpFl8?YFT51cpHuB9{E%V}RA>$;LM(5%N3qe*7v*ErRM$-NMPE}$WP?un zsoo&u(C`^HkIE8`)i%oF7PgVS7akWLKP9kVr#>>t7uG;Bw)!qlAI2{qa_KF^H|t8f zo((%E6LhQ9bv*3rcCCO-!x7!Ei7g=((wdGrS(vXZ_#8Z&qLd4SEpn|gsQPH;XLon1 zderapMO)>(mQC(-Up~in7sv`Q5WMLU2e-a_PIp%X9WMLU2 z|NRMBSjNbI8$!PJ%x5QLfq+yO8ToHW$XDCQXCY)^86#6d_O$7ik+I*hoQ$7obu1Dw zhmn#A?OZkFQUYEenolRel4d$39m%DOHST?rV3X{hh2;7Ro>L!{bgYF}fr=&N6m_0=X$U+wnm ztN$IjYc&VoXOWAi^lb)@u0md_qrQv5Z_>ySsV-x4%mBh}Kh!Qgmr-OcqRsB8?~+aV zt53pCsTyR}3%2W}G)(=eb@g2tGP(mB!B1H45os#J%r+g7GBaz6Y~ifKE-oy)T}>Q$ ze8A)_Z-3Mf(4zCzlbr6VbTqTMP8p3P(MZC%To>>{eF!hnO^{~#G7U*zuyf>`OC(zG zONF1oHkUT*PaoX8FKv%C2=vE?pxY!@Spzp=xcaW!_vasDWqdpF?y+gh6v-X+{gt4cm$RdA{g z2=|>v*m^bZT=^k=_^crG(NC8zqC=~>fVlp4&`Xz?gT4WuFS{T&;-dbU62`&FF_h1M<5lD1IT5=_JA0RHe_G}@Z=8t_!MF^u(3PR=0?yp zrJp`4uu4otfR;{j^zA|qeBY5io{3K&8k~(Wl3uAX-W)gc^nCNAUq3$iv~)4PEY>ai z@fqMAtCQY{{{pjF`Y^sU=FVQiog7;&h&(CnOvp&;LhHnlGRID&&OQ0GbULA!k)eCSXgld%10xGBRBf3`TTM5 zHK`)0Fw3v}eR?bDo38B_zlHt2oj-w(ntqXR)>BQ=k)#5~D)lE-ug7t;I4+KvIIXXQ zJAJuSe(v(AdQIs5o!_)fI?1?tD%3_|y+k+R^5HL@kBeGG){`2NNNS|ctg5&J?c_iw*@HevFJ~1ChYxkE-MxGN zzJsovU)k>}@7P>fX)N2*cHqfl=Z9MA@E3qy+m7ck4Qf~a7>4nMUkQSGQ2rH0TO9$IIk zmEPviVnm&UjmB!w9{$3G!G9Kj+Gz=6B^jL0Yh<(cboRFGK6qefW9Pokw!@uUc6J{; zxUXl^rky<-H>@3ew9xx0A<`d$C{C9Ae{(3KAEVbnapK8{K#Gv9AK3S;Q&ir@$AYm|<6t z3E+jI6`)NnH30u%3DyD6J%IoF0j!{N9W69I?g6~w0labv7H9&!&Rno*u?N;YfNO!T zA+8TK1#{2&~^uS)6d?JnSwDAocW298|YMs9b{0Z$qTsV4Y3rwQML zAq!+dHwncr0WZA{eib-;QLK*me?}X?{LBY2DzDO)Hs;j6vcUrF5&p z2MRq}nX8(>6ii6o%4gXLy_B;qM@{De)XT5SkEQ39)*aIRb-8MKNZh%D7uWF&^Ro0G z>kHUpCDKRhi`n(1QvQZwPE&?`2_mHX94eulNDs{o)Fs0q-zX>K2TKeKyYK%1@yBem From 6612cbc1d8c2eb5f19b1c270d158243ac78e6007 Mon Sep 17 00:00:00 2001 From: zhouwg Date: Mon, 24 Mar 2025 12:48:25 +0800 Subject: [PATCH 69/76] ggml-qnn: refine the entire ggml-qnn.cpp to make code more clear --- ggml/src/ggml-qnn/ggml-qnn.cpp | 208 ++++++++++--------- ggml/src/ggml-qnn/kernels/ggmlop_ap_skel.c | 8 +- ggml/src/ggml-qnn/kernels/ggmlop_ap_skel.h | 8 +- ggml/src/ggml-qnn/kernels/ggmlop_cdsp.c | 12 +- ggml/src/ggml-qnn/kernels/ggmlop_cdsp_skel.c | 8 +- ggml/src/ggml-qnn/kernels/libggmlop_skel.so | Bin 13704 -> 13736 bytes 6 files changed, 123 insertions(+), 121 deletions(-) diff --git a/ggml/src/ggml-qnn/ggml-qnn.cpp b/ggml/src/ggml-qnn/ggml-qnn.cpp index 70bdc625fe37b..3357251f290e0 100644 --- a/ggml/src/ggml-qnn/ggml-qnn.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn.cpp @@ -5,15 +5,20 @@ * https://www.qualcomm.com/developer/software/qualcomm-ai-engine-direct-sdk * https://developer.qualcomm.com/software/hexagon-dsp-sdk/tools * - * this single-source-file or self-contained implementation of ggml-qnn backend has 10 sections: + * there are three tech approaches to implement the ggml-hexagon backend for Qualcomm's Hexagon NPU: + * - general approach through Qualcomm QNN SDK:offload ggml op to QNN, then QNN will transfer to Hexagon cDSP + * - general approach through Qualcomm Hexagon SDK:offload ggml op to Hexagon cDSP directly + * - special approach through Qualcomm QNN SDK:mapping the entire ggml cgraph to a single QNN graph + * + * this single-source-file or self-contained implementation of ggml-hexagon backend has 10 sections: * section-1 forward/prototype declaration, global vars, macros, data structures * section-2 ggml-qnn internal troubleshooting function/class * section-3 helper function for WoA(Windows on ARM) * section-4 general helper function * section-5 QNN helper function * section-6 Hexagon DSP helper function - * section-7 ggml-qnn backend helper function / class - * section-8 implementation of ggml-qnn backend according to ggml's backend subsystem + * section-7 backend helper function / class + * section-8 implementation of ggml-hexagon backend according to specification in ggml backend subsystem * section-9 implementation of general approach through QNN and Hexagon DSP * section-10 implementation of special approach through QNN:mapping the entire ggml cgraph to a single QNN graph * @@ -131,6 +136,8 @@ class qnn_instance; struct qnn_parameter; struct ggml_backend_qnn_context; +typedef int (*pfn_mallopt)(int, int); +typedef int (*pfn_android_mallopt)(int, void *, size_t); typedef void (* ggmlqnn_op_func_t)(ggml_backend_qnn_context * ctx, ggml_tensor * op); typedef int (* notify_callback_fn)(void * context, int domain, int session, remote_rpc_status_flags_t status); typedef int (* ggmlhexagon_op_func_t)(remote_handle64 handle, const dsptensor * src0, const dsptensor * src1, dsptensor * dst); @@ -276,11 +283,11 @@ using _pfn_QnnSaver_initialize = decltype(QnnSaver_initialize); using _pfn_QnnInterface_getProviders = decltype(QnnInterface_getProviders); using _pfn_QnnSystemInterface_getProviders = decltype(QnnSystemInterface_getProviders); -//QNN resource management for the first technical approach(general approach in ggml-sycl or ggml-cann) +//QNN resource management for the general approach through QNN(similar to ggml-sycl or ggml-cann) using qnn_ptensors_t = std::vector< Qnn_Tensor_t *>; using qnn_singlenode_res_t = std::tuple; -//QNN resource management for the second technical approach(mapping the entire cgraph to a single QNN graph) +//QNN resource management for the special approach through QNN(mapping the entire cgraph to a single QNN graph) using qnn_tensors_t = std::vector< Qnn_Tensor_t >; using qnn_tensor_pair_t = std::tuple< ggml_tensor *, Qnn_Tensor_t *>; using qnn_tensor_pairs_t = std::vector< qnn_tensor_pair_t >; @@ -360,17 +367,19 @@ struct ggml_backend_qnn_context { QNN_SYSTEM_INTERFACE_VER_TYPE raw_system_interface; struct qcom_socinfo socinfo; - //QNN resource management for the first technical approach(general approach in ggml-sycl or ggml-cann) + //QNN resource management for the general approach through QNN(similar to ggml-sycl or ggml-cann) std::map qnn_singlenode_graph_map; - //QNN resource management for the second technical approach(mapping the entire cgraph to a single QNN graph) + //QNN resource management for the special approach through QNN(mapping the entire cgraph to a single QNN graph) std::map qnn_multinode_graph_map; + //quantize data -> fp32 std::unique_ptr work_data; std::vector> tasks; size_t work_size; size_t desired_size; int n_threads; + //hexagon resource management for the general approach through Hexagaon cDSP size_t rpc_mempool_len; void * rpc_mempool; remote_handle64 ggmlop_handle; @@ -381,7 +390,6 @@ struct qnn_op_caps { ggml_op op; const char * qnn_op_name; const size_t input_param_count; - const char * qnn_param_name; }; struct qnn_parameter { @@ -562,7 +570,7 @@ static domain hexagon_supported_domains[] = { }; static constexpr const qnn_op_caps ggmlqnn_k_op_caps[] = { - {true, GGML_OP_NONE, nullptr, 0, nullptr}, + {true, GGML_OP_NONE, nullptr, 0}, {false, GGML_OP_DUP}, {true, GGML_OP_ADD, QNN_OP_ELEMENT_WISE_ADD, 2}, {false, GGML_OP_ADD1}, @@ -985,12 +993,10 @@ static void ggmlqnn_get_timestring(char * p_currenttime) { } //fix some tricky memory issue -typedef int (*pfn_mallopt)(int, int); -typedef int (*pfn_android_mallopt)(int, void *, size_t); static void ggmlqnn_disable_android_tags(int disable) { if (0 == disable) return; - +#if defined(__ANDROID__) void * lib_handle = dlopen("libc.so", RTLD_LAZY); if (nullptr != lib_handle) { int api_level = android_get_device_api_level(); @@ -1013,6 +1019,7 @@ static void ggmlqnn_disable_android_tags(int disable) { } dlclose(lib_handle); } +#endif } // ================================================================================================= @@ -1338,32 +1345,6 @@ static const char * ggmlqnn_get_qnnerror_string(Qnn_ErrorHandle_t qnn_error_code } } -static Qnn_OpConfig_t ggmlqnn_create_op_config(const char * name, const char * package, const char * type, - Qnn_Param_t * params, uint32_t num_params, - Qnn_Tensor_t * inputs, uint32_t num_inputs, - Qnn_Tensor_t * outputs, uint32_t num_outputs) { - - char opcfg_name[GGML_MAX_NAME] = {}; - - //ensure the opcfg name is unique - if (nullptr == name) { - snprintf(opcfg_name, GGML_MAX_NAME, "opcfg_%-8d", ggmlqnn_get_idx(QNN_OPCFG_INDEX)); - } else { - snprintf(opcfg_name, GGML_MAX_NAME, "opcfg_%s_%-8d", name, ggmlqnn_get_idx(QNN_OPCFG_INDEX)); - } - GGMLQNN_LOG_DEBUG("create qnn opconfig %s", opcfg_name); - ggmlqnn_inc_idx(QNN_OPCFG_INDEX); - - Qnn_OpConfigV1_t v1 = {opcfg_name, package, type, - num_params, params, - num_inputs, inputs, - num_outputs, outputs - }; - Qnn_OpConfig_t opcfg = {QNN_OPCONFIG_VERSION_1, {v1}}; - - return opcfg; -} - // ================================================================================================= // section-6: Hexagon DSP helper function // ================================================================================================= @@ -2050,7 +2031,7 @@ static int ggmlhexagon_init_dsp(ggml_backend_qnn_context * ctx) { ggmlop_domain_uri = (char *)malloc(ggmlop_domain_uri_len); snprintf(ggmlop_domain_uri, ggmlop_domain_uri_len, "%s%s", ggmlop_URI, uri); GGMLQNN_LOG_INFO("ggmlop domain uri:%s\n", ggmlop_domain_uri); - hexagon_error = ggmlop_open(ggmlop_domain_uri, &ctx->ggmlop_handle); + hexagon_error = ggmlop_dsp_open(ggmlop_domain_uri, &ctx->ggmlop_handle); if (AEE_SUCCESS == hexagon_error) { GGMLQNN_LOG_INFO("succeed to open domain %d(%s)", domain_id, ggmlhexagon_get_dsp_name(domain_id)); GGMLQNN_LOG_INFO("only support GGML_OP_ADD on cDSP currently\n"); @@ -2082,7 +2063,7 @@ static void ggmlhexagon_close_cdsp(ggml_backend_qnn_context * ctx) { int hexagon_error = AEE_SUCCESS; GGMLQNN_LOG_DEBUG("enter %s", __func__); if (-1 != ctx->ggmlop_handle) { - hexagon_error = ggmlop_close(ctx->ggmlop_handle); + hexagon_error = ggmlop_dsp_close(ctx->ggmlop_handle); if (AEE_SUCCESS != hexagon_error) { GGMLQNN_LOG_WARN("error 0x%x: failed to close ggmlop handle", hexagon_error); } else { @@ -2109,18 +2090,18 @@ static void ggmlhexagon_compute(ggml_backend_qnn_context * ctx, struct ggml_tens void * wdata = nullptr; ggml_tensor * src0 = op->src[0]; - //TODO: src1 might-be nullptr + //src1 might-be nullptr for some ggml op ggml_tensor * src1 = op->src[1]; ggml_tensor * dst = op; ggml_type src0_type = src0->type; switch (op->op) { case GGML_OP_ADD: - op_func = ggmlop_add; + op_func = ggmlop_dsp_add; break; case GGML_OP_MUL_MAT: { wdata = ggmlqnn_type_trait(ctx, op); - op_func = ggmlop_mulmat; + op_func = ggmlop_dsp_mulmat; break; } default: @@ -2128,11 +2109,11 @@ static void ggmlhexagon_compute(ggml_backend_qnn_context * ctx, struct ggml_tens } if ((GGML_OP_MUL_MAT == op->op) && (src0_type != GGML_TYPE_F32)) { - dsptensor_0.data = wdata; - dsptensor_0.data_len = ctx->desired_size; + dsptensor_0.data = wdata; + dsptensor_0.data_len = ctx->desired_size; } else { - dsptensor_0.data = src0->data; - dsptensor_0.data_len= ggml_nbytes(src0); + dsptensor_0.data = src0->data; + dsptensor_0.data_len = ggml_nbytes(src0); } dsptensor_1.data = src1->data; @@ -2183,7 +2164,7 @@ static void ggmlhexagon_compute(ggml_backend_qnn_context * ctx, struct ggml_tens } // ================================================================================================= -// section-7:ggml-qnn backend helper function / class +// section-7: backend helper function / class // ================================================================================================= static const char * ggmlqnn_get_socmodel_desc(uint32_t soc_model) { switch (soc_model) { @@ -3510,7 +3491,7 @@ int qnn_instance::qnn_init(const QnnSaver_Config_t ** saver_config) { } } - auto qnnstatus = QNN_SUCCESS; + Qnn_ErrorHandle_t qnnstatus = QNN_SUCCESS; if (_device_id == QNN_BACKEND_NPU) { //TODO: remove duplicated code between here and function htp_print_info const QnnDevice_PlatformInfo_t * p_info = nullptr; @@ -3587,7 +3568,6 @@ int qnn_instance::qnn_init(const QnnSaver_Config_t ** saver_config) { } #if defined(__ANDROID__) || defined(__linux__) - //_rpc_lib_handle = dlopen("libcdsprpc.so", RTLD_NOW | RTLD_LOCAL); std::filesystem::path full_path(std::string(g_qnn_params.qnn_runtimelib_path) + "libcdsprpc.so"); full_path /= std::filesystem::path("libcdsprpc.so").filename(); _rpc_lib_handle = dlopen(full_path.string().c_str(), RTLD_NOW | RTLD_LOCAL); @@ -4182,51 +4162,30 @@ static uint8_t * ggmlqnn_create_rpc_buffer(qnn_instance * instance, const ggml_t return qnn_rpcbuffer; } -static void ggmlqnn_load_cfg() { - //this function can be called in various scenarios - static bool initialized = false; - if (initialized) { - GGMLQNN_LOG_INFO("qnn cfg file already loadded\n"); - return; - } - char time_string[GGML_QNN_TMPBUF_LEN]; - memset(time_string, 0, GGML_QNN_TMPBUF_LEN); - ggmlqnn_get_timestring(time_string); - GGMLQNN_LOG_DEBUG("program running start time:%s", time_string); - ggmlqnn_disable_android_tags(1); +static Qnn_OpConfig_t ggmlqnn_create_op_config(const char * name, const char * package, const char * type, + Qnn_Param_t * params, uint32_t num_params, + Qnn_Tensor_t * inputs, uint32_t num_inputs, + Qnn_Tensor_t * outputs, uint32_t num_outputs) { - std::string cfg_filename = std::string(g_qnn_params.qnn_runtimelib_path) + std::string(g_qnn_params.qnn_cfgfilename); - GGMLQNN_LOG_INFO("load ggml-qnn config from %s", cfg_filename.c_str()); - qnn_cfg qnncfg_instance; - qnncfg_instance.load(cfg_filename); - qnncfg_instance.dump([](const std::string & section, const std::string & key, const std::string value) { - std::ostringstream tmposs; - tmposs << "section[" << std::setw(10) << std::left << section << "],[" << std::setw(25) << std::left << key << "] = [" << value << "]" << std::endl; - GGMLQNN_LOG_INFO("%s", tmposs.str().c_str()); - }); - std::string precision_mode; - qnncfg_instance.get_intvalue("general", "print_qnn_internal_log", g_qnn_params.print_qnn_internal_log, 0); - qnncfg_instance.get_intvalue("general", "enable_perf", g_qnn_params.enable_perf, 0); - qnncfg_instance.get_intvalue("general", "print_tensors_info", g_qnn_params.print_tensors_info, 0); - qnncfg_instance.get_intvalue("general", "dump_op_info", g_qnn_params.dump_op_info, 0); - qnncfg_instance.get_intvalue("general", "inference_approach", g_qnn_params.inference_approach, 0); - qnncfg_instance.get_intvalue("general", "qnn_backend", g_qnn_params.qnn_backend, 2); - qnncfg_instance.get_intvalue("npu", "hvx_threads", g_qnn_params.hvx_threads, 4); - qnncfg_instance.get_intvalue("npu", "vtcm_size_in_mb", g_qnn_params.vtcm_size_in_mb, 8); - qnncfg_instance.get_intvalue("npu", "enable_dlbc", g_qnn_params.enable_dlbc, 0); - qnncfg_instance.get_stringvalue("npu", "precision_mode", precision_mode, "fp32"); - GGMLQNN_LOG_INFO("print_qnn_internal_log=%d", g_qnn_params.print_qnn_internal_log); - GGMLQNN_LOG_INFO("inference_approach=%d(%s)", g_qnn_params.inference_approach, - ggmlqnn_get_inference_approach_name(g_qnn_params.inference_approach)); - GGMLQNN_LOG_INFO("qnn_backend=%d", g_qnn_params.qnn_backend); - GGMLQNN_LOG_INFO("npu inference precision mode=%s", precision_mode.c_str()); - GGMLQNN_LOG_INFO("qnn runtime lib path=%s", g_qnn_params.qnn_runtimelib_path); - if (precision_mode.find("fp16") != std::string::npos) { - g_qnn_params.precision_mode = 1; + char opcfg_name[GGML_MAX_NAME] = {}; + + //ensure the opcfg name is unique + if (nullptr == name) { + snprintf(opcfg_name, GGML_MAX_NAME, "opcfg_%-8d", ggmlqnn_get_idx(QNN_OPCFG_INDEX)); } else { - g_qnn_params.precision_mode = 0; + snprintf(opcfg_name, GGML_MAX_NAME, "opcfg_%s_%-8d", name, ggmlqnn_get_idx(QNN_OPCFG_INDEX)); } - initialized = true; + GGMLQNN_LOG_DEBUG("create qnn opconfig %s", opcfg_name); + ggmlqnn_inc_idx(QNN_OPCFG_INDEX); + + Qnn_OpConfigV1_t v1 = {opcfg_name, package, type, + num_params, params, + num_inputs, inputs, + num_outputs, outputs + }; + Qnn_OpConfig_t opcfg = {QNN_OPCONFIG_VERSION_1, {v1}}; + + return opcfg; } static Qnn_Tensor_t * ggmlqnn_create_general_tensor(qnn_instance * instance, Qnn_GraphHandle_t graph_handle, @@ -4349,8 +4308,55 @@ static Qnn_Tensor_t * ggmlqnn_create_compute_tensor(qnn_instance * instance, Qnn return p_qnn_tensor; } +static void ggmlqnn_load_cfg() { + //this function can be called in various scenarios + static bool initialized = false; + if (initialized) { + GGMLQNN_LOG_INFO("qnn cfg file already loadded\n"); + return; + } + char time_string[GGML_QNN_TMPBUF_LEN]; + memset(time_string, 0, GGML_QNN_TMPBUF_LEN); + ggmlqnn_get_timestring(time_string); + GGMLQNN_LOG_DEBUG("program running start time:%s", time_string); + ggmlqnn_disable_android_tags(1); + + std::string cfg_filename = std::string(g_qnn_params.qnn_runtimelib_path) + std::string(g_qnn_params.qnn_cfgfilename); + GGMLQNN_LOG_INFO("load ggml-qnn config from %s", cfg_filename.c_str()); + qnn_cfg qnncfg_instance; + qnncfg_instance.load(cfg_filename); + qnncfg_instance.dump([](const std::string & section, const std::string & key, const std::string value) { + std::ostringstream tmposs; + tmposs << "section[" << std::setw(10) << std::left << section << "],[" << std::setw(25) << std::left << key << "] = [" << value << "]" << std::endl; + GGMLQNN_LOG_INFO("%s", tmposs.str().c_str()); + }); + std::string precision_mode; + qnncfg_instance.get_intvalue("general", "print_qnn_internal_log", g_qnn_params.print_qnn_internal_log, 0); + qnncfg_instance.get_intvalue("general", "enable_perf", g_qnn_params.enable_perf, 0); + qnncfg_instance.get_intvalue("general", "print_tensors_info", g_qnn_params.print_tensors_info, 0); + qnncfg_instance.get_intvalue("general", "dump_op_info", g_qnn_params.dump_op_info, 0); + qnncfg_instance.get_intvalue("general", "inference_approach", g_qnn_params.inference_approach, 0); + qnncfg_instance.get_intvalue("general", "qnn_backend", g_qnn_params.qnn_backend, 2); + qnncfg_instance.get_intvalue("npu", "hvx_threads", g_qnn_params.hvx_threads, 4); + qnncfg_instance.get_intvalue("npu", "vtcm_size_in_mb", g_qnn_params.vtcm_size_in_mb, 8); + qnncfg_instance.get_intvalue("npu", "enable_dlbc", g_qnn_params.enable_dlbc, 0); + qnncfg_instance.get_stringvalue("npu", "precision_mode", precision_mode, "fp32"); + GGMLQNN_LOG_INFO("print_qnn_internal_log=%d", g_qnn_params.print_qnn_internal_log); + GGMLQNN_LOG_INFO("inference_approach=%d(%s)", g_qnn_params.inference_approach, + ggmlqnn_get_inference_approach_name(g_qnn_params.inference_approach)); + GGMLQNN_LOG_INFO("qnn_backend=%d", g_qnn_params.qnn_backend); + GGMLQNN_LOG_INFO("npu inference precision mode=%s", precision_mode.c_str()); + GGMLQNN_LOG_INFO("qnn runtime lib path=%s", g_qnn_params.qnn_runtimelib_path); + if (precision_mode.find("fp16") != std::string::npos) { + g_qnn_params.precision_mode = 1; + } else { + g_qnn_params.precision_mode = 0; + } + initialized = true; +} + // ================================================================================================= -// section-8: implementation of ggml-qnn backend +// section-8: implementation of ggml-hexagon backend according to ggml backend subsystem // ================================================================================================= static bool ggmlqnn_same_types(const ggml_backend_qnn_context * ctx, const ggml_tensor * op_tensor) { GGML_UNUSED(ctx); @@ -4375,7 +4381,6 @@ static bool ggmlqnn_same_types(const ggml_backend_qnn_context * ctx, const ggml_ static bool ggmlhexagon_can_handle_op(const ggml_backend_qnn_context * ctx, const struct ggml_tensor * op_tensor) { struct ggml_tensor * src0 = op_tensor->src[0]; struct ggml_tensor * src1 = op_tensor->src[1]; - const int64_t ne00 = op_tensor->src[0]->ne[0]; uint32_t src0_rank = ggml_n_dims(src0); uint32_t src1_rank = 0; @@ -4387,14 +4392,11 @@ static bool ggmlhexagon_can_handle_op(const ggml_backend_qnn_context * ctx, cons if (op_tensor->op != GGML_OP_ADD) return false; - //ggmlqnn_dump_op_info(op_tensor); + ggmlqnn_dump_op_info(op_tensor); if (!ggml_are_same_shape(src0, src1)) { return false; } - if (ne00 < 32) - return false; - return ggmlqnn_same_types(ctx, op_tensor); } @@ -5057,7 +5059,7 @@ struct ggml_backend_qnn_reg_context { static const char * ggml_backend_qnn_reg_get_name(ggml_backend_reg_t reg) { GGML_UNUSED(reg); - return "ggml-qnn"; + return "ggml-hexagon"; } static size_t ggml_backend_qnn_reg_get_device_count(ggml_backend_reg_t reg) { @@ -5264,7 +5266,7 @@ ggml_backend_t ggml_backend_qnn_init(size_t device, const char * qnn_lib_path) { GGML_BACKEND_DL_IMPL(ggml_backend_qnn_reg) // ================================================================================================= -// section-9: general approach: offload GGML op to QNN backend or offload GGML op to Hexagon DSP directly +// section-9: general approach: offload GGML op to QNN backend or offload GGML op to Hexagon cDSP directly // ================================================================================================= static inline uint32_t ggmlqnn_get_tensor_data_size(const ggml_tensor * tensor) { /* @@ -5296,8 +5298,8 @@ static inline bool ggmlqnn_is_valid_params(ggml_backend_qnn_context * ctx, const } /* - * provide a general skeleton to offload ggml op to QNN backend or Hexagon cDSP: peform element-wise operation on 1/2 - * input tensors and 1 output tensors + * provide a general skeleton to offload ggml op to QNN backend or Hexagon cDSP: perform element-wise + * operation on 1/2 input tensors and 1 output tensors */ static void ggmlqnn_compute_elementwise(ggml_backend_qnn_context * ctx, ggml_tensor * op) { Qnn_ErrorHandle_t error = QNN_SUCCESS; @@ -5675,7 +5677,7 @@ static void ggmlqnn_compute_mul_mat_4d(ggml_backend_qnn_context * ctx, ggml_tens operation when offloading mulmat to QNN backend. this implementation will handle transpose in func ggmlqnn_compute_create_general_tensor() - * @param ctx the context of ggml-qnn backend + * @param ctx the context of backend * @param op the destination tensor where the result of the matrix multiplication will be stored. * * @note the logic of ggmlqnn_compute_mul_mat is similar to ggmlqnn_compute_op_two_tensors but much more complicated diff --git a/ggml/src/ggml-qnn/kernels/ggmlop_ap_skel.c b/ggml/src/ggml-qnn/kernels/ggmlop_ap_skel.c index 1e1ce6488d25e..6f2c37e4087cc 100644 --- a/ggml/src/ggml-qnn/kernels/ggmlop_ap_skel.c +++ b/ggml/src/ggml-qnn/kernels/ggmlop_ap_skel.c @@ -288,10 +288,10 @@ __QAIC_SLIM_EXPORT const Interface __QAIC_SLIM(ggmlop_slim) = {4,&(methodArrays[ #ifdef __cplusplus extern "C" { #endif -__QAIC_STUB_EXPORT int __QAIC_STUB(ggmlop_open)(const char* uri, remote_handle64* h) __QAIC_STUB_ATTRIBUTE { +__QAIC_STUB_EXPORT int __QAIC_STUB(ggmlop_dsp_open)(const char* uri, remote_handle64* h) __QAIC_STUB_ATTRIBUTE { return __QAIC_REMOTE(remote_handle64_open)(uri, h); } -__QAIC_STUB_EXPORT int __QAIC_STUB(ggmlop_close)(remote_handle64 h) __QAIC_STUB_ATTRIBUTE { +__QAIC_STUB_EXPORT int __QAIC_STUB(ggmlop_dsp_close)(remote_handle64 h) __QAIC_STUB_ATTRIBUTE { return __QAIC_REMOTE(remote_handle64_close)(h); } static __inline int _stub_unpack(_ATTRIBUTE_UNUSED remote_arg* _praROutPost, _ATTRIBUTE_UNUSED remote_arg* _ppraROutPost[1], _ATTRIBUTE_UNUSED void* _primROut, _ATTRIBUTE_UNUSED uint32_t _rout0[1], _ATTRIBUTE_UNUSED uint64_t _rout1[4], _ATTRIBUTE_UNUSED uint64_t _rout2[4], _ATTRIBUTE_UNUSED uint32_t _rout3[1], _ATTRIBUTE_UNUSED uint32_t _rout4[1], _ATTRIBUTE_UNUSED char* _rout5[1], _ATTRIBUTE_UNUSED uint32_t _rout5Len[1]) { @@ -419,11 +419,11 @@ static __inline int _stub_method(remote_handle64 _handle, uint32_t _mid, uint64_ _allocator_deinit(_al); return _nErr; } -__QAIC_STUB_EXPORT int __QAIC_STUB(ggmlop_add)(remote_handle64 _handle, const dsptensor* src0, const dsptensor* src1, dsptensor* dst) __QAIC_STUB_ATTRIBUTE { +__QAIC_STUB_EXPORT int __QAIC_STUB(ggmlop_dsp_add)(remote_handle64 _handle, const dsptensor* src0, const dsptensor* src1, dsptensor* dst) __QAIC_STUB_ATTRIBUTE { uint32_t _mid = 2; return _stub_method(_handle, _mid, (uint64_t*)src0, (uint64_t*)src1, (uint64_t*)dst); } -__QAIC_STUB_EXPORT int __QAIC_STUB(ggmlop_mulmat)(remote_handle64 _handle, const dsptensor* src0, const dsptensor* src1, dsptensor* dst) __QAIC_STUB_ATTRIBUTE { +__QAIC_STUB_EXPORT int __QAIC_STUB(ggmlop_dsp_mulmat)(remote_handle64 _handle, const dsptensor* src0, const dsptensor* src1, dsptensor* dst) __QAIC_STUB_ATTRIBUTE { uint32_t _mid = 3; return _stub_method(_handle, _mid, (uint64_t*)src0, (uint64_t*)src1, (uint64_t*)dst); } diff --git a/ggml/src/ggml-qnn/kernels/ggmlop_ap_skel.h b/ggml/src/ggml-qnn/kernels/ggmlop_ap_skel.h index 0301f8f78f8d2..1273cb76b1797 100644 --- a/ggml/src/ggml-qnn/kernels/ggmlop_ap_skel.h +++ b/ggml/src/ggml-qnn/kernels/ggmlop_ap_skel.h @@ -268,7 +268,7 @@ struct dsptensor { * @param h, resulting handle * @retval, 0 on success */ -__QAIC_HEADER_EXPORT int __QAIC_HEADER(ggmlop_open)(const char* uri, remote_handle64* h) __QAIC_HEADER_ATTRIBUTE; +__QAIC_HEADER_EXPORT int __QAIC_HEADER(ggmlop_dsp_open)(const char* uri, remote_handle64* h) __QAIC_HEADER_ATTRIBUTE; /** * Closes a handle. If this is the last handle to close, the session * is closed as well, releasing all the allocated resources. @@ -276,9 +276,9 @@ __QAIC_HEADER_EXPORT int __QAIC_HEADER(ggmlop_open)(const char* uri, remote_hand * @param h, the handle to close * @retval, 0 on success, should always succeed */ -__QAIC_HEADER_EXPORT int __QAIC_HEADER(ggmlop_close)(remote_handle64 h) __QAIC_HEADER_ATTRIBUTE; -__QAIC_HEADER_EXPORT int __QAIC_HEADER(ggmlop_add)(remote_handle64 _h, const dsptensor * src0, const dsptensor * src1, dsptensor * dst) __QAIC_HEADER_ATTRIBUTE; -__QAIC_HEADER_EXPORT int __QAIC_HEADER(ggmlop_mulmat)(remote_handle64 _h, const dsptensor * src0, const dsptensor * src1, dsptensor * dst) __QAIC_HEADER_ATTRIBUTE; +__QAIC_HEADER_EXPORT int __QAIC_HEADER(ggmlop_dsp_close)(remote_handle64 h) __QAIC_HEADER_ATTRIBUTE; +__QAIC_HEADER_EXPORT int __QAIC_HEADER(ggmlop_dsp_add)(remote_handle64 _h, const dsptensor * src0, const dsptensor * src1, dsptensor * dst) __QAIC_HEADER_ATTRIBUTE; +__QAIC_HEADER_EXPORT int __QAIC_HEADER(ggmlop_dsp_mulmat)(remote_handle64 _h, const dsptensor * src0, const dsptensor * src1, dsptensor * dst) __QAIC_HEADER_ATTRIBUTE; #ifndef ggmlop_URI #define ggmlop_URI "file:///libggmlop_skel.so?ggmlop_skel_handle_invoke&_modver=1.0&_idlver=0.0.1" #endif /*ggmlop_URI*/ diff --git a/ggml/src/ggml-qnn/kernels/ggmlop_cdsp.c b/ggml/src/ggml-qnn/kernels/ggmlop_cdsp.c index bddafa29ea81e..b5cfd8810cfe6 100644 --- a/ggml/src/ggml-qnn/kernels/ggmlop_cdsp.c +++ b/ggml/src/ggml-qnn/kernels/ggmlop_cdsp.c @@ -9,7 +9,7 @@ #define ggml_tensor dsptensor -int ggmlop_open(const char*uri, remote_handle64* handle) { +int ggmlop_dsp_open(const char*uri, remote_handle64* handle) { void *tptr = NULL; FARF(HIGH, "uri %s", uri); tptr = (void *)malloc(1); @@ -18,7 +18,7 @@ int ggmlop_open(const char*uri, remote_handle64* handle) { return 0; } -int ggmlop_close(remote_handle64 handle) { +int ggmlop_dsp_close(remote_handle64 handle) { if (handle) free((void*)handle); return 0; @@ -279,9 +279,9 @@ static void ggml_compute_forward_add_f32( } } -int ggmlop_add(remote_handle64 h, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) +int ggmlop_dsp_add(remote_handle64 h, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - FARF(HIGH, "=============== DSP: ggmlop_add "); + FARF(HIGH, "=============== DSP: ggmlop_dsp_add "); switch (src0->type) { case GGML_TYPE_F32: { @@ -349,8 +349,8 @@ int ggmlop_add(remote_handle64 h, const ggml_tensor * src0, const ggml_tensor * } -int ggmlop_mulmat(remote_handle64 h, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - FARF(HIGH, "=============== DSP: ggmlop_mulmat "); +int ggmlop_dsp_mulmat(remote_handle64 h, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + FARF(HIGH, "=============== DSP: ggmlop_dsp_mulmat "); GGML_TENSOR_BINARY_OP_LOCALS diff --git a/ggml/src/ggml-qnn/kernels/ggmlop_cdsp_skel.c b/ggml/src/ggml-qnn/kernels/ggmlop_cdsp_skel.c index 33d47174bf5ef..9d6b64fd6b570 100644 --- a/ggml/src/ggml-qnn/kernels/ggmlop_cdsp_skel.c +++ b/ggml/src/ggml-qnn/kernels/ggmlop_cdsp_skel.c @@ -573,13 +573,13 @@ static __inline int _skel_method_2(int (*_pfn)(const char*, remote_handle64*), u __QAIC_SKEL_EXPORT int __QAIC_SKEL(ggmlop_skel_handle_invoke)(remote_handle64 _h, uint32_t _sc, remote_arg* _pra) __QAIC_SKEL_ATTRIBUTE { switch(REMOTE_SCALARS_METHOD(_sc)){ case 0: - return _skel_method_2(__QAIC_IMPL(ggmlop_open), _sc, _pra); + return _skel_method_2(__QAIC_IMPL(ggmlop_dsp_open), _sc, _pra); case 1: - return _skel_method_1(__QAIC_IMPL(ggmlop_close), _sc, _pra); + return _skel_method_1(__QAIC_IMPL(ggmlop_dsp_close), _sc, _pra); case 2: - return _skel_method(__QAIC_IMPL(ggmlop_add), _h, _sc, _pra); + return _skel_method(__QAIC_IMPL(ggmlop_dsp_add), _h, _sc, _pra); case 3: - return _skel_method(__QAIC_IMPL(ggmlop_mulmat), _h, _sc, _pra); + return _skel_method(__QAIC_IMPL(ggmlop_dsp_mulmat), _h, _sc, _pra); } return AEE_EUNSUPPORTED; } diff --git a/ggml/src/ggml-qnn/kernels/libggmlop_skel.so b/ggml/src/ggml-qnn/kernels/libggmlop_skel.so index 8dcdba3c2fd271b8a8ec61bc77f7b443bf6a9c1c..8695cbf36949a45dfd7ce3d7cec38ad6f8fea6c4 100755 GIT binary patch delta 2409 zcmb7_e@t6d6vxjk{XzNJ@8lzGYNtS3;f>W7cY>Nz|ng3w?$AC^_LL7ciUmtsU|9O)0&i$Nw z-+TARx#x@=nmRP9i1=8k*J>^%G8cdHWVwj8r4enn66KJcb`sGH^f=T>f|_9W*ztfk z(BG;y6VDtxLk1oJE6MTvy>ak3cpEJ9hp;{m(bKR_%BJs#qG+nZA6(J^mic$6f)~uN z{6fAUYR3l~6a9AD0oH`aSLoOQZqPCJ?*<3mM31ALLuu43!bHOuIfY%|8Fpa6EPik< zF~Cm!;0YU1B*8X1YWUzA27X(|JOL@Mo9cFSclSp6`oo=r{o&ptk=~BiXs|DQwChz0 z!`Bx+&~dmk(j@~aFWPy?{z%_om;9{8eY(~nucWkO>0a;a?~*2Slit+Cp z9ae!|A~Hc!!v(t zx;$cOb}2Z;n+lN^9v|Av$@ZwcW!Vslyl%}CnR3-y?c$k0W^izx6tz9eZJVsJ71XAx zo;mg;XLFU?Ce^>dZJsq`k=q$#>qQ<@{K5pS{4{A=zA#xBwD+8eZf2kOiiQf8%m4fh zZJL)CZO@9h{M%L@Js89~7F&9@#ZQei%(TTAdY5scl-vzf&nX?5)s zPst|NU}>nOCtl;JIQjbKOBAPQQILk*&|sV#8F(fC(b$qJEX3E+?|vnf0}a7+Sia}3 z$$78Yse{l zq~lNyq!Ib9@|c~2h=XS|c3b&Cm=pwHQJn)J@qblrcuHe`d5SEi&Ow>!+s>xir?A=U ztAvRS`5MK(DV`Q2B{4sB#wzoi)ZePChNY6sUMa{0EW9$7l*HA-Dz-RX?I@NHl%I;S z+XaW9bee{p0`o>OUxS?m^SxpI0rn1_58xdtlxso)GuvVAFZa_9a;XkAgXe zalcO>i7vyzGiFCM2CRX3_n8w(5D8zE3~EI`>G);v$BH)LEIJ6j1AY#}1kQZ`+)%2f z5#~3+ZQ!UdD#X+BfN4D5WbKub=3sWwwfHmP= zT{ZCCpw7YDx?x}wZ$(nL-)>-s9H=OXmKwZ1u;y~m`sOYJw}SQ6G6d!uq1B2Pz*;IyVssg-FV|PV`r>s9tgmiAg5!!>ew{eDU%~on znuX--^JwLGe>PRXaU2d7hsi;^z{6YYq-OAKjjCzGdJuI_B3d$zLjO@D-h| z^70kEQFT`MwM<01GM6ZUdIyT!eT-K5({7SDfx`wYZ_kog=GQ3Q@!-Ek2*6d>9I?V&7MgieY4 z!Oavv3;%=RL%t1K@;4|Lx=8#n1G~Z129|h4Me%|nq8)&2N~fJjXhh~&e5K5=5j^Z9 z>Pp6)bjb9A9upriu>9d0e8uwkmVw^rtF13nTU&QmZ(sDKuHJ!Gibh-DZg1*o>1s_y zx{q{qH@(7P>kI3fT3S-x-o94Otnl-v);hM^YPsIl=aI>$!~(;`S8Vy-99RXqTmIdn z+gYsaIVyd8rA*n_Z_lsD)B`DH*A09rWniqw$i+y*$irC2Z`pTw|H49D^AQE%ks4Xh z?RNg%eqRyabmXcmUUh8s>eD-itqE8qo9KerY5cS^|A1Z*m&Kz`aZYTDmifKd@+pvc zv1K;8SQ2L@Y4Nuy%fgwdf{3ewV-K=MzoEf`h49_muxXZOoqN?hKav)XbwuzbiMo#B z#PM-ToNhQQylCQUCjQaHb0(gB$nEmJUrqeW8t-aICd{unH2%)SmmhMQyzqvJ7uew{ zah$}*sjuT<*Hcb47eC~lu{lbRtzL|m2XFLNt7rM3cfh}=t|L(%*!0e85B5`n zVucYJEW(H+$en>l@(#xT^hTAs#0^CU(okGI1mW{V;`fR&Y$sHRBg$5B0tGEK zeF{Ia4CTBEr&=>;RILtO4WIO-dE;X!Zjg#dKyD2Ymkrmw!5T?q8va{^fNdmQhPT{yj*W^$klV##DAMu$-8kPqy=p3rv2au;s;6i3c8b%T=(k+veh`9ztBiZQ1K4 zKODnr4i9YwUr#Ysqzar8S;tNj4;fe3~`?m${gXm&|d=i#iJWKo)EZL7?zP&Bz&@LCw zt_;|do5D4fLA$J$kDDrkE;+$6n)z5|sY8y-QgM8yvO}pa_|dAMdY_-EDpP%YxT>t+ O&Ss*I4WE{(u73gdVuRTL From f81a739fc2322356e24169c84254ca970713077b Mon Sep 17 00:00:00 2001 From: zhouwg Date: Mon, 24 Mar 2025 21:52:56 +0800 Subject: [PATCH 70/76] ggml-qnn: refine the entire ggml-qnn.cpp to make code more clear --- ggml/src/ggml-qnn/ggml-qnn.cpp | 177 ++++--- ggml/src/ggml-qnn/kernels/ggmlop_cdsp.c | 536 +++++++++++++------- ggml/src/ggml-qnn/kernels/libggmlop_skel.so | Bin 13736 -> 13672 bytes 3 files changed, 460 insertions(+), 253 deletions(-) diff --git a/ggml/src/ggml-qnn/ggml-qnn.cpp b/ggml/src/ggml-qnn/ggml-qnn.cpp index 3357251f290e0..847191b7dbc0c 100644 --- a/ggml/src/ggml-qnn/ggml-qnn.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn.cpp @@ -367,7 +367,7 @@ struct ggml_backend_qnn_context { QNN_SYSTEM_INTERFACE_VER_TYPE raw_system_interface; struct qcom_socinfo socinfo; - //QNN resource management for the general approach through QNN(similar to ggml-sycl or ggml-cann) + //QNN resource management for the general approach through QNN(similar to ggml-sycl or ggml-opencl) std::map qnn_singlenode_graph_map; //QNN resource management for the special approach through QNN(mapping the entire cgraph to a single QNN graph) std::map qnn_multinode_graph_map; @@ -379,7 +379,7 @@ struct ggml_backend_qnn_context { size_t desired_size; int n_threads; - //hexagon resource management for the general approach through Hexagaon cDSP + //hexagon resource management for the general approach through Hexagaon cDSP(similar to ggml-sycl or ggml-opencl) size_t rpc_mempool_len; void * rpc_mempool; remote_handle64 ggmlop_handle; @@ -719,10 +719,14 @@ static void ggmlqnn_print_tensors_info(const char * func_name, const ggml_backen if (nullptr != func_name && nullptr != ctx) { GGMLQNN_LOG_DEBUG("call %s in dev %s\n", func_name, ctx->name); } - GGMLQNN_LOG_DEBUG("%-6s: type = %i (%s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi, %5zi)", - src0->name, - src0->type, ggml_type_name(src0->type), src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], - src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3]); + if (nullptr != src0) { + GGMLQNN_LOG_DEBUG( + "%-6s: type = %i (%s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi, %5zi)", + src0->name, + src0->type, ggml_type_name(src0->type), src0->ne[0], src0->ne[1], src0->ne[2], + src0->ne[3], + src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3]); + } if (nullptr != src1) { GGMLQNN_LOG_DEBUG( "%-6s: type = %i (%s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi, %5zi)", @@ -874,7 +878,7 @@ static const char * dlerror(void) { // ================================================================================================= // section-4: general helper function // ================================================================================================= -//ensure every QNN tensor/opcfg name is unique +//ensure every QNN tensor/opcfg name is unique, threadsafe is not required at the moment static void ggmlqnn_reset_idx() { g_qnntensor_idx = 0; g_qnnopcfg_idx = 0; @@ -1472,7 +1476,7 @@ static int ggmlhexagon_get_domains_info(const char * domain_type, int * num_doma } } *domains_info = req.sys.domains; - *num_domains = req.sys.num_domains; + *num_domains = req.sys.num_domains; } else { hexagon_err = AEE_EUNSUPPORTED; goto bail; @@ -1498,9 +1502,9 @@ static int ggmlhexagon_get_dsp_support(int * domain) { } if (0 == dsp_capability_domain.capability) { - dsp_capability_domain.domain = HEXAGON_ADSP; + dsp_capability_domain.domain = HEXAGON_ADSP; dsp_capability_domain.attribute_ID = DOMAIN_SUPPORT; - dsp_capability_domain.capability = 0; + dsp_capability_domain.capability = 0; hexagon_error = remote_handle_control(DSPRPC_GET_DSP_INFO, &dsp_capability_domain, sizeof(struct remote_dsp_capability)); if(dsp_capability_domain.capability) { *domain = HEXAGON_ADSP; @@ -1538,9 +1542,9 @@ static int ggmlhexagon_get_vtcm_info(int domain, uint32_t * capability, uint32_t * since the ADSP does not have a dedicated VTCM, we expect the output to be 0 */ struct remote_dsp_capability dsp_capability_vtcm_dsp; - dsp_capability_vtcm_dsp.domain = (uint32_t)domain; + dsp_capability_vtcm_dsp.domain = (uint32_t)domain; dsp_capability_vtcm_dsp.attribute_ID = attr; - dsp_capability_vtcm_dsp.capability = (uint32_t)0; + dsp_capability_vtcm_dsp.capability = (uint32_t)0; hexagon_error = remote_handle_control(DSPRPC_GET_DSP_INFO, &dsp_capability_vtcm_dsp, sizeof(struct remote_dsp_capability)); if ((hexagon_error & 0xFF) == (AEE_EUNSUPPORTEDAPI & 0xFF)) { GGMLQNN_LOG_DEBUG("FastRPC Capability API is not supported on this device"); @@ -1607,9 +1611,9 @@ static bool ggmlhexagon_is_async_fastrpc_supported(int domain) { * Async fastrpc is supported only on CDSP */ struct remote_dsp_capability dsp_capability_async_support; - dsp_capability_async_support.domain = (uint32_t)domain; + dsp_capability_async_support.domain = (uint32_t)domain; dsp_capability_async_support.attribute_ID = ASYNC_FASTRPC_SUPPORT; - dsp_capability_async_support.capability = (uint32_t)0; + dsp_capability_async_support.capability = (uint32_t)0; hexagon_error = remote_handle_control(DSPRPC_GET_DSP_INFO, &dsp_capability_async_support, sizeof(struct remote_dsp_capability)); if ((hexagon_error & 0xFF) == (AEE_EUNSUPPORTEDAPI & 0xFF)) { GGMLQNN_LOG_WARN("FastRPC Capability API is not supported on this device"); @@ -1643,16 +1647,17 @@ static void ggmlhexagon_set_rpc_latency(int domain, int qos, int latency) { if (remote_handle_control) { struct remote_rpc_control_latency data; #if 1 - data.enable = RPC_PM_QOS; + data.enable = RPC_PM_QOS; data.latency = 300; #else data.enable = RPC_POLL_QOS; data.latency = 1000; #endif - data.enable = qos; - data.latency = latency; + data.enable = qos; + data.latency = latency; hexagon_error = remote_handle64_control(DSPRPC_GET_DSP_INFO, DSPRPC_CONTROL_LATENCY, (void*)&data, sizeof(data)); - if (hexagon_error != AEE_SUCCESS){ + if (hexagon_error != AEE_SUCCESS) { + //FIXME: why set rpc latency failure GGMLQNN_LOG_WARN("failed with error 0x%x", hexagon_error); goto bail; } else { @@ -1676,9 +1681,9 @@ static bool ggmlhexagon_is_status_notification_supported(int domain) { * DSP User PD status notification Support */ struct remote_dsp_capability dsp_capability_status_notification_support; - dsp_capability_status_notification_support.domain = (uint32_t)domain; + dsp_capability_status_notification_support.domain = (uint32_t)domain; dsp_capability_status_notification_support.attribute_ID = STATUS_NOTIFICATION_SUPPORT; - dsp_capability_status_notification_support.capability = (uint32_t)0; + dsp_capability_status_notification_support.capability = (uint32_t)0; hexagon_error = remote_handle_control(DSPRPC_GET_DSP_INFO, &dsp_capability_status_notification_support, sizeof(struct remote_dsp_capability)); if ((hexagon_error & 0xFF) == (AEE_EUNSUPPORTEDAPI & 0xFF)) { GGMLQNN_LOG_WARN("FastRPC Capability API is not supported on this device"); @@ -1718,9 +1723,9 @@ static int ggmlhexagon_get_hmx_support_info(int domain, uint32_t * capability, u * HMX is supported on CDSP only */ struct remote_dsp_capability dsp_capability_hmx_dsp; - dsp_capability_hmx_dsp.domain = (uint32_t)domain; + dsp_capability_hmx_dsp.domain = (uint32_t)domain; dsp_capability_hmx_dsp.attribute_ID = attr; - dsp_capability_hmx_dsp.capability = (uint32_t)0; + dsp_capability_hmx_dsp.capability = (uint32_t)0; hexagon_error = remote_handle_control(DSPRPC_GET_DSP_INFO, &dsp_capability_hmx_dsp, sizeof(struct remote_dsp_capability)); if ((hexagon_error & 0xFF) == (AEE_EUNSUPPORTEDAPI & 0xFF)) { GGMLQNN_LOG_DEBUG("FastRPC Capability API is not supported on this device"); @@ -1755,9 +1760,9 @@ static int ggmlhexagon_get_hex_arch_ver(int domain, uint32_t * capability) { * Query the Hexagon processor architecture version information */ struct remote_dsp_capability dsp_capability_arch_ver; - dsp_capability_arch_ver.domain = (uint32_t)domain; + dsp_capability_arch_ver.domain = (uint32_t)domain; dsp_capability_arch_ver.attribute_ID = ARCH_VER; - dsp_capability_arch_ver.capability = (uint32_t)0; + dsp_capability_arch_ver.capability = (uint32_t)0; hexagon_error = remote_handle_control(DSPRPC_GET_DSP_INFO, &dsp_capability_arch_ver, sizeof(struct remote_dsp_capability)); if ((hexagon_error & 0xFF) == (AEE_EUNSUPPORTEDAPI & 0xFF)) { GGMLQNN_LOG_DEBUG("FastRPC Capability API is not supported on this device"); @@ -1774,7 +1779,7 @@ static int ggmlhexagon_get_hex_arch_ver(int domain, uint32_t * capability) { GGMLQNN_LOG_DEBUG("remote_dsp_capability interface is not supported on this device"); } - bail: +bail: return hexagon_error; } @@ -1801,9 +1806,9 @@ static int ggmlhexagon_get_hvx_support_info(int domain, uint32_t * capability, u * HVX is supported on CDSP only */ struct remote_dsp_capability dsp_capability_hvx_dsp; - dsp_capability_hvx_dsp.domain = (uint32_t)domain; + dsp_capability_hvx_dsp.domain = (uint32_t)domain; dsp_capability_hvx_dsp.attribute_ID = attr; - dsp_capability_hvx_dsp.capability = (uint32_t)0; + dsp_capability_hvx_dsp.capability = (uint32_t)0; hexagon_error = remote_handle_control(DSPRPC_GET_DSP_INFO, &dsp_capability_hvx_dsp, sizeof(struct remote_dsp_capability)); if ((hexagon_error & 0xFF)==(AEE_EUNSUPPORTEDAPI & 0xFF)) { GGMLQNN_LOG_DEBUG("FastRPC Capability API is not supported on this device"); @@ -1834,8 +1839,8 @@ static int ggmlhexagon_request_status_notifications(int domain_id, void * contex struct remote_rpc_notif_register notif; bool status_notification_support; - notif.context = context; - notif.domain = domain_id; + notif.context = context; + notif.domain = domain_id; notif.notifier_fn = call_back_fn; status_notification_support = ggmlhexagon_is_status_notification_supported(domain_id); @@ -1851,7 +1856,7 @@ static int ggmlhexagon_request_status_notifications(int domain_id, void * contex return hexagon_error; } -//TODO:not work on cDSP currently +//TODO:not work on cDSP currently, this function will affect the performance of cDSP static AEEResult ggmlhexagon_set_clocks(remote_handle64 handle, int32 power_level, int32 latency, int32 dcvs_enabled) { #if 0 GGMLQNN_LOG_DEBUG("----------- entering power set clocks"); @@ -1936,8 +1941,8 @@ static int ggmlhexagon_init_dsp(ggml_backend_qnn_context * ctx) { return 2; } - if (domain_id == -1) { - if (domain_type != NULL) { + if (-1 == domain_id) { + if (NULL != domain_type) { if ((strcmp(domain_type, "NSP") != 0 && strcmp(domain_type, "HPASS") != 0)) { GGMLQNN_LOG_WARN("invalid domain_type %s. possible values are NSP or HPASS", domain_type); goto bail; @@ -2065,7 +2070,7 @@ static void ggmlhexagon_close_cdsp(ggml_backend_qnn_context * ctx) { if (-1 != ctx->ggmlop_handle) { hexagon_error = ggmlop_dsp_close(ctx->ggmlop_handle); if (AEE_SUCCESS != hexagon_error) { - GGMLQNN_LOG_WARN("error 0x%x: failed to close ggmlop handle", hexagon_error); + GGMLQNN_LOG_WARN("error 0x%x: failed to close ggmlop dsp handle", hexagon_error); } else { ctx->ggmlop_handle = -1; } @@ -2119,6 +2124,7 @@ static void ggmlhexagon_compute(ggml_backend_qnn_context * ctx, struct ggml_tens dsptensor_1.data = src1->data; dsptensor_2.data = dst->data; + //make compiler happy dsptensor_0.ne[0] = src0->ne[0]; dsptensor_0.ne[1] = src0->ne[1]; dsptensor_0.ne[2] = src0->ne[2]; @@ -2153,13 +2159,17 @@ static void ggmlhexagon_compute(ggml_backend_qnn_context * ctx, struct ggml_tens dsptensor_1.data_len = ggml_nbytes(src1); dsptensor_2.data_len = ggml_nbytes(dst); + if ((GGML_OP_MUL_MAT == op->op) && (src0_type != GGML_TYPE_F32)) { + dsptensor_0.data_len = ctx->desired_size; + } + dsptensor_0.type = src0->type; dsptensor_1.type = src1->type; dsptensor_2.type = dst->type; hexagon_error = op_func(ctx->ggmlop_handle, &dsptensor_0, &dsptensor_1, &dsptensor_2); if (AEE_SUCCESS != hexagon_error) { - GGMLQNN_LOG_WARN("ggmlop computation fail on cdsp"); + GGMLQNN_LOG_WARN("ggmlop %s computation fail on cdsp", ggml_op_name(op->op)); } } @@ -2206,11 +2216,11 @@ static const char * ggmlqnn_get_htparch_desc(size_t htp_arch) { static const char * ggmlqnn_get_inference_approach_name(int inference_approach) { switch (inference_approach) { - case 0: + case QNN_GENERAL: return "QNN_GENERAL"; - case 1: + case DIRECT_USE_CDSP: return "DIRECT_USE_CDSP"; - case 2: + case QNN_SINGLEGRAPH: return "QNN_SINGLEGRAPH"; default: return "unknown approach"; @@ -2437,9 +2447,6 @@ static void ggmlqnn_get_graphkey_from_cgraph(const ggml_cgraph * cgraph, std::st return; } - //output += "cgraph_" + std::to_string(ggml_time_us()); - //return; - bool is_start = true; for (int i = 0; i < cgraph->n_nodes; ++i) { auto * op = cgraph->nodes[i]; @@ -3108,7 +3115,7 @@ Qnn_MemHandle_t qnn_instance::register_rpcmem(void * p_data, const uint32_t ran return _qnn_rpc_buffer_to_handles[p_data]; } - auto mem_fd = rpcmem_to_fd(p_data); + int32_t mem_fd = rpcmem_to_fd(p_data); if (mem_fd == -1) { GGMLQNN_LOG_WARN("failed to get file descriptor"); return nullptr; @@ -3121,7 +3128,7 @@ Qnn_MemHandle_t qnn_instance::register_rpcmem(void * p_data, const uint32_t ran {{mem_fd}} }; Qnn_MemHandle_t handle = nullptr; - auto error = _qnn_interface.qnn_mem_register(_qnn_context_handle, &descriptor, /*numDescriptors=*/1, &handle); + Qnn_ErrorHandle_t error = _qnn_interface.qnn_mem_register(_qnn_context_handle, &descriptor, /*numDescriptors=*/1, &handle); if (error != QNN_SUCCESS) { GGMLQNN_LOG_WARN("failed to register shared memory, error %d, %s", QNN_GET_ERROR_CODE(error), strerror(error)); return nullptr; @@ -3497,7 +3504,7 @@ int qnn_instance::qnn_init(const QnnSaver_Config_t ** saver_config) { const QnnDevice_PlatformInfo_t * p_info = nullptr; qcom_socinfo soc_info = {}; qnnstatus = _qnn_raw_interface.deviceGetPlatformInfo(nullptr, &p_info); - if (qnnstatus == QNN_SUCCESS) { + if (QNN_SUCCESS == qnnstatus) { GGMLQNN_LOG_INFO("device counts %d\n", p_info->v1.numHwDevices); QnnDevice_HardwareDeviceInfo_t * infos = p_info->v1.hwDevices; QnnHtpDevice_OnChipDeviceInfoExtension_t chipinfo = {}; @@ -3518,8 +3525,8 @@ int qnn_instance::qnn_init(const QnnSaver_Config_t ** saver_config) { } QnnHtpDevice_CustomConfig_t soc_customconfig; - soc_customconfig.option = QNN_HTP_DEVICE_CONFIG_OPTION_SOC; - soc_customconfig.socModel = soc_info.soc_model; + soc_customconfig.option = QNN_HTP_DEVICE_CONFIG_OPTION_SOC; + soc_customconfig.socModel = soc_info.soc_model; QnnDevice_Config_t soc_devconfig; soc_devconfig.option = QNN_DEVICE_CONFIG_OPTION_CUSTOM; soc_devconfig.customConfig = &soc_customconfig; @@ -3590,9 +3597,8 @@ int qnn_instance::qnn_init(const QnnSaver_Config_t ** saver_config) { _pfn_rpc_mem_alloc = reinterpret_cast(dlsym(_rpc_lib_handle,"rpcmem_alloc")); _pfn_rpc_mem_free = reinterpret_cast(dlsym(_rpc_lib_handle, "rpcmem_free")); _pfn_rpc_mem_to_fd = reinterpret_cast(dlsym(_rpc_lib_handle,"rpcmem_to_fd")); - if (nullptr == _pfn_rpc_mem_alloc || nullptr == _pfn_rpc_mem_free - || nullptr == _pfn_rpc_mem_to_fd) { - GGMLQNN_LOG_WARN("unable to access symbols in QNN RPC lib. dlerror(): %s", dlerror()); + if (nullptr == _pfn_rpc_mem_alloc || nullptr == _pfn_rpc_mem_free || nullptr == _pfn_rpc_mem_to_fd) { + GGMLQNN_LOG_WARN("unable to access symbols in QNN RPC lib, dlerror(): %s", dlerror()); dlclose(_rpc_lib_handle); return 8; } @@ -3613,13 +3619,13 @@ int qnn_instance::qnn_init(const QnnSaver_Config_t ** saver_config) { if (_backend_name.find("Htp") != std::string::npos) { htp_print_info(); - htp_probe_rpc_meminfo(); if (0 != htp_init_perfinfra()) { GGMLQNN_LOG_WARN("initialize HTP performance failure"); } #if 1 + //FIXME: ht_set_rpc_polling + htp_set_high_performance_mode should be equivalent to htp_enter_performance_mode if (0 != htp_set_rpc_polling()) { GGMLQNN_LOG_WARN("set RPC polling failure"); } @@ -3627,13 +3633,14 @@ int qnn_instance::qnn_init(const QnnSaver_Config_t ** saver_config) { GGMLQNN_LOG_WARN("set HTP high performance mode failure"); } #else - htp_set_memory_grow_size(); htp_enter_performance_mode(); #endif + htp_set_memory_grow_size(); + if (enable_qnn_rpc()) { - GGMLQNN_LOG_INFO("NPU RPC feature enabled"); + GGMLQNN_LOG_INFO("NPU RPC feature enabled with QNN-NPU backend"); } else { - GGMLQNN_LOG_INFO("NPU RPC feature disabled"); + GGMLQNN_LOG_INFO("NPU RPC feature disabled with QNN-NPU backend"); } } @@ -3657,7 +3664,7 @@ int qnn_instance::qnn_finalize() { if (nullptr != _pfn_rpc_mem_deinit) _pfn_rpc_mem_deinit(); - if (dlclose(_rpc_lib_handle) != 0) { + if (0 != dlclose(_rpc_lib_handle)) { GGMLQNN_LOG_WARN("failed to unload qualcomm's rpc lib, error:%s\n", dlerror()); } else { GGMLQNN_LOG_DEBUG("succeed to close rpcmem lib\n"); @@ -3713,10 +3720,9 @@ int qnn_instance::qnn_finalize() { } unload_backend(); - unload_system(); - GGMLQNN_LOG_DEBUG("leave %s\n", __func__); + GGMLQNN_LOG_DEBUG("leave %s\n", __func__); return ret_status; } @@ -3727,7 +3733,7 @@ int qnn_instance::init_qnn_graph(const std::string & graph_name, QNNBackend devi GGMLQNN_LOG_DEBUG("[%s][%s]created", ggml_backend_qnn_get_devname(device), graph_name.c_str()); Qnn_ErrorHandle_t error = QNN_SUCCESS; - if (device == QNN_BACKEND_NPU) { + if (QNN_BACKEND_NPU == device) { QnnHtpGraph_CustomConfig_t hvx_config; hvx_config.option = QNN_HTP_GRAPH_CONFIG_OPTION_NUM_HVX_THREADS; hvx_config.numHvxThreads = hvx_threads; @@ -3781,7 +3787,7 @@ int qnn_instance::init_qnn_graph(const std::string & graph_name, QNNBackend devi } else { error = _qnn_interface.qnn_graph_create(_qnn_context_handle, graph_name.c_str(), nullptr, &_qnn_graph_handle); } - if (error != QNN_SUCCESS) { + if (QNN_SUCCESS != error) { GGMLQNN_LOG_ERROR("[%s][%s]failed to create qnn graph, error: %s", ggml_backend_qnn_get_devname(device), graph_name.c_str(), ggmlqnn_get_qnnerror_string(error)); @@ -3789,7 +3795,7 @@ int qnn_instance::init_qnn_graph(const std::string & graph_name, QNNBackend devi } GGMLQNN_LOG_DEBUG("[%s]create graph %s succeed", ggml_backend_qnn_get_devname(device), graph_name.c_str()); - if (device == QNN_BACKEND_NPU) { + if (QNN_BACKEND_NPU == device) { htp_set_n_hvx_threads(hvx_threads); } return QNN_SUCCESS; @@ -3797,7 +3803,7 @@ int qnn_instance::init_qnn_graph(const std::string & graph_name, QNNBackend devi int qnn_instance::init_qnn_graph(const char * graph_name, bool debug, uint8_t do_node_validation, const QnnGraph_Config_t ** graph_configs) { - int result = 0; + Qnn_ErrorHandle_t result = 0; if (nullptr == graph_name) { GGMLQNN_LOG_WARN("graph name is null\n"); @@ -3813,19 +3819,19 @@ int qnn_instance::init_qnn_graph(const char * graph_name, bool debug, uint8_t do GGMLQNN_LOG_WARN("node validation disabled, backend will not perform op validation prior to adding node\n"); } - _graph_name = graph_name; - _debug_tensor = debug; - _do_node_validations = do_node_validation; + _graph_name = graph_name; + _debug_tensor = debug; + _do_node_validations = do_node_validation; result = _qnn_raw_interface.graphCreate(_qnn_context_handle, graph_name, graph_configs, &_qnn_graph_handle); - if (result != QNN_GRAPH_NO_ERROR || nullptr == _qnn_graph_handle) { + if (QNN_GRAPH_NO_ERROR != result || nullptr == _qnn_graph_handle) { GGMLQNN_LOG_WARN("failed to create graph in qnn context\n"); return 3; } else { - GGMLQNN_LOG_INFO("succeed to create graph %s, %p\n", graph_name, _qnn_graph_handle); + GGMLQNN_LOG_DEBUG("succeed to create graph %s, %p\n", graph_name, _qnn_graph_handle); } return 0; @@ -3848,8 +3854,8 @@ int qnn_instance::finalize_qnn_graph() { int qnn_instance::htp_init_perfinfra() { QnnDevice_Infrastructure_t device_infra = nullptr; - int error = _qnn_raw_interface.deviceGetInfrastructure(&device_infra); - if (error != QNN_SUCCESS) { + Qnn_ErrorHandle_t error = _qnn_raw_interface.deviceGetInfrastructure(&device_infra); + if (QNN_SUCCESS != error) { GGMLQNN_LOG_WARN("failed to get qnn device infra\n"); return 1; } @@ -3964,8 +3970,8 @@ void qnn_instance::htp_set_memory_grow_size(size_t size) { &grow_size_config, nullptr, }; - Qnn_ErrorHandle_t ret = _qnn_htp_perfinfra->setMemoryConfig(_qnn_htp_device_id, _qnn_htp_core_id, memory_config); - if (ret != QNN_SUCCESS) { + Qnn_ErrorHandle_t result = _qnn_htp_perfinfra->setMemoryConfig(_qnn_htp_device_id, _qnn_htp_core_id, memory_config); + if (QNN_SUCCESS != result) { GGMLQNN_LOG_WARN("failed to set HTP memory config"); } else { GGMLQNN_LOG_INFO("succeed to set HTP memory config"); @@ -3984,8 +3990,8 @@ void qnn_instance::htp_set_n_hvx_threads(size_t n_threads) { }; const QnnGraph_Config_t * graph_configs[] = {&hvx_thread_config, nullptr}; - Qnn_ErrorHandle_t ret = _qnn_raw_interface.graphSetConfig(_qnn_graph_handle, graph_configs); - if (ret != QNN_SUCCESS) { + Qnn_ErrorHandle_t result = _qnn_raw_interface.graphSetConfig(_qnn_graph_handle, graph_configs); + if (QNN_SUCCESS != result) { GGMLQNN_LOG_WARN("failed to set QNN graph config: set hvx threads %d", n_threads); } else { GGMLQNN_LOG_INFO("succeed to set QNN graph config: set hvx threads %d", n_threads); @@ -4319,7 +4325,7 @@ static void ggmlqnn_load_cfg() { memset(time_string, 0, GGML_QNN_TMPBUF_LEN); ggmlqnn_get_timestring(time_string); GGMLQNN_LOG_DEBUG("program running start time:%s", time_string); - ggmlqnn_disable_android_tags(1); + ggmlqnn_disable_android_tags(0); std::string cfg_filename = std::string(g_qnn_params.qnn_runtimelib_path) + std::string(g_qnn_params.qnn_cfgfilename); GGMLQNN_LOG_INFO("load ggml-qnn config from %s", cfg_filename.c_str()); @@ -4382,14 +4388,22 @@ static bool ggmlhexagon_can_handle_op(const ggml_backend_qnn_context * ctx, cons struct ggml_tensor * src0 = op_tensor->src[0]; struct ggml_tensor * src1 = op_tensor->src[1]; const int64_t ne00 = op_tensor->src[0]->ne[0]; - uint32_t src0_rank = ggml_n_dims(src0); + uint32_t src0_rank = 0; uint32_t src1_rank = 0; + if (nullptr != src0) { + src0_rank = ggml_n_dims(src0); + } else { + GGMLQNN_LOG_DEBUG("op name %s\n", ggml_op_name(op_tensor->op)); + } if (nullptr != src1) { src1_rank = ggml_n_dims(src1); + } else { + GGMLQNN_LOG_DEBUG("op name %s\n", ggml_op_name(op_tensor->op)); } //FIXME: mulmat on cDSP doesn't work as expected - if (op_tensor->op != GGML_OP_ADD) + bool support = ((op_tensor->op == GGML_OP_ADD) || (op_tensor->op == GGML_OP_MUL_MAT)); + if (!support) return false; ggmlqnn_dump_op_info(op_tensor); @@ -4397,7 +4411,12 @@ static bool ggmlhexagon_can_handle_op(const ggml_backend_qnn_context * ctx, cons return false; } - return ggmlqnn_same_types(ctx, op_tensor); + support = ggmlqnn_same_types(ctx, op_tensor); + if (!support) { + return false; + } + + return (src0_rank <= 2); } static bool ggmlqnn_can_handle_op(const ggml_backend_qnn_context * ctx, const struct ggml_tensor * op_tensor) { @@ -4406,7 +4425,13 @@ static bool ggmlqnn_can_handle_op(const ggml_backend_qnn_context * ctx, const st } if (DIRECT_USE_CDSP == g_qnn_params.inference_approach) { - return ggmlhexagon_can_handle_op(ctx, op_tensor); + //return ggmlhexagon_can_handle_op(ctx, op_tensor); + //bool support = ((op_tensor->op == GGML_OP_ADD) || (op_tensor->op == GGML_OP_MUL_MAT)); + //FIXME: mulmat on cDSP doesn't work as expected + bool support = (op_tensor->op == GGML_OP_ADD); + if (!support) + return false; + } if (!ggmlqnn_k_op_caps[ggmlqnn_get_op_index(op_tensor)].supported) { @@ -4445,7 +4470,7 @@ static bool ggmlqnn_can_handle_op(const ggml_backend_qnn_context * ctx, const st return false; } - if ((src0_rank != 2) || (src1_rank != 2)) //TODO: 3D and 4D matrix + if ((src0_rank != 2) || (src1_rank != 2)) //TODO: 3D and 4D matrix mul return false; return ggmlqnn_same_types(ctx, op_tensor); diff --git a/ggml/src/ggml-qnn/kernels/ggmlop_cdsp.c b/ggml/src/ggml-qnn/kernels/ggmlop_cdsp.c index b5cfd8810cfe6..fa00d9bc5614f 100644 --- a/ggml/src/ggml-qnn/kernels/ggmlop_cdsp.c +++ b/ggml/src/ggml-qnn/kernels/ggmlop_cdsp.c @@ -1,3 +1,24 @@ +/* +* Copyright (c) 2023-2025 The ggml authors +* +* Permission is hereby granted, free of charge, to any person obtaining a copy +* of this software and associated documentation files (the "Software"), to +* deal in the Software without restriction, including without limitation the +* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +* sell copies of the Software, and to permit persons to whom the Software is +* furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +* IN THE SOFTWARE. +*/ #include #include #include @@ -7,45 +28,32 @@ #include "HAP_farf.h" #include "ggmlop_ap_skel.h" -#define ggml_tensor dsptensor - -int ggmlop_dsp_open(const char*uri, remote_handle64* handle) { - void *tptr = NULL; - FARF(HIGH, "uri %s", uri); - tptr = (void *)malloc(1); - *handle = (remote_handle64)tptr; - assert(*handle); - return 0; -} - -int ggmlop_dsp_close(remote_handle64 handle) { - if (handle) - free((void*)handle); - return 0; -} - -static void ggml_dump_tensor(struct ggml_tensor * tensor) { - FARF(HIGH, "ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi, %5zi)\n", - tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3], - tensor->nb[0], tensor->nb[1], tensor->nb[2], tensor->nb[3]); -} - -static void ggml_abort(const char * file, int line, const char * fmt, ...) { - //abort(); - return; -} +#define ggml_tensor dsptensor #define GGML_MAX_DIMS 4 #define GGML_UNUSED(x) (void)(x) +#define UNUSED GGML_UNUSED #define GGML_PAD(x, n) (((x) + (n) - 1) & ~((n) - 1)) #define GGML_ABORT(...) ggml_abort(__FILE__, __LINE__, __VA_ARGS__) #define GGML_ASSERT(x) if (!(x)) GGML_ABORT("GGML_ASSERT(%s) failed", #x) #define MIN(a, b) ((a) < (b) ? (a) : (b)) #define MAX(a, b) ((a) > (b) ? (a) : (b)) +#define SWAP(x, y, T) do { T SWAP = x; (x) = y; (y) = SWAP; } while (0) +#if UINTPTR_MAX == 0xFFFFFFFF +#define GGML_MEM_ALIGN 4 +#else +#define GGML_MEM_ALIGN 16 +#endif + #define GGML_RESTRICT #define static_assert(a, b) do { } while (0) +typedef uint16_t ggml_fp16_t; +typedef struct { uint16_t bits; } ggml_bf16_t; +typedef double ggml_float; + + #define GGML_TENSOR_LOCALS_1(type, prefix, pointer, array) \ const type prefix##0 = (pointer)->array[0]; \ GGML_UNUSED(prefix##0); @@ -125,10 +133,124 @@ enum ggml_type { GGML_TYPE_COUNT = 39, }; +static void ggml_vec_dot_f32(int n, float * GGML_RESTRICT s, size_t bs, const float * GGML_RESTRICT x, size_t bx, const float * GGML_RESTRICT y, size_t by, int nrc); +static void ggml_vec_dot_f16(int n, float * GGML_RESTRICT s, size_t bs, ggml_fp16_t * GGML_RESTRICT x, size_t bx, ggml_fp16_t * GGML_RESTRICT y, size_t by, int nrc); +static void ggml_vec_dot_bf16(int n, float * GGML_RESTRICT s, size_t bs, ggml_bf16_t * GGML_RESTRICT x, size_t bx, ggml_bf16_t * GGML_RESTRICT y, size_t by, int nrc); + +typedef void (*ggml_vec_dot_t) (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT x, size_t bx, + const void * GGML_RESTRICT y, size_t by, int nrc); +typedef void (*ggml_from_float_t)(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); + +typedef void (*ggml_to_float_t) (const void * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); +typedef void (*ggml_from_float_t)(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); + +struct ggml_type_traits { + const char * type_name; + int64_t blck_size; + int64_t blck_size_interleave; // interleave elements in blocks + size_t type_size; + bool is_quantized; + ggml_to_float_t to_float; + ggml_from_float_t from_float_ref; +}; + +struct ggml_type_traits_cpu { + ggml_from_float_t from_float; + ggml_vec_dot_t vec_dot; + enum ggml_type vec_dot_type; + int64_t nrows; // number of rows to process simultaneously +}; + +static const struct ggml_type_traits_cpu type_traits_cpu[1] = { + [GGML_TYPE_F32] = { + .vec_dot = (ggml_vec_dot_t) ggml_vec_dot_f32, + .vec_dot_type = GGML_TYPE_F32, + .nrows = 1, + }, +}; + +static const struct ggml_type_traits type_traits[1] = { + [GGML_TYPE_F32] = { + .type_name = "f32", + .blck_size = 1, + .type_size = sizeof(float), + .is_quantized = false, + }, + +}; + +static const struct ggml_type_traits_cpu * ggml_get_type_traits_cpu(enum ggml_type type) { + return &type_traits_cpu[type]; +} + +static void ggml_vec_dot_f32(int n, float * GGML_RESTRICT s, size_t bs, const float * GGML_RESTRICT x, size_t bx, const float * GGML_RESTRICT y, size_t by, int nrc) { + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + ggml_float sumf = 0.0; + for (int i = 0; i < n; ++i) { + sumf += (ggml_float)(x[i]*y[i]); + } + *s = sumf; +} + +static const struct ggml_type_traits * ggml_get_type_traits(enum ggml_type type) { + return &type_traits[type]; +} + +static int64_t ggml_blck_size(enum ggml_type type) { + return type_traits[type].blck_size; +} + +static size_t ggml_type_size(enum ggml_type type) { + return type_traits[type].type_size; +} + +static size_t ggml_row_size(enum ggml_type type, int64_t ne) { + assert(ne % ggml_blck_size(type) == 0); + return ggml_type_size(type)*ne/ggml_blck_size(type); +} + +static size_t ggml_nbytes(const struct ggml_tensor * tensor) { + size_t nbytes; + const size_t blck_size = ggml_blck_size(tensor->type); + if (blck_size == 1) { + nbytes = ggml_type_size(tensor->type); + for (int i = 0; i < GGML_MAX_DIMS; ++i) { + nbytes += (tensor->ne[i] - 1)*tensor->nb[i]; + } + } + else { + nbytes = tensor->ne[0]*tensor->nb[0]/blck_size; + for (int i = 1; i < GGML_MAX_DIMS; ++i) { + nbytes += (tensor->ne[i] - 1)*tensor->nb[i]; + } + } + + return nbytes; +} + +static size_t ggml_nbytes_pad(const struct ggml_tensor * tensor) { + return GGML_PAD(ggml_nbytes(tensor), GGML_MEM_ALIGN); +} + +static double ggml_type_sizef(enum ggml_type type) { + return ((double)(type_traits[type].type_size))/type_traits[type].blck_size; +} + +static const char * ggml_type_name(enum ggml_type type) { + return type < GGML_TYPE_COUNT ? type_traits[type].type_name : "NONE"; +} + +static bool ggml_is_quantized(enum ggml_type type) { + return type_traits[type].is_quantized; +} + static bool ggml_is_empty(const struct ggml_tensor * tensor) { for (int i = 0; i < GGML_MAX_DIMS; ++i) { if (tensor->ne[i] == 0) { - // empty if any dimension has no elements return true; } } @@ -161,12 +283,16 @@ static int64_t ggml_nrows(const struct ggml_tensor * tensor) { return tensor->ne[1]*tensor->ne[2]*tensor->ne[3]; } +static bool ggml_is_transposed(const struct ggml_tensor * tensor) { + return tensor->nb[0] > tensor->nb[1]; +} + static bool ggml_is_contiguous_n(const struct ggml_tensor * tensor, int n) { - size_t next_nb = sizeof(float);//ggml_type_size(tensor->type); - if (tensor->ne[0] != 1/*ggml_blck_size(tensor->type)*/ && tensor->nb[0] != next_nb) { + size_t next_nb = ggml_type_size(tensor->type); + if (tensor->ne[0] != ggml_blck_size(tensor->type) && tensor->nb[0] != next_nb) { return false; } - next_nb *= tensor->ne[0]/1/*ggml_blck_size(tensor->type)*/; + next_nb *= tensor->ne[0]/ggml_blck_size(tensor->type); for (int i = 1; i < GGML_MAX_DIMS; i++) { if (tensor->ne[i] != 1) { if (i > n) { @@ -193,24 +319,34 @@ static bool ggml_is_contiguous(const struct ggml_tensor * tensor) { inline static void ggml_vec_add_f32 (const int n, float * z, const float * x, const float * y) { for (int i = 0; i < n; ++i) z[i] = x[i] + y[i]; } -static void ggml_compute_forward_add_f32( - const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_dump_tensor(const ggml_tensor * tensor) { + FARF(HIGH, "ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi, %5zi)\n", + tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3], + tensor->nb[0], tensor->nb[1], tensor->nb[2], tensor->nb[3]); +} - ggml_dump_tensor(src0); - ggml_dump_tensor(src1); -#if 1 - float * a = (float*)src0->data; - float * b = (float*)src1->data; - float * c = (float*)dst->data; - //TODO: Hexagon SIMD - for (size_t idx = 0; idx < src0->data_len; idx++) { - *c = *a + *b; - a++; - b++; - c++; - } +static void ggml_abort(const char * file, int line, const char * fmt, ...) { + //abort(); return; -#endif +} + +int ggmlop_dsp_open(const char*uri, remote_handle64* handle) { + void *tptr = NULL; + FARF(HIGH, "uri %s", uri); + tptr = (void *)malloc(1); + *handle = (remote_handle64)tptr; + assert(*handle); + return 0; +} + +int ggmlop_dsp_close(remote_handle64 handle) { + if (handle) + free((void*)handle); + return 0; +} + +static void ggml_compute_forward_add_f32( + const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(ggml_can_repeat(src1, src0) && ggml_are_same_shape(src0, dst)); const int ith = 0; @@ -230,6 +366,23 @@ static void ggml_compute_forward_add_f32( const int ir0 = dr*ith; const int ir1 = MIN(ir0 + dr, nr); + ggml_dump_tensor(src0); + ggml_dump_tensor(src1); + +#if 1 //naive algorithm, can works with llama-cli + float * a = (float*)src0->data; + float * b = (float*)src1->data; + float * c = (float*)dst->data; + //TODO: Hexagon SIMD + for (size_t idx = 0; idx < src0->data_len; idx++) { + *c = *a + *b; + a++; + b++; + c++; + } + return; +#endif + if (nb10 == sizeof(float)) { for (int ir = ir0; ir < ir1; ++ir) { // src1 is broadcastable across src0 and dst in i1, i2, i3 @@ -247,11 +400,7 @@ static void ggml_compute_forward_add_f32( float * src1_ptr = (float *) ((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11); for (int64_t r = 0; r < nr0; ++r) { -#ifdef GGML_USE_ACCELERATE - vDSP_vadd(src0_ptr + r*ne10, 1, src1_ptr, 1, dst_ptr + r*ne10, 1, ne10); -#else ggml_vec_add_f32(ne10, dst_ptr + r*ne10, src0_ptr + r*ne10, src1_ptr); -#endif } } } else { @@ -290,55 +439,8 @@ int ggmlop_dsp_add(remote_handle64 h, const ggml_tensor * src0, const ggml_tenso } else { GGML_ABORT("fatal error"); } - } break; - case GGML_TYPE_F16: - { - if (src1->type == GGML_TYPE_F16) { - //ggml_compute_forward_add_f16_f16(dst); - } - else if (src1->type == GGML_TYPE_F32) { - //ggml_compute_forward_add_f16_f32(dst); - } - else { - GGML_ABORT("fatal error"); - } - } break; - case GGML_TYPE_BF16: - { - if (src1->type == GGML_TYPE_BF16) { - //ggml_compute_forward_add_bf16_bf16(dst); - } - else if (src1->type == GGML_TYPE_F32) { - //ggml_compute_forward_add_bf16_f32(dst); - } - else { - GGML_ABORT("fatal error"); - } - } break; - case GGML_TYPE_Q4_0: - case GGML_TYPE_Q4_1: - case GGML_TYPE_Q5_0: - case GGML_TYPE_Q5_1: - case GGML_TYPE_Q8_0: - case GGML_TYPE_Q2_K: - case GGML_TYPE_Q3_K: - case GGML_TYPE_Q4_K: - case GGML_TYPE_Q5_K: - case GGML_TYPE_Q6_K: - case GGML_TYPE_TQ1_0: - case GGML_TYPE_TQ2_0: - case GGML_TYPE_IQ2_XXS: - case GGML_TYPE_IQ2_XS: - case GGML_TYPE_IQ3_XXS: - case GGML_TYPE_IQ1_S: - case GGML_TYPE_IQ1_M: - case GGML_TYPE_IQ4_NL: - case GGML_TYPE_IQ4_XS: - case GGML_TYPE_IQ3_S: - case GGML_TYPE_IQ2_S: - { - //ggml_compute_forward_add_q_f32(dst); - } break; + break; + } default: { GGML_ABORT("fatal error"); @@ -349,124 +451,204 @@ int ggmlop_dsp_add(remote_handle64 h, const ggml_tensor * src0, const ggml_tenso } -int ggmlop_dsp_mulmat(remote_handle64 h, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - FARF(HIGH, "=============== DSP: ggmlop_dsp_mulmat "); +static void ggml_compute_forward_mul_mat_one_chunk( + const struct ggml_tensor * src0, + const struct ggml_tensor * src1, + struct ggml_tensor * dst, + const enum ggml_type type, + const int64_t num_rows_per_vec_dot, + const int64_t ir0_start, + const int64_t ir0_end, + const int64_t ir1_start, + const int64_t ir1_end) { GGML_TENSOR_BINARY_OP_LOCALS - ggml_dump_tensor(src0); - ggml_dump_tensor(src1); + const bool src1_cont = ggml_is_contiguous(src1); + + ggml_vec_dot_t const vec_dot = type_traits_cpu[type].vec_dot; + enum ggml_type const vec_dot_type = type_traits_cpu[type].vec_dot_type; + + // broadcast factors + const int64_t r2 = ne12 / ne02; + const int64_t r3 = ne13 / ne03; + + //printf("ir0_start = %6lld, ir0_end = %6lld, ir1_start = %6lld, ir1_end = %6lld\n", ir0_start, ir0_end, ir1_start, ir1_end); + + // threads with no work simply yield (not sure if it helps) + if (ir0_start >= ir0_end || ir1_start >= ir1_end) { + return; + } + + //FIXME:hardcode to src1->data + const void * wdata = src1->data; + const size_t row_size = ggml_row_size(vec_dot_type, ne10); + + assert(ne12 % ne02 == 0); + assert(ne13 % ne03 == 0); + + // block-tiling attempt + const int64_t blck_0 = 16; + const int64_t blck_1 = 16; + + const size_t src1_col_stride = src1_cont || src1->type != vec_dot_type ? row_size : nb11; + + // attempt to reduce false-sharing (does not seem to make a difference) + // 16 * 2, accounting for mmla kernels + float tmp[32]; + + for (int64_t iir1 = ir1_start; iir1 < ir1_end; iir1 += blck_1) { + for (int64_t iir0 = ir0_start; iir0 < ir0_end; iir0 += blck_0) { + for (int64_t ir1 = iir1; ir1 < iir1 + blck_1 && ir1 < ir1_end; ir1 += num_rows_per_vec_dot) { + const int64_t i13 = (ir1 / (ne12 * ne1)); + const int64_t i12 = (ir1 - i13 * ne12 * ne1) / ne1; + const int64_t i11 = (ir1 - i13 * ne12 * ne1 - i12 * ne1); + + // broadcast src0 into src1 + const int64_t i03 = i13 / r3; + const int64_t i02 = i12 / r2; + + const int64_t i1 = i11; + const int64_t i2 = i12; + const int64_t i3 = i13; + + const char * src0_row = (const char*)src0->data + (0 + i02 * nb02 + i03 * nb03); + + // desc: when src1 is not a contiguous memory block we have to calculate the offset using the strides + // if it is, then we have either copied the data to params->wdata and made it contiguous or we are using + // the original src1 data pointer, so we should index using the indices directly + // TODO: this is a bit of a hack, we should probably have a better way to handle this + const char * src1_col = (const char*)wdata + + (src1_cont || src1->type != vec_dot_type + ? (i11 + i12 * ne11 + i13 * ne12 * ne11) * row_size + : (i11 * nb11 + i12 * nb12 + i13 * nb13)); + float * dst_col = (float*)((char*)dst->data + (i1 * nb1 + i2 * nb2 + i3 * nb3)); - const int vec_dot_type = 0; - int64_t const vec_dot_num_rows = 1; + //for (int64_t ir0 = iir0; ir0 < iir0 + blck_0 && ir0 < ir0_end; ++ir0) { + // vec_dot(ne00, &dst_col[ir0], src0_row + ir0*nb01, src1_col); + //} + + for (int64_t ir0 = iir0; ir0 < iir0 + blck_0 && ir0 < ir0_end; ir0 += num_rows_per_vec_dot) { + vec_dot(ne00, &tmp[ir0 - iir0], (num_rows_per_vec_dot > 1 ? 16 : 0), src0_row + ir0 * nb01, (num_rows_per_vec_dot > 1 ? nb01 : 0), src1_col, (num_rows_per_vec_dot > 1 ? src1_col_stride : 0), num_rows_per_vec_dot); + } + + for (int cn = 0; cn < num_rows_per_vec_dot; ++cn) { + memcpy(&dst_col[iir0 + cn * nb1 / nb0], tmp + (cn * 16), (MIN(iir0 + blck_0, ir0_end) - iir0) * sizeof(float)); + } + } + } + } +} + +int ggmlop_dsp_mulmat(remote_handle64 h, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + + GGML_TENSOR_BINARY_OP_LOCALS + + enum ggml_type const vec_dot_type = type_traits_cpu[src0->type].vec_dot_type; + ggml_from_float_t const from_float = type_traits_cpu[vec_dot_type].from_float; + int64_t const vec_dot_num_rows = type_traits_cpu[src0->type].nrows; GGML_ASSERT(ne0 == ne01); GGML_ASSERT(ne1 == ne11); GGML_ASSERT(ne2 == ne12); GGML_ASSERT(ne3 == ne13); - GGML_ASSERT(nb00 == sizeof(float)); - GGML_ASSERT(nb10 == sizeof(float)); + // we don't support permuted src0 or src1 + GGML_ASSERT(nb00 == ggml_type_size(src0->type)); + GGML_ASSERT(nb10 == ggml_type_size(src1->type)); + // dst cannot be transposed or permuted GGML_ASSERT(nb0 == sizeof(float)); GGML_ASSERT(nb0 <= nb1); GGML_ASSERT(nb1 <= nb2); GGML_ASSERT(nb2 <= nb3); +#if 1 //naive algorithm for fp32, can pass various case in UT + { + ggml_dump_tensor(src0); + ggml_dump_tensor(src1); + + float * a = (float*)src0->data; + float * b = (float*)src1->data; + float * c = (float*)dst->data; + int M = src0->ne[1]; + int K = src0->ne[0]; + int N = src1->ne[1]; + float sum = 0; + for (int i = 0; i < M; i++) { + for (int j = 0; j < N; j++) { + sum = 0; + for (int h = 0; h < K; h++) { + sum += a[i * K + h] * b[h * N + j]; + } + c[i * N + j] = sum; + } + } + return 0; + } +#endif + + // This is the size of the first dimension of the result, so we can iterate that way. (see the ASSERT above, these are the same numbers) const int64_t nr0 = ne0; + + // This is the size of the rest of the dimensions of the result const int64_t nr1 = ne1 * ne2 * ne3; + // Now select a reasonable chunk size. int chunk_size = 16; - int nth = 1; + // We need to step up the size if it's small if (nr0 == 1 || nr1 == 1) { chunk_size = 64; } + // distribute the work across the inner or outer loop based on which one is larger + // The number of chunks in the 0/1 dim. + // CEIL(nr0/chunk_size) int64_t nchunk0 = (nr0 + chunk_size - 1) / chunk_size; int64_t nchunk1 = (nr1 + chunk_size - 1) / chunk_size; - if (nchunk0 * nchunk1 < nth * 4) { - nchunk0 = nr0 > nr1 ? nth : 1; - nchunk1 = nr0 > nr1 ? 1 : nth; + // If the chunking is poor for the number of threads on this setup, scrap the whole plan. Re-chunk it by thread. + // Also, chunking by thread was measured to have perform better on NUMA systems. See https://github.com/ggml-org/llama.cpp/pull/6915 + // In theory, chunking should be just as useful on NUMA and non NUMA systems, but testing disagreed with that. + if (nchunk0 * nchunk1 < 4) { + // distribute the thread work across the inner or outer loop based on which one is larger + nchunk0 = 1; // parallelize by src0 rows + nchunk1 = 1; // parallelize by src1 rows } + // The number of elements in each chunk const int64_t dr0 = (nr0 + nchunk0 - 1) / nchunk0; const int64_t dr1 = (nr1 + nchunk1 - 1) / nchunk1; + // The first chunk comes from our thread_id, the rest will get auto-assigned. int current_chunk = 0; - const int64_t ith0 = current_chunk % nchunk0; - const int64_t ith1 = current_chunk / nchunk0; - - const int64_t ir0_start = dr0 * ith0; - const int64_t ir0_end = MIN(ir0_start + dr0, nr0); - - const int64_t ir1_start = dr1 * ith1; - const int64_t ir1_end = MIN(ir1_start + dr1, nr1); - - int64_t num_rows_per_vec_dot = vec_dot_num_rows; - - const int src1_cont = ggml_is_contiguous(src1); - const int64_t r2 = ne12 / ne02; - const int64_t r3 = ne13 / ne03; - - const void * wdata = src1->data; - const size_t row_size = sizeof(float) * ne10; - assert(ne12 % ne02 == 0); - assert(ne13 % ne03 == 0); + while (current_chunk < nchunk0 * nchunk1) { + const int64_t ith0 = current_chunk % nchunk0; + const int64_t ith1 = current_chunk / nchunk0; - const int64_t blck_0 = 16; - const int64_t blck_1 = 16; + const int64_t ir0_start = dr0 * ith0; + const int64_t ir0_end = MIN(ir0_start + dr0, nr0); - float tmp[32]; + const int64_t ir1_start = dr1 * ith1; + const int64_t ir1_end = MIN(ir1_start + dr1, nr1); - for (int64_t iir1 = ir1_start; iir1 < ir1_end; iir1 += blck_1) { - for (int64_t iir0 = ir0_start; iir0 < ir0_end; iir0 += blck_0) { - for (int64_t ir1 = iir1; - ir1 < iir1 + blck_1 && ir1 < ir1_end; ir1 += num_rows_per_vec_dot) { - const int64_t i13 = (ir1 / (ne12 * ne1)); - const int64_t i12 = (ir1 - i13 * ne12 * ne1) / ne1; - const int64_t i11 = (ir1 - i13 * ne12 * ne1 - i12 * ne1); + // dot kernels can handle 1 row and col at a time, but mmla kernels can process 2 rows and cols + int64_t num_rows_per_vec_dot = vec_dot_num_rows; - const int64_t i03 = i13 / r3; - const int64_t i02 = i12 / r2; - - const int64_t i1 = i11; - const int64_t i2 = i12; - const int64_t i3 = i13; - - const char * src0_row = (const char *)src0->data + (0 + i02 * nb02 + i03 * nb03); - - const char * src1_col = (const char *)wdata + - (src1_cont || src1->type != vec_dot_type - ? (i11 + i12 * ne11 + i13 * ne12 * ne11) * row_size - : (i11 * nb11 + i12 * nb12 + i13 * nb13)); - float * dst_col = (float *)((char *) dst->data + - (i1 * nb1 + i2 * nb2 + i3 * nb3)); - - - for (int64_t ir0 = iir0; - ir0 < iir0 + blck_0 && ir0 < ir0_end; ir0 += num_rows_per_vec_dot) { - - float sumf = 0.0; - const float * GGML_RESTRICT x = (float*)src0_row + ir0 * nb01; - const float * GGML_RESTRICT y = (float*)src1_col; - float * GGML_RESTRICT s = &tmp[ir0 - iir0]; - for (int i = 0; i < ne00; i++) { - sumf += x[i] * y[i]; - } - *s = sumf; - - } + // these checks are needed to avoid crossing dim1 boundaries + // can be optimized, but the logic would become more complicated, so keeping it like this for simplicity + if ((nr0 % 2 != 0) || (ne11 % 2 != 0) || ((ir0_end - ir0_start) % 2 != 0) || ((ir1_end - ir1_start) % 2 != 0)) { + num_rows_per_vec_dot = 1; + } + ggml_compute_forward_mul_mat_one_chunk(src0, src1, dst, src0->type, num_rows_per_vec_dot, ir0_start, ir0_end, ir1_start, ir1_end); - for (int cn = 0; cn < num_rows_per_vec_dot; ++cn) { - memcpy(&dst_col[iir0 + cn * nb1 / nb0], tmp + (cn * 16), - (MIN(iir0 + blck_0, ir0_end) - iir0) * sizeof(float)); - } - } + if (1 >= nchunk0 * nchunk1) { + break; } + current_chunk++; } - return 0; + return 0; } diff --git a/ggml/src/ggml-qnn/kernels/libggmlop_skel.so b/ggml/src/ggml-qnn/kernels/libggmlop_skel.so index 8695cbf36949a45dfd7ce3d7cec38ad6f8fea6c4..b92e38c950500e09eb9e76cd961c2de86de2898d 100755 GIT binary patch delta 3151 zcma);e{56N701tePW&s0@sHR}91|RSB*CLJBmtoeMK5UzBm_AE&%Niq`|I8F?!>dRT_=6gpjsBtcesf<+-i#__p(fMS?#dm42eFur^vk zzb1;nO$xv8w`qW7{u|VX205_YAO{1zh7Q&z>P=J#P7BAEWUv)nk-=PF#q1(#g0YjC z)=G~MDS1Q_=Hv)l5m1Q{zm^Cv(JruJAbKvr2HKPL!o3;H9rlUqav+wgdqNU>avtg| zMiFaTIE01QC>QF4^0-*AAy^`!0%$(85Nd&9bc=#fCe~sJET%!&PSQb()H`mWNIB}f zRWhh|0!}>&SsY1|c-q7db2oazXm`U$6hPS^{OOcG>k@y@{VKxSH;ff8D@wGXG5CHS z4&>+Tr^+YT=k6hA+2`RvqU@)p`40Q3kz8OO3(4{>`#c4R#=e~NC;ixBzby%uxFS!z z;}K&9t9>N0u;l3PDO@xiy!&VP6c=&BR{gUIKCWxRWei5xwP?*7VE;R&BYVXN|F$m*T{Y((re zxtFiOTd1YEiSwyDVcZVDyEbK1&C^~{ zolV}mbq&6@pmI!Y)x@cnT5NNQ@@kJ=dm-d)a^5XOoK5?&N)4`dpTcoM-L8$CI^LQ$ z$FUBy&(pd?R;35=jzoM=8&PQxhhL${c4g$0O%p1|=PBKx>8dkTFSSH$b6h~$i5^RX z<+DPK+FnsiQEp_sI(){Z(c`3?==mbN(8e#}8FKv+bS`P5>hVXUxkT)Zvu7$bV^n(B z_|NVy7uw8dCqJyeC215s^5Vjd#<)C;_F-qKx86`{sV~(^e}Ww%5(CzcPFql( z9}sUkU1H2OWR4Cj>{xZ<;`rMGVwL@>r3uL07foFpUq2wsjw$)%fcV5wZ;tja?2z6o z96#GHf`$JmU&?#Exw&gSib>laDr}(W?U4I|8|8{dR;Sh;s_qx{&Ykk7edz6V^Dp|6 zJ&pB=MpxXd0E^IZU7xVHr=&*lvBM$0a95h6y{V3RMX+d}yscNfUbI1O?-idEm6v)^ zsbPBPCM-|KzE!BShc0MjFW}*96XnIF{OA%K?nJ8>LZO814ow~uM?L$@wW;DN1edf(SHzZ*=DeDkb-v&;Pwwk@ZjTr* zamRe#8Qtlny7NmXWX^Vv9!2)$1U!NVr^->-bc*Gi#hIJaDd)-@m=GR%SpE%gTIE_gw%l00TaZdw^d!2he_p1&;i zFaMXin5jZ1DZ38rd1_b3^WD3iuBi@G9(ca9V@05{v!;5bFVww%rSE@ws8|tL9SFp3 zR{TZU%~N90B_^1ha*}!I$-yNVi5-&uJ?j$x8WdnkEV{%35N%rI&XlI)+k&*JGwR>4PE zJe$S0!MvO-R=Tqw0W#SE$h_!zz?s(^A2|J`@Wm}si-OFXU;{YwBK9aa^QN~OoOzG> zK3Kzsr*E2v!3w^x^u6m1Fn@~tjqt_nXB_WK-W+XYiO@L|`21hr1gF56SIK#BKk8Ym zvXvxUiv|}NcSrE`G-E}a;I!}-d^>|V>^m83CvpFQKgI5^3P`cas_#moOP4QBuih{s z6>}OApRZmgofONfmzgv4va2Nt>)TkV#D{{hfUnvj6qY zoIU4v?z#7#d+y^qcRy?G+kV2#$zt|wjVhBdRi?NyT|LAzHY9HQ&cVVZ5qvJFPS4mJ z4P%K+$Cdz}19gHD7{^M%t9AH6F`#}`NDTf``?XQ{2(US9PyL++J`J1!PWbnXIascR zu{BU6uo>*{;E_TddLjUv@K+!+>{Z}opTsfNj1HFv>ve1{aHNp^{3+M~TrveyeHpil zF<};C6#$7$&#GY11AXB>j<} zZCJm4>!yyDExUHE-?(k>#%)FYUcQmj?AWy9Df>QYzN%seJYq!2pcGJ~BtU>?m^ouu zK&AynN;-tJFrc}mhUFx18k)*<&=ijt)|u(Vdnwans5V^G;=Q6Yif}wA4m1;lIWr?@ znzTY)m2Sog4w)I74NWfOk+6IsL;9`yzQc@&9m6tZlTRqUkS#Pe%I`=H>-$I!`}{V^ z;klkAndXXej^wa=7f2>gP~IUqJV%-2a4g&wmcuOJ316lPKGfhU$rKor4@iy^@1;vZ ztRe9w$K;G(53r8Zk-V=zhs*}04Y6x9tpawcES-!j;(<4w`y85EvG`RLq5yF?HZmo$sCB3H2$v!YS`2;5Sqhc0V zhe2qu8#2cKxnEMWDRZj@7+7msS(d7lWUjWR)sB|DjB{@7^5`!!Tbq=od&6778vg$<^ab zt*rLz%Hje8x#rHaDs6xhj(dt46=&*I#hF&4323TW$E-DqbAh|Z$%K=hWMST(X4xgG zts2MfE_9M5D^5+BOUwpO7GB#kPZ$tm2Nh=}xCwkScpP{)cpCU1xEcHocnr9VKKP*C zE$YyR4>%dvS8w$4WdouXI=((-;tnRDaw%|vIW%Q%F=^1DID4O+Y$64b5skjBOZBD; z+yphqL^MvI-dCUM4cf@e!8Zzijlg%#ky%H~$1D(YF+Cu-x;=cOYCNVXW!xaB#uF~4 zjPId#E~D2@9DVY22@~mXMBB_deR=n8an%Prl@2R^%=iSe9k|j{#iP?+goPHUqQPt{ zsZgpK29?ULGDR$LDnwx*{rNizsUZK9rb8IWK%_~=`_yMJgVXK!3FM8VZ z+&8tARzHKIS#VTW#Rm>q4~f%BE_Y}GDi|6rE6#KQXG73-@_8|xK3V0ZwesO~@X#4C z3$0r{Ayjl9xNlAOYMNs^IqVfz<9V-7NcLu6x6i`K7!zLij=GAy6;{tkQfhXfQL3xapEi&lVD%0Dv?e~l52s9UAEgGk>&XG%I~cMJefLF0R1I4F>Pvh;J)jAUR-}73LdRQGUq((r zYBTHn^d~*yDGYktdQkjyphsjv`QiF%v+Y%DPx&6`mzk`kT4AI8Dr8lKfF;;$AI4@r z=sGBBDr^1fi&{q@7MBTmIUfUz4XV5!E-!E3UFxqjxNydVVs8PCS!KD#ol~IBjYNPV zQdak(r$8{A%@w*|c|`uzD~GFffp0&#ZSyrSuVys!Y?pA#V}dj4vF>GM!poi;|G8~5 zV$((@z&2!_H3*KK_D4#^y0^ZcY@(UR3wHZ39C4cZN>xI$T_bF;=T<1@P zQ~};v?RR4hW9Vj+H`-MdwSJ$Kd99avMD_ccfX~fvSHyqK!3jYyy1UW07JUflF(s77VbFMljRxKJP&>nym&tZlWcnVQTk#G*RPejC3Zx zR9}Z0ZoufQ-7kF+|JA%K>RqU5y}1t!>$vLQIgEAxUnwd-h2WW<1Fs z*e5+{IHj+IC5G+ZUWk#&^1(_wzAYZDbJ2Op|VVEq?zVJB@lT=&{OW{g9 zs*~_3H*FUv!w>|%{pAm!8PzzhQohAN&F^CFO)a0`~z^&P#Y4{M#w@4}d99!5H-S4WXdNsD_GjQD z4{-u>1D^*T;Pbx!0t^t z9*a_}0zQX@7yLfpW~}G(puLVA1)lmsKLtGH z{p(GwN!IND4kKgOk(uI%HohH4#u!K7+`4?vCd_49t=#n(O zFiS_b0g+EyRG6hcf@|xD^hDuo9o<$$r=`Qt(?a!1Zx+thM{Xs?NyCLEJ*^m#Hc@&| zXwpzTg;|oN$fBe3Pt+={F3Qpb4_9XY(?#hVS0O!LJX Date: Mon, 24 Mar 2025 22:37:35 +0800 Subject: [PATCH 71/76] ggml-qnn: add build script for libggmlop_skel.so --- ggml/src/ggml-qnn/kernels/Makefile | 24 ++++++++++++++++++++ ggml/src/ggml-qnn/kernels/ggmlop_cdsp.c | 23 ++++--------------- ggml/src/ggml-qnn/kernels/libggmlop_skel.so | Bin 13672 -> 0 bytes scripts/build-run-android.sh | 23 +++++++++++++++++++ 4 files changed, 51 insertions(+), 19 deletions(-) create mode 100755 ggml/src/ggml-qnn/kernels/Makefile delete mode 100755 ggml/src/ggml-qnn/kernels/libggmlop_skel.so diff --git a/ggml/src/ggml-qnn/kernels/Makefile b/ggml/src/ggml-qnn/kernels/Makefile new file mode 100755 index 0000000000000..879bd3444ee1b --- /dev/null +++ b/ggml/src/ggml-qnn/kernels/Makefile @@ -0,0 +1,24 @@ +HEXAGON_SDK_PATH=/opt/qcom/Hexagon_SDK/6.2.0.1 + +TARGET=libggmlop_skel.so + +INCS=-I${HEXAGON_SDK_PATH}/incs -I${HEXAGON_SDK_PATH}/libs/qprintf/inc -I${HEXAGON_SDK_PATH}/incs/stddef -I${HEXAGON_SDK_PATH}/ipc/fastrpc/incs -I${HEXAGON_SDK_PATH}/ipc/fastrpc/rpcmem/inc -I${HEXAGON_SDK_PATH}/utils/examples -I${HEXAGON_SDK_PATH}/ipc/fastrpc/rtld/ship/inc -I${HEXAGON_SDK_PATH}/libs/atomic/inc -I${HEXAGON_SDK_PATH}/utils/sim_utils/inc + +CFLAGS=-mv75 -c -Ofast -Wall -Wstrict-prototypes -fno-zero-initialized-in-bss -fdata-sections -fpic -D__V_DYNAMIC__ -mhvx -mhvx-length=128B ${INCS} + +LDFLAGS=-mv75 -Wl,--defsym=ISDB_TRUSTED_FLAG=2 -Wl,--defsym=ISDB_SECURE_FLAG=2 -Wl,--no-threads -fpic -shared -Wl,-Bsymbolic -Wl,--wrap=malloc -Wl,--wrap=calloc -Wl,--wrap=free -Wl,--wrap=realloc -Wl,--wrap=memalign -lc -Wl,-soname=${TARGET} + +SRCS = ggmlop_cdsp.c ggmlop_cdsp_skel.c +OBJS = $(patsubst %.c, %.o, $(SRCS)) + +ALL:$(OBJS) + ${HEXAGON_SDK_PATH}/tools/HEXAGON_Tools/8.8.06/Tools/bin/hexagon-clang ${LDFLAGS} -o ${TARGET} -Wl,--start-group ${OBJS} -Wl,--end-group + @ls -l ${TARGET} + +%.o:%.c + @echo "${HEXAGON_SDK_PATH}/tools/HEXAGON_Tools/8.8.06/Tools/bin/hexagon-clang ${CFLAGS} -D__FILENAME__=\"$<\" -o $@ -c $< " + ${HEXAGON_SDK_PATH}/tools/HEXAGON_Tools/8.8.06/Tools/bin/hexagon-clang ${CFLAGS} -D__FILENAME__=\"$<\" -o $@ -c $< + @echo "\n" + +clean: + rm -f *.o diff --git a/ggml/src/ggml-qnn/kernels/ggmlop_cdsp.c b/ggml/src/ggml-qnn/kernels/ggmlop_cdsp.c index fa00d9bc5614f..bdb39fdf1f2a3 100644 --- a/ggml/src/ggml-qnn/kernels/ggmlop_cdsp.c +++ b/ggml/src/ggml-qnn/kernels/ggmlop_cdsp.c @@ -349,9 +349,6 @@ static void ggml_compute_forward_add_f32( const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(ggml_can_repeat(src1, src0) && ggml_are_same_shape(src0, dst)); - const int ith = 0; - const int nth = 1; - const int nr = ggml_nrows(src0); GGML_TENSOR_BINARY_OP_LOCALS @@ -359,17 +356,16 @@ static void ggml_compute_forward_add_f32( GGML_ASSERT( nb0 == sizeof(float)); GGML_ASSERT(nb00 == sizeof(float)); - // rows per thread - const int dr = (nr + nth - 1)/nth; + const int dr = nr; // row range for this thread - const int ir0 = dr*ith; + const int ir0 = 0; const int ir1 = MIN(ir0 + dr, nr); ggml_dump_tensor(src0); ggml_dump_tensor(src1); -#if 1 //naive algorithm, can works with llama-cli +#if 1 //naive algorithm for fp32, can works with llama-cli float * a = (float*)src0->data; float * b = (float*)src1->data; float * c = (float*)dst->data; @@ -473,9 +469,6 @@ static void ggml_compute_forward_mul_mat_one_chunk( const int64_t r2 = ne12 / ne02; const int64_t r3 = ne13 / ne03; - //printf("ir0_start = %6lld, ir0_end = %6lld, ir1_start = %6lld, ir1_end = %6lld\n", ir0_start, ir0_end, ir1_start, ir1_end); - - // threads with no work simply yield (not sure if it helps) if (ir0_start >= ir0_end || ir1_start >= ir1_end) { return; } @@ -514,20 +507,12 @@ static void ggml_compute_forward_mul_mat_one_chunk( const char * src0_row = (const char*)src0->data + (0 + i02 * nb02 + i03 * nb03); - // desc: when src1 is not a contiguous memory block we have to calculate the offset using the strides - // if it is, then we have either copied the data to params->wdata and made it contiguous or we are using - // the original src1 data pointer, so we should index using the indices directly - // TODO: this is a bit of a hack, we should probably have a better way to handle this const char * src1_col = (const char*)wdata + (src1_cont || src1->type != vec_dot_type ? (i11 + i12 * ne11 + i13 * ne12 * ne11) * row_size : (i11 * nb11 + i12 * nb12 + i13 * nb13)); float * dst_col = (float*)((char*)dst->data + (i1 * nb1 + i2 * nb2 + i3 * nb3)); - //for (int64_t ir0 = iir0; ir0 < iir0 + blck_0 && ir0 < ir0_end; ++ir0) { - // vec_dot(ne00, &dst_col[ir0], src0_row + ir0*nb01, src1_col); - //} - for (int64_t ir0 = iir0; ir0 < iir0 + blck_0 && ir0 < ir0_end; ir0 += num_rows_per_vec_dot) { vec_dot(ne00, &tmp[ir0 - iir0], (num_rows_per_vec_dot > 1 ? 16 : 0), src0_row + ir0 * nb01, (num_rows_per_vec_dot > 1 ? nb01 : 0), src1_col, (num_rows_per_vec_dot > 1 ? src1_col_stride : 0), num_rows_per_vec_dot); } @@ -574,7 +559,7 @@ int ggmlop_dsp_mulmat(remote_handle64 h, const ggml_tensor * src0, const ggml_te int M = src0->ne[1]; int K = src0->ne[0]; int N = src1->ne[1]; - float sum = 0; + float sum = 0; for (int i = 0; i < M; i++) { for (int j = 0; j < N; j++) { sum = 0; diff --git a/ggml/src/ggml-qnn/kernels/libggmlop_skel.so b/ggml/src/ggml-qnn/kernels/libggmlop_skel.so deleted file mode 100755 index b92e38c950500e09eb9e76cd961c2de86de2898d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 13672 zcmeHOe|%fjb-#LY96Je)Y|B6Li$4E|ane|p9oun&15dIP+rcCzG1P!Rm1Rk`<;a%M z4_Bs)xJhV%DJBZgu_f-}lrgrjR-ZM4tb-3P#$ka`ma;6%Xx*BS?NeaPX`F79c6i_O zemKuc7_=R2|Hy^wbH4YSd+v{S-+f2-p6s+YZ_?>>Oig^$JMD~uW0F>|qG7o*4ThdQKKTpAaog`0ZMqo9O{xS{5qhvQ|Fv-n? za~ZRNWU&RT23T0g*wYK-{+|%5JYkgec%sg3f4>6- zuE3zf=f%<)bU1z9?tYKk(dX@7TiH1n_Jqh-xK?6O%N2Dxy1e~RyVv6gg(IEXW?epi z$fFj}QdAmsc>4$ZJ3Nj^(90a#!~Td5mV%ycZz${uI=X_+J`Ys7{Ekj%$P@ZM4BX@N zx@brtuYb;X{Q-<0(}4eB`Ab1(09HL7Dhm5|I8cZ_dq3W=DxtY^y4?)D@nViFp^zs? zb0t~qbNYOKSJE6Xyn8OZaYVaav)r7w5szz$lIhV=qGw8pmNe}jImqJ;Q5pItruPvF@Yi!w2PS|<+0(?p$t6Ywc0)t6R@?5-Ke51e4pWk1NDJ}~o( z7nyhZ3*Don($7q8#P#6R6ejqVVz*Pm+Y&9ELkw^cY8uM-c( zGprdgCLWaJq@zuwpCLB&E*FR6q|;0I%_Mv}3BR6%PbT59%h*fyewc*MBw_SdZC}!V zH3^Sj#@fCU&tXoVz&z?E2$;yFp0rKwoz@3A+z8i+c{FH!XV4@W$aIX@V;-UvLLj2@l2##IMEE zAJV-yPv{KjPXF2Hs~rQG=<`k{?5Uq8?yjf$dOLfM30MmlOWK*kM+A&BXTi$xgm5Bj z{(_8mzbfpGKO>G64~^}K+pMXv2K4(Iz?g&RF7$Ud`gdJJ+WU((rM;gnx&qC@qVZeD zt{XQ9B~t&YEc0W_1#ch&`sL#Mfo!2Ku-u*gi;l_W%riSLy+~LGS%LUiWVvnmbSBo| zLh+$^4)_fgzJ0Z@H{K#Xdf+RYGp7XMu}GE}3M{wfOdFw>DLx)&mfQ|rAiYQ9w~PA^ z47V(tBHe7%UnqtH1|b6dY)OB$xIfN@X#8yF?ZW;8nc|~y(qAZq182=g%v6ss6Ca6W zSTf!-imd%1Aw5BL(q4}p$rV`}6CR0t5S?k4<`MHp^2KOdhxcZ9t8PltJ^jp)QZX%| zv!`7Q|9qx>0qSw~)c18Fi#~Y&%-z;8E=Kd6bN%w0=3k$26V}Anv4W(OiRbrGVB*hL zVXbiX&b>xFyOP$4(UoT=Mh#Y$_M8QCC0HXjyfHC)LC8DSZei`kZ4;wK*fU`BzJC|9 z4D9}y#paH6;Z*{@gU78|+!u`lrQmx`S&Iu>j4?gtB}MEEJbAKp`23Gk2F_dR-z0gd zXf4(^R_#6K;07+*CgP{ZC*oW^H~lR5d%?SI2g!!;FroV*?iYXq*{0vmbFD-zCIYT{PZx1p}CIX6*Hcx(jr>xVfTi}Jwwf2fJ8V^Y^Kjmt*k z5!lZBGcSzRsbg!A$7XHCxc(XA0=0lj;{sm-es)~I9I!gB)!K1E_LO#9z-4Gh8W;AJ zQQ)D|)GzcKo8mR+O#3h{JDzzibPPgAfH|we%b{00DP$S2 z4(o-HI2$`UT8QUX(4vp&$B&K{HR@wJiy?75i|b9?H*>e{CFAHbMD2zIvm51Sk9ni# zk2Q^baU|Xd8%5|V-@?UifsMjOF1A@Tjy741qld>v;suJVdCVxuwpxs%TOha9v$k7N zb`EQr*8LLnEnu-4^2CYJ6s&bU);^WBjaiGoK5pDQ?~dZ15;o)Atr^!(an{^v!iF*H z%0kRVKKSU{)|HRxA7#~9mDx0ZYYjD-V=eY!;bdxzs~4w8$9$eEAet|5hHYZJfqR3S zKDU$mqdclVX)UI?)HOHt)?wUC31F$?Xp z@_f?Xz-`8!P^T+>_sdt`6P`J#llFuhw8Lxm1j1=adx8;VZJIrSaP3w01S(Ja750Ru z&^Ou>4uNX-gx|$}P%P+vz5U>LUV`=k0b|I=ST?9*QT707f5UtFyV?hiIoYqZ4;*x6 z&DjI27+3Nh5SwiIbnpMY%KK0Bvv~ip_UZk{yBJM_-yqsjCh9)m+gs5dFXksM2`p=& zd4%T#W893jj`uIR9$Ek4xfkDd=m#?JPKkDIf_%>7Hqd|N+n2)5Iw3E?{Y&(;j(;LtTvHz75zP7NK5FTT`zNd$9@f z!=<{?#VC8rHWYdJ75LM_l1N?$3mehzv?jhC{$g7J%F@LVs{oo08LhuG)IGa#S6PAZ z$C33Ne;dw1-LsPc!vg4&Txm0^!}>)T(|xh2S2P!{`Hw*wH`}=seOiVrpid;v+0xXz zM8p_T#_&y;gr2r7zaIMe#_K(TP}R* zjzAs0PseUbYb=Sfp`sRDZ0Nr-Z!GCyL&a?@R$wf7uAqg*c=#zLIYrMFWEFphN4q+a z8`a&JchPt!p7F_%dOf~vub-hDtJ7|w-*Ba>^x+O@U(YUP|wZcvfaxsr57wT6{T=Qa3s%~r6R*o~}~ z&Bv)t_8hck4l@4=x;Nm&+JciKi!xs2Ah65l><%&ccrFxlRWP?R>}27=0FKClE;Ez* z<_?7!9e&X%oj<_(J6XSn^~i=CUbheDZX4BUq*Lb#0Fzp9jb599Ycp_d2CmJ(wHdfJ z1J`EY_k0F)=-WPlz7Rqy5K%7yM{rF-pM`K~RHBHz{1T`ygYkS^Wbi5$a? zFJs?ExgK<1p=38e{Lg1*{3}4|4iDy}d%CAns^5p@i;9*@n7w&#jJ;WPnnp`!M1erN0#A7#atm??asXKoOFAiNnA-l4_*XxY3-n2t;l~DJhft4FE-KH(dWI>I z?%b%HY*ByGQqrXRw@K8ad`ii1k0l+bNMqnofd_v9a;i&xSNo>}<`ir|Jxb|NB0Wly z96-21#$8^YXG3{;`4tfsH>(%$SKM-!IQsl<@|6*rQKI_Is=q8-mH!@6JEbjD_yiWq zr_JoPNK-xE+)mvepBdbg`{ZN<7;*D9XX`}_J3vaq4t9|)J<N=P1WxePkJ{pCuiLzxdFoS~kwP#rPLDQ_^B2fVIw3QyQ>^7CcQzlGL5`&K^V7CuAsSWRwES7@+rW7Bq< z)zP}`_U)~9n`2Y6rD0=bl5o4d{`PJ5%d&odS-29~&B!`n> znU#6^-HP(us%1U|z08sS78YbV1@mC}vrR?)enj>l_=zHjoC8j;&)Mno6crVji;C)t zXk3UEayi`ou%l~jWl`yX-|JpgRO($@xrPsfgKPMYdnitjq6V4CJ=l*0CsTw> znP^93rl7~?B$2`ed|_si0-;P28}>v|AO%vHf_~`)$>iyg!?z?;&OWaT433t_6m(?3 zOs=p$7y^(BY%#P?4977PY#-dWrn`LGL}Eo>i1eNd1;U+zLs9{{&3Gb zi^)5LZ=}TM=p~QtvcIO6yiNEhivOWU{2`_2ES5dAp;??M$tf;~l3s}vBR^Bd+3aDN zmHwu%pUymygpUAI|CWO4;JyAeXb&*8MEHB4XnspGQQUjVC#=IC#d@Jkw; z1^KH<`1?utY!d!)68=dNz5q;fNy&icsRPjBYyeu%_aRd}?NVS`zdqnBHlOjp>UyR4 z*eYOcysZUT8}E7>ur{8y6IdIsdJnLOhfVWH_I3jcSmtVc>j*Ht541ig{`H%r&tHxY z{x&eJcY5V0-u4WzHeUA<@O_X|GH?c_!w(O2{Tn#c0RFxz$9&|_4-+NP{NJL%q;{7E zll=E5X+O3uHSUz|59E(6Hqd4hg|V8N9geO@zsun;DOJ>u3_^IrJ%0E0aM0V|9kK+2 z&cV>!GKza8*=1W~(o~7qRePgj(>BYkc8As8(6q(j&=%RZ*iax_pJO(}0!|9jMYOI` zK8r)tR>T!<#a%*3x=~Ox=yD+h>-DW$w>g@dwzo=c*$Bz)I2xL_S}o0vt(!J&x3@Z4 zE!Ji`b?0*Q?%cRrv_nzOYW%VsMvOxIuZi$qA3^pz&%NNaiSmDzSkQ$PD4;Lu^N zm7hGnK@2#Ie0(7HZ~oTNwn(30Y!7q(U4&{``GiYfQ2OeU9-Yrn-iJ)v0NEft+Q;dW zOl71e;typyw^5VIehvk6wnKX4Cs016U=Yb3*<7jcq$i*PWjepJf~dYshrM# zDDMGJn7+BH-g@xV2Ri?t{aoGC0j0z%^kz_1Pdz7?RAfYGtdwlXsvdn0C^t%nT9TOY zGGUuFdXitG?B$AhRyI|;-*TP zFWD&z0qv5XRf0@iptbc1vew>(GR6ip I1**OO1x>QY9{>OV diff --git a/scripts/build-run-android.sh b/scripts/build-run-android.sh index 4675c5fcad307..d3c47e0473bcd 100755 --- a/scripts/build-run-android.sh +++ b/scripts/build-run-android.sh @@ -33,6 +33,15 @@ function show_pwd() } +function check_hexagon_sdk() +{ + if [ ! -d ${HEXAGON_SDK_PATH} ]; then + echo -e "HEXAGON_SDK_PATH ${HEXAGON_SDK_PATH} not exist, pls install it accordingly...\n" + exit 0 + fi +} + + function check_and_download_qnn_sdk() { is_qnn_sdk_exist=1 @@ -98,6 +107,16 @@ function check_and_download_ndk() } +function build_dsp +{ + cd ggml/src/ggml-qnn/kernels/ + show_pwd + make clean + make + cd - +} + + function build_arm64 { cmake -H. -B./out/android -DCMAKE_BUILD_TYPE=Release -DGGML_OPENMP=OFF -DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK}/build/cmake/android.toolchain.cmake -DANDROID_ABI=arm64-v8a -DANDROID_PLATFORM=latest -DCMAKE_C_FLAGS=-march=armv8.7-a -DGGML_QNN=ON -DQNN_SDK_PATH=${QNN_SDK_PATH} -DHEXAGON_SDK_PATH=${HEXAGON_SDK_PATH} @@ -106,6 +125,8 @@ function build_arm64 show_pwd cd - + + build_dsp } @@ -158,6 +179,7 @@ function build_ggml_qnn() show_pwd check_and_download_ndk check_and_download_qnn_sdk + check_hexagon_sdk dump_vars remove_temp_dir build_arm64 @@ -314,6 +336,7 @@ show_pwd check_and_download_ndk check_and_download_qnn_sdk +check_hexagon_sdk if [ $# == 0 ]; then show_usage From d59037488af55e8c4e56fedfafabcd58cc470185 Mon Sep 17 00:00:00 2001 From: zhouwg Date: Tue, 25 Mar 2025 09:30:19 +0800 Subject: [PATCH 72/76] ggml-qnn: remove redundant functions in this PR and make codes more clear --- ggml/include/ggml-qnn.h | 4 +- ggml/src/ggml-qnn/CMakeLists.txt | 2 - ggml/src/ggml-qnn/ggml-qnn.cpp | 391 +++++++------------------------ 3 files changed, 82 insertions(+), 315 deletions(-) diff --git a/ggml/include/ggml-qnn.h b/ggml/include/ggml-qnn.h index 2ff2bef9dcf7d..63d136c5e52b9 100644 --- a/ggml/include/ggml-qnn.h +++ b/ggml/include/ggml-qnn.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023-2024 The ggml authors + * Copyright (c) 2023-2025 The ggml authors * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to @@ -29,7 +29,7 @@ extern "C" { #endif #define GGML_QNN_MAX_DEVICES 3 -#define GGML_QNN_BACKEND_NAME "qnn" +#define GGML_QNN_BACKEND_NAME "hexagon" enum QNNBackend { QNN_BACKEND_CPU, diff --git a/ggml/src/ggml-qnn/CMakeLists.txt b/ggml/src/ggml-qnn/CMakeLists.txt index d5a16ffd4e1a1..4fb3a8b6d4b47 100644 --- a/ggml/src/ggml-qnn/CMakeLists.txt +++ b/ggml/src/ggml-qnn/CMakeLists.txt @@ -31,8 +31,6 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Android") include_directories(${HEXAGON_SDK_PATH}/ipc/fastrpc/incs) include_directories(${HEXAGON_SDK_PATH}/ipc/fastrpc/rpcmem/inc) include_directories(${HEXAGON_SDK_PATH}/ipc/fastrpc/remote/ship/android_Debug_aarch64) - include_directories(${HEXAGON_SDK_PATH}/incs/qnx) - include_directories(${HEXAGON_SDK_PATH}/libs/common/qnx/ship/android_Debug_aarch64) include_directories(${HEXAGON_SDK_PATH}/utils/examples) include_directories(${HEXAGON_SDK_PATH}/ipc/fastrpc/rtld/ship/android_aarch64) include_directories(${HEXAGON_SDK_PATH}/libs/atomic/inc) diff --git a/ggml/src/ggml-qnn/ggml-qnn.cpp b/ggml/src/ggml-qnn/ggml-qnn.cpp index 847191b7dbc0c..fd1ee54a8cf28 100644 --- a/ggml/src/ggml-qnn/ggml-qnn.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn.cpp @@ -1,16 +1,11 @@ /* - * Copyright (c) 2023-2024 The ggml authors + * Copyright (c) 2023-2025 The ggml authors * * Qualcomm QNN SDK and reference tech guides could be found at: * https://www.qualcomm.com/developer/software/qualcomm-ai-engine-direct-sdk * https://developer.qualcomm.com/software/hexagon-dsp-sdk/tools * - * there are three tech approaches to implement the ggml-hexagon backend for Qualcomm's Hexagon NPU: - * - general approach through Qualcomm QNN SDK:offload ggml op to QNN, then QNN will transfer to Hexagon cDSP - * - general approach through Qualcomm Hexagon SDK:offload ggml op to Hexagon cDSP directly - * - special approach through Qualcomm QNN SDK:mapping the entire ggml cgraph to a single QNN graph - * - * this single-source-file or self-contained implementation of ggml-hexagon backend has 10 sections: + * this single-source-file or self-contained implementation of ggml-hexagon backend has 9 sections: * section-1 forward/prototype declaration, global vars, macros, data structures * section-2 ggml-qnn internal troubleshooting function/class * section-3 helper function for WoA(Windows on ARM) @@ -20,7 +15,6 @@ * section-7 backend helper function / class * section-8 implementation of ggml-hexagon backend according to specification in ggml backend subsystem * section-9 implementation of general approach through QNN and Hexagon DSP - * section-10 implementation of special approach through QNN:mapping the entire ggml cgraph to a single QNN graph * * currently provide following ggml op' implementation through QNN: * - GGML_OP_ADD/GGML_OP_SUB/GGML_OP_MUL/GGML_OP_DIV/GGML_OP_LOG/GGML_OP_SQRT: @@ -136,60 +130,34 @@ class qnn_instance; struct qnn_parameter; struct ggml_backend_qnn_context; -typedef int (*pfn_mallopt)(int, int); -typedef int (*pfn_android_mallopt)(int, void *, size_t); -typedef void (* ggmlqnn_op_func_t)(ggml_backend_qnn_context * ctx, ggml_tensor * op); -typedef int (* notify_callback_fn)(void * context, int domain, int session, remote_rpc_status_flags_t status); -typedef int (* ggmlhexagon_op_func_t)(remote_handle64 handle, const dsptensor * src0, const dsptensor * src1, dsptensor * dst); - -static void * ggmlqnn_type_trait(ggml_backend_qnn_context * ctx, ggml_tensor * op); -static void ggmlqnn_dump_tensor(const ggml_tensor * tensor, const char * name); -static enum ggml_status ggmlqnn_backend_graph_compute_special(ggml_backend_t backend, struct ggml_cgraph * cgraph); -static void ggmlqnn_log_internal(ggml_log_level level, const char * file, const char * func, int line, const char * format, ...); -static inline bool ggmlqnn_is_valid_params(ggml_backend_qnn_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); -static Qnn_Tensor_t * ggmlqnn_create_general_tensor(qnn_instance * instance, Qnn_GraphHandle_t graph_handle, - const ggml_tensor * tensor, const char * name, - Qnn_TensorType_t qnn_tensor_type, - Qnn_DataType_t qnn_data_type, - uint32_t rank, uint32_t * dims, - void * data, uint32_t data_size, - bool b_transpose = false); - - -//function prototypes for all op functions in the general approach -//general op function for elment-wise operation on 1/2 input tensors and 1 output tensor -static void ggmlqnn_compute_elementwise(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -static void ggmlqnn_compute_mul_mat(ggml_backend_qnn_context * ctx, ggml_tensor * dst); - -//todo by AI experts -static void ggmlqnn_compute_repeat(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -static void ggmlqnn_compute_leaky_relu(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -static void ggmlqnn_compute_concat(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -static void ggmlqnn_compute_arange(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -static void ggmlqnn_compute_sqr(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -static void ggmlqnn_compute_clamp(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -static void ggmlqnn_compute_scale(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -static void ggmlqnn_compute_argsort(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -static void ggmlqnn_compute_norm(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -static void ggmlqnn_compute_group_norm(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -static void ggmlqnn_compute_acc(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -static void ggmlqnn_compute_sum_rows(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -static void ggmlqnn_compute_pad(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -static void ggmlqnn_compute_pool2d(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -static void ggmlqnn_compute_dup(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -static void ggmlqnn_compute_rms_norm(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -static void ggmlqnn_compute_cpy(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -static void ggmlqnn_compute_rope(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -static void ggmlqnn_compute_im2col(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -static void ggmlqnn_compute_softmax(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -static void ggmlqnn_compute_get_rows(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -static void ggmlqnn_compute_upsample_nearest2d(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -static void ggmlqnn_compute_timestep_embedding(ggml_backend_qnn_context * ctx, ggml_tensor * dst); -static void ggmlqnn_compute_diag_mask(ggml_backend_qnn_context * ctx, ggml_tensor * dst, float value); - -//function prototypes for all op functions in the special approach("mapping the entire cgraph to a single QNN graph") -static void ggmlqnn_graph_addnode(ggml_backend_qnn_context * ctx, struct ggml_cgraph * cgraph, - Qnn_GraphHandle_t graph_handle, std::string & graph_name, ggml_tensor * op, bool is_reuse_graph = false); +static void * ggmlqnn_type_trait(ggml_backend_qnn_context * ctx, ggml_tensor * op); +static void ggmlqnn_compute_elementwise(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggmlqnn_compute_mul_mat(ggml_backend_qnn_context * ctx, ggml_tensor * dst); + +static void ggmlqnn_compute_repeat(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggmlqnn_compute_leaky_relu(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggmlqnn_compute_concat(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggmlqnn_compute_arange(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggmlqnn_compute_sqr(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggmlqnn_compute_clamp(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggmlqnn_compute_scale(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggmlqnn_compute_argsort(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggmlqnn_compute_norm(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggmlqnn_compute_group_norm(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggmlqnn_compute_acc(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggmlqnn_compute_sum_rows(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggmlqnn_compute_pad(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggmlqnn_compute_pool2d(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggmlqnn_compute_dup(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggmlqnn_compute_rms_norm(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggmlqnn_compute_cpy(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggmlqnn_compute_rope(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggmlqnn_compute_im2col(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggmlqnn_compute_softmax(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggmlqnn_compute_get_rows(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggmlqnn_compute_upsample_nearest2d(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggmlqnn_compute_timestep_embedding(ggml_backend_qnn_context * ctx, ggml_tensor * dst); +static void ggmlqnn_compute_diag_mask(ggml_backend_qnn_context * ctx, ggml_tensor * dst, float value); #if 0//def NDEBUG #define GGMLQNN_DEBUG 0 @@ -283,17 +251,14 @@ using _pfn_QnnSaver_initialize = decltype(QnnSaver_initialize); using _pfn_QnnInterface_getProviders = decltype(QnnInterface_getProviders); using _pfn_QnnSystemInterface_getProviders = decltype(QnnSystemInterface_getProviders); -//QNN resource management for the general approach through QNN(similar to ggml-sycl or ggml-cann) +//QNN resource management for the general approach through QNN +using qnn_tensors_t = std::vector< Qnn_Tensor_t >; using qnn_ptensors_t = std::vector< Qnn_Tensor_t *>; using qnn_singlenode_res_t = std::tuple; -//QNN resource management for the special approach through QNN(mapping the entire cgraph to a single QNN graph) -using qnn_tensors_t = std::vector< Qnn_Tensor_t >; -using qnn_tensor_pair_t = std::tuple< ggml_tensor *, Qnn_Tensor_t *>; -using qnn_tensor_pairs_t = std::vector< qnn_tensor_pair_t >; -using qnn_cgraph_node_t = std::tuple; -using qnn_cgraph_nodes_t = std::vector; -using qnn_multinode_res_t = std::tuple; +typedef void (* ggmlqnn_op_func_t)(ggml_backend_qnn_context * ctx, ggml_tensor * op); +typedef int (* notify_callback_fn)(void * context, int domain, int session, remote_rpc_status_flags_t status); +typedef int (* ggmlhexagon_op_func_t)(remote_handle64 handle, const dsptensor * src0, const dsptensor * src1, dsptensor * dst); enum qnn_index_type { QNN_TENSOR_INDEX = 0, @@ -306,9 +271,9 @@ enum qnn_profile_level { PROFILE_DETAIL = 2, }; -//0: general approach through QNN -//1: general approach through Hexagon cDSP -//2: special approach through QNN:mapping entire ggml cgraph to QNN graph +//0: general approach through QNN:offload ggmlop to QNN +//1: general approach through Hexagon cDSP:offload ggmlop to Hexagon cDSP directly +//2: special approach through QNN:mapping entire ggml cgraph to a single QNN graph enum inference_approach { QNN_GENERAL = 0, DIRECT_USE_CDSP = 1, @@ -357,7 +322,6 @@ struct qcom_socinfo { struct ggml_backend_qnn_context { int device; - int threads; char name[GGML_MAX_NAME]; char desc[GGML_MAX_NAME]; char lib[GGML_MAX_NAME]; @@ -367,10 +331,8 @@ struct ggml_backend_qnn_context { QNN_SYSTEM_INTERFACE_VER_TYPE raw_system_interface; struct qcom_socinfo socinfo; - //QNN resource management for the general approach through QNN(similar to ggml-sycl or ggml-opencl) + //QNN resource management for the general approach through QNN std::map qnn_singlenode_graph_map; - //QNN resource management for the special approach through QNN(mapping the entire cgraph to a single QNN graph) - std::map qnn_multinode_graph_map; //quantize data -> fp32 std::unique_ptr work_data; @@ -379,7 +341,7 @@ struct ggml_backend_qnn_context { size_t desired_size; int n_threads; - //hexagon resource management for the general approach through Hexagaon cDSP(similar to ggml-sycl or ggml-opencl) + //Hexagon resource management for the general approach through Hexagaon cDSP size_t rpc_mempool_len; void * rpc_mempool; remote_handle64 ggmlop_handle; @@ -516,7 +478,6 @@ static struct qcom_socinfo g_qnn_soc_info_table[] = { // HTA - Choose a quantized model. Quantized models are required when running on the HTA backend static struct ggml_backend_qnn_context g_qnn_mgr[GGML_QNN_MAX_DEVICES] = { [QNN_BACKEND_CPU] = {.device = 0, - .threads = 1, .name = "qnn-cpu", .desc = "Qualcomm Kryo CPU", #if !defined(__ANDROID__) && !defined(__linux__) @@ -531,7 +492,6 @@ static struct ggml_backend_qnn_context g_qnn_mgr[GGML_QNN_MAX_DEVICES] = { .socinfo = {}}, [QNN_BACKEND_GPU] = {.device = 1, - .threads = 1, .name = "qnn-gpu", .desc = "Qualcomm Adreno GPU", #if !defined(__ANDROID__) && !defined(__linux__) @@ -546,7 +506,6 @@ static struct ggml_backend_qnn_context g_qnn_mgr[GGML_QNN_MAX_DEVICES] = { .socinfo = {}}, [QNN_BACKEND_NPU] = {.device = 2, - .threads = 1, .name = "qnn-npu", .desc = "Qualcomm NPU(Hexagon Tensor Processor)", #if !defined(__ANDROID__) && !defined(__linux__) @@ -996,36 +955,6 @@ static void ggmlqnn_get_timestring(char * p_currenttime) { p_tm->tm_hour, p_tm->tm_min, p_tm->tm_sec); } -//fix some tricky memory issue -static void ggmlqnn_disable_android_tags(int disable) { - if (0 == disable) - return; -#if defined(__ANDROID__) - void * lib_handle = dlopen("libc.so", RTLD_LAZY); - if (nullptr != lib_handle) { - int api_level = android_get_device_api_level(); - GGMLQNN_LOG_INFO("device_api_level=%d", api_level); - if (api_level >= 31) { //ANDROID 12 - pfn_mallopt mallopt = reinterpret_cast(dlsym(lib_handle, "mallopt")); - if (mallopt) { - mallopt(M_BIONIC_SET_HEAP_TAGGING_LEVEL, M_HEAP_TAGGING_LEVEL_NONE); - } - return; - } else if (api_level >= 30) { //ANDROID 11 - /* android_get_device_api_level() < 31 */ - pfn_android_mallopt android_mallopt = reinterpret_cast(dlsym( - lib_handle, "android_mallopt")); - if (android_mallopt) { - int android_malloc_tag_level = 0; - int tmp = 0; - android_mallopt(8, &tmp, sizeof(tmp)); - } - } - dlclose(lib_handle); - } -#endif -} - // ================================================================================================= // section-5: QNN helper function // ================================================================================================= @@ -2876,8 +2805,6 @@ class qnn_instance { void * alloc_rpcmem_internal(size_t bytes, size_t alignment); - void htp_print_info(); - void htp_probe_rpc_meminfo(); void print_backend_info(); @@ -3500,7 +3427,6 @@ int qnn_instance::qnn_init(const QnnSaver_Config_t ** saver_config) { Qnn_ErrorHandle_t qnnstatus = QNN_SUCCESS; if (_device_id == QNN_BACKEND_NPU) { - //TODO: remove duplicated code between here and function htp_print_info const QnnDevice_PlatformInfo_t * p_info = nullptr; qcom_socinfo soc_info = {}; qnnstatus = _qnn_raw_interface.deviceGetPlatformInfo(nullptr, &p_info); @@ -3618,23 +3544,13 @@ int qnn_instance::qnn_init(const QnnSaver_Config_t ** saver_config) { } if (_backend_name.find("Htp") != std::string::npos) { - htp_print_info(); htp_probe_rpc_meminfo(); if (0 != htp_init_perfinfra()) { GGMLQNN_LOG_WARN("initialize HTP performance failure"); } -#if 1 - //FIXME: ht_set_rpc_polling + htp_set_high_performance_mode should be equivalent to htp_enter_performance_mode - if (0 != htp_set_rpc_polling()) { - GGMLQNN_LOG_WARN("set RPC polling failure"); - } - if (0 != htp_set_high_performance_mode()) { - GGMLQNN_LOG_WARN("set HTP high performance mode failure"); - } -#else + htp_enter_performance_mode(); -#endif htp_set_memory_grow_size(); if (enable_qnn_rpc()) { @@ -3875,37 +3791,6 @@ int qnn_instance::htp_init_perfinfra() { return 0; } -void qnn_instance::htp_print_info() { - const QnnDevice_PlatformInfo_t * p_info = nullptr; - _qnn_raw_interface.deviceGetPlatformInfo(nullptr, &p_info); - GGMLQNN_LOG_INFO("HTP device counts %d", p_info->v1.numHwDevices); - QnnDevice_HardwareDeviceInfo_t * infos = p_info->v1.hwDevices; - for (size_t i = 0; i < p_info->v1.numHwDevices; i++) { - GGMLQNN_LOG_INFO("HTP deviceID:%d, deviceType:%d, numCores %d", infos[i].v1.deviceId, - infos[i].v1.deviceType, infos[i].v1.numCores); - QnnDevice_DeviceInfoExtension_t devinfo = infos[i].v1.deviceInfoExtension; - QnnHtpDevice_OnChipDeviceInfoExtension_t chipinfo = devinfo->onChipDevice; - QnnHtpDevice_Arch_t htp_arch = chipinfo.arch; - GGMLQNN_LOG_INFO("HTP_TYPE:%d(%s)", devinfo->devType, - (devinfo->devType == QNN_HTP_DEVICE_TYPE_ON_CHIP) ? "QNN_HTP_DEVICE_TYPE_ON_CHIP" : "QNN_HTP_DEVICE_TYPE_UNKNOWN"); - GGMLQNN_LOG_INFO("qualcomm soc_model:%d(%s), htp_arch:%d(%s), vtcm_size:%d MB，" \ - "dlbc_support:%d, signedpd_support:%d", \ - chipinfo.socModel, ggmlqnn_get_socmodel_desc(chipinfo.socModel), \ - htp_arch, ggmlqnn_get_htparch_desc(htp_arch), chipinfo.vtcmSize, \ - chipinfo.dlbcSupport, chipinfo.signedPdSupport); - struct qcom_socinfo * socinfo = ggmlqnn_get_socinfo_from_socmodel(chipinfo.socModel); - g_qnn_mgr[QNN_BACKEND_NPU].socinfo = { chipinfo.socModel, htp_arch, chipinfo.vtcmSize, {}}; - if (nullptr != socinfo) { - memcpy(g_qnn_mgr[QNN_BACKEND_NPU].socinfo.soc_desc, socinfo->soc_desc, sizeof(socinfo->soc_desc)); - GGMLQNN_LOG_INFO("soc info:%s", socinfo->soc_desc); - } else { - memcpy(g_qnn_mgr[QNN_BACKEND_NPU].socinfo.soc_desc, "unknown", 7); - GGMLQNN_LOG_INFO("soc info:unknown"); - } - } - _qnn_raw_interface.deviceFreePlatformInfo(nullptr, p_info); -} - void qnn_instance::htp_probe_rpc_meminfo() { size_t candidate_size = 0; uint8_t * rpc_buffer = nullptr; @@ -3998,58 +3883,6 @@ void qnn_instance::htp_set_n_hvx_threads(size_t n_threads) { } } -int qnn_instance::htp_set_rpc_polling() { - if (_qnn_rpc_pollingtime > 0) { - QnnHtpPerfInfrastructure_PowerConfig_t rpc_pollingtime; - memset(&rpc_pollingtime, 0, sizeof(rpc_pollingtime)); - rpc_pollingtime.option = QNN_HTP_PERF_INFRASTRUCTURE_POWER_CONFIGOPTION_RPC_POLLING_TIME; - rpc_pollingtime.rpcPollingTimeConfig = _qnn_rpc_pollingtime; - const QnnHtpPerfInfrastructure_PowerConfig_t * power_configs[] = {&rpc_pollingtime, nullptr}; - if (_qnn_htp_perfinfra) { - _qnn_htp_perfinfra->setPowerConfig(_qnn_htp_powerconfig_id, power_configs); - } - } - return 0; -} - -int qnn_instance::htp_set_high_performance_mode() { - if (nullptr == _qnn_htp_perfinfra) { - GGMLQNN_LOG_DEBUG("perf intra is null\n"); - return 1; - } - - QnnHtpPerfInfrastructure_PowerConfig_t power_config; - memset(&power_config, 0, sizeof(power_config)); - power_config.option = QNN_HTP_PERF_INFRASTRUCTURE_POWER_CONFIGOPTION_DCVS_V3; - power_config.dcvsV3Config.dcvsEnable = 0; - power_config.dcvsV3Config.setDcvsEnable = 1; - power_config.dcvsV3Config.contextId = _qnn_htp_powerconfig_id; - power_config.dcvsV3Config.powerMode = QNN_HTP_PERF_INFRASTRUCTURE_POWERMODE_PERFORMANCE_MODE; - power_config.dcvsV3Config.setSleepLatency = 1; // True to consider Latency parameter otherwise False - power_config.dcvsV3Config.setBusParams = 1; // True to consider Bus parameter otherwise False - power_config.dcvsV3Config.setCoreParams = 1; // True to consider Core parameter otherwise False - power_config.dcvsV3Config.sleepDisable = 0; // True to consider sleep/LPM modes, False to enable - power_config.dcvsV3Config.setSleepDisable = 0; // True to consider sleep disable/enable parameter otherwise False - // set Sleep latency parameter - uint32_t latencyValue = 40; - power_config.dcvsV3Config.sleepLatency = latencyValue; // range 40-2000 micro sec - // set Bus Clock Parameters (refer QnnHtpPerfInfrastructure_VoltageCorner_t enum) - power_config.dcvsV3Config.busVoltageCornerMin = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; - power_config.dcvsV3Config.busVoltageCornerTarget = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; - power_config.dcvsV3Config.busVoltageCornerMax = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; - // set Core Clock Parameters (refer QnnHtpPerfInfrastructure_VoltageCorner_t enum) - power_config.dcvsV3Config.coreVoltageCornerMin = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; - power_config.dcvsV3Config.coreVoltageCornerTarget = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; - power_config.dcvsV3Config.coreVoltageCornerMax = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; - // set power config with different performance parameters - const QnnHtpPerfInfrastructure_PowerConfig_t * power_configs[] = {&power_config, nullptr}; - - _qnn_htp_perfinfra->setPowerConfig(_qnn_htp_powerconfig_id, power_configs); - - return 0; -} - -//TODO: merge code between this function and htp_set_rpc_polling,htp_set_high_performance_mode void qnn_instance::htp_enter_performance_mode() { QnnHtpPerfInfrastructure_PowerConfig_t dcvs_v3_config = { .option = QNN_HTP_PERF_INFRASTRUCTURE_POWER_CONFIGOPTION_DCVS_V3, @@ -4194,6 +4027,13 @@ static Qnn_OpConfig_t ggmlqnn_create_op_config(const char * name, const char * p return opcfg; } +static Qnn_Tensor_t * ggmlqnn_create_general_tensor(qnn_instance * instance, Qnn_GraphHandle_t graph_handle, + const ggml_tensor * tensor, const char * name, + Qnn_TensorType_t qnn_tensor_type, + Qnn_DataType_t qnn_data_type, + uint32_t rank, uint32_t * dims, + void * data, uint32_t data_size, + bool b_transpose = false); static Qnn_Tensor_t * ggmlqnn_create_general_tensor(qnn_instance * instance, Qnn_GraphHandle_t graph_handle, const ggml_tensor * tensor, const char * name, Qnn_TensorType_t qnn_tensor_type, @@ -4325,8 +4165,6 @@ static void ggmlqnn_load_cfg() { memset(time_string, 0, GGML_QNN_TMPBUF_LEN); ggmlqnn_get_timestring(time_string); GGMLQNN_LOG_DEBUG("program running start time:%s", time_string); - ggmlqnn_disable_android_tags(0); - std::string cfg_filename = std::string(g_qnn_params.qnn_runtimelib_path) + std::string(g_qnn_params.qnn_cfgfilename); GGMLQNN_LOG_INFO("load ggml-qnn config from %s", cfg_filename.c_str()); qnn_cfg qnncfg_instance; @@ -4393,30 +4231,46 @@ static bool ggmlhexagon_can_handle_op(const ggml_backend_qnn_context * ctx, cons if (nullptr != src0) { src0_rank = ggml_n_dims(src0); } else { - GGMLQNN_LOG_DEBUG("op name %s\n", ggml_op_name(op_tensor->op)); + //GGMLQNN_LOG_DEBUG("op name %s\n", ggml_op_name(op_tensor->op)); } if (nullptr != src1) { src1_rank = ggml_n_dims(src1); } else { - GGMLQNN_LOG_DEBUG("op name %s\n", ggml_op_name(op_tensor->op)); + //GGMLQNN_LOG_DEBUG("op name %s\n", ggml_op_name(op_tensor->op)); } - //FIXME: mulmat on cDSP doesn't work as expected - bool support = ((op_tensor->op == GGML_OP_ADD) || (op_tensor->op == GGML_OP_MUL_MAT)); + //TODO: remove this filter in the future, mulmat on cDSP doesn't work as expected + //bool support = ((op_tensor->op == GGML_OP_ADD) || (op_tensor->op == GGML_OP_MUL_MAT)); + bool support = (op_tensor->op == GGML_OP_ADD); if (!support) return false; - ggmlqnn_dump_op_info(op_tensor); - if (!ggml_are_same_shape(src0, src1)) { - return false; - } + switch (op_tensor->op) { + case GGML_OP_ADD: + { + if (!ggml_are_same_shape(src0, src1)) { + return false; + } + return ggmlqnn_same_types(ctx, op_tensor); + } - support = ggmlqnn_same_types(ctx, op_tensor); - if (!support) { - return false; - } + case GGML_OP_MUL_MAT: + { + ggmlqnn_dump_op_info(op_tensor); + if (src0_rank != src1_rank) + return false; - return (src0_rank <= 2); + //TODO: remove this filter in the future + if (src0_rank != 2) + return false; + + return (src0->type == GGML_TYPE_F32 || ggml_is_quantized(src0->type)) + && (src1->type == GGML_TYPE_F32) && (op_tensor->type == GGML_TYPE_F32); + + } + default: + return false; + } } static bool ggmlqnn_can_handle_op(const ggml_backend_qnn_context * ctx, const struct ggml_tensor * op_tensor) { @@ -4425,13 +4279,7 @@ static bool ggmlqnn_can_handle_op(const ggml_backend_qnn_context * ctx, const st } if (DIRECT_USE_CDSP == g_qnn_params.inference_approach) { - //return ggmlhexagon_can_handle_op(ctx, op_tensor); - //bool support = ((op_tensor->op == GGML_OP_ADD) || (op_tensor->op == GGML_OP_MUL_MAT)); - //FIXME: mulmat on cDSP doesn't work as expected - bool support = (op_tensor->op == GGML_OP_ADD); - if (!support) - return false; - + return ggmlhexagon_can_handle_op(ctx, op_tensor); } if (!ggmlqnn_k_op_caps[ggmlqnn_get_op_index(op_tensor)].supported) { @@ -4640,7 +4488,6 @@ static bool ggmlqnn_compute_forward(ggml_backend_t backend, struct ggml_tensor * return true; } -//TODO: refine this data structure struct ggml_backend_qnn_buffer_context { ~ggml_backend_qnn_buffer_context() { if (buffer) { @@ -4814,20 +4661,6 @@ static void ggml_backend_qnn_free(ggml_backend_t backend) { } ctx->qnn_singlenode_graph_map.clear(); - std::map::iterator multinode_graph_it; - for (multinode_graph_it = ctx->qnn_multinode_graph_map.begin(); - multinode_graph_it != ctx->qnn_multinode_graph_map.end(); multinode_graph_it++) { - auto & graph_res = multinode_graph_it->second; - Qnn_GraphHandle_t & graph_handle = std::get<0>(graph_res); - qnn_ptensors_t & ptensors = std::get<2>(graph_res); - for (auto tensor_it = ptensors.begin(); tensor_it != ptensors.end(); ++tensor_it) { - free_qnn_tensor(*tensor_it); - } - GGML_UNUSED(graph_handle); - GGMLQNN_LOG_DEBUG("clean up graph:%s", multinode_graph_it->first.c_str()); - } - ctx->qnn_multinode_graph_map.clear(); - instance->qnn_finalize(); delete instance; g_qnn_mgr[ctx->device].instance = nullptr; @@ -5071,7 +4904,7 @@ void ggml_backend_qnn_set_n_threads(ggml_backend_t backend, int n_threads) { GGML_ASSERT(ggml_backend_is_qnn(backend)); struct ggml_backend_qnn_context * ctx = (struct ggml_backend_qnn_context *)backend->context; - ctx->threads = n_threads; + ctx->n_threads = n_threads; } int ggml_backend_qnn_get_device_count() { @@ -5260,11 +5093,7 @@ ggml_backend_t ggml_backend_qnn_init(size_t device, const char * qnn_lib_path) { if (nullptr == instance) return nullptr; - if (QNN_SINGLEGRAPH == g_qnn_params.inference_approach) { - ggml_backend_qnn_interface.graph_compute = ggmlqnn_backend_graph_compute_special; - } else { - ggml_backend_qnn_interface.graph_compute = ggmlqnn_backend_graph_compute_general; - } + ggml_backend_qnn_interface.graph_compute = ggmlqnn_backend_graph_compute_general; ggml_backend_t qnn_backend = new ggml_backend{ /* .guid = */ ggml_backend_qnn_guid(), @@ -6003,63 +5832,3 @@ static void ggmlqnn_compute_rope(ggml_backend_qnn_context * ctx, ggml_tensor * d GGML_UNUSED(ctx); GGML_UNUSED(dst); } - -// ================================================================================================= -// section-10: special approach: mapping ggml computational cgraph to QNN graph -// ================================================================================================= -// TODO: remove duplicated codes between section-9 and section-10 -// TODO: the graph algorithm in this section is naive, should optimized by AI experts -// details: https://github.com/ggml-org/llama.cpp/pull/12326#issuecomment-2712838649 -// ref: https://github.com/kantv-ai/kantv/blob/kantv-poc-with-qnn/core/ggml/jni/Inception_v3.cpp#L20634 -static enum ggml_status ggmlqnn_backend_graph_compute_special(ggml_backend_t backend, struct ggml_cgraph * cgraph) { - enum ggml_status ggml_result = GGML_STATUS_SUCCESS; - Qnn_ErrorHandle_t qnn_error = QNN_SUCCESS; - qnn_perf op_perf = qnn_perf("ggmlqnn_backend_graph_compute_special"); - qnn_instance * instance = nullptr; - Qnn_GraphHandle_t graph_handle = nullptr; - ggml_backend_qnn_context * ctx = (ggml_backend_qnn_context *) backend->context; - instance = ctx->instance; - QNN_INTERFACE_VER_TYPE qnn_raw_interface = ctx->raw_interface; - op_perf.start(); - - //now we got the entire ggml cgraph or a ggml cgraph which contains multiple nodes - GGMLQNN_LOG_DEBUG("qnn device %d(%s)", ctx->device, ggml_backend_qnn_get_devname(ctx->device)); - GGMLQNN_LOG_DEBUG("cgraph->n_nodes %d", cgraph->n_nodes); - int num_nodes = std::min(5, cgraph->n_nodes); - //for (int i = 0; i < cgraph->n_nodes; i++) { - for (int i = 0; i < num_nodes; i++) { - ggml_tensor * node = cgraph->nodes[i]; - GGMLQNN_LOG_DEBUG("%s: op %s (%s)\n", __func__, node->name, ggml_op_name(node->op)); - } - - //now we'll offload the ggml cgraph to a single QNN graph - std::string graph_name; - ggmlqnn_get_graphkey_from_cgraph(cgraph, graph_name); - if (graph_name == "") - return GGML_STATUS_SUCCESS; - if (ctx->qnn_multinode_graph_map.find(graph_name) != ctx->qnn_multinode_graph_map.end()) { - GGMLQNN_LOG_DEBUG("graph name %s already create", graph_name.c_str()); - //retrieve computational resource from cached QNN graph - qnn_multinode_res_t &graph_res = ctx->qnn_multinode_graph_map[graph_name]; - graph_handle = std::get<0>(graph_res); - } else { - //create QNN graph - GGMLQNN_LOG_INFO("graph name %s", graph_name.c_str()); - qnn_error = instance->init_qnn_graph(graph_name, static_cast(ctx->device), 8, 4); - if (QNN_SUCCESS != qnn_error) { - GGMLQNN_LOG_WARN("can't create qnn graph handle with graph name %s, error = %d(%s)\n", graph_name.c_str(), qnn_error, - ggmlqnn_get_qnnerror_string(qnn_error)); - return ggml_result; - } - graph_handle = instance->get_qnn_graph_handle(); - //TBD: compose a single QNN graph - - //finalize QNN graph - CHECK_QNN_API(qnn_error, qnn_raw_interface.graphFinalize(graph_handle, nullptr, nullptr)); - - //TBD: cache QNN graph - } - //TBD: exec QNN graph - - return ggml_result; -} From 0703cbcf920e47ca3f3164b4edca1a5454d41607 Mon Sep 17 00:00:00 2001 From: zhouwg Date: Tue, 25 Mar 2025 22:29:28 +0800 Subject: [PATCH 73/76] ggml-qnn: original ggml_compute_forward_add and ggml_compute_forward_mul_mat works fine on Hexagon cDSP at the first time --- ggml/src/ggml-qnn/ggml-qnn.cpp | 34 +-- ggml/src/ggml-qnn/kernels/ggmlop_ap_skel.c | 64 ++--- ggml/src/ggml-qnn/kernels/ggmlop_ap_skel.h | 9 +- ggml/src/ggml-qnn/kernels/ggmlop_cdsp.c | 279 +++++++++++++------ ggml/src/ggml-qnn/kernels/ggmlop_cdsp_skel.c | 70 ++--- 5 files changed, 268 insertions(+), 188 deletions(-) diff --git a/ggml/src/ggml-qnn/ggml-qnn.cpp b/ggml/src/ggml-qnn/ggml-qnn.cpp index fd1ee54a8cf28..909650124a9eb 100644 --- a/ggml/src/ggml-qnn/ggml-qnn.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn.cpp @@ -179,8 +179,6 @@ static void ggmlqnn_compute_diag_mask(ggml_backend_qnn_context * ctx, ggml_ten #endif #define GGMLQNN_DUMP_TENSOR(tensor) ggmlqnn_dump_tensor(tensor, #tensor) -#define GGMLQNN_MEM_ADD(alignment) (sizeof (size_t) + alignment) -#define GGMLQNN_MEM_MASK(alignment) ((uintptr_t)alignment - 1) #define QNN_VER_PTR(x) (&((x).v1)) #define RPCMEM_DEFAULT_FLAGS 1 #define RPCMEM_HEAP_ID_SYSTEM 25 @@ -4230,18 +4228,13 @@ static bool ggmlhexagon_can_handle_op(const ggml_backend_qnn_context * ctx, cons uint32_t src1_rank = 0; if (nullptr != src0) { src0_rank = ggml_n_dims(src0); - } else { - //GGMLQNN_LOG_DEBUG("op name %s\n", ggml_op_name(op_tensor->op)); } if (nullptr != src1) { src1_rank = ggml_n_dims(src1); - } else { - //GGMLQNN_LOG_DEBUG("op name %s\n", ggml_op_name(op_tensor->op)); } - //TODO: remove this filter in the future, mulmat on cDSP doesn't work as expected - //bool support = ((op_tensor->op == GGML_OP_ADD) || (op_tensor->op == GGML_OP_MUL_MAT)); - bool support = (op_tensor->op == GGML_OP_ADD); + //TODO: only support offload GGML_OP_ADD and GGML_OP_MUL_MAT to cDSP directly + bool support = ((op_tensor->op == GGML_OP_ADD) || (op_tensor->op == GGML_OP_MUL_MAT)); if (!support) return false; @@ -4251,21 +4244,17 @@ static bool ggmlhexagon_can_handle_op(const ggml_backend_qnn_context * ctx, cons if (!ggml_are_same_shape(src0, src1)) { return false; } - return ggmlqnn_same_types(ctx, op_tensor); + return (src0->type == GGML_TYPE_F32) && (src1->type == GGML_TYPE_F32) && (op_tensor->type == GGML_TYPE_F32); } case GGML_OP_MUL_MAT: { ggmlqnn_dump_op_info(op_tensor); - if (src0_rank != src1_rank) - return false; - //TODO: remove this filter in the future - if (src0_rank != 2) + if (src1_rank != 2) return false; - return (src0->type == GGML_TYPE_F32 || ggml_is_quantized(src0->type)) - && (src1->type == GGML_TYPE_F32) && (op_tensor->type == GGML_TYPE_F32); + return (src0->type == GGML_TYPE_F32) && (src1->type == GGML_TYPE_F32) && (op_tensor->type == GGML_TYPE_F32); } default: @@ -5110,6 +5099,8 @@ ggml_backend_t ggml_backend_qnn_init(size_t device, const char * qnn_lib_path) { ggml_backend_qnn_free(qnn_backend); return nullptr; } + //ensure test-backend-ops get the correct backend name when inference approach is 1(DIRECT_USE_CDSP) + memcpy(g_qnn_mgr[device].name, "Hexagon-cDSP", strlen("Hexagon-cDSP")); } GGMLQNN_LOG_INFO("leave %s\n", __func__); @@ -5564,11 +5555,6 @@ static void ggmlqnn_compute_mul_mat(ggml_backend_qnn_context * ctx, ggml_tensor const enum ggml_type src0_type = src0->type; const uint32_t src0_rank = ggml_n_dims(src0); const uint32_t src1_rank = ggml_n_dims(src1); - GGML_ASSERT(src0_rank == src1_rank); - GGML_ASSERT(src0_rank >= 2); //QNN SDK's limitation, make QNN SDK happy - if (4 == src0_rank) { - return ggmlqnn_compute_mul_mat_4d(ctx, op); - } ggmlqnn_print_tensors_info(__func__, ctx, src0, src1, dst); @@ -5584,6 +5570,12 @@ static void ggmlqnn_compute_mul_mat(ggml_backend_qnn_context * ctx, ggml_tensor return; } + GGML_ASSERT(src0_rank == src1_rank); + GGML_ASSERT(src0_rank >= 2); //QNN SDK's limitation, make QNN SDK happy + if (4 == src0_rank) { + return ggmlqnn_compute_mul_mat_4d(ctx, op); + } + void * wdata = ggmlqnn_type_trait(ctx, op); const size_t desired_size = ctx->desired_size; diff --git a/ggml/src/ggml-qnn/kernels/ggmlop_ap_skel.c b/ggml/src/ggml-qnn/kernels/ggmlop_ap_skel.c index 6f2c37e4087cc..82de512150bf8 100644 --- a/ggml/src/ggml-qnn/kernels/ggmlop_ap_skel.c +++ b/ggml/src/ggml-qnn/kernels/ggmlop_ap_skel.c @@ -270,13 +270,13 @@ struct Interface { #define __QAIC_SLIM_EXPORT #endif -static const Type types[5]; -static const Type* const typeArrays[6] = {&(types[0]),&(types[1]),&(types[1]),&(types[0]),&(types[0]),&(types[3])}; -static const StructType structTypes[1] = {{0x6,&(typeArrays[0]),0x58,0x4,0x50,0x8,0x4,0x8}}; -static const Type types[5] = {{0x4,{{(const uintptr_t)0,(const uintptr_t)1}}, 2,0x4},{0x20,{{(const uintptr_t)&(types[2]),(const uintptr_t)0x4}}, 8,0x8},{0x8,{{(const uintptr_t)0,(const uintptr_t)1}}, 2,0x8},{SLIM_IFPTR32(0x8,0x10),{{(const uintptr_t)&(types[4]),(const uintptr_t)0x0}}, 9,SLIM_IFPTR32(0x4,0x8)},{0x4,{{(const uintptr_t)0,(const uintptr_t)1}}, 2,0x4}}; -static const Parameter parameters[5] = {{SLIM_IFPTR32(0x8,0x10),{{(const uintptr_t)0x0,0}}, 4,SLIM_IFPTR32(0x4,0x8),0,0},{SLIM_IFPTR32(0x4,0x8),{{(const uintptr_t)0xdeadc0de,(const uintptr_t)0}}, 0,SLIM_IFPTR32(0x4,0x8),3,0},{SLIM_IFPTR32(0x4,0x8),{{(const uintptr_t)0xdeadc0de,(const uintptr_t)0}}, 0,SLIM_IFPTR32(0x4,0x8),0,0},{SLIM_IFPTR32(0x58,0x60),{{(const uintptr_t)&(structTypes[0]),0}}, 22,0x8,0,0},{SLIM_IFPTR32(0x58,0x60),{{(const uintptr_t)&(structTypes[0]),0}}, 22,0x8,3,0}}; +static const Type types[4]; +static const Type* const typeArrays[6] = {&(types[0]),&(types[1]),&(types[1]),&(types[0]),&(types[0]),&(types[2])}; +static const StructType structTypes[1] = {{0x6,&(typeArrays[0]),0x30,0x4,0x2c,0x4,0x4,0x4}}; +static const Type types[4] = {{0x4,{{(const uintptr_t)0,(const uintptr_t)1}}, 2,0x4},{0x10,{{(const uintptr_t)&(types[0]),(const uintptr_t)0x4}}, 8,0x4},{SLIM_IFPTR32(0x8,0x10),{{(const uintptr_t)&(types[3]),(const uintptr_t)0x0}}, 9,SLIM_IFPTR32(0x4,0x8)},{0x4,{{(const uintptr_t)0,(const uintptr_t)1}}, 2,0x4}}; +static const Parameter parameters[5] = {{SLIM_IFPTR32(0x8,0x10),{{(const uintptr_t)0x0,0}}, 4,SLIM_IFPTR32(0x4,0x8),0,0},{SLIM_IFPTR32(0x4,0x8),{{(const uintptr_t)0xdeadc0de,(const uintptr_t)0}}, 0,SLIM_IFPTR32(0x4,0x8),3,0},{SLIM_IFPTR32(0x4,0x8),{{(const uintptr_t)0xdeadc0de,(const uintptr_t)0}}, 0,SLIM_IFPTR32(0x4,0x8),0,0},{SLIM_IFPTR32(0x34,0x40),{{(const uintptr_t)&(structTypes[0]),0}}, 22,SLIM_IFPTR32(0x4,0x8),0,0},{SLIM_IFPTR32(0x34,0x40),{{(const uintptr_t)&(structTypes[0]),0}}, 22,SLIM_IFPTR32(0x4,0x8),3,0}}; static const Parameter* const parameterArrays[6] = {(&(parameters[3])),(&(parameters[3])),(&(parameters[4])),(&(parameters[0])),(&(parameters[1])),(&(parameters[2]))}; -static const Method methods[3] = {{REMOTE_SCALARS_MAKEX(0,0,0x2,0x0,0x0,0x1),0x4,0x0,2,2,(&(parameterArrays[3])),0x4,0x1},{REMOTE_SCALARS_MAKEX(0,0,0x0,0x0,0x1,0x0),0x0,0x0,1,1,(&(parameterArrays[5])),0x1,0x0},{REMOTE_SCALARS_MAKEX(0,0,0x3,0x2,0x0,0x0),0xb4,0x50,3,3,(&(parameterArrays[0])),0x8,0x8}}; +static const Method methods[3] = {{REMOTE_SCALARS_MAKEX(0,0,0x2,0x0,0x0,0x1),0x4,0x0,2,2,(&(parameterArrays[3])),0x4,0x1},{REMOTE_SCALARS_MAKEX(0,0,0x0,0x0,0x1,0x0),0x0,0x0,1,1,(&(parameterArrays[5])),0x1,0x0},{REMOTE_SCALARS_MAKEX(0,0,0x3,0x2,0x0,0x0),0x64,0x2c,3,3,(&(parameterArrays[0])),0x4,0x4}}; static const Method* const methodArrays[4] = {&(methods[0]),&(methods[1]),&(methods[2]),&(methods[2])}; static const char strings[68] = "mulmat\0flags\0close\0src1\0data\0type\0src0\0open\0dst\0add\0uri\0op\0nb\0ne\0h\0"; static const uint16_t methodStrings[49] = {0,34,29,62,59,56,7,24,19,29,62,59,56,7,24,44,29,62,59,56,7,24,48,34,29,62,59,56,7,24,19,29,62,59,56,7,24,44,29,62,59,56,7,24,39,52,65,13,65}; @@ -294,20 +294,20 @@ __QAIC_STUB_EXPORT int __QAIC_STUB(ggmlop_dsp_open)(const char* uri, remote_hand __QAIC_STUB_EXPORT int __QAIC_STUB(ggmlop_dsp_close)(remote_handle64 h) __QAIC_STUB_ATTRIBUTE { return __QAIC_REMOTE(remote_handle64_close)(h); } -static __inline int _stub_unpack(_ATTRIBUTE_UNUSED remote_arg* _praROutPost, _ATTRIBUTE_UNUSED remote_arg* _ppraROutPost[1], _ATTRIBUTE_UNUSED void* _primROut, _ATTRIBUTE_UNUSED uint32_t _rout0[1], _ATTRIBUTE_UNUSED uint64_t _rout1[4], _ATTRIBUTE_UNUSED uint64_t _rout2[4], _ATTRIBUTE_UNUSED uint32_t _rout3[1], _ATTRIBUTE_UNUSED uint32_t _rout4[1], _ATTRIBUTE_UNUSED char* _rout5[1], _ATTRIBUTE_UNUSED uint32_t _rout5Len[1]) { +static __inline int _stub_unpack(_ATTRIBUTE_UNUSED remote_arg* _praROutPost, _ATTRIBUTE_UNUSED remote_arg* _ppraROutPost[1], _ATTRIBUTE_UNUSED void* _primROut, _ATTRIBUTE_UNUSED uint32_t _rout0[1], _ATTRIBUTE_UNUSED uint32_t _rout1[4], _ATTRIBUTE_UNUSED uint32_t _rout2[4], _ATTRIBUTE_UNUSED uint32_t _rout3[1], _ATTRIBUTE_UNUSED uint32_t _rout4[1], _ATTRIBUTE_UNUSED char* _rout5[1], _ATTRIBUTE_UNUSED uint32_t _rout5Len[1]) { int _nErr = 0; remote_arg* _praROutPostStart = _praROutPost; remote_arg** _ppraROutPostStart = _ppraROutPost; _ppraROutPost = &_praROutPost; _COPY(_rout0, 0, _primROut, 0, 4); - _COPY(_rout1, 0, _primROut, 8, 32); - _COPY(_rout2, 0, _primROut, 40, 32); - _COPY(_rout3, 0, _primROut, 72, 4); - _COPY(_rout4, 0, _primROut, 76, 4); + _COPY(_rout1, 0, _primROut, 4, 16); + _COPY(_rout2, 0, _primROut, 20, 16); + _COPY(_rout3, 0, _primROut, 36, 4); + _COPY(_rout4, 0, _primROut, 40, 4); _ppraROutPostStart[0] += (_praROutPost - _praROutPostStart) +1; return _nErr; } -static __inline int _stub_pack(_ATTRIBUTE_UNUSED _allocator* _al, _ATTRIBUTE_UNUSED remote_arg* _praIn, _ATTRIBUTE_UNUSED remote_arg* _ppraIn[1], _ATTRIBUTE_UNUSED remote_arg* _praROut, _ATTRIBUTE_UNUSED remote_arg* _ppraROut[1], _ATTRIBUTE_UNUSED remote_arg* _praHIn, _ATTRIBUTE_UNUSED remote_arg* _ppraHIn[1], _ATTRIBUTE_UNUSED remote_arg* _praHROut, _ATTRIBUTE_UNUSED remote_arg* _ppraHROut[1], _ATTRIBUTE_UNUSED void* _primIn, _ATTRIBUTE_UNUSED void* _primROut, _ATTRIBUTE_UNUSED uint32_t _rout0[1], _ATTRIBUTE_UNUSED uint64_t _rout1[4], _ATTRIBUTE_UNUSED uint64_t _rout2[4], _ATTRIBUTE_UNUSED uint32_t _rout3[1], _ATTRIBUTE_UNUSED uint32_t _rout4[1], _ATTRIBUTE_UNUSED char* _rout5[1], _ATTRIBUTE_UNUSED uint32_t _rout5Len[1]) { +static __inline int _stub_pack(_ATTRIBUTE_UNUSED _allocator* _al, _ATTRIBUTE_UNUSED remote_arg* _praIn, _ATTRIBUTE_UNUSED remote_arg* _ppraIn[1], _ATTRIBUTE_UNUSED remote_arg* _praROut, _ATTRIBUTE_UNUSED remote_arg* _ppraROut[1], _ATTRIBUTE_UNUSED remote_arg* _praHIn, _ATTRIBUTE_UNUSED remote_arg* _ppraHIn[1], _ATTRIBUTE_UNUSED remote_arg* _praHROut, _ATTRIBUTE_UNUSED remote_arg* _ppraHROut[1], _ATTRIBUTE_UNUSED void* _primIn, _ATTRIBUTE_UNUSED void* _primROut, _ATTRIBUTE_UNUSED uint32_t _rout0[1], _ATTRIBUTE_UNUSED uint32_t _rout1[4], _ATTRIBUTE_UNUSED uint32_t _rout2[4], _ATTRIBUTE_UNUSED uint32_t _rout3[1], _ATTRIBUTE_UNUSED uint32_t _rout4[1], _ATTRIBUTE_UNUSED char* _rout5[1], _ATTRIBUTE_UNUSED uint32_t _rout5Len[1]) { int _nErr = 0; remote_arg* _praInStart = _praIn; remote_arg** _ppraInStart = _ppraIn; @@ -322,7 +322,7 @@ static __inline int _stub_pack(_ATTRIBUTE_UNUSED _allocator* _al, _ATTRIBUTE_UNU _ppraROutStart[0] += (_praROut - _praROutStart) +1; return _nErr; } -static __inline int _stub_pack_1(_ATTRIBUTE_UNUSED _allocator* _al, _ATTRIBUTE_UNUSED remote_arg* _praIn, _ATTRIBUTE_UNUSED remote_arg* _ppraIn[1], _ATTRIBUTE_UNUSED remote_arg* _praROut, _ATTRIBUTE_UNUSED remote_arg* _ppraROut[1], _ATTRIBUTE_UNUSED remote_arg* _praHIn, _ATTRIBUTE_UNUSED remote_arg* _ppraHIn[1], _ATTRIBUTE_UNUSED remote_arg* _praHROut, _ATTRIBUTE_UNUSED remote_arg* _ppraHROut[1], _ATTRIBUTE_UNUSED void* _primIn, _ATTRIBUTE_UNUSED void* _primROut, _ATTRIBUTE_UNUSED uint32_t _in0[1], _ATTRIBUTE_UNUSED uint64_t _in1[4], _ATTRIBUTE_UNUSED uint64_t _in2[4], _ATTRIBUTE_UNUSED uint32_t _in3[1], _ATTRIBUTE_UNUSED uint32_t _in4[1], _ATTRIBUTE_UNUSED char* _in5[1], _ATTRIBUTE_UNUSED uint32_t _in5Len[1]) { +static __inline int _stub_pack_1(_ATTRIBUTE_UNUSED _allocator* _al, _ATTRIBUTE_UNUSED remote_arg* _praIn, _ATTRIBUTE_UNUSED remote_arg* _ppraIn[1], _ATTRIBUTE_UNUSED remote_arg* _praROut, _ATTRIBUTE_UNUSED remote_arg* _ppraROut[1], _ATTRIBUTE_UNUSED remote_arg* _praHIn, _ATTRIBUTE_UNUSED remote_arg* _ppraHIn[1], _ATTRIBUTE_UNUSED remote_arg* _praHROut, _ATTRIBUTE_UNUSED remote_arg* _ppraHROut[1], _ATTRIBUTE_UNUSED void* _primIn, _ATTRIBUTE_UNUSED void* _primROut, _ATTRIBUTE_UNUSED uint32_t _in0[1], _ATTRIBUTE_UNUSED uint32_t _in1[4], _ATTRIBUTE_UNUSED uint32_t _in2[4], _ATTRIBUTE_UNUSED uint32_t _in3[1], _ATTRIBUTE_UNUSED uint32_t _in4[1], _ATTRIBUTE_UNUSED char* _in5[1], _ATTRIBUTE_UNUSED uint32_t _in5Len[1]) { int _nErr = 0; remote_arg* _praInStart = _praIn; remote_arg** _ppraInStart = _ppraIn; @@ -331,38 +331,38 @@ static __inline int _stub_pack_1(_ATTRIBUTE_UNUSED _allocator* _al, _ATTRIBUTE_U _ppraIn = &_praIn; _ppraROut = &_praROut; _COPY(_primIn, 0, _in0, 0, 4); - _COPY(_primIn, 8, _in1, 0, 32); - _COPY(_primIn, 40, _in2, 0, 32); - _COPY(_primIn, 72, _in3, 0, 4); - _COPY(_primIn, 76, _in4, 0, 4); - _COPY(_primIn, 80, _in5Len, 0, 4); + _COPY(_primIn, 4, _in1, 0, 16); + _COPY(_primIn, 20, _in2, 0, 16); + _COPY(_primIn, 36, _in3, 0, 4); + _COPY(_primIn, 40, _in4, 0, 4); + _COPY(_primIn, 44, _in5Len, 0, 4); _praIn[0].buf.pv = (void*) _in5[0]; _praIn[0].buf.nLen = (4 * _in5Len[0]); _ppraInStart[0] += (_praIn - _praInStart) + 1; _ppraROutStart[0] += (_praROut - _praROutStart) +0; return _nErr; } -static __inline void _count(int _numIn[1], int _numROut[1], int _numInH[1], int _numROutH[1], _ATTRIBUTE_UNUSED uint32_t _rout0[1], _ATTRIBUTE_UNUSED uint64_t _rout1[4], _ATTRIBUTE_UNUSED uint64_t _rout2[4], _ATTRIBUTE_UNUSED uint32_t _rout3[1], _ATTRIBUTE_UNUSED uint32_t _rout4[1], _ATTRIBUTE_UNUSED char* _rout5[1], _ATTRIBUTE_UNUSED uint32_t _rout5Len[1]) { +static __inline void _count(int _numIn[1], int _numROut[1], int _numInH[1], int _numROutH[1], _ATTRIBUTE_UNUSED uint32_t _rout0[1], _ATTRIBUTE_UNUSED uint32_t _rout1[4], _ATTRIBUTE_UNUSED uint32_t _rout2[4], _ATTRIBUTE_UNUSED uint32_t _rout3[1], _ATTRIBUTE_UNUSED uint32_t _rout4[1], _ATTRIBUTE_UNUSED char* _rout5[1], _ATTRIBUTE_UNUSED uint32_t _rout5Len[1]) { _numIn[0] += 0; _numROut[0] += 1; _numInH[0] += 0; _numROutH[0] += 0; } -static __inline void _count_1(int _numIn[1], int _numROut[1], int _numInH[1], int _numROutH[1], _ATTRIBUTE_UNUSED uint32_t _in0[1], _ATTRIBUTE_UNUSED uint64_t _in1[4], _ATTRIBUTE_UNUSED uint64_t _in2[4], _ATTRIBUTE_UNUSED uint32_t _in3[1], _ATTRIBUTE_UNUSED uint32_t _in4[1], _ATTRIBUTE_UNUSED char* _in5[1], _ATTRIBUTE_UNUSED uint32_t _in5Len[1]) { +static __inline void _count_1(int _numIn[1], int _numROut[1], int _numInH[1], int _numROutH[1], _ATTRIBUTE_UNUSED uint32_t _in0[1], _ATTRIBUTE_UNUSED uint32_t _in1[4], _ATTRIBUTE_UNUSED uint32_t _in2[4], _ATTRIBUTE_UNUSED uint32_t _in3[1], _ATTRIBUTE_UNUSED uint32_t _in4[1], _ATTRIBUTE_UNUSED char* _in5[1], _ATTRIBUTE_UNUSED uint32_t _in5Len[1]) { _numIn[0] += 1; _numROut[0] += 0; _numInH[0] += 0; _numROutH[0] += 0; } -static __inline int _stub_method(remote_handle64 _handle, uint32_t _mid, uint64_t _in0[SLIM_IFPTR32(11, 12)], uint64_t _in1[SLIM_IFPTR32(11, 12)], uint64_t _rout2[SLIM_IFPTR32(11, 12)]) { +static __inline int _stub_method(remote_handle64 _handle, uint32_t _mid, uintptr_t _in0[SLIM_IFPTR32(13, 8)], uintptr_t _in1[SLIM_IFPTR32(13, 8)], uintptr_t _rout2[SLIM_IFPTR32(13, 8)]) { remote_arg* _pra = 0; int _numIn[1] = {0}; int _numROut[1] = {0}; int _numInH[1] = {0}; int _numROutH[1] = {0}; _allocator _al[1] = {{0}}; - uint64_t _primIn[23]= {0}; - uint64_t _primROut[10]= {0}; + uint32_t _primIn[25]= {0}; + uint32_t _primROut[11]= {0}; remote_arg* _praIn = 0; remote_arg* _praROut = 0; remote_arg* _praROutPost = 0; @@ -378,9 +378,9 @@ static __inline int _stub_method(remote_handle64 _handle, uint32_t _mid, uint64_ _numROut[0] = 0; _numInH[0] = 0; _numROutH[0] = 0; - _count_1(_numIn, _numROut, _numInH, _numROutH, (uint32_t*)&(((uint32_t*)_in0)[0]), (uint64_t*)&(((uint64_t*)_in0)[1]), (uint64_t*)&(((uint64_t*)_in0)[5]), (uint32_t*)&(((uint32_t*)_in0)[18]), (uint32_t*)&(((uint32_t*)_in0)[19]), SLIM_IFPTR32((char**)&(((uint32_t*)_in0)[20]), (char**)&(((uint64_t*)_in0)[10])), SLIM_IFPTR32((uint32_t*)&(((uint32_t*)_in0)[21]), (uint32_t*)&(((uint32_t*)_in0)[22]))); - _count_1(_numIn, _numROut, _numInH, _numROutH, (uint32_t*)&(((uint32_t*)_in1)[0]), (uint64_t*)&(((uint64_t*)_in1)[1]), (uint64_t*)&(((uint64_t*)_in1)[5]), (uint32_t*)&(((uint32_t*)_in1)[18]), (uint32_t*)&(((uint32_t*)_in1)[19]), SLIM_IFPTR32((char**)&(((uint32_t*)_in1)[20]), (char**)&(((uint64_t*)_in1)[10])), SLIM_IFPTR32((uint32_t*)&(((uint32_t*)_in1)[21]), (uint32_t*)&(((uint32_t*)_in1)[22]))); - _count(_numIn, _numROut, _numInH, _numROutH, (uint32_t*)&(((uint32_t*)_rout2)[0]), (uint64_t*)&(((uint64_t*)_rout2)[1]), (uint64_t*)&(((uint64_t*)_rout2)[5]), (uint32_t*)&(((uint32_t*)_rout2)[18]), (uint32_t*)&(((uint32_t*)_rout2)[19]), SLIM_IFPTR32((char**)&(((uint32_t*)_rout2)[20]), (char**)&(((uint64_t*)_rout2)[10])), SLIM_IFPTR32((uint32_t*)&(((uint32_t*)_rout2)[21]), (uint32_t*)&(((uint32_t*)_rout2)[22]))); + _count_1(_numIn, _numROut, _numInH, _numROutH, (uint32_t*)&(((uint32_t*)_in0)[0]), (uint32_t*)&(((uint32_t*)_in0)[1]), (uint32_t*)&(((uint32_t*)_in0)[5]), (uint32_t*)&(((uint32_t*)_in0)[9]), (uint32_t*)&(((uint32_t*)_in0)[10]), SLIM_IFPTR32((char**)&(((uint32_t*)_in0)[11]), (char**)&(((uint64_t*)_in0)[6])), SLIM_IFPTR32((uint32_t*)&(((uint32_t*)_in0)[12]), (uint32_t*)&(((uint32_t*)_in0)[14]))); + _count_1(_numIn, _numROut, _numInH, _numROutH, (uint32_t*)&(((uint32_t*)_in1)[0]), (uint32_t*)&(((uint32_t*)_in1)[1]), (uint32_t*)&(((uint32_t*)_in1)[5]), (uint32_t*)&(((uint32_t*)_in1)[9]), (uint32_t*)&(((uint32_t*)_in1)[10]), SLIM_IFPTR32((char**)&(((uint32_t*)_in1)[11]), (char**)&(((uint64_t*)_in1)[6])), SLIM_IFPTR32((uint32_t*)&(((uint32_t*)_in1)[12]), (uint32_t*)&(((uint32_t*)_in1)[14]))); + _count(_numIn, _numROut, _numInH, _numROutH, (uint32_t*)&(((uint32_t*)_rout2)[0]), (uint32_t*)&(((uint32_t*)_rout2)[1]), (uint32_t*)&(((uint32_t*)_rout2)[5]), (uint32_t*)&(((uint32_t*)_rout2)[9]), (uint32_t*)&(((uint32_t*)_rout2)[10]), SLIM_IFPTR32((char**)&(((uint32_t*)_rout2)[11]), (char**)&(((uint64_t*)_rout2)[6])), SLIM_IFPTR32((uint32_t*)&(((uint32_t*)_rout2)[12]), (uint32_t*)&(((uint32_t*)_rout2)[14]))); if(_numIn[0]>=255){ _QAIC_FARF(RUNTIME_ERROR, "ERROR: Unsupported number of input buffers\n"); return AEE_EUNSUPPORTED; @@ -405,13 +405,13 @@ static __inline int _stub_method(remote_handle64 _handle, uint32_t _mid, uint64_ } if(_praHROut == 0) (_praHROut = _praHIn + _numInH[0] + 0); - _TRY(_nErr, _stub_pack_1(_al, (_praIn + 0), _ppraIn, (_praROut + 0), _ppraROut, _praHIn, _ppraHIn, _praHROut, _ppraHROut, ((char*)_primIn + 0), 0, (uint32_t*)&(((uint32_t*)_in0)[0]), (uint64_t*)&(((uint64_t*)_in0)[1]), (uint64_t*)&(((uint64_t*)_in0)[5]), (uint32_t*)&(((uint32_t*)_in0)[18]), (uint32_t*)&(((uint32_t*)_in0)[19]), SLIM_IFPTR32((char**)&(((uint32_t*)_in0)[20]), (char**)&(((uint64_t*)_in0)[10])), SLIM_IFPTR32((uint32_t*)&(((uint32_t*)_in0)[21]), (uint32_t*)&(((uint32_t*)_in0)[22])))); - _TRY(_nErr, _stub_pack_1(_al, (_praIn + 0), _ppraIn, (_praROut + 0), _ppraROut, _praHIn, _ppraHIn, _praHROut, _ppraHROut, ((char*)_primIn + 88), 0, (uint32_t*)&(((uint32_t*)_in1)[0]), (uint64_t*)&(((uint64_t*)_in1)[1]), (uint64_t*)&(((uint64_t*)_in1)[5]), (uint32_t*)&(((uint32_t*)_in1)[18]), (uint32_t*)&(((uint32_t*)_in1)[19]), SLIM_IFPTR32((char**)&(((uint32_t*)_in1)[20]), (char**)&(((uint64_t*)_in1)[10])), SLIM_IFPTR32((uint32_t*)&(((uint32_t*)_in1)[21]), (uint32_t*)&(((uint32_t*)_in1)[22])))); - _TRY(_nErr, _stub_pack(_al, (_praIn + 0), _ppraIn, (_praROut + 0), _ppraROut, _praHIn, _ppraHIn, _praHROut, _ppraHROut, ((char*)_primIn + 176), ((char*)_primROut + 0), (uint32_t*)&(((uint32_t*)_rout2)[0]), (uint64_t*)&(((uint64_t*)_rout2)[1]), (uint64_t*)&(((uint64_t*)_rout2)[5]), (uint32_t*)&(((uint32_t*)_rout2)[18]), (uint32_t*)&(((uint32_t*)_rout2)[19]), SLIM_IFPTR32((char**)&(((uint32_t*)_rout2)[20]), (char**)&(((uint64_t*)_rout2)[10])), SLIM_IFPTR32((uint32_t*)&(((uint32_t*)_rout2)[21]), (uint32_t*)&(((uint32_t*)_rout2)[22])))); + _TRY(_nErr, _stub_pack_1(_al, (_praIn + 0), _ppraIn, (_praROut + 0), _ppraROut, _praHIn, _ppraHIn, _praHROut, _ppraHROut, ((char*)_primIn + 0), 0, (uint32_t*)&(((uint32_t*)_in0)[0]), (uint32_t*)&(((uint32_t*)_in0)[1]), (uint32_t*)&(((uint32_t*)_in0)[5]), (uint32_t*)&(((uint32_t*)_in0)[9]), (uint32_t*)&(((uint32_t*)_in0)[10]), SLIM_IFPTR32((char**)&(((uint32_t*)_in0)[11]), (char**)&(((uint64_t*)_in0)[6])), SLIM_IFPTR32((uint32_t*)&(((uint32_t*)_in0)[12]), (uint32_t*)&(((uint32_t*)_in0)[14])))); + _TRY(_nErr, _stub_pack_1(_al, (_praIn + 0), _ppraIn, (_praROut + 0), _ppraROut, _praHIn, _ppraHIn, _praHROut, _ppraHROut, ((char*)_primIn + 48), 0, (uint32_t*)&(((uint32_t*)_in1)[0]), (uint32_t*)&(((uint32_t*)_in1)[1]), (uint32_t*)&(((uint32_t*)_in1)[5]), (uint32_t*)&(((uint32_t*)_in1)[9]), (uint32_t*)&(((uint32_t*)_in1)[10]), SLIM_IFPTR32((char**)&(((uint32_t*)_in1)[11]), (char**)&(((uint64_t*)_in1)[6])), SLIM_IFPTR32((uint32_t*)&(((uint32_t*)_in1)[12]), (uint32_t*)&(((uint32_t*)_in1)[14])))); + _TRY(_nErr, _stub_pack(_al, (_praIn + 0), _ppraIn, (_praROut + 0), _ppraROut, _praHIn, _ppraHIn, _praHROut, _ppraHROut, ((char*)_primIn + 96), ((char*)_primROut + 0), (uint32_t*)&(((uint32_t*)_rout2)[0]), (uint32_t*)&(((uint32_t*)_rout2)[1]), (uint32_t*)&(((uint32_t*)_rout2)[5]), (uint32_t*)&(((uint32_t*)_rout2)[9]), (uint32_t*)&(((uint32_t*)_rout2)[10]), SLIM_IFPTR32((char**)&(((uint32_t*)_rout2)[11]), (char**)&(((uint64_t*)_rout2)[6])), SLIM_IFPTR32((uint32_t*)&(((uint32_t*)_rout2)[12]), (uint32_t*)&(((uint32_t*)_rout2)[14])))); _QAIC_ASSERT(_nErr, (_numInH[0] + 0) <= 15); _QAIC_ASSERT(_nErr, (_numROutH[0] + 0) <= 15); _TRY_FARF(_nErr, __QAIC_REMOTE(remote_handle64_invoke)(_handle, REMOTE_SCALARS_MAKEX(0, _mid, (_numIn[0] + 1), (_numROut[0] + 1), (_numInH[0] + 0), (_numROutH[0] + 0)), _pra)); - _TRY(_nErr, _stub_unpack((_praROutPost + 0), _ppraROutPost, ((char*)_primROut + 0), (uint32_t*)&(((uint32_t*)_rout2)[0]), (uint64_t*)&(((uint64_t*)_rout2)[1]), (uint64_t*)&(((uint64_t*)_rout2)[5]), (uint32_t*)&(((uint32_t*)_rout2)[18]), (uint32_t*)&(((uint32_t*)_rout2)[19]), SLIM_IFPTR32((char**)&(((uint32_t*)_rout2)[20]), (char**)&(((uint64_t*)_rout2)[10])), SLIM_IFPTR32((uint32_t*)&(((uint32_t*)_rout2)[21]), (uint32_t*)&(((uint32_t*)_rout2)[22])))); + _TRY(_nErr, _stub_unpack((_praROutPost + 0), _ppraROutPost, ((char*)_primROut + 0), (uint32_t*)&(((uint32_t*)_rout2)[0]), (uint32_t*)&(((uint32_t*)_rout2)[1]), (uint32_t*)&(((uint32_t*)_rout2)[5]), (uint32_t*)&(((uint32_t*)_rout2)[9]), (uint32_t*)&(((uint32_t*)_rout2)[10]), SLIM_IFPTR32((char**)&(((uint32_t*)_rout2)[11]), (char**)&(((uint64_t*)_rout2)[6])), SLIM_IFPTR32((uint32_t*)&(((uint32_t*)_rout2)[12]), (uint32_t*)&(((uint32_t*)_rout2)[14])))); _QAIC_CATCH(_nErr) {} _CATCH_FARF(_nErr) { _QAIC_FARF(RUNTIME_ERROR, "ERROR 0x%x: handle=0x%"PRIx64", scalar=0x%x, method ID=%d: %s failed\n", _nErr , _handle, REMOTE_SCALARS_MAKEX(0, _mid, (_numIn[0] + 1), (_numROut[0] + 1), (_numInH[0] + 0), (_numROutH[0] + 0)), _mid, __func__); @@ -421,9 +421,9 @@ static __inline int _stub_method(remote_handle64 _handle, uint32_t _mid, uint64_ } __QAIC_STUB_EXPORT int __QAIC_STUB(ggmlop_dsp_add)(remote_handle64 _handle, const dsptensor* src0, const dsptensor* src1, dsptensor* dst) __QAIC_STUB_ATTRIBUTE { uint32_t _mid = 2; - return _stub_method(_handle, _mid, (uint64_t*)src0, (uint64_t*)src1, (uint64_t*)dst); + return _stub_method(_handle, _mid, (uintptr_t*)src0, (uintptr_t*)src1, (uintptr_t*)dst); } __QAIC_STUB_EXPORT int __QAIC_STUB(ggmlop_dsp_mulmat)(remote_handle64 _handle, const dsptensor* src0, const dsptensor* src1, dsptensor* dst) __QAIC_STUB_ATTRIBUTE { uint32_t _mid = 3; - return _stub_method(_handle, _mid, (uint64_t*)src0, (uint64_t*)src1, (uint64_t*)dst); + return _stub_method(_handle, _mid, (uintptr_t*)src0, (uintptr_t*)src1, (uintptr_t*)dst); } diff --git a/ggml/src/ggml-qnn/kernels/ggmlop_ap_skel.h b/ggml/src/ggml-qnn/kernels/ggmlop_ap_skel.h index 1273cb76b1797..8e05d06f1c2ba 100644 --- a/ggml/src/ggml-qnn/kernels/ggmlop_ap_skel.h +++ b/ggml/src/ggml-qnn/kernels/ggmlop_ap_skel.h @@ -6,7 +6,6 @@ #include #include - #ifndef __QAIC_HEADER #define __QAIC_HEADER(ff) ff #endif //__QAIC_HEADER @@ -240,8 +239,8 @@ typedef struct _cstring1_s { typedef struct dsptensor dsptensor; struct dsptensor { int32_t type; - int64_t ne[4]; - int64_t nb[4]; + int32_t ne[4]; + int32_t nb[4]; int32_t op; int32_t flags; void * data; @@ -277,8 +276,8 @@ __QAIC_HEADER_EXPORT int __QAIC_HEADER(ggmlop_dsp_open)(const char* uri, remote_ * @retval, 0 on success, should always succeed */ __QAIC_HEADER_EXPORT int __QAIC_HEADER(ggmlop_dsp_close)(remote_handle64 h) __QAIC_HEADER_ATTRIBUTE; -__QAIC_HEADER_EXPORT int __QAIC_HEADER(ggmlop_dsp_add)(remote_handle64 _h, const dsptensor * src0, const dsptensor * src1, dsptensor * dst) __QAIC_HEADER_ATTRIBUTE; -__QAIC_HEADER_EXPORT int __QAIC_HEADER(ggmlop_dsp_mulmat)(remote_handle64 _h, const dsptensor * src0, const dsptensor * src1, dsptensor * dst) __QAIC_HEADER_ATTRIBUTE; +__QAIC_HEADER_EXPORT int __QAIC_HEADER(ggmlop_dsp_add)(remote_handle64 _h, const dsptensor* src0, const dsptensor* src1, dsptensor* dst) __QAIC_HEADER_ATTRIBUTE; +__QAIC_HEADER_EXPORT int __QAIC_HEADER(ggmlop_dsp_mulmat)(remote_handle64 _h, const dsptensor* src0, const dsptensor* src1, dsptensor* dst) __QAIC_HEADER_ATTRIBUTE; #ifndef ggmlop_URI #define ggmlop_URI "file:///libggmlop_skel.so?ggmlop_skel_handle_invoke&_modver=1.0&_idlver=0.0.1" #endif /*ggmlop_URI*/ diff --git a/ggml/src/ggml-qnn/kernels/ggmlop_cdsp.c b/ggml/src/ggml-qnn/kernels/ggmlop_cdsp.c index bdb39fdf1f2a3..85d0ad8c8e29e 100644 --- a/ggml/src/ggml-qnn/kernels/ggmlop_cdsp.c +++ b/ggml/src/ggml-qnn/kernels/ggmlop_cdsp.c @@ -25,20 +25,35 @@ #include #include #include + #include "HAP_farf.h" +#include "HAP_compute_res.h" +#include "hexagon_types.h" +#include "AEEStdErr.h" + #include "ggmlop_ap_skel.h" +// ================================================================================================= +// section-1: forward/prototype declaration,global vars,macros,data structures +// ================================================================================================= #define ggml_tensor dsptensor #define GGML_MAX_DIMS 4 + #define GGML_UNUSED(x) (void)(x) + #define UNUSED GGML_UNUSED + #define GGML_PAD(x, n) (((x) + (n) - 1) & ~((n) - 1)) + #define GGML_ABORT(...) ggml_abort(__FILE__, __LINE__, __VA_ARGS__) + #define GGML_ASSERT(x) if (!(x)) GGML_ABORT("GGML_ASSERT(%s) failed", #x) + #define MIN(a, b) ((a) < (b) ? (a) : (b)) #define MAX(a, b) ((a) > (b) ? (a) : (b)) #define SWAP(x, y, T) do { T SWAP = x; (x) = y; (y) = SWAP; } while (0) + #if UINTPTR_MAX == 0xFFFFFFFF #define GGML_MEM_ALIGN 4 #else @@ -49,9 +64,39 @@ #define static_assert(a, b) do { } while (0) -typedef uint16_t ggml_fp16_t; +typedef double ggml_float; +typedef uint16_t ggml_fp16_t; typedef struct { uint16_t bits; } ggml_bf16_t; -typedef double ggml_float; + +static void ggmlhexagon_log_internal(int level, const char * file, const char * func, int line, const char * format, ...); + +enum ggmlhexagon_log_level { + GGMLHEXAGON_LOG_LEVEL_NONE = 0, + GGMLHEXAGON_LOG_LEVEL_DEBUG = 1, + GGMLHEXAGON_LOG_LEVEL_INFO = 2, + GGMLHEXAGON_LOG_LEVEL_WARN = 3, + GGMLHEXAGON_LOG_LEVEL_ERROR = 4, + GGMLHEXAGON_LOG_LEVEL_CONT = 5, +}; + +#if 0//def NDEBUG +#define GGMLQNN_DEBUG 0 +#else +#define GGMLQNN_DEBUG 1 +#endif + +#define GGMLHEXAGON_LOGBUF_LEN 4096 +#define GGMLHEXAGON_TMPBUF_LEN 256 + +#define GGMLHEXAGON_LOG_ERROR(...) ggmlhexagon_log_internal(GGMLHEXAGON_LOG_LEVEL_ERROR, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) +#define GGMLHEXAGON_LOG_WARN(...) ggmlhexagon_log_internal(GGMLHEXAGON_LOG_LEVEL_WARN , __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) +#define GGMLHEXAGON_LOG_INFO(...) ggmlhexagon_log_internal(GGMLHEXAGON_LOG_LEVEL_INFO , __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) +#if GGMLQNN_DEBUG +#define GGMLHEXAGON_LOG_DEBUG(...) ggmlhexagon_log_internal(GGMLHEXAGON_LOG_LEVEL_DEBUG, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) +#else +#define GGMLHEXAGON_LOG_DEBUG(...) +#endif +#define GGMLQNN_DUMP_TENSOR(tensor) ggmlhexagon_dump_tensor(tensor, #tensor) #define GGML_TENSOR_LOCALS_1(type, prefix, pointer, array) \ @@ -133,6 +178,7 @@ enum ggml_type { GGML_TYPE_COUNT = 39, }; + static void ggml_vec_dot_f32(int n, float * GGML_RESTRICT s, size_t bs, const float * GGML_RESTRICT x, size_t bx, const float * GGML_RESTRICT y, size_t by, int nrc); static void ggml_vec_dot_f16(int n, float * GGML_RESTRICT s, size_t bs, ggml_fp16_t * GGML_RESTRICT x, size_t bx, ggml_fp16_t * GGML_RESTRICT y, size_t by, int nrc); static void ggml_vec_dot_bf16(int n, float * GGML_RESTRICT s, size_t bs, ggml_bf16_t * GGML_RESTRICT x, size_t bx, ggml_bf16_t * GGML_RESTRICT y, size_t by, int nrc); @@ -179,19 +225,54 @@ static const struct ggml_type_traits type_traits[1] = { }; +// ================================================================================================= +// section-2: ggml-hexagon kernel's internal troubleshooting function +// ================================================================================================= +static void ggmlhexagon_log_internal(int level, const char *file, const char *func, int line, const char *format, ...) { + return; + static char s_ggmlhexagon_log_internal_buf[GGMLHEXAGON_LOGBUF_LEN]; + va_list args; + va_start(args, format); + int len_prefix = snprintf(s_ggmlhexagon_log_internal_buf, GGMLHEXAGON_LOGBUF_LEN, "[%s, %d]: ", + func, line); + int len = vsnprintf(s_ggmlhexagon_log_internal_buf + len_prefix, + GGMLHEXAGON_LOGBUF_LEN - len_prefix, format, args); + if (len < (GGMLHEXAGON_LOGBUF_LEN - len_prefix)) { + + FARF(ALWAYS, "%s\n", s_ggmlhexagon_log_internal_buf); + } + va_end(args); +} + +static void ggmlhexagon_dump_tensor(const ggml_tensor * tensor) { + GGMLHEXAGON_LOG_DEBUG("ne = %5d x %5d x %5d x %5d , nb = (%5zi, %5zi, %5zi, %5zi)\n", + tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3], + tensor->nb[0], tensor->nb[1], tensor->nb[2], tensor->nb[3]); +} + +static void ggml_abort(const char * file, int line, const char * fmt, ...) { + GGMLHEXAGON_LOG_DEBUG("enter ggml_abort"); + //abort(); + return; +} + +// ================================================================================================= +// section-3: ggml-hexagon kernel's helper function(tiny ggml-dsp, ported from original ggml) +// ================================================================================================= static const struct ggml_type_traits_cpu * ggml_get_type_traits_cpu(enum ggml_type type) { return &type_traits_cpu[type]; } -static void ggml_vec_dot_f32(int n, float * GGML_RESTRICT s, size_t bs, const float * GGML_RESTRICT x, size_t bx, const float * GGML_RESTRICT y, size_t by, int nrc) { +static void ggml_vec_dot_f32(int n, float * GGML_RESTRICT s, size_t bs, const float * GGML_RESTRICT x, + size_t bx, const float *GGML_RESTRICT y, size_t by, int nrc) { assert(nrc == 1); - UNUSED(nrc); - UNUSED(bx); - UNUSED(by); - UNUSED(bs); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); ggml_float sumf = 0.0; for (int i = 0; i < n; ++i) { - sumf += (ggml_float)(x[i]*y[i]); + sumf += (ggml_float) (x[i] * y[i]); } *s = sumf; } @@ -269,7 +350,6 @@ static bool ggml_can_repeat(const struct ggml_tensor * t0, const struct ggml_ten static bool ggml_are_same_shape(const struct ggml_tensor * t0, const struct ggml_tensor * t1) { static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function"); - return (t0->ne[0] == t1->ne[0]) && (t0->ne[1] == t1->ne[1]) && @@ -317,17 +397,8 @@ static bool ggml_is_contiguous(const struct ggml_tensor * tensor) { return ggml_is_contiguous_0(tensor); } -inline static void ggml_vec_add_f32 (const int n, float * z, const float * x, const float * y) { for (int i = 0; i < n; ++i) z[i] = x[i] + y[i]; } - -static void ggml_dump_tensor(const ggml_tensor * tensor) { - FARF(HIGH, "ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi, %5zi)\n", - tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3], - tensor->nb[0], tensor->nb[1], tensor->nb[2], tensor->nb[3]); -} - -static void ggml_abort(const char * file, int line, const char * fmt, ...) { - //abort(); - return; +inline static void ggml_vec_add_f32 (const int n, float * z, const float * x, const float * y) { + for (int i = 0; i < n; ++i) z[i] = x[i] + y[i]; } int ggmlop_dsp_open(const char*uri, remote_handle64* handle) { @@ -345,40 +416,31 @@ int ggmlop_dsp_close(remote_handle64 handle) { return 0; } +// ================================================================================================= +// section-4: ggml-hexagon kernel function +// ================================================================================================= static void ggml_compute_forward_add_f32( - const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + struct ggml_tensor * src0, + struct ggml_tensor * src1, + struct ggml_tensor * dst) { + GGMLHEXAGON_LOG_DEBUG("enter %s", __func__ ); + memcpy(dst->ne, src1->ne, 16); + memcpy(dst->nb, src1->nb, 16); + GGML_ASSERT(ggml_can_repeat(src1, src0) && ggml_are_same_shape(src0, dst)); - const int nr = ggml_nrows(src0); + const int ith = 0; + const int nth = 1; + const int nr = ggml_nrows(src0); GGML_TENSOR_BINARY_OP_LOCALS GGML_ASSERT( nb0 == sizeof(float)); GGML_ASSERT(nb00 == sizeof(float)); - const int dr = nr; - - // row range for this thread - const int ir0 = 0; + const int dr = (nr + nth - 1)/nth; + const int ir0 = dr*ith; const int ir1 = MIN(ir0 + dr, nr); - - ggml_dump_tensor(src0); - ggml_dump_tensor(src1); - -#if 1 //naive algorithm for fp32, can works with llama-cli - float * a = (float*)src0->data; - float * b = (float*)src1->data; - float * c = (float*)dst->data; - //TODO: Hexagon SIMD - for (size_t idx = 0; idx < src0->data_len; idx++) { - *c = *a + *b; - a++; - b++; - c++; - } - return; -#endif - if (nb10 == sizeof(float)) { for (int ir = ir0; ir < ir1; ++ir) { // src1 is broadcastable across src0 and dst in i1, i2, i3 @@ -394,9 +456,12 @@ static void ggml_compute_forward_add_f32( float * dst_ptr = (float *) ((char *) dst->data + i03*nb3 + i02*nb2 + i01*nb1 ); float * src0_ptr = (float *) ((char *) src0->data + i03*nb03 + i02*nb02 + i01*nb01); float * src1_ptr = (float *) ((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11); - for (int64_t r = 0; r < nr0; ++r) { +#ifdef GGML_USE_ACCELERATE + vDSP_vadd(src0_ptr + r*ne10, 1, src1_ptr, 1, dst_ptr + r*ne10, 1, ne10); +#else ggml_vec_add_f32(ne10, dst_ptr + r*ne10, src0_ptr + r*ne10, src1_ptr); +#endif } } } else { @@ -422,11 +487,12 @@ static void ggml_compute_forward_add_f32( } } } + GGMLHEXAGON_LOG_DEBUG("leave %s", __func__ ); } int ggmlop_dsp_add(remote_handle64 h, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - FARF(HIGH, "=============== DSP: ggmlop_dsp_add "); + GGMLHEXAGON_LOG_DEBUG("enter ggmlop_dsp_add\n"); switch (src0->type) { case GGML_TYPE_F32: { @@ -442,21 +508,34 @@ int ggmlop_dsp_add(remote_handle64 h, const ggml_tensor * src0, const ggml_tenso GGML_ABORT("fatal error"); } } - + GGMLHEXAGON_LOG_DEBUG("leave ggmlop_dsp_add\n"); return 0; } - static void ggml_compute_forward_mul_mat_one_chunk( const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst, const enum ggml_type type, - const int64_t num_rows_per_vec_dot, - const int64_t ir0_start, - const int64_t ir0_end, - const int64_t ir1_start, - const int64_t ir1_end) { + const int32_t num_rows_per_vec_dot, + const int32_t ir0_start, + const int32_t ir0_end, + const int32_t ir1_start, + const int32_t ir1_end) { + ggmlhexagon_dump_tensor(src0); + ggmlhexagon_dump_tensor(src1); + ggmlhexagon_dump_tensor(dst); + + dst->ne[0] = src0->ne[1]; + dst->ne[1] = src1->ne[1]; + dst->ne[2] = src1->ne[2]; + dst->ne[3] = src1->ne[3]; + + dst->nb[0] = ggml_type_size(src1->type); + dst->nb[1] = dst->nb[0] * (dst->ne[0] / ggml_blck_size(src1->type)); + dst->nb[2] = dst->nb[1] * dst->ne[1]; + dst->nb[3] = dst->nb[2] * dst->ne[2]; + ggmlhexagon_dump_tensor(dst); GGML_TENSOR_BINARY_OP_LOCALS @@ -466,8 +545,8 @@ static void ggml_compute_forward_mul_mat_one_chunk( enum ggml_type const vec_dot_type = type_traits_cpu[type].vec_dot_type; // broadcast factors - const int64_t r2 = ne12 / ne02; - const int64_t r3 = ne13 / ne03; + const int32_t r2 = ne12 / ne02; + const int32_t r3 = ne13 / ne03; if (ir0_start >= ir0_end || ir1_start >= ir1_end) { return; @@ -481,8 +560,8 @@ static void ggml_compute_forward_mul_mat_one_chunk( assert(ne13 % ne03 == 0); // block-tiling attempt - const int64_t blck_0 = 16; - const int64_t blck_1 = 16; + const int32_t blck_0 = 16; + const int32_t blck_1 = 16; const size_t src1_col_stride = src1_cont || src1->type != vec_dot_type ? row_size : nb11; @@ -490,30 +569,38 @@ static void ggml_compute_forward_mul_mat_one_chunk( // 16 * 2, accounting for mmla kernels float tmp[32]; - for (int64_t iir1 = ir1_start; iir1 < ir1_end; iir1 += blck_1) { - for (int64_t iir0 = ir0_start; iir0 < ir0_end; iir0 += blck_0) { - for (int64_t ir1 = iir1; ir1 < iir1 + blck_1 && ir1 < ir1_end; ir1 += num_rows_per_vec_dot) { - const int64_t i13 = (ir1 / (ne12 * ne1)); - const int64_t i12 = (ir1 - i13 * ne12 * ne1) / ne1; - const int64_t i11 = (ir1 - i13 * ne12 * ne1 - i12 * ne1); + for (int32_t iir1 = ir1_start; iir1 < ir1_end; iir1 += blck_1) { + for (int32_t iir0 = ir0_start; iir0 < ir0_end; iir0 += blck_0) { + for (int32_t ir1 = iir1; ir1 < iir1 + blck_1 && ir1 < ir1_end; ir1 += num_rows_per_vec_dot) { + const int32_t i13 = (ir1 / (ne12 * ne1)); + const int32_t i12 = (ir1 - i13 * ne12 * ne1) / ne1; + const int32_t i11 = (ir1 - i13 * ne12 * ne1 - i12 * ne1); // broadcast src0 into src1 - const int64_t i03 = i13 / r3; - const int64_t i02 = i12 / r2; + const int32_t i03 = i13 / r3; + const int32_t i02 = i12 / r2; - const int64_t i1 = i11; - const int64_t i2 = i12; - const int64_t i3 = i13; + const int32_t i1 = i11; + const int32_t i2 = i12; + const int32_t i3 = i13; const char * src0_row = (const char*)src0->data + (0 + i02 * nb02 + i03 * nb03); + // desc: when src1 is not a contiguous memory block we have to calculate the offset using the strides + // if it is, then we have either copied the data to params->wdata and made it contiguous or we are using + // the original src1 data pointer, so we should index using the indices directly + // TODO: this is a bit of a hack, we should probably have a better way to handle this const char * src1_col = (const char*)wdata + (src1_cont || src1->type != vec_dot_type ? (i11 + i12 * ne11 + i13 * ne12 * ne11) * row_size : (i11 * nb11 + i12 * nb12 + i13 * nb13)); float * dst_col = (float*)((char*)dst->data + (i1 * nb1 + i2 * nb2 + i3 * nb3)); - for (int64_t ir0 = iir0; ir0 < iir0 + blck_0 && ir0 < ir0_end; ir0 += num_rows_per_vec_dot) { + //for (int32_t ir0 = iir0; ir0 < iir0 + blck_0 && ir0 < ir0_end; ++ir0) { + // vec_dot(ne00, &dst_col[ir0], src0_row + ir0*nb01, src1_col); + //} + + for (int32_t ir0 = iir0; ir0 < iir0 + blck_0 && ir0 < ir0_end; ir0 += num_rows_per_vec_dot) { vec_dot(ne00, &tmp[ir0 - iir0], (num_rows_per_vec_dot > 1 ? 16 : 0), src0_row + ir0 * nb01, (num_rows_per_vec_dot > 1 ? nb01 : 0), src1_col, (num_rows_per_vec_dot > 1 ? src1_col_stride : 0), num_rows_per_vec_dot); } @@ -525,13 +612,27 @@ static void ggml_compute_forward_mul_mat_one_chunk( } } -int ggmlop_dsp_mulmat(remote_handle64 h, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + int ggmlop_dsp_mulmat(remote_handle64 h, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggmlhexagon_dump_tensor(src0); + ggmlhexagon_dump_tensor(src1); + ggmlhexagon_dump_tensor(dst); + + dst->ne[0] = src0->ne[1]; + dst->ne[1] = src1->ne[1]; + dst->ne[2] = src1->ne[2]; + dst->ne[3] = src1->ne[3]; + + dst->nb[0] = ggml_type_size(src1->type); + dst->nb[1] = dst->nb[0] * (dst->ne[0] / ggml_blck_size(src1->type)); + dst->nb[2] = dst->nb[1] * dst->ne[1]; + dst->nb[3] = dst->nb[2] * dst->ne[2]; + ggmlhexagon_dump_tensor(dst); GGML_TENSOR_BINARY_OP_LOCALS enum ggml_type const vec_dot_type = type_traits_cpu[src0->type].vec_dot_type; ggml_from_float_t const from_float = type_traits_cpu[vec_dot_type].from_float; - int64_t const vec_dot_num_rows = type_traits_cpu[src0->type].nrows; + int32_t const vec_dot_num_rows = type_traits_cpu[src0->type].nrows; GGML_ASSERT(ne0 == ne01); GGML_ASSERT(ne1 == ne11); @@ -548,10 +649,10 @@ int ggmlop_dsp_mulmat(remote_handle64 h, const ggml_tensor * src0, const ggml_te GGML_ASSERT(nb1 <= nb2); GGML_ASSERT(nb2 <= nb3); -#if 1 //naive algorithm for fp32, can pass various case in UT - { - ggml_dump_tensor(src0); - ggml_dump_tensor(src1); +#if 0 //naive algorithm for fp32, can pass various case in UT + { + //ggml_dump_tensor(src0); + //ggml_dump_tensor(src1); float * a = (float*)src0->data; float * b = (float*)src1->data; @@ -574,10 +675,10 @@ int ggmlop_dsp_mulmat(remote_handle64 h, const ggml_tensor * src0, const ggml_te #endif // This is the size of the first dimension of the result, so we can iterate that way. (see the ASSERT above, these are the same numbers) - const int64_t nr0 = ne0; + const int32_t nr0 = ne0; // This is the size of the rest of the dimensions of the result - const int64_t nr1 = ne1 * ne2 * ne3; + const int32_t nr1 = ne1 * ne2 * ne3; // Now select a reasonable chunk size. int chunk_size = 16; @@ -590,8 +691,8 @@ int ggmlop_dsp_mulmat(remote_handle64 h, const ggml_tensor * src0, const ggml_te // distribute the work across the inner or outer loop based on which one is larger // The number of chunks in the 0/1 dim. // CEIL(nr0/chunk_size) - int64_t nchunk0 = (nr0 + chunk_size - 1) / chunk_size; - int64_t nchunk1 = (nr1 + chunk_size - 1) / chunk_size; + int32_t nchunk0 = (nr0 + chunk_size - 1) / chunk_size; + int32_t nchunk1 = (nr1 + chunk_size - 1) / chunk_size; // If the chunking is poor for the number of threads on this setup, scrap the whole plan. Re-chunk it by thread. // Also, chunking by thread was measured to have perform better on NUMA systems. See https://github.com/ggml-org/llama.cpp/pull/6915 @@ -603,24 +704,24 @@ int ggmlop_dsp_mulmat(remote_handle64 h, const ggml_tensor * src0, const ggml_te } // The number of elements in each chunk - const int64_t dr0 = (nr0 + nchunk0 - 1) / nchunk0; - const int64_t dr1 = (nr1 + nchunk1 - 1) / nchunk1; + const int32_t dr0 = (nr0 + nchunk0 - 1) / nchunk0; + const int32_t dr1 = (nr1 + nchunk1 - 1) / nchunk1; // The first chunk comes from our thread_id, the rest will get auto-assigned. int current_chunk = 0; while (current_chunk < nchunk0 * nchunk1) { - const int64_t ith0 = current_chunk % nchunk0; - const int64_t ith1 = current_chunk / nchunk0; + const int32_t ith0 = current_chunk % nchunk0; + const int32_t ith1 = current_chunk / nchunk0; - const int64_t ir0_start = dr0 * ith0; - const int64_t ir0_end = MIN(ir0_start + dr0, nr0); + const int32_t ir0_start = dr0 * ith0; + const int32_t ir0_end = MIN(ir0_start + dr0, nr0); - const int64_t ir1_start = dr1 * ith1; - const int64_t ir1_end = MIN(ir1_start + dr1, nr1); + const int32_t ir1_start = dr1 * ith1; + const int32_t ir1_end = MIN(ir1_start + dr1, nr1); // dot kernels can handle 1 row and col at a time, but mmla kernels can process 2 rows and cols - int64_t num_rows_per_vec_dot = vec_dot_num_rows; + int32_t num_rows_per_vec_dot = vec_dot_num_rows; // these checks are needed to avoid crossing dim1 boundaries // can be optimized, but the logic would become more complicated, so keeping it like this for simplicity @@ -635,5 +736,5 @@ int ggmlop_dsp_mulmat(remote_handle64 h, const ggml_tensor * src0, const ggml_te current_chunk++; } - return 0; + return 0; } diff --git a/ggml/src/ggml-qnn/kernels/ggmlop_cdsp_skel.c b/ggml/src/ggml-qnn/kernels/ggmlop_cdsp_skel.c index 9d6b64fd6b570..58bf1a846742f 100644 --- a/ggml/src/ggml-qnn/kernels/ggmlop_cdsp_skel.c +++ b/ggml/src/ggml-qnn/kernels/ggmlop_cdsp_skel.c @@ -7,7 +7,6 @@ #include #include -#include "version_note.h" #include "ggmlop_ap_skel.h" typedef struct _heap _heap; @@ -270,13 +269,13 @@ struct Interface { #define __QAIC_SLIM_EXPORT #endif -static const Type types[5]; -static const Type* const typeArrays[6] = {&(types[0]),&(types[1]),&(types[1]),&(types[0]),&(types[0]),&(types[3])}; -static const StructType structTypes[1] = {{0x6,&(typeArrays[0]),0x58,0x4,0x50,0x8,0x4,0x8}}; -static const Type types[5] = {{0x4,{{(const uintptr_t)0,(const uintptr_t)1}}, 2,0x4},{0x20,{{(const uintptr_t)&(types[2]),(const uintptr_t)0x4}}, 8,0x8},{0x8,{{(const uintptr_t)0,(const uintptr_t)1}}, 2,0x8},{SLIM_IFPTR32(0x8,0x10),{{(const uintptr_t)&(types[4]),(const uintptr_t)0x0}}, 9,SLIM_IFPTR32(0x4,0x8)},{0x4,{{(const uintptr_t)0,(const uintptr_t)1}}, 2,0x4}}; -static const Parameter parameters[5] = {{SLIM_IFPTR32(0x8,0x10),{{(const uintptr_t)0x0,0}}, 4,SLIM_IFPTR32(0x4,0x8),0,0},{SLIM_IFPTR32(0x4,0x8),{{(const uintptr_t)0xdeadc0de,(const uintptr_t)0}}, 0,SLIM_IFPTR32(0x4,0x8),3,0},{SLIM_IFPTR32(0x4,0x8),{{(const uintptr_t)0xdeadc0de,(const uintptr_t)0}}, 0,SLIM_IFPTR32(0x4,0x8),0,0},{SLIM_IFPTR32(0x58,0x60),{{(const uintptr_t)&(structTypes[0]),0}}, 22,0x8,0,0},{SLIM_IFPTR32(0x58,0x60),{{(const uintptr_t)&(structTypes[0]),0}}, 22,0x8,3,0}}; +static const Type types[4]; +static const Type* const typeArrays[6] = {&(types[0]),&(types[1]),&(types[1]),&(types[0]),&(types[0]),&(types[2])}; +static const StructType structTypes[1] = {{0x6,&(typeArrays[0]),0x30,0x4,0x2c,0x4,0x4,0x4}}; +static const Type types[4] = {{0x4,{{(const uintptr_t)0,(const uintptr_t)1}}, 2,0x4},{0x10,{{(const uintptr_t)&(types[0]),(const uintptr_t)0x4}}, 8,0x4},{SLIM_IFPTR32(0x8,0x10),{{(const uintptr_t)&(types[3]),(const uintptr_t)0x0}}, 9,SLIM_IFPTR32(0x4,0x8)},{0x4,{{(const uintptr_t)0,(const uintptr_t)1}}, 2,0x4}}; +static const Parameter parameters[5] = {{SLIM_IFPTR32(0x8,0x10),{{(const uintptr_t)0x0,0}}, 4,SLIM_IFPTR32(0x4,0x8),0,0},{SLIM_IFPTR32(0x4,0x8),{{(const uintptr_t)0xdeadc0de,(const uintptr_t)0}}, 0,SLIM_IFPTR32(0x4,0x8),3,0},{SLIM_IFPTR32(0x4,0x8),{{(const uintptr_t)0xdeadc0de,(const uintptr_t)0}}, 0,SLIM_IFPTR32(0x4,0x8),0,0},{SLIM_IFPTR32(0x34,0x40),{{(const uintptr_t)&(structTypes[0]),0}}, 22,SLIM_IFPTR32(0x4,0x8),0,0},{SLIM_IFPTR32(0x34,0x40),{{(const uintptr_t)&(structTypes[0]),0}}, 22,SLIM_IFPTR32(0x4,0x8),3,0}}; static const Parameter* const parameterArrays[6] = {(&(parameters[3])),(&(parameters[3])),(&(parameters[4])),(&(parameters[0])),(&(parameters[1])),(&(parameters[2]))}; -static const Method methods[3] = {{REMOTE_SCALARS_MAKEX(0,0,0x2,0x0,0x0,0x1),0x4,0x0,2,2,(&(parameterArrays[3])),0x4,0x1},{REMOTE_SCALARS_MAKEX(0,0,0x0,0x0,0x1,0x0),0x0,0x0,1,1,(&(parameterArrays[5])),0x1,0x0},{REMOTE_SCALARS_MAKEX(0,0,0x3,0x2,0x0,0x0),0xb4,0x50,3,3,(&(parameterArrays[0])),0x8,0x8}}; +static const Method methods[3] = {{REMOTE_SCALARS_MAKEX(0,0,0x2,0x0,0x0,0x1),0x4,0x0,2,2,(&(parameterArrays[3])),0x4,0x1},{REMOTE_SCALARS_MAKEX(0,0,0x0,0x0,0x1,0x0),0x0,0x0,1,1,(&(parameterArrays[5])),0x1,0x0},{REMOTE_SCALARS_MAKEX(0,0,0x3,0x2,0x0,0x0),0x64,0x2c,3,3,(&(parameterArrays[0])),0x4,0x4}}; static const Method* const methodArrays[4] = {&(methods[0]),&(methods[1]),&(methods[2]),&(methods[2])}; static const char strings[68] = "mulmat\0flags\0close\0src1\0data\0type\0src0\0open\0dst\0add\0uri\0op\0nb\0ne\0h\0"; static const uint16_t methodStrings[49] = {0,34,29,62,59,56,7,24,19,29,62,59,56,7,24,44,29,62,59,56,7,24,48,34,29,62,59,56,7,24,19,29,62,59,56,7,24,44,29,62,59,56,7,24,39,52,65,13,65}; @@ -289,20 +288,20 @@ extern "C" { #endif _ATTRIBUTE_VISIBILITY uint32_t ggmlop_skel_handle_invoke_qaic_version = 10048; _ATTRIBUTE_VISIBILITY char ggmlop_skel_handle_invoke_uri[77+1]="file:///libggmlop_skel.so?ggmlop_skel_handle_invoke&_modver=1.0&_idlver=0.0.1"; -static __inline int _skel_pack(_ATTRIBUTE_UNUSED remote_arg* _praROutPost, _ATTRIBUTE_UNUSED remote_arg* _ppraROutPost[1], _ATTRIBUTE_UNUSED void* _primROut, _ATTRIBUTE_UNUSED uint32_t _rout0[1], _ATTRIBUTE_UNUSED uint64_t _rout1[4], _ATTRIBUTE_UNUSED uint64_t _rout2[4], _ATTRIBUTE_UNUSED uint32_t _rout3[1], _ATTRIBUTE_UNUSED uint32_t _rout4[1], _ATTRIBUTE_UNUSED char* _rout5[1], _ATTRIBUTE_UNUSED uint32_t _rout5Len[1]) { +static __inline int _skel_pack(_ATTRIBUTE_UNUSED remote_arg* _praROutPost, _ATTRIBUTE_UNUSED remote_arg* _ppraROutPost[1], _ATTRIBUTE_UNUSED void* _primROut, _ATTRIBUTE_UNUSED uint32_t _rout0[1], _ATTRIBUTE_UNUSED uint32_t _rout1[4], _ATTRIBUTE_UNUSED uint32_t _rout2[4], _ATTRIBUTE_UNUSED uint32_t _rout3[1], _ATTRIBUTE_UNUSED uint32_t _rout4[1], _ATTRIBUTE_UNUSED char* _rout5[1], _ATTRIBUTE_UNUSED uint32_t _rout5Len[1]) { int _nErr = 0; remote_arg* _praROutPostStart = _praROutPost; remote_arg** _ppraROutPostStart = _ppraROutPost; _ppraROutPost = &_praROutPost; _COPY(_primROut, 0, _rout0, 0, 4); - _COPY(_primROut, 8, _rout1, 0, 32); - _COPY(_primROut, 40, _rout2, 0, 32); - _COPY(_primROut, 72, _rout3, 0, 4); - _COPY(_primROut, 76, _rout4, 0, 4); + _COPY(_primROut, 4, _rout1, 0, 16); + _COPY(_primROut, 20, _rout2, 0, 16); + _COPY(_primROut, 36, _rout3, 0, 4); + _COPY(_primROut, 40, _rout4, 0, 4); _ppraROutPostStart[0] += (_praROutPost - _praROutPostStart) +1; return _nErr; } -static __inline int _skel_unpack(_ATTRIBUTE_UNUSED _allocator* _al, _ATTRIBUTE_UNUSED remote_arg* _praIn, _ATTRIBUTE_UNUSED remote_arg* _ppraIn[1], _ATTRIBUTE_UNUSED remote_arg* _praROut, _ATTRIBUTE_UNUSED remote_arg* _ppraROut[1], _ATTRIBUTE_UNUSED remote_arg* _praHIn, _ATTRIBUTE_UNUSED remote_arg* _ppraHIn[1], _ATTRIBUTE_UNUSED remote_arg* _praHROut, _ATTRIBUTE_UNUSED remote_arg* _ppraHROut[1], _ATTRIBUTE_UNUSED void* _primIn, _ATTRIBUTE_UNUSED void* _primROut, _ATTRIBUTE_UNUSED uint32_t _rout0[1], _ATTRIBUTE_UNUSED uint64_t _rout1[4], _ATTRIBUTE_UNUSED uint64_t _rout2[4], _ATTRIBUTE_UNUSED uint32_t _rout3[1], _ATTRIBUTE_UNUSED uint32_t _rout4[1], _ATTRIBUTE_UNUSED char* _rout5[1], _ATTRIBUTE_UNUSED uint32_t _rout5Len[1]) { +static __inline int _skel_unpack(_ATTRIBUTE_UNUSED _allocator* _al, _ATTRIBUTE_UNUSED remote_arg* _praIn, _ATTRIBUTE_UNUSED remote_arg* _ppraIn[1], _ATTRIBUTE_UNUSED remote_arg* _praROut, _ATTRIBUTE_UNUSED remote_arg* _ppraROut[1], _ATTRIBUTE_UNUSED remote_arg* _praHIn, _ATTRIBUTE_UNUSED remote_arg* _ppraHIn[1], _ATTRIBUTE_UNUSED remote_arg* _praHROut, _ATTRIBUTE_UNUSED remote_arg* _ppraHROut[1], _ATTRIBUTE_UNUSED void* _primIn, _ATTRIBUTE_UNUSED void* _primROut, _ATTRIBUTE_UNUSED uint32_t _rout0[1], _ATTRIBUTE_UNUSED uint32_t _rout1[4], _ATTRIBUTE_UNUSED uint32_t _rout2[4], _ATTRIBUTE_UNUSED uint32_t _rout3[1], _ATTRIBUTE_UNUSED uint32_t _rout4[1], _ATTRIBUTE_UNUSED char* _rout5[1], _ATTRIBUTE_UNUSED uint32_t _rout5Len[1]) { int _nErr = 0; remote_arg* _praInStart = _praIn; remote_arg** _ppraInStart = _ppraIn; @@ -318,7 +317,7 @@ static __inline int _skel_unpack(_ATTRIBUTE_UNUSED _allocator* _al, _ATTRIBUTE_U _QAIC_CATCH(_nErr) {} return _nErr; } -static __inline int _skel_unpack_1(_ATTRIBUTE_UNUSED _allocator* _al, _ATTRIBUTE_UNUSED remote_arg* _praIn, _ATTRIBUTE_UNUSED remote_arg* _ppraIn[1], _ATTRIBUTE_UNUSED remote_arg* _praROut, _ATTRIBUTE_UNUSED remote_arg* _ppraROut[1], _ATTRIBUTE_UNUSED remote_arg* _praHIn, _ATTRIBUTE_UNUSED remote_arg* _ppraHIn[1], _ATTRIBUTE_UNUSED remote_arg* _praHROut, _ATTRIBUTE_UNUSED remote_arg* _ppraHROut[1], _ATTRIBUTE_UNUSED void* _primIn, _ATTRIBUTE_UNUSED void* _primROut, _ATTRIBUTE_UNUSED uint32_t _in0[1], _ATTRIBUTE_UNUSED uint64_t _in1[4], _ATTRIBUTE_UNUSED uint64_t _in2[4], _ATTRIBUTE_UNUSED uint32_t _in3[1], _ATTRIBUTE_UNUSED uint32_t _in4[1], _ATTRIBUTE_UNUSED char* _in5[1], _ATTRIBUTE_UNUSED uint32_t _in5Len[1]) { +static __inline int _skel_unpack_1(_ATTRIBUTE_UNUSED _allocator* _al, _ATTRIBUTE_UNUSED remote_arg* _praIn, _ATTRIBUTE_UNUSED remote_arg* _ppraIn[1], _ATTRIBUTE_UNUSED remote_arg* _praROut, _ATTRIBUTE_UNUSED remote_arg* _ppraROut[1], _ATTRIBUTE_UNUSED remote_arg* _praHIn, _ATTRIBUTE_UNUSED remote_arg* _ppraHIn[1], _ATTRIBUTE_UNUSED remote_arg* _praHROut, _ATTRIBUTE_UNUSED remote_arg* _ppraHROut[1], _ATTRIBUTE_UNUSED void* _primIn, _ATTRIBUTE_UNUSED void* _primROut, _ATTRIBUTE_UNUSED uint32_t _in0[1], _ATTRIBUTE_UNUSED uint32_t _in1[4], _ATTRIBUTE_UNUSED uint32_t _in2[4], _ATTRIBUTE_UNUSED uint32_t _in3[1], _ATTRIBUTE_UNUSED uint32_t _in4[1], _ATTRIBUTE_UNUSED char* _in5[1], _ATTRIBUTE_UNUSED uint32_t _in5Len[1]) { int _nErr = 0; remote_arg* _praInStart = _praIn; remote_arg** _ppraInStart = _ppraIn; @@ -327,11 +326,11 @@ static __inline int _skel_unpack_1(_ATTRIBUTE_UNUSED _allocator* _al, _ATTRIBUTE _ppraIn = &_praIn; _ppraROut = &_praROut; _COPY(_in0, 0, _primIn, 0, 4); - _COPY(_in1, 0, _primIn, 8, 32); - _COPY(_in2, 0, _primIn, 40, 32); - _COPY(_in3, 0, _primIn, 72, 4); - _COPY(_in4, 0, _primIn, 76, 4); - _COPY(_in5Len, 0, _primIn, 80, 4); + _COPY(_in1, 0, _primIn, 4, 16); + _COPY(_in2, 0, _primIn, 20, 16); + _COPY(_in3, 0, _primIn, 36, 4); + _COPY(_in4, 0, _primIn, 40, 4); + _COPY(_in5Len, 0, _primIn, 44, 4); _QAIC_ASSERT(_nErr, ((_praIn[0].buf.nLen / 4)) >= (size_t)(_in5Len[0])); _in5[0] = _praIn[0].buf.pv; _ppraInStart[0] += (_praIn - _praInStart) + 1; @@ -341,12 +340,12 @@ static __inline int _skel_unpack_1(_ATTRIBUTE_UNUSED _allocator* _al, _ATTRIBUTE } static __inline int _skel_method(int (*_pfn)(remote_handle64, const dsptensor*, const dsptensor*, dsptensor*), remote_handle64 _h, uint32_t _sc, remote_arg* _pra) { remote_arg* _praEnd = 0; - uint64_t _in0[SLIM_IFPTR32(11, 12)] = {0}; - uint64_t _in1[SLIM_IFPTR32(11, 12)] = {0}; - uint64_t _rout2[SLIM_IFPTR32(11, 12)] = {0}; - uint64_t* _primIn= 0; + uintptr_t _in0[SLIM_IFPTR32(13, 8)] = {0}; + uintptr_t _in1[SLIM_IFPTR32(13, 8)] = {0}; + uintptr_t _rout2[SLIM_IFPTR32(13, 8)] = {0}; + uint32_t* _primIn= 0; int _numIn[1] = {0}; - uint64_t* _primROut= 0; + uint32_t* _primROut= 0; int _numInH[1] = {0}; int _numROut[1] = {0}; remote_arg* _praIn = 0; @@ -368,9 +367,9 @@ static __inline int _skel_method(int (*_pfn)(remote_handle64, const dsptensor*, _QAIC_ASSERT(_nErr, REMOTE_SCALARS_OUTHANDLES(_sc)==0); _QAIC_ASSERT(_nErr, (_pra + ((1 + 1) + (((0 + 0) + 0) + 0))) <= _praEnd); _numIn[0] = (REMOTE_SCALARS_INBUFS(_sc) - 1); - _QAIC_ASSERT(_nErr, _pra[0].buf.nLen >= 180); + _QAIC_ASSERT(_nErr, _pra[0].buf.nLen >= 100); _primIn = _pra[0].buf.pv; - _QAIC_ASSERT(_nErr, _pra[(_numIn[0] + 1)].buf.nLen >= 80); + _QAIC_ASSERT(_nErr, _pra[(_numIn[0] + 1)].buf.nLen >= 44); _primROut = _pra[(_numIn[0] + 1)].buf.pv; _numInH[0] = REMOTE_SCALARS_INHANDLES(_sc); _numROut[0] = REMOTE_SCALARS_OUTBUFS(_sc); @@ -384,11 +383,11 @@ static __inline int _skel_method(int (*_pfn)(remote_handle64, const dsptensor*, } if(_praHROut == 0) (_praHROut = _praHIn + _numInH[0] + 0); - _TRY(_nErr, _skel_unpack_1(_al, (_praIn + 0), _ppraIn, (_praROut + 0), _ppraROut, _praHIn, _ppraHIn, _praHROut, _ppraHROut, ((char*)_primIn + 0), 0, (uint32_t*)&(((uint32_t*)_in0)[0]), (uint64_t*)&(((uint64_t*)_in0)[1]), (uint64_t*)&(((uint64_t*)_in0)[5]), (uint32_t*)&(((uint32_t*)_in0)[18]), (uint32_t*)&(((uint32_t*)_in0)[19]), SLIM_IFPTR32((char**)&(((uint32_t*)_in0)[20]), (char**)&(((uint64_t*)_in0)[10])), SLIM_IFPTR32((uint32_t*)&(((uint32_t*)_in0)[21]), (uint32_t*)&(((uint32_t*)_in0)[22])))); - _TRY(_nErr, _skel_unpack_1(_al, (_praIn + 0), _ppraIn, (_praROut + 0), _ppraROut, _praHIn, _ppraHIn, _praHROut, _ppraHROut, ((char*)_primIn + 88), 0, (uint32_t*)&(((uint32_t*)_in1)[0]), (uint64_t*)&(((uint64_t*)_in1)[1]), (uint64_t*)&(((uint64_t*)_in1)[5]), (uint32_t*)&(((uint32_t*)_in1)[18]), (uint32_t*)&(((uint32_t*)_in1)[19]), SLIM_IFPTR32((char**)&(((uint32_t*)_in1)[20]), (char**)&(((uint64_t*)_in1)[10])), SLIM_IFPTR32((uint32_t*)&(((uint32_t*)_in1)[21]), (uint32_t*)&(((uint32_t*)_in1)[22])))); - _TRY(_nErr, _skel_unpack(_al, (_praIn + 0), _ppraIn, (_praROut + 0), _ppraROut, _praHIn, _ppraHIn, _praHROut, _ppraHROut, ((char*)_primIn + 176), ((char*)_primROut + 0), (uint32_t*)&(((uint32_t*)_rout2)[0]), (uint64_t*)&(((uint64_t*)_rout2)[1]), (uint64_t*)&(((uint64_t*)_rout2)[5]), (uint32_t*)&(((uint32_t*)_rout2)[18]), (uint32_t*)&(((uint32_t*)_rout2)[19]), SLIM_IFPTR32((char**)&(((uint32_t*)_rout2)[20]), (char**)&(((uint64_t*)_rout2)[10])), SLIM_IFPTR32((uint32_t*)&(((uint32_t*)_rout2)[21]), (uint32_t*)&(((uint32_t*)_rout2)[22])))); + _TRY(_nErr, _skel_unpack_1(_al, (_praIn + 0), _ppraIn, (_praROut + 0), _ppraROut, _praHIn, _ppraHIn, _praHROut, _ppraHROut, ((char*)_primIn + 0), 0, (uint32_t*)&(((uint32_t*)_in0)[0]), (uint32_t*)&(((uint32_t*)_in0)[1]), (uint32_t*)&(((uint32_t*)_in0)[5]), (uint32_t*)&(((uint32_t*)_in0)[9]), (uint32_t*)&(((uint32_t*)_in0)[10]), SLIM_IFPTR32((char**)&(((uint32_t*)_in0)[11]), (char**)&(((uint64_t*)_in0)[6])), SLIM_IFPTR32((uint32_t*)&(((uint32_t*)_in0)[12]), (uint32_t*)&(((uint32_t*)_in0)[14])))); + _TRY(_nErr, _skel_unpack_1(_al, (_praIn + 0), _ppraIn, (_praROut + 0), _ppraROut, _praHIn, _ppraHIn, _praHROut, _ppraHROut, ((char*)_primIn + 48), 0, (uint32_t*)&(((uint32_t*)_in1)[0]), (uint32_t*)&(((uint32_t*)_in1)[1]), (uint32_t*)&(((uint32_t*)_in1)[5]), (uint32_t*)&(((uint32_t*)_in1)[9]), (uint32_t*)&(((uint32_t*)_in1)[10]), SLIM_IFPTR32((char**)&(((uint32_t*)_in1)[11]), (char**)&(((uint64_t*)_in1)[6])), SLIM_IFPTR32((uint32_t*)&(((uint32_t*)_in1)[12]), (uint32_t*)&(((uint32_t*)_in1)[14])))); + _TRY(_nErr, _skel_unpack(_al, (_praIn + 0), _ppraIn, (_praROut + 0), _ppraROut, _praHIn, _ppraHIn, _praHROut, _ppraHROut, ((char*)_primIn + 96), ((char*)_primROut + 0), (uint32_t*)&(((uint32_t*)_rout2)[0]), (uint32_t*)&(((uint32_t*)_rout2)[1]), (uint32_t*)&(((uint32_t*)_rout2)[5]), (uint32_t*)&(((uint32_t*)_rout2)[9]), (uint32_t*)&(((uint32_t*)_rout2)[10]), SLIM_IFPTR32((char**)&(((uint32_t*)_rout2)[11]), (char**)&(((uint64_t*)_rout2)[6])), SLIM_IFPTR32((uint32_t*)&(((uint32_t*)_rout2)[12]), (uint32_t*)&(((uint32_t*)_rout2)[14])))); _TRY(_nErr, _pfn(_h, (const dsptensor*)_in0, (const dsptensor*)_in1, (dsptensor*)_rout2)); - _TRY(_nErr, _skel_pack((_praROutPost + 0), _ppraROutPost, ((char*)_primROut + 0), (uint32_t*)&(((uint32_t*)_rout2)[0]), (uint64_t*)&(((uint64_t*)_rout2)[1]), (uint64_t*)&(((uint64_t*)_rout2)[5]), (uint32_t*)&(((uint32_t*)_rout2)[18]), (uint32_t*)&(((uint32_t*)_rout2)[19]), SLIM_IFPTR32((char**)&(((uint32_t*)_rout2)[20]), (char**)&(((uint64_t*)_rout2)[10])), SLIM_IFPTR32((uint32_t*)&(((uint32_t*)_rout2)[21]), (uint32_t*)&(((uint32_t*)_rout2)[22])))); + _TRY(_nErr, _skel_pack((_praROutPost + 0), _ppraROutPost, ((char*)_primROut + 0), (uint32_t*)&(((uint32_t*)_rout2)[0]), (uint32_t*)&(((uint32_t*)_rout2)[1]), (uint32_t*)&(((uint32_t*)_rout2)[5]), (uint32_t*)&(((uint32_t*)_rout2)[9]), (uint32_t*)&(((uint32_t*)_rout2)[10]), SLIM_IFPTR32((char**)&(((uint32_t*)_rout2)[11]), (char**)&(((uint64_t*)_rout2)[6])), SLIM_IFPTR32((uint32_t*)&(((uint32_t*)_rout2)[12]), (uint32_t*)&(((uint32_t*)_rout2)[14])))); _QAIC_CATCH(_nErr) {} _allocator_deinit(_al); return _nErr; @@ -583,14 +582,3 @@ __QAIC_SKEL_EXPORT int __QAIC_SKEL(ggmlop_skel_handle_invoke)(remote_handle64 _h } return AEE_EUNSUPPORTED; } - -/* Library version needs to be added in the name member of note_type structure in below format - * "lib.ver.1.0.0." + "" + ":" + "" - */ -const lib_ver_note_t so_ver __attribute__ ((section (".note.lib.ver"))) - __attribute__ ((visibility ("default"))) = { - 100, - 0, - 0, - "lib.ver.1.0.0.libggmlop_skel.so:4.5.0", - }; From 9ab1ea5be7d2b7310a3e4664804df2b8f91366c6 Mon Sep 17 00:00:00 2001 From: zhouwg Date: Tue, 25 Mar 2025 23:08:39 +0800 Subject: [PATCH 74/76] ggml-qnn: modify build-run-android.sh to verify mulmat and validate mulmat performance on cDSP easily --- scripts/build-run-android.sh | 46 ++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/scripts/build-run-android.sh b/scripts/build-run-android.sh index d3c47e0473bcd..2e4143b5acca3 100755 --- a/scripts/build-run-android.sh +++ b/scripts/build-run-android.sh @@ -236,6 +236,38 @@ function run_test-ops() } +function run_test-op() +{ + prepare_run_on_phone test-backend-ops + + qnnbackendname=qnn-cpu + case $qnnbackend in + 0) + qnnbackendname=qnn-cpu + ;; + 1) + qnnbackendname=qnn-gpu + ;; + 2) + qnnbackendname=qnn-npu + ;; + *) + qnnbackendname=qnn-cpu + ;; + esac + + #debug + echo "adb shell cd ${REMOTE_PATH} \ + && export LD_LIBRARY_PATH=${REMOTE_PATH} \ + && ${REMOTE_PATH}/test-backend-ops test -o $opname -b $qnnbackendname " + + echo "\n" + adb shell "cd ${REMOTE_PATH} \ + && export LD_LIBRARY_PATH=${REMOTE_PATH} \ + && ${REMOTE_PATH}/test-backend-ops test -o $opname -b $qnnbackendname " + +} + function print_oplist() { @@ -325,6 +357,7 @@ function show_usage() echo " $0 build" echo " $0 updateqnnlib" echo " $0 run_testops" + echo " $0 run_testop [ADD/MUL_MAT] [0 (QNN_CPU) / 1 (QNN_GPU) / 2 (QNN_NPU)]" echo " $0 run_llamacli" echo " $0 run_llamabench" @@ -370,6 +403,19 @@ elif [ $# == 1 ]; then show_usage exit 1 fi +elif [ $# == 3 ]; then + opname=$2 +#TODO: check opname in oplist +#opname can be found via print_oplist: + + qnnbackend=$3 + if [ ${qnnbackend} -gt 3 ]; then + show_usage + exit 1 + fi + + run_test-op + exit 0 else show_usage exit 1 From ae6402e2b1e22f8107005e793c942010fc3b13a2 Mon Sep 17 00:00:00 2001 From: zhouwg Date: Wed, 26 Mar 2025 19:41:58 +0800 Subject: [PATCH 75/76] ggml-qnn: make host code(ggml-qnn.cpp) more clear and more stable --- ggml/src/ggml-qnn/ggml-qnn.cpp | 167 +++++++++++++++++++++++---------- scripts/ggml-qnn.cfg | 12 ++- 2 files changed, 126 insertions(+), 53 deletions(-) diff --git a/ggml/src/ggml-qnn/ggml-qnn.cpp b/ggml/src/ggml-qnn/ggml-qnn.cpp index 909650124a9eb..c28d01d134b29 100644 --- a/ggml/src/ggml-qnn/ggml-qnn.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn.cpp @@ -23,7 +23,7 @@ * this is a complicated skeleton, can expand other ggml ops accordingly * * currently provide following ggml op' implementation through Hexagon DSP: - * - GGML_OP_ADD: + * - GGML_OP_ADD & GGML_OP_MUL_MAT: * this is a skeleton, can expand other ggml ops accordingly * * Permission is hereby granted, free of charge, to any person obtaining a copy @@ -343,6 +343,7 @@ struct ggml_backend_qnn_context { size_t rpc_mempool_len; void * rpc_mempool; remote_handle64 ggmlop_handle; + int domain_id; }; struct qnn_op_caps { @@ -363,6 +364,8 @@ struct qnn_parameter { int enable_dlbc; int inference_approach; // 0: QNN_GENERAL 1: DIRECT_USE_CDSP 2: QNN_SINGELGRAPH int qnn_backend; // 0: QNN-CPU backend 1: QNN-GPU backend 2: QNN-NPU backend + int enable_mulmat_cdsp; // enable/disable offload mulmat to cDSP + int enable_q_mulmat; // enable/disable offload fp32 & all quantized type mulmat to cDSP const char * qnn_cfgfilename; const char * qnn_runtimelib_path; }; @@ -381,6 +384,8 @@ static struct qnn_parameter g_qnn_params = { .enable_dlbc = 1, .inference_approach = 0, .qnn_backend = 2, //default is QNN-NPU backend + .enable_mulmat_cdsp = 0, + .enable_q_mulmat = 0, .qnn_cfgfilename = "ggml-qnn.cfg", #if defined(__ANDROID__) //Android command line program @@ -1451,7 +1456,7 @@ static int ggmlhexagon_get_dsp_support(int * domain) { return hexagon_error; } -static int ggmlhexagon_get_vtcm_info(int domain, uint32_t * capability, uint32_t attr) { +static int ggmlhexagon_get_vtcm_info(int domain, uint32_t attr, uint32_t * capability) { int hexagon_error = AEE_SUCCESS; *capability = 0; @@ -1633,7 +1638,7 @@ static bool ggmlhexagon_is_status_notification_supported(int domain) { return false; } -static int ggmlhexagon_get_hmx_support_info(int domain, uint32_t * capability, uint32_t attr) { +static int ggmlhexagon_get_hmx_support_info(int domain, uint32_t attr, uint32_t * capability) { int hexagon_error = AEE_SUCCESS; *capability = 0; @@ -1679,7 +1684,7 @@ static int ggmlhexagon_get_hmx_support_info(int domain, uint32_t * capability, u return hexagon_error; } -static int ggmlhexagon_get_hex_arch_ver(int domain, uint32_t * capability) { +static int ggmlhexagon_get_hvx_arch_ver(int domain, uint32_t * capability) { int hexagon_error = AEE_SUCCESS; *capability = 0; if(remote_handle_control) { @@ -1696,7 +1701,7 @@ static int ggmlhexagon_get_hex_arch_ver(int domain, uint32_t * capability) { hexagon_error = AEE_SUCCESS; goto bail; } else if (hexagon_error == AEE_SUCCESS) { - *capability = dsp_capability_arch_ver.capability; + *capability = dsp_capability_arch_ver.capability & 0xFF; } else { GGMLQNN_LOG_DEBUG("get_hex_arch_ver failed with error 0x%x", hexagon_error); goto bail; @@ -1710,7 +1715,7 @@ static int ggmlhexagon_get_hex_arch_ver(int domain, uint32_t * capability) { return hexagon_error; } -static int ggmlhexagon_get_hvx_support_info(int domain, uint32_t * capability, uint32_t attr) +static int ggmlhexagon_get_hvx_support_info(int domain, uint32_t attr, uint32_t * capability) { int hexagon_error = AEE_SUCCESS; *capability = 0; @@ -1834,6 +1839,58 @@ static AEEResult ggmlhexagon_set_clocks(remote_handle64 handle, int32 power_leve return AEE_SUCCESS; } +static void ggmlhexagon_probe_dspinfo(ggml_backend_qnn_context * ctx, size_t * rpcmem_capacity) { + size_t candidate_size = 0; + uint8_t * rpc_buffer = nullptr; + const int SIZE_IN_MB = (1 << 20); + size_t probe_slots[] = {1024, 1536, 2048 - 48, 2048}; + size_t probe_counts = sizeof(probe_slots) / sizeof(size_t); + for (size_t idx = 0; idx < probe_counts; idx++) { + rpc_buffer = static_cast(rpcmem_alloc(RPCMEM_HEAP_ID_SYSTEM, RPCMEM_DEFAULT_FLAGS, (probe_slots[idx] * SIZE_IN_MB))); + if (nullptr == rpc_buffer) { + GGMLQNN_LOG_DEBUG("alloc rpcmem %d (MB) failure, %s\n", probe_slots[idx], strerror(errno)); + break; + } else { + candidate_size = probe_slots[idx]; + rpcmem_free(rpc_buffer); + rpc_buffer = nullptr; + } + } + + *rpcmem_capacity = candidate_size; + GGMLQNN_LOG_INFO("capacity of rpc ion memory %d MB\n", *rpcmem_capacity); + + uint32_t dsp_version = 0; + ggmlhexagon_get_hvx_arch_ver(ctx->domain_id, &dsp_version); + + if (dsp_version == 0x68 || dsp_version == 0x69 || dsp_version == 0x73 || dsp_version == 0x75 || dsp_version == 0x79) { + GGMLQNN_LOG_DEBUG("dsp arch version 0x%x", dsp_version); + } else { + GGMLQNN_LOG_WARN("error: dsp arch version 0x%x is not supported", dsp_version); + } + + uint32_t vtcm_count = 0; + uint32_t vtcm_page = 0; + ggmlhexagon_get_vtcm_info(ctx->domain_id, VTCM_COUNT, &vtcm_count); + ggmlhexagon_get_vtcm_info(ctx->domain_id, VTCM_PAGE, &vtcm_page); + GGMLQNN_LOG_DEBUG("vtcm_count %d", vtcm_count); + GGMLQNN_LOG_DEBUG("vtcm_page %d", vtcm_page); + + uint32_t hmx_depth = 0; + uint32_t hmx_spatial = 0; + ggmlhexagon_get_hmx_support_info(ctx->domain_id, HMX_SUPPORT_DEPTH, &hmx_depth); + ggmlhexagon_get_hmx_support_info(ctx->domain_id, HMX_SUPPORT_SPATIAL, &hmx_spatial); + GGMLQNN_LOG_DEBUG("hmx_depth %d", hmx_depth); + GGMLQNN_LOG_DEBUG("hmx_spatial %d", hmx_spatial); + + uint32_t hvx_support_128b = 0; + ggmlhexagon_get_hvx_support_info(ctx->domain_id, HVX_SUPPORT_128B, &hvx_support_128b); + GGMLQNN_LOG_DEBUG("hvx_support_128b %d", hvx_support_128b); + + GGMLQNN_LOG_DEBUG("unsigned pd supported %d", ggmlhexagon_get_unsignedpd_support()); + GGMLQNN_LOG_DEBUG("async fastrpc supported %d", ggmlhexagon_is_async_fastrpc_supported(ctx->domain_id)); +} + static int ggmlhexagon_init_dsp(ggml_backend_qnn_context * ctx) { int hexagon_error = AEE_SUCCESS; @@ -1931,6 +1988,7 @@ static int ggmlhexagon_init_dsp(ggml_backend_qnn_context * ctx) { } } + ctx->domain_id = domain_id; GGMLQNN_LOG_INFO("using Hexagon domain %d(%s)", domain_id, ggmlhexagon_get_dsp_name(domain_id)); GGMLQNN_LOG_INFO("unsignedpd_enabled %d", is_unsignedpd_enabled); if (is_unsignedpd_enabled) { @@ -1966,7 +2024,9 @@ static int ggmlhexagon_init_dsp(ggml_backend_qnn_context * ctx) { hexagon_error = ggmlop_dsp_open(ggmlop_domain_uri, &ctx->ggmlop_handle); if (AEE_SUCCESS == hexagon_error) { GGMLQNN_LOG_INFO("succeed to open domain %d(%s)", domain_id, ggmlhexagon_get_dsp_name(domain_id)); - GGMLQNN_LOG_INFO("only support GGML_OP_ADD on cDSP currently\n"); + GGMLQNN_LOG_INFO("only support GGML_OP_ADD and GGML_OP_MUL_MAT on cDSP currently\n"); + size_t rpcmem_size = 0; + ggmlhexagon_probe_dspinfo(ctx, &rpcmem_size); ggmlhexagon_set_clocks(ctx->ggmlop_handle, HAP_DCVS_V2_DUTY_CYCLE_MODE, 40, 1); ggmlhexagon_set_rpc_latency(domain_id, RPC_POLL_QOS, 1000); } else { @@ -1983,9 +2043,10 @@ static int ggmlhexagon_init_dsp(ggml_backend_qnn_context * ctx) { if (ctx->rpc_mempool) { rpcmem_free(ctx->rpc_mempool); - ctx->rpc_mempool = nullptr; - ctx->rpc_mempool_len = 0; - ctx->ggmlop_handle = -1; + ctx->rpc_mempool = nullptr; + ctx->rpc_mempool_len = 0; + ctx->ggmlop_handle = -1; + ctx->domain_id = -1; } return -1; @@ -2005,8 +2066,9 @@ static void ggmlhexagon_close_cdsp(ggml_backend_qnn_context * ctx) { if (ctx->rpc_mempool) { rpcmem_free(ctx->rpc_mempool); - ctx->rpc_mempool = nullptr; - ctx->rpc_mempool_len = 0; + ctx->rpc_mempool = nullptr; + ctx->rpc_mempool_len = 0; + ctx->domain_id = -1; } GGMLQNN_LOG_DEBUG("leave %s", __func__); } @@ -2019,20 +2081,15 @@ static void ggmlhexagon_compute(ggml_backend_qnn_context * ctx, struct ggml_tens int hexagon_error = AEE_SUCCESS; ggmlhexagon_op_func_t op_func = nullptr; - void * wdata = nullptr; - ggml_tensor * src0 = op->src[0]; - //src1 might-be nullptr for some ggml op ggml_tensor * src1 = op->src[1]; ggml_tensor * dst = op; - ggml_type src0_type = src0->type; switch (op->op) { case GGML_OP_ADD: op_func = ggmlop_dsp_add; break; case GGML_OP_MUL_MAT: { - wdata = ggmlqnn_type_trait(ctx, op); op_func = ggmlop_dsp_mulmat; break; } @@ -2040,18 +2097,12 @@ static void ggmlhexagon_compute(ggml_backend_qnn_context * ctx, struct ggml_tens return; } - if ((GGML_OP_MUL_MAT == op->op) && (src0_type != GGML_TYPE_F32)) { - dsptensor_0.data = wdata; - dsptensor_0.data_len = ctx->desired_size; - } else { - dsptensor_0.data = src0->data; - dsptensor_0.data_len = ggml_nbytes(src0); - } + dsptensor_0.data = src0->data; + dsptensor_0.data_len = ggml_nbytes(src0); - dsptensor_1.data = src1->data; - dsptensor_2.data = dst->data; + dsptensor_1.data = src1->data; + dsptensor_2.data = dst->data; - //make compiler happy dsptensor_0.ne[0] = src0->ne[0]; dsptensor_0.ne[1] = src0->ne[1]; dsptensor_0.ne[2] = src0->ne[2]; @@ -2086,10 +2137,6 @@ static void ggmlhexagon_compute(ggml_backend_qnn_context * ctx, struct ggml_tens dsptensor_1.data_len = ggml_nbytes(src1); dsptensor_2.data_len = ggml_nbytes(dst); - if ((GGML_OP_MUL_MAT == op->op) && (src0_type != GGML_TYPE_F32)) { - dsptensor_0.data_len = ctx->desired_size; - } - dsptensor_0.type = src0->type; dsptensor_1.type = src1->type; dsptensor_2.type = dst->type; @@ -4179,10 +4226,12 @@ static void ggmlqnn_load_cfg() { qnncfg_instance.get_intvalue("general", "dump_op_info", g_qnn_params.dump_op_info, 0); qnncfg_instance.get_intvalue("general", "inference_approach", g_qnn_params.inference_approach, 0); qnncfg_instance.get_intvalue("general", "qnn_backend", g_qnn_params.qnn_backend, 2); - qnncfg_instance.get_intvalue("npu", "hvx_threads", g_qnn_params.hvx_threads, 4); - qnncfg_instance.get_intvalue("npu", "vtcm_size_in_mb", g_qnn_params.vtcm_size_in_mb, 8); - qnncfg_instance.get_intvalue("npu", "enable_dlbc", g_qnn_params.enable_dlbc, 0); - qnncfg_instance.get_stringvalue("npu", "precision_mode", precision_mode, "fp32"); + qnncfg_instance.get_intvalue("qnn", "hvx_threads", g_qnn_params.hvx_threads, 4); + qnncfg_instance.get_intvalue("qnn", "vtcm_size_in_mb", g_qnn_params.vtcm_size_in_mb, 8); + qnncfg_instance.get_intvalue("qnn", "enable_dlbc", g_qnn_params.enable_dlbc, 0); + qnncfg_instance.get_stringvalue("qnn", "precision_mode", precision_mode, "fp32"); + qnncfg_instance.get_intvalue("cdsp", "enable_mulmat_cdsp", g_qnn_params.enable_mulmat_cdsp, 0); + qnncfg_instance.get_intvalue("cdsp", "enable_q_mulmat", g_qnn_params.enable_q_mulmat, 0); GGMLQNN_LOG_INFO("print_qnn_internal_log=%d", g_qnn_params.print_qnn_internal_log); GGMLQNN_LOG_INFO("inference_approach=%d(%s)", g_qnn_params.inference_approach, ggmlqnn_get_inference_approach_name(g_qnn_params.inference_approach)); @@ -4226,6 +4275,8 @@ static bool ggmlhexagon_can_handle_op(const ggml_backend_qnn_context * ctx, cons const int64_t ne00 = op_tensor->src[0]->ne[0]; uint32_t src0_rank = 0; uint32_t src1_rank = 0; + bool support = false; + if (nullptr != src0) { src0_rank = ggml_n_dims(src0); } @@ -4233,32 +4284,39 @@ static bool ggmlhexagon_can_handle_op(const ggml_backend_qnn_context * ctx, cons src1_rank = ggml_n_dims(src1); } - //TODO: only support offload GGML_OP_ADD and GGML_OP_MUL_MAT to cDSP directly - bool support = ((op_tensor->op == GGML_OP_ADD) || (op_tensor->op == GGML_OP_MUL_MAT)); + if (g_qnn_params.enable_mulmat_cdsp) + support = ((op_tensor->op == GGML_OP_ADD) || (op_tensor->op == GGML_OP_MUL_MAT)); + else + support = (op_tensor->op == GGML_OP_ADD); if (!support) return false; + ggmlqnn_dump_op_info(op_tensor); switch (op_tensor->op) { case GGML_OP_ADD: { if (!ggml_are_same_shape(src0, src1)) { return false; } + return (src0->type == GGML_TYPE_F32) && (src1->type == GGML_TYPE_F32) && (op_tensor->type == GGML_TYPE_F32); } - case GGML_OP_MUL_MAT: { ggmlqnn_dump_op_info(op_tensor); - if (src1_rank != 2) + //TODO:3d&4d matrix mulmat on cDSP + if (src0_rank != 2) return false; - return (src0->type == GGML_TYPE_F32) && (src1->type == GGML_TYPE_F32) && (op_tensor->type == GGML_TYPE_F32); - + if (g_qnn_params.enable_q_mulmat) + return (src0->type == GGML_TYPE_F32 || ggml_is_quantized(src0->type)) + && (src1->type == GGML_TYPE_F32) && (op_tensor->type == GGML_TYPE_F32); + else + return (src0->type == GGML_TYPE_F32) && (src1->type == GGML_TYPE_F32) && (op_tensor->type == GGML_TYPE_F32); } default: - return false; + return ggmlqnn_same_types(ctx, op_tensor); } } @@ -4597,8 +4655,6 @@ static ggml_backend_buffer_t ggml_backend_qnn_buffer_type_alloc_buffer( if (nullptr == ctx->buffer) { GGMLQNN_LOG_WARN("%s: failed to allocate %d MiB\n", __func__, size / (1 << 20)); return nullptr; - } else { - GGMLQNN_LOG_DEBUG("%s: allocate %d MiB\n", __func__, size_aligned / (1 << 20)); } return ggml_backend_buffer_init(buft, ggml_backend_qnn_buffer_interface, ctx, size); @@ -4729,10 +4785,16 @@ static void ggml_backend_qnn_device_get_memory(ggml_backend_dev_t dev, size_t * *total = ggmlqnn_get_system_total_memory_in_bytes(); *free = ggmlqnn_get_system_free_memory_in_bytes(); } else if (QNN_BACKEND_NPU == ctx->device) { - size_t rpc_ion_memsize = ctx->instance->get_rpcmem_capacity(); - size_t rpc_ion_usage = ctx->instance->get_rpcmem_usage(); - GGMLQNN_LOG_DEBUG("rpc memsize %d", rpc_ion_memsize); - GGMLQNN_LOG_DEBUG("rpc usage %d", rpc_ion_usage); + size_t rpc_ion_memsize = 0; + size_t rpc_ion_usage = 0; + if (DIRECT_USE_CDSP != g_qnn_params.inference_approach) { + rpc_ion_memsize = ctx->instance->get_rpcmem_capacity(); + rpc_ion_usage = ctx->instance->get_rpcmem_usage(); + } else { + ggmlhexagon_probe_dspinfo(ctx, &rpc_ion_memsize); + } + GGMLQNN_LOG_DEBUG("rpc memsize %d M", rpc_ion_memsize); + GGMLQNN_LOG_DEBUG("rpc usage %d M", rpc_ion_usage); *total = rpc_ion_memsize * (1 << 20); *free = (rpc_ion_memsize - rpc_ion_usage) * (1 << 20); } @@ -5078,9 +5140,12 @@ ggml_backend_t ggml_backend_qnn_init(size_t device, const char * qnn_lib_path) { return g_qnn_mgr[device].backend; } - qnn_instance * instance = ggmlqnn_init_qnn_instance(device, qnn_lib_path); - if (nullptr == instance) - return nullptr; + //don't initialize QNN when inference approach is offload ggml op to Hexagon cDSP directly + if (DIRECT_USE_CDSP != g_qnn_params.inference_approach) { + qnn_instance * instance = ggmlqnn_init_qnn_instance(device, qnn_lib_path); + if (nullptr == instance) + return nullptr; + } ggml_backend_qnn_interface.graph_compute = ggmlqnn_backend_graph_compute_general; diff --git a/scripts/ggml-qnn.cfg b/scripts/ggml-qnn.cfg index 9d47dba7a596a..513aecfc64862 100644 --- a/scripts/ggml-qnn.cfg +++ b/scripts/ggml-qnn.cfg @@ -1,7 +1,7 @@ [general] #0: QNN-CPU backend #1: QNN-GPU backend -#2: QNN-NPU backend +#2: QNN-NPU backend / Hexagon cDSP #3: default ggml backend qnn_backend = 2 @@ -22,8 +22,16 @@ dump_op_info = 0 # 2: special approach through QNN: mapping entire ggml cgraph to QNN graph inference_approach = 1 -[npu] +#inference approach through QNN +[qnn] hvx_threads = 4 vtcm_size_in_mb = 8 enable_dlbc = 1 precision_mode = "fp16" + +#inference approach through cDSP +[cdsp] +#enable/disable offload mulmat to cDSP +enable_mulmat_cdsp = 0 +#enable/disable offload fp32 & all quantized type mulmat to cDSP +enable_q_mulmat = 0 From 5134179f890f929b74453908cbcdb938e9b1893e Mon Sep 17 00:00:00 2001 From: zhouwg Date: Wed, 26 Mar 2025 22:42:03 +0800 Subject: [PATCH 76/76] ggml-qnn: refine code according to self code-review and make code more clear --- ggml/src/ggml-qnn/ggml-qnn.cpp | 95 +++++++++++++++++----------------- scripts/ggml-qnn.cfg | 12 ++--- 2 files changed, 53 insertions(+), 54 deletions(-) diff --git a/ggml/src/ggml-qnn/ggml-qnn.cpp b/ggml/src/ggml-qnn/ggml-qnn.cpp index c28d01d134b29..f882552a7fc1b 100644 --- a/ggml/src/ggml-qnn/ggml-qnn.cpp +++ b/ggml/src/ggml-qnn/ggml-qnn.cpp @@ -14,17 +14,17 @@ * section-6 Hexagon DSP helper function * section-7 backend helper function / class * section-8 implementation of ggml-hexagon backend according to specification in ggml backend subsystem - * section-9 implementation of general approach through QNN and Hexagon DSP + * section-9 implementation of hwaccel approach through QNN and Hexagon DSP * * currently provide following ggml op' implementation through QNN: * - GGML_OP_ADD/GGML_OP_SUB/GGML_OP_MUL/GGML_OP_DIV/GGML_OP_LOG/GGML_OP_SQRT: - * this is a simple skeleton, can expand other ggml ops according to expertise + * this is a simple hwaccel skeleton, can expand other ggml ops according to expertise * - GGML_OP_MUL_MAT: - * this is a complicated skeleton, can expand other ggml ops accordingly + * this is a complicated hwaccel skeleton, can expand other ggml ops accordingly * * currently provide following ggml op' implementation through Hexagon DSP: * - GGML_OP_ADD & GGML_OP_MUL_MAT: - * this is a skeleton, can expand other ggml ops accordingly + * this is a hwaccel skeleton, can expand other ggml ops accordingly * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to @@ -230,7 +230,7 @@ static void ggmlqnn_compute_diag_mask(ggml_backend_qnn_context * ctx, ggml_ten #define GGMLQNN_CHECK_PARAMS(ctx, src0, src1, dst) \ do { \ - if (g_qnn_params.inference_approach != DIRECT_USE_CDSP) { \ + if (g_qnn_params.hwaccel_approach != HWACCEL_CDSP) { \ if (!ggmlqnn_is_valid_params((ctx), (src0), (src1), (dst))) { \ return; \ } \ @@ -270,12 +270,12 @@ enum qnn_profile_level { }; //0: general approach through QNN:offload ggmlop to QNN -//1: general approach through Hexagon cDSP:offload ggmlop to Hexagon cDSP directly -//2: special approach through QNN:mapping entire ggml cgraph to a single QNN graph -enum inference_approach { - QNN_GENERAL = 0, - DIRECT_USE_CDSP = 1, - QNN_SINGLEGRAPH = 2, +//1: special approach through QNN-SINGLEGRAPH:mapping entire ggml cgraph to a single QNN graph +//2: general approach through Hexagon cDSP:offload ggmlop to Hexagon cDSP directly +enum hwaccel_approach_type { + HWACCEL_QNN = 0, + HWACCEL_QNN_SINGLEGRAPH = 1, + HWACCEL_CDSP = 2, }; enum hexagon_dsp_type { @@ -362,7 +362,7 @@ struct qnn_parameter { int hvx_threads; int vtcm_size_in_mb; int enable_dlbc; - int inference_approach; // 0: QNN_GENERAL 1: DIRECT_USE_CDSP 2: QNN_SINGELGRAPH + int hwaccel_approach; // 0: HWACCEL_QNN 1: HWACCEL_QNN_SINGLEGRAPH 2: HWACCEL_CDSP int qnn_backend; // 0: QNN-CPU backend 1: QNN-GPU backend 2: QNN-NPU backend int enable_mulmat_cdsp; // enable/disable offload mulmat to cDSP int enable_q_mulmat; // enable/disable offload fp32 & all quantized type mulmat to cDSP @@ -382,8 +382,8 @@ static struct qnn_parameter g_qnn_params = { .hvx_threads = 4, .vtcm_size_in_mb = 8, .enable_dlbc = 1, - .inference_approach = 0, - .qnn_backend = 2, //default is QNN-NPU backend + .hwaccel_approach = HWACCEL_CDSP, + .qnn_backend = QNN_BACKEND_NPU, .enable_mulmat_cdsp = 0, .enable_q_mulmat = 0, .qnn_cfgfilename = "ggml-qnn.cfg", @@ -1578,13 +1578,12 @@ static void ggmlhexagon_set_rpc_latency(int domain, int qos, int latency) { if (remote_handle_control) { struct remote_rpc_control_latency data; -#if 1 - data.enable = RPC_PM_QOS; - data.latency = 300; -#else - data.enable = RPC_POLL_QOS; - data.latency = 1000; -#endif +/* + qos | latency + ----------------------- + RPC_PM_QOS | 300 + RPC_POLL_QOS | 1000 +*/ data.enable = qos; data.latency = latency; hexagon_error = remote_handle64_control(DSPRPC_GET_DSP_INFO, DSPRPC_CONTROL_LATENCY, (void*)&data, sizeof(data)); @@ -1926,7 +1925,7 @@ static int ggmlhexagon_init_dsp(ggml_backend_qnn_context * ctx) { } if (-1 == domain_id) { - if (NULL != domain_type) { + if (nullptr != domain_type) { if ((strcmp(domain_type, "NSP") != 0 && strcmp(domain_type, "HPASS") != 0)) { GGMLQNN_LOG_WARN("invalid domain_type %s. possible values are NSP or HPASS", domain_type); goto bail; @@ -2188,16 +2187,16 @@ static const char * ggmlqnn_get_htparch_desc(size_t htp_arch) { } } -static const char * ggmlqnn_get_inference_approach_name(int inference_approach) { - switch (inference_approach) { - case QNN_GENERAL: - return "QNN_GENERAL"; - case DIRECT_USE_CDSP: - return "DIRECT_USE_CDSP"; - case QNN_SINGLEGRAPH: - return "QNN_SINGLEGRAPH"; +static const char * ggmlqnn_get_hwaccel_approach_name(int hwaccle_approach) { + switch (hwaccle_approach) { + case HWACCEL_QNN: + return "HWACCEL_QNN"; + case HWACCEL_QNN_SINGLEGRAPH: + return "HWACCEL_QNN_SINGLEGRAPH"; + case HWACCEL_CDSP: + return "HWACCEL_CDSP"; default: - return "unknown approach"; + return "unknown hwaccel approach"; } } @@ -3996,7 +3995,7 @@ void qnn_instance::htp_enter_performance_mode() { } static void ggmlqnn_set_runtime_path(size_t device, const std::string & path) { - if ((QNN_BACKEND_NPU == device) || (DIRECT_USE_CDSP == g_qnn_params.inference_approach)) { + if ((QNN_BACKEND_NPU == device) || (HWACCEL_CDSP == g_qnn_params.hwaccel_approach)) { if (0 == setenv("LD_LIBRARY_PATH", (path + ":/vendor/dsp/cdsp:/vendor/lib64:/vendor/dsp/dsp:/vendor/dsp/images").c_str(), @@ -4224,7 +4223,7 @@ static void ggmlqnn_load_cfg() { qnncfg_instance.get_intvalue("general", "enable_perf", g_qnn_params.enable_perf, 0); qnncfg_instance.get_intvalue("general", "print_tensors_info", g_qnn_params.print_tensors_info, 0); qnncfg_instance.get_intvalue("general", "dump_op_info", g_qnn_params.dump_op_info, 0); - qnncfg_instance.get_intvalue("general", "inference_approach", g_qnn_params.inference_approach, 0); + qnncfg_instance.get_intvalue("general", "hwaccel_approach", g_qnn_params.hwaccel_approach, 0); qnncfg_instance.get_intvalue("general", "qnn_backend", g_qnn_params.qnn_backend, 2); qnncfg_instance.get_intvalue("qnn", "hvx_threads", g_qnn_params.hvx_threads, 4); qnncfg_instance.get_intvalue("qnn", "vtcm_size_in_mb", g_qnn_params.vtcm_size_in_mb, 8); @@ -4233,8 +4232,8 @@ static void ggmlqnn_load_cfg() { qnncfg_instance.get_intvalue("cdsp", "enable_mulmat_cdsp", g_qnn_params.enable_mulmat_cdsp, 0); qnncfg_instance.get_intvalue("cdsp", "enable_q_mulmat", g_qnn_params.enable_q_mulmat, 0); GGMLQNN_LOG_INFO("print_qnn_internal_log=%d", g_qnn_params.print_qnn_internal_log); - GGMLQNN_LOG_INFO("inference_approach=%d(%s)", g_qnn_params.inference_approach, - ggmlqnn_get_inference_approach_name(g_qnn_params.inference_approach)); + GGMLQNN_LOG_INFO("hwaccel_approach=%d(%s)", g_qnn_params.hwaccel_approach, + ggmlqnn_get_hwaccel_approach_name(g_qnn_params.hwaccel_approach)); GGMLQNN_LOG_INFO("qnn_backend=%d", g_qnn_params.qnn_backend); GGMLQNN_LOG_INFO("npu inference precision mode=%s", precision_mode.c_str()); GGMLQNN_LOG_INFO("qnn runtime lib path=%s", g_qnn_params.qnn_runtimelib_path); @@ -4325,7 +4324,7 @@ static bool ggmlqnn_can_handle_op(const ggml_backend_qnn_context * ctx, const st return true; } - if (DIRECT_USE_CDSP == g_qnn_params.inference_approach) { + if (HWACCEL_CDSP == g_qnn_params.hwaccel_approach) { return ggmlhexagon_can_handle_op(ctx, op_tensor); } @@ -4686,7 +4685,7 @@ static void ggml_backend_qnn_free(ggml_backend_t backend) { ggml_backend_qnn_context * ctx = (ggml_backend_qnn_context *)backend->context; GGMLQNN_LOG_DEBUG("device idx %d, name:%s", ctx->device, g_qnn_mgr[ctx->device].name); - if (DIRECT_USE_CDSP == g_qnn_params.inference_approach) { + if (HWACCEL_CDSP == g_qnn_params.hwaccel_approach) { ggmlhexagon_close_cdsp(ctx); } @@ -4787,7 +4786,7 @@ static void ggml_backend_qnn_device_get_memory(ggml_backend_dev_t dev, size_t * } else if (QNN_BACKEND_NPU == ctx->device) { size_t rpc_ion_memsize = 0; size_t rpc_ion_usage = 0; - if (DIRECT_USE_CDSP != g_qnn_params.inference_approach) { + if (HWACCEL_CDSP != g_qnn_params.hwaccel_approach) { rpc_ion_memsize = ctx->instance->get_rpcmem_capacity(); rpc_ion_usage = ctx->instance->get_rpcmem_usage(); } else { @@ -5013,8 +5012,8 @@ ggml_backend_reg_t ggml_backend_qnn_reg() { //case-2: normal scenario, such as llama-cli or UI applicaton ggmlqnn_load_cfg(); - GGMLQNN_LOG_INFO("inference approach=%d(%s)", g_qnn_params.inference_approach, - ggmlqnn_get_inference_approach_name(g_qnn_params.inference_approach)); + GGMLQNN_LOG_INFO("inference approach=%d(%s)", g_qnn_params.hwaccel_approach, + ggmlqnn_get_hwaccel_approach_name(g_qnn_params.hwaccel_approach)); GGMLQNN_LOG_INFO("user's specified qnn_backend=%d", g_qnn_params.qnn_backend); GGMLQNN_LOG_INFO("user's specified qnn runtime lib path=%s", g_qnn_params.qnn_runtimelib_path); if (g_qnn_params.qnn_backend >= GGML_QNN_MAX_DEVICES) { @@ -5053,7 +5052,7 @@ ggml_backend_reg_t ggml_backend_qnn_reg() { } const char * ggml_backend_qnn_get_devname(size_t dev_num) { - if (DIRECT_USE_CDSP == g_qnn_params.inference_approach) { + if (HWACCEL_CDSP == g_qnn_params.hwaccel_approach) { if (dev_num == QNN_BACKEND_GGML) return "ggml"; else @@ -5076,8 +5075,8 @@ const char * ggml_backend_qnn_get_devname(size_t dev_num) { static qnn_instance * ggmlqnn_init_qnn_instance(size_t device, const char * qnn_lib_path) { int result = 0; - GGMLQNN_LOG_INFO("inference approach=%d(%s)", g_qnn_params.inference_approach, - ggmlqnn_get_inference_approach_name(g_qnn_params.inference_approach)); + GGMLQNN_LOG_INFO("inference approach=%d(%s)", g_qnn_params.hwaccel_approach, + ggmlqnn_get_hwaccel_approach_name(g_qnn_params.hwaccel_approach)); qnn_instance * instance = nullptr; instance = new qnn_instance(qnn_lib_path, g_qnn_mgr[device].lib, ""); @@ -5141,7 +5140,7 @@ ggml_backend_t ggml_backend_qnn_init(size_t device, const char * qnn_lib_path) { } //don't initialize QNN when inference approach is offload ggml op to Hexagon cDSP directly - if (DIRECT_USE_CDSP != g_qnn_params.inference_approach) { + if (HWACCEL_CDSP != g_qnn_params.hwaccel_approach) { qnn_instance * instance = ggmlqnn_init_qnn_instance(device, qnn_lib_path); if (nullptr == instance) return nullptr; @@ -5157,14 +5156,14 @@ ggml_backend_t ggml_backend_qnn_init(size_t device, const char * qnn_lib_path) { }; g_qnn_mgr[device].backend = qnn_backend; - if (DIRECT_USE_CDSP == g_qnn_params.inference_approach) { + if (HWACCEL_CDSP == g_qnn_params.hwaccel_approach) { int result = ggmlhexagon_init_dsp(&g_qnn_mgr[device]); if (0 != result) { GGMLQNN_LOG_INFO("init hexagon dsp failure"); ggml_backend_qnn_free(qnn_backend); return nullptr; } - //ensure test-backend-ops get the correct backend name when inference approach is 1(DIRECT_USE_CDSP) + //ensure test-backend-ops get the correct backend name when inference approach is 1(HWACCEL_CDSP) memcpy(g_qnn_mgr[device].name, "Hexagon-cDSP", strlen("Hexagon-cDSP")); } @@ -5237,7 +5236,7 @@ static void ggmlqnn_compute_elementwise(ggml_backend_qnn_context * ctx, ggml_ten qnn_perf op_perf = qnn_perf(graph_name); op_perf.start(); - if (DIRECT_USE_CDSP == g_qnn_params.inference_approach) { + if (HWACCEL_CDSP == g_qnn_params.hwaccel_approach) { ggmlhexagon_compute(ctx, op); op_perf.info(); return; @@ -5629,7 +5628,7 @@ static void ggmlqnn_compute_mul_mat(ggml_backend_qnn_context * ctx, ggml_tensor qnn_perf op_perf = qnn_perf(graph_name); op_perf.start(); - if (DIRECT_USE_CDSP == g_qnn_params.inference_approach) { + if (HWACCEL_CDSP == g_qnn_params.hwaccel_approach) { ggmlhexagon_compute(ctx, op); op_perf.info(); return; diff --git a/scripts/ggml-qnn.cfg b/scripts/ggml-qnn.cfg index 513aecfc64862..17bd8f6a4b1ca 100644 --- a/scripts/ggml-qnn.cfg +++ b/scripts/ggml-qnn.cfg @@ -17,19 +17,19 @@ print_tensors_info = 0 # enable/disable dump op info in handle_op dump_op_info = 0 -# 0: general approach through QNN -# 1: general approach through Hexagon cDSP -# 2: special approach through QNN: mapping entire ggml cgraph to QNN graph -inference_approach = 1 +# 0: hwaccel approach through QNN +# 1: hwaccel approach through QNN-SINGLEGRAPH: mapping entire ggml cgraph to a single QNN graph +# 2: hwaccel approach through Hexagon cDSP +hwaccel_approach = 2 -#inference approach through QNN +#hwaccel approach through QNN [qnn] hvx_threads = 4 vtcm_size_in_mb = 8 enable_dlbc = 1 precision_mode = "fp16" -#inference approach through cDSP +#hwaccel approach through cDSP [cdsp] #enable/disable offload mulmat to cDSP enable_mulmat_cdsp = 0