PR: Refine ggml-hexagon backend(Qualcomm Hexagon NPU backend) for latest ggml,whisper.cpp,llama.cpp

CMakeLists.txt

@@ -7,6 +7,16 @@ set(CMAKE_WARN_UNUSED_CLI YES)
 
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
+if(CMAKE_SYSTEM_NAME STREQUAL "Android")
+    set(TARGET_SNAPDRAGON8GEN3    ON)
+    if(TARGET_SNAPDRAGON8GEN3)
+       #works fine on Snapdragon 8Gen3 with 1.5x(45+ tokens/second)-3x(70+ tokens/second) performance gain through the default ggml backend
+       add_definitions(-march=armv8.7-a)
+       add_definitions(-mcpu=cortex-x1)
+       add_definitions(-mtune=cortex-x1)
+   endif()
+endif()
+
 if (NOT XCODE AND NOT MSVC AND NOT CMAKE_BUILD_TYPE)
     set(CMAKE_BUILD_TYPE Release CACHE STRING "Build type" FORCE)
     set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo")
@@ -119,6 +129,7 @@ llama_option_depr(WARNING     LLAMA_RPC                 GGML_RPC)
 llama_option_depr(WARNING     LLAMA_SYCL                GGML_SYCL)
 llama_option_depr(WARNING     LLAMA_SYCL_F16            GGML_SYCL_F16)
 llama_option_depr(WARNING     LLAMA_CANN                GGML_CANN)
+llama_option_depr(WARNING     LLAMA_QNN                 GGML_QNN)
 
 if (NOT MSVC)
     if (LLAMA_SANITIZE_THREAD)

ggml/CMakeLists.txt

@@ -198,6 +198,7 @@ option(GGML_OPENCL_EMBED_KERNELS            "ggml: embed kernels"
 option(GGML_OPENCL_USE_ADRENO_KERNELS       "ggml: use optimized kernels for Adreno"          ON)
 set   (GGML_OPENCL_TARGET_VERSION "300" CACHE STRING
                                             "gmml: OpenCL API version to target")
+option(GGML_QNN                             "ggml: use QNN"                                   OFF)
 
 # toolchain for vulkan-shaders-gen
 set   (GGML_VULKAN_SHADERS_GEN_TOOLCHAIN "" CACHE FILEPATH "ggml: toolchain file for vulkan-shaders-gen")
@@ -263,6 +264,7 @@ set(GGML_PUBLIC_HEADERS
     include/ggml-rpc.h
     include/ggml-sycl.h
     include/ggml-vulkan.h
+    include/ggml-qnn.h
     include/gguf.h)
 
 set_target_properties(ggml PROPERTIES PUBLIC_HEADER "${GGML_PUBLIC_HEADERS}")

ggml/include/ggml-qnn.h

@@ -0,0 +1,53 @@
+ /*
+ * Copyright (c) 2023-2024 The ggml authors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+#pragma once
+
+#include "ggml.h"
+#include "ggml-backend.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define GGML_QNN_MAX_DEVICES    3
+#define GGML_QNN_BACKEND_NAME   "qnn"
+
+enum QNNBackend {
+    QNN_BACKEND_CPU,
+    QNN_BACKEND_GPU,
+    QNN_BACKEND_NPU,
+    QNN_BACKEND_GGML, //"fake" QNN backend for compare performance between QNN backend and cpu backend
+};
+
+GGML_BACKEND_API ggml_backend_t ggml_backend_qnn_init(size_t dev_num, const char * qnn_lib_path);
+
+GGML_BACKEND_API bool           ggml_backend_is_qnn(ggml_backend_t backend);
+
+GGML_BACKEND_API int            ggml_backend_qnn_get_device_count(void);
+
+GGML_BACKEND_API ggml_backend_reg_t ggml_backend_qnn_reg(void);
+
+const char * ggml_backend_qnn_get_devname(size_t dev_num);
+
+#ifdef __cplusplus
+}
+#endif

ggml/src/CMakeLists.txt

@@ -310,6 +310,7 @@ ggml_add_backend(RPC)
 ggml_add_backend(SYCL)
 ggml_add_backend(Vulkan)
 ggml_add_backend(OpenCL)
+ggml_add_backend(QNN)
 
 foreach (target ggml-base ggml)
     target_include_directories(${target} PUBLIC    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../include> $<INSTALL_INTERFACE:include>)

ggml/src/ggml-backend-reg.cpp

@@ -65,6 +65,10 @@
 #include "ggml-kompute.h"
 #endif
 
+#ifdef GGML_USE_QNN
+#include "ggml-qnn.h"
+#endif
+
 // disable C++17 deprecation warning for std::codecvt_utf8
 #if defined(__clang__)
 #    pragma clang diagnostic push
@@ -187,6 +191,9 @@ struct ggml_backend_registry {
 #ifdef GGML_USE_KOMPUTE
         register_backend(ggml_backend_kompute_reg());
 #endif
+#ifdef GGML_USE_QNN
+        register_backend(ggml_backend_qnn_reg());
+#endif
 #ifdef GGML_USE_CPU
         register_backend(ggml_backend_cpu_reg());
 #endif
@@ -577,6 +584,7 @@ void ggml_backend_load_all_from_path(const char * dir_path) {
     ggml_backend_load_best("vulkan", silent, dir_path);
     ggml_backend_load_best("opencl", silent, dir_path);
     ggml_backend_load_best("musa", silent, dir_path);
+    ggml_backend_load_best("qnn", silent, dir_path);
     ggml_backend_load_best("cpu", silent, dir_path);
     // check the environment variable GGML_BACKEND_PATH to load an out-of-tree backend
     const char * backend_path = std::getenv("GGML_BACKEND_PATH");

ggml/src/ggml-qnn/CMakeLists.txt

@@ -0,0 +1,59 @@
+message(STATUS "Using QNN backend")
+message("CMAKE_SYSTEM_NAME : ${CMAKE_SYSTEM_NAME}")
+
+if(NOT DEFINED QNN_SDK_PATH)
+    message(FATAL_ERROR "QNN_SDK_PATH not defined")
+endif()
+
+if(NOT DEFINED HEXAGON_SDK_PATH)
+    message(FATAL_ERROR "HEXAGON_SDK_PATH not defined")
+endif()
+
+message("QNN_SDK_PATH:     ${QNN_SDK_PATH}")
+message("HEXAGON_SDK_PATH: ${HEXAGON_SDK_PATH}")
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Android")
+    find_library(LOG_LIB log)
+
+    add_library(cdsprpc
+        SHARED
+        IMPORTED)
+    set_target_properties(cdsprpc
+        PROPERTIES
+        IMPORTED_LOCATION
+        ${HEXAGON_SDK_PATH}/ipc/fastrpc/remote/ship/android_aarch64/libcdsprpc.so)
+
+    set(QNN_LINK_LIBRARIES ${LOG_LIB} cdsprpc)
+    set(QNN_DEFAULT_LIB_SEARCH_PATH "/data/local/tmp/" CACHE STRING "customized library search path for QNN backend")
+
+    include_directories(${HEXAGON_SDK_PATH}/incs)
+    include_directories(${HEXAGON_SDK_PATH}/incs/stddef)
+    include_directories(${HEXAGON_SDK_PATH}/ipc/fastrpc/incs)
+    include_directories(${HEXAGON_SDK_PATH}/ipc/fastrpc/rpcmem/inc)
+    include_directories(${HEXAGON_SDK_PATH}/ipc/fastrpc/remote/ship/android_Debug_aarch64)
+    include_directories(${HEXAGON_SDK_PATH}/incs/qnx)
+    include_directories(${HEXAGON_SDK_PATH}/libs/common/qnx/ship/android_Debug_aarch64)
+    include_directories(${HEXAGON_SDK_PATH}/utils/examples)
+    include_directories(${HEXAGON_SDK_PATH}/ipc/fastrpc/rtld/ship/android_aarch64)
+    include_directories(${HEXAGON_SDK_PATH}/libs/atomic/inc)
+    include_directories(${HEXAGON_SDK_PATH}/libs/atomic/android_Debug_aarch64/ship)
+    include_directories(${CMAKE_SOURCE_DIR}/ggml/src/ggml-qnn/)
+    include_directories(${CMAKE_SOURCE_DIR}/ggml/src/ggml-qnn/kernels/)
+
+elseif(CMAKE_SYSTEM_NAME STREQUAL "Windows")
+    set(QNN_DEFAULT_LIB_SEARCH_PATH "C:\\" CACHE STRING "customized library search path for QNN backend")
+else()
+    message(FATAL_ERROR "QNN now only available on Android and Windows(Windows on ARM)")
+endif()
+
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DGGML_USE_QNN")
+set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3")
+
+file(GLOB QNN_SOURCES "${CMAKE_CURRENT_LIST_DIR}/*.cpp" "${CMAKE_CURRENT_LIST_DIR}/kernels/ggmlop_ap_skel.c")
+ggml_add_backend_library(ggml-qnn ${QNN_SOURCES})
+
+target_include_directories(ggml-qnn PRIVATE ${QNN_SDK_PATH}/include/QNN ${HEXAGON_SDK_PATH} ${CMAKE_CURRENT_LIST_DIR})
+target_link_libraries(ggml-qnn PRIVATE ${QNN_LINK_LIBRARIES})
+
+string(REGEX REPLACE "/$" "" QNN_DEFAULT_LIB_SEARCH_PATH "${QNN_DEFAULT_LIB_SEARCH_PATH}")
+target_compile_definitions(ggml-qnn PRIVATE QNN_DEFAULT_LIB_SEARCH_PATH="${QNN_DEFAULT_LIB_SEARCH_PATH}/")