feat: op perf opt

ggml/include/ggml-qnn.h

@@ -1,24 +1,11 @@
 #pragma once
 
 #include "ggml-backend.h"
-#include "ggml.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-#define GGML_QNN_NAME        "qnn"
-#define GGML_QNN_MAX_DEVICES QNN_BACKEND_COUNT
-
-enum QNNBackend {
-    QNN_BACKEND_CPU = 0,
-    QNN_BACKEND_GPU,
-    QNN_BACKEND_NPU,
-    QNN_BACKEND_COUNT,
-};
-
-GGML_API bool ggml_backend_is_qnn(ggml_backend_t backend);
-
 GGML_API ggml_backend_reg_t ggml_backend_qnn_reg(void);
 
 #ifdef __cplusplus

ggml/src/ggml-qnn/CMakeLists.txt

@@ -1,9 +1,13 @@
 message(STATUS "Using QNN backend")
 
+option(GGML_HEXAGON_NPU_ONLY "ggml-qnn: Only use Hexagon NPU" OFF)
+option(GGML_QNN_ENABLE_HEXAGON_BACKEND "ggml-qnn: Enable Hexagon custom package" ${GGML_HEXAGON_NPU_ONLY})
+
 if(CMAKE_SYSTEM_NAME STREQUAL "Android")
     find_library(LOG_LIB log)
     set(QNN_LINK_LIBRARIES ${LOG_LIB})
     set(QNN_DEFAULT_LIB_SEARCH_PATH "/data/local/tmp/" CACHE STRING "customized library search path for QNN backend")
+    add_compile_options(-g -O0)
 elseif(CMAKE_SYSTEM_NAME STREQUAL "Windows" OR CMAKE_SYSTEM_NAME STREQUAL "Linux")
     set(QNN_DEFAULT_LIB_SEARCH_PATH "" CACHE STRING "customized library search path for QNN backend")
 else()
@@ -21,15 +25,22 @@ if(NOT DEFINED GGML_QNN_SDK_PATH)
 endif()
 
 message("CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}")
+message("CMAKE_CXX_FLAGS_DEBUG: ${CMAKE_CXX_FLAGS_DEBUG}")
 message("CMAKE_CXX_FLAGS_RELEASE: ${CMAKE_CXX_FLAGS_RELEASE}")
 message("QNN_SDK_PATH: ${GGML_QNN_SDK_PATH}")
 
-file(GLOB QNN_SOURCES "${CMAKE_CURRENT_LIST_DIR}/*.cpp")
+file(GLOB QNN_SOURCES "${CMAKE_CURRENT_LIST_DIR}/qnn/*.cpp")
+file(GLOB COMMON_SOURCES "${CMAKE_CURRENT_LIST_DIR}/*.cpp")
 ggml_add_backend_library(ggml-qnn
     ${QNN_SOURCES}
+    ${COMMON_SOURCES}
 )
 
-target_include_directories(ggml-qnn PRIVATE ${GGML_QNN_SDK_PATH}/include/QNN ${CMAKE_CURRENT_LIST_DIR})
+target_include_directories(ggml-qnn PRIVATE
+    ${GGML_QNN_SDK_PATH}/include/QNN
+    ${CMAKE_CURRENT_LIST_DIR}/qnn
+    ${CMAKE_CURRENT_LIST_DIR}
+)
 target_link_libraries(ggml-qnn PRIVATE ${QNN_LINK_LIBRARIES})
 
 if(NOT "${QNN_DEFAULT_LIB_SEARCH_PATH}" STREQUAL "")
@@ -52,3 +63,99 @@ if(GGML_QNN_ENABLE_PERFORMANCE_TRACKING)
 else()
     message("GGML_QNN_ENABLE_PERFORMANCE_TRACKING is disabled")
 endif()
+
+add_subdirectory(shared)
+
+if(GGML_HEXAGON_NPU_ONLY)
+    message("GGML_HEXAGON_NPU_ONLY is enabled")
+    add_compile_definitions(GGML_HEXAGON_NPU_ONLY)
+    set(GGML_QNN_ENABLE_HEXAGON_BACKEND ON)
+else()
+    message("GGML_HEXAGON_NPU_ONLY is disabled")
+endif()
+
+if(GGML_QNN_ENABLE_HEXAGON_BACKEND)
+    message("GGML_QNN_ENABLE_HEXAGON_BACKEND is enabled")
+    add_subdirectory(npu)
+    target_link_libraries(hexagon-npu-host runtime-common)
+    target_link_libraries(ggml-qnn PRIVATE hexagon-npu-host)
+else()
+    message("GGML_QNN_ENABLE_HEXAGON_BACKEND is disabled")
+    target_link_libraries(ggml-qnn PRIVATE runtime-common)
+endif()
+
+# Copy QNN dynamic libraries
+set(QNN_DYNAMIC_LIBS "")
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Android" OR CMAKE_SYSTEM_NAME STREQUAL "Linux")
+    if(CMAKE_SYSTEM_NAME STREQUAL "Android")
+        # Android
+        set(QNN_SDK_LIB_PATH "${GGML_QNN_SDK_PATH}/lib/aarch64-android")
+    elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+        # Linux x86_64
+        set(QNN_SDK_LIB_PATH "${GGML_QNN_SDK_PATH}/lib/x86_64-linux-clang")
+    else()
+        # Linux aarch64
+        set(QNN_SDK_LIB_PATH "${GGML_QNN_SDK_PATH}/lib/aarch64-oe-linux-gcc11.2")
+    endif()
+
+    list(APPEND QNN_DYNAMIC_LIBS "${QNN_SDK_LIB_PATH}/libQnnSystem.so")
+    list(APPEND QNN_DYNAMIC_LIBS "${QNN_SDK_LIB_PATH}/libQnnCpu.so")
+    list(APPEND QNN_DYNAMIC_LIBS "${QNN_SDK_LIB_PATH}/libQnnGpu.so")
+    list(APPEND QNN_DYNAMIC_LIBS "${QNN_SDK_LIB_PATH}/libQnnHtp.so")
+    file(GLOB HTP_STUB_LIBS "${QNN_SDK_LIB_PATH}/libQnnHtp*.so")
+    list(APPEND QNN_DYNAMIC_LIBS ${HTP_STUB_LIBS})
+
+    if(CMAKE_SYSTEM_NAME STREQUAL "Android")
+        file(GLOB HTP_SKEL_LIBS "${GGML_QNN_SDK_PATH}/lib/hexagon-*/unsigned/libQnnHtp*Skel.so")
+        list(APPEND QNN_DYNAMIC_LIBS ${HTP_SKEL_LIBS})
+
+        if(CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64")
+            if(EXISTS "${CMAKE_ANDROID_NDK}/prebuilt/android-arm64/gdbserver/gdbserver")
+                list(APPEND QNN_DYNAMIC_LIBS "${CMAKE_ANDROID_NDK}/prebuilt/android-arm64/gdbserver/gdbserver")
+                message("old ndk, copy gdbserver")
+            else()
+                file(GLOB LLDB_SERVER "${CMAKE_ANDROID_NDK}/toolchains/llvm/prebuilt/linux-x86_64/lib64/clang/*/lib/linux/aarch64/lldb-server")
+                list(APPEND QNN_DYNAMIC_LIBS ${LLDB_SERVER})
+                message("new ndk, copy lldb-server")
+            endif()
+
+            file(GLOB OMP_LIBS "${CMAKE_ANDROID_NDK}/toolchains/llvm/prebuilt/linux-x86_64/lib64/clang/*/lib/linux/aarch64/libomp.so")
+            file(GLOB ASAN_LIBS "${CMAKE_ANDROID_NDK}/toolchains/llvm/prebuilt/linux-x86_64/lib64/clang/*/lib/linux/libclang_rt.asan-aarch64-android.so")
+            list(APPEND QNN_DYNAMIC_LIBS ${OMP_LIBS})
+            list(APPEND QNN_DYNAMIC_LIBS ${ASAN_LIBS})
+        endif()
+    else()
+        # Linux
+        list(APPEND QNN_DYNAMIC_LIBS "${QNN_SDK_LIB_PATH}/libHtpPrepare.so")
+    endif()
+elseif(CMAKE_SYSTEM_NAME STREQUAL "Windows")
+    if(CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+        # x86_64
+        set(QNN_SDK_LIB_PATH "${GGML_QNN_SDK_PATH}/lib/x86_64-windows-msvc")
+    else()
+        # aarch64
+        set(QNN_SDK_LIB_PATH "${GGML_QNN_SDK_PATH}/lib/aarch64-windows-msvc")
+    endif()
+
+    list(APPEND QNN_DYNAMIC_LIBS "${QNN_SDK_LIB_PATH}/QnnSystem.dll")
+    list(APPEND QNN_DYNAMIC_LIBS "${QNN_SDK_LIB_PATH}/QnnCpu.dll")
+    list(APPEND QNN_DYNAMIC_LIBS "${QNN_SDK_LIB_PATH}/QnnGpu.dll")
+    list(APPEND QNN_DYNAMIC_LIBS "${QNN_SDK_LIB_PATH}/QnnHtp.dll")
+    file(GLOB HTP_STUB_LIBS "${QNN_SDK_LIB_PATH}/QnnHtp*.dll")
+
+    if(CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+        list(APPEND QNN_DYNAMIC_LIBS "${QNN_SDK_LIB_PATH}/HtpPrepare.dll")
+    endif()
+
+    list(APPEND QNN_DYNAMIC_LIBS ${HTP_STUB_LIBS})
+endif()
+
+foreach(QNN_DYNAMIC_LIB ${QNN_DYNAMIC_LIBS})
+    message("Copy: ${QNN_DYNAMIC_LIB} -> ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}")
+    add_custom_command(
+        TARGET ggml-qnn POST_BUILD
+        COMMAND ${CMAKE_COMMAND} -E copy
+        ${QNN_DYNAMIC_LIB}
+        ${CMAKE_RUNTIME_OUTPUT_DIRECTORY})
+endforeach()

ggml/src/ggml-qnn/npu/CMakeLists.txt

@@ -0,0 +1,147 @@
+enable_language(ASM)
+cmake_policy(SET CMP0115 OLD)
+
+if(DEFINED ENV{HEXAGON_SDK_ROOT})
+    set(HEXAGON_SDK_ROOT $ENV{HEXAGON_SDK_ROOT})
+    message("HEXAGON_SDK_ROOT: ${HEXAGON_SDK_ROOT}")
+else()
+    message(FATAL_ERROR "HEXAGON_SDK_ROOT not defined")
+endif()
+
+if(HEXAGON_SDK_ROOT)
+    include(${HEXAGON_SDK_ROOT}/build/cmake/hexagon_fun.cmake)
+else()
+    include(${HEXAGON_CMAKE_ROOT}/hexagon_fun.cmake)
+endif()
+
+# Base Include dirs for the Project
+set(common_incs
+    ${CMAKE_CURRENT_BINARY_DIR}/
+    ${HEXAGON_SDK_ROOT}/incs/
+    ${HEXAGON_SDK_ROOT}/incs/stddef/
+    ${HEXAGON_SDK_ROOT}/incs/HAP/
+    ${HEXAGON_SDK_ROOT}/rtos/qurt/
+    ${HEXAGON_SDK_ROOT}/utils/examples/
+)
+
+include_directories(${common_incs})
+
+if(${CMAKE_SYSTEM_NAME} MATCHES "Android|Linux|Windows")
+    # host build
+    file(GLOB common_srcs "${CMAKE_CURRENT_LIST_DIR}/common/*.cpp")
+    file(GLOB host_srcs "${CMAKE_CURRENT_LIST_DIR}/host/*.cpp")
+    set(stub_srcs "${CMAKE_CURRENT_BINARY_DIR}/npu_device_stub.c")
+    add_library(hexagon-npu-host STATIC
+        ${common_srcs}
+        ${host_srcs}
+        ${stub_srcs}
+    )
+
+    # disable warnings for the stub
+    set_source_files_properties(
+        ${stub_srcs}
+        PROPERTIES
+        COMPILE_FLAGS "-w"
+    )
+
+    build_idl(idl/hexagon_npu.idl hexagon-npu-host)
+
+    # Add compile definitions to the target
+    target_compile_definitions(hexagon-npu-host PUBLIC
+        VERIFY_PRINT_ERROR
+        GGML_QNN_ENABLE_HEXAGON_BACKEND
+    )
+
+    target_include_directories(hexagon-npu-host PRIVATE
+        ${HEXAGON_SDK_ROOT}/ipc/fastrpc/rpcmem/inc/
+        ${QNN_SDK_ROOT}/include/QNN/
+        ${CMAKE_CURRENT_LIST_DIR}/host/
+        ${CMAKE_CURRENT_LIST_DIR}/
+    )
+
+    target_include_directories(hexagon-npu-host PUBLIC
+        ${HEXAGON_SDK_ROOT}/incs/ # TODO: this is for rpc-mem
+    )
+
+    if(NOT ${CMAKE_SYSTEM_NAME} MATCHES "Windows")
+        set_target_properties(hexagon-npu-host PROPERTIES OUTPUT_NAME "hexagon_npu")
+    endif()
+
+    if(${CMAKE_SYSTEM_NAME} MATCHES "Android|Linux")
+        target_link_options(hexagon-npu-host PUBLIC -pie)
+    endif()
+
+    link_options(hexagon-npu-host)
+
+    if(${CMAKE_SYSTEM_NAME} MATCHES "Android")
+        set(PREBUILT_LIB_DIR "android_aarch64")
+    elseif(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
+        set(PREBUILT_LIB_DIR "UbuntuARM_aarch64")
+    else()
+        # Windows
+        set(PREBUILT_LIB_DIR "windows_aarch64")
+    endif()
+
+    choose_dsprpc("3" dsprpc) # cdsprpc
+    link_custom_library(hexagon-npu-host ${dsprpc})
+else()
+    # hexagon npu build
+    cmake_minimum_required(VERSION 3.14.3)
+    project(hexagon_npu C CXX ASM)
+
+    # check if QNN_SDK_ROOT is set
+    if(NOT DEFINED ENV{QNN_SDK_ROOT})
+        message(FATAL_ERROR "QNN_SDK_ROOT not defined")
+    endif()
+
+    set(QNN_SDK_ROOT $ENV{QNN_SDK_ROOT})
+    message("QNN_SDK_ROOT: ${QNN_SDK_ROOT}")
+    include_directories(
+        ${QNN_SDK_ROOT}/include/QNN/
+    )
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17")
+
+    file(GLOB common_srcs "${CMAKE_CURRENT_LIST_DIR}/common/*.cpp")
+    file(GLOB device_srcs "${CMAKE_CURRENT_LIST_DIR}/device/*.cpp")
+    set(skel_srcs "${CMAKE_CURRENT_BINARY_DIR}/npu_device_skel.c")
+    add_library(hexagon_npu_skel_OBJS OBJECT
+        ${common_srcs}
+        ${device_srcs}
+        ${skel_srcs}
+    )
+
+    if(CMAKE_BUILD_TYPE MATCHES "Debug|Dbg")
+        message("Debug build, enable all logging")
+        target_compile_definitions(hexagon_npu_skel_OBJS PUBLIC
+            _DEBUG
+            DEBUG_LOGGING
+        )
+    else()
+        message("Release build, disable debug logging")
+        target_compile_definitions(hexagon_npu_skel_OBJS PUBLIC
+            NDEBUG
+            RELEASE_LOGGING
+        )
+    endif()
+
+    build_idl(idl/hexagon_npu.idl hexagon_npu_skel_OBJS)
+
+    # disable warnings for the skel
+    set_source_files_properties(
+        ${skel_srcs}
+        PROPERTIES
+        COMPILE_FLAGS "-w"
+    )
+
+    add_library(hexagon_npu_skel SHARED $<TARGET_OBJECTS:hexagon_npu_skel_OBJS>)
+
+    target_link_libraries(hexagon_npu_skel
+        ${HEXAGON_LIB_DIR}/${HEXAGON_ARCH}/G0/pic/libc++abi.a
+        ${HEXAGON_LIB_DIR}/${HEXAGON_ARCH}/G0/pic/libc++.a
+    )
+    set_target_properties(hexagon_npu_skel PROPERTIES OUTPUT_NAME "hexagon_npu_skel_${HEXAGON_ARCH}")
+
+    copy_binaries(hexagon_npu_skel)
+endif()
+
+# vim: set noet fenc=utf-8 ff=unix ft=cmake :