ggml-qnn: pr to upstream

Author: zhouwg

cmake/aarch64-w64-mingw32.cmake

@@ -9,8 +9,8 @@ set( CMAKE_CXX_COMPILER  clang++ )
 set( CMAKE_C_COMPILER_TARGET   ${target} )
 set( CMAKE_CXX_COMPILER_TARGET ${target} )
 
-#set( arch_c_flags "-march=armv8.7-a -fvectorize -ffp-model=fast -fno-finite-math-only" )
-#set( warn_c_flags "-Wno-format -Wno-unused-variable -Wno-unused-function -Wno-gnu-zero-variadic-macro-arguments" )
+set( arch_c_flags "-march=armv8.7-a -fvectorize -ffp-model=fast -fno-finite-math-only" )
+set( warn_c_flags "-Wno-format -Wno-unused-variable -Wno-unused-function -Wno-gnu-zero-variadic-macro-arguments" )
 
 set( CMAKE_C_FLAGS_INIT   "${arch_c_flags} ${warn_c_flags}" )
 set( CMAKE_CXX_FLAGS_INIT "${arch_c_flags} ${warn_c_flags}" )

cmake/arm64-windows-cygwin.cmake

@@ -14,7 +14,7 @@ GGUF_MODEL_NAME=/sdcard/qwen1_5-1_8b-chat-q4_0.gguf
 #https://www.qualcomm.com/developer/software/qualcomm-ai-engine-direct-sdk
 #https://developer.qualcomm.com/software/hexagon-dsp-sdk/tools
 QNN_SDK_URL=https://www.qualcomm.com/developer/software/qualcomm-ai-engine-direct-sdk
-QNN_SDK_PATH=/opt/qcom/aistack/qairt/2.31.0.250130/
+QNN_SDK_PATH=/opt/qcom/aistack/qairt/2.32.0.250228/
 
 #default is QNN NPU
 qnnbackend=2
@@ -97,11 +97,14 @@ function check_qnn_libs()
 {
     #reuse the cached qnn libs on Android phone
     adb shell ls ${REMOTE_PATH}/libQnnCpu.so
+    adb shell ls ${REMOTE_PATH}/libQnnGpu.so
+    adb shell ls ${REMOTE_PATH}/libQnnHtp.so
     if [ $? -eq 0 ]; then
         printf "QNN libs already exist on Android phone\n"
     else
         update_qnn_libs
     fi
+    update_qnn_cfg
 }
 
 
@@ -119,6 +122,12 @@ function update_qnn_libs()
 }
 
 
+function update_qnn_cfg()
+{
+    adb push ./scripts/ggml-qnn.cfg ${REMOTE_PATH}/
+}
+
+
 function build_ggml_qnn()
 {
     show_pwd

cmake/arm64-windows-llvm.cmake

@@ -0,0 +1,9 @@
+[general]
+# enable/disable QNN's internal log
+print_qnn_internal_log = 0
+# 0: general approach,similar to ggml-sycl or ggml-cann
+# 1: mapping entire ggml cgraph to QNN graph
+inference_approach = 0
+
+[npu]
+npu_inference_datatype = "fp16"

ggml/src/ggml-qnn/ggml-qnn.cpp

@@ -14,10 +14,6 @@
 #include "ggml-backend.h"
 #include "ggml-cpp.h"
 
-#ifdef GGML_USE_QNN
-#include "ggml-qnn.h"
-#endif
-
 #include <algorithm>
 #include <array>
 #include <cassert>
@@ -9714,19 +9710,8 @@ struct llama_context * llama_init_from_model(
         // add ACCEL backends (such as BLAS)
         for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {
             ggml_backend_dev_t dev = ggml_backend_dev_get(i);
-
-#ifdef GGML_USE_QNN // avoid side-effect to other backends
-            if (QNN_BACKEND_GGML == model->params.main_gpu) {
-                break;
-            }
-#endif
             if (ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_ACCEL) {
-                ggml_backend_t backend = nullptr;
-#ifndef GGML_USE_QNN
-                backend = ggml_backend_dev_init(dev, nullptr);
-#else
-                backend = ggml_backend_dev_init(dev, reinterpret_cast<const char *>(model->params.main_gpu));
-#endif
+                ggml_backend_t backend = ggml_backend_dev_init(dev, nullptr);
                 if (backend == nullptr) {
                     LLAMA_LOG_ERROR("%s: failed to initialize %s backend\n", __func__, ggml_backend_dev_name(dev));
                     llama_free(ctx);

scripts/build-run-android.sh

@@ -137,7 +137,6 @@ llama_target_and_test(test-chat-template.cpp)
 # llama_target_and_test(test-opt.cpp) # SLOW
 llama_target_and_test(test-gguf.cpp)
 llama_target_and_test(test-backend-ops.cpp)
-llama_target_and_test(ggml-qnn-ut.cpp)
 
 llama_target_and_test(test-model-load-cancel.cpp  LABEL "model")
 llama_target_and_test(test-autorelease.cpp        LABEL "model")