PoC: Add Qualcomm mobile SoC native backend for GGML

.gitignore

@@ -71,3 +71,5 @@ prebuilts/toolchain/emsdk/
 prebuilts/toolchain/*.zip
 prebuilts/toolchain/*.xz
 prebuilts/toolchain/*.gz
+
+crash.log

cdeosplayer/cdeosplayer-lib/src/main/java/cdeos/media/player/CDEUtils.java

@@ -260,18 +260,25 @@ public class CDEUtils {
      public static final int  ASR_MODE_TRANSCRIPTION_RECORD = 3; // transcription + audio record
 
      //keep sync with ggml-jni.h
-     public static final int BECHMARK_ASR      = 0;
-     public static final int BECHMARK_MEMCPY   = 1;
-     public static final int BECHMARK_MULMAT   = 2;
-     public static final int BECHMARK_FULL     = 3;
-     public static final int BENCHMARK_MATRIX  = 4;
-     public static final int BENCHMARK_LLM     = 5;
-     public static final int BENCHMARK_STABLEDIFFUSION= 6;
-     public static final int BENCHMARK_QNN_SAMPLE     = 7;
-     public static final int BENCHMARK_QNN_SAVER      = 8;
-     public static final int BENCHMARK_QNN_MATRIX     = 9;
-     public static final int BENCHMARK_QNN_GGML       = 10;
-     public static final int BENCHMAKR_QNN_COMPLEX    = 11;
+     public static final int BENCHMARK_ASR      = 0;
+     public static final int BENCHMARK_MEMCPY   = 1;
+     public static final int BENCHMARK_MULMAT   = 2;
+     public static final int BENCHMARK_FULL     = 3;
+     public static final int BENCHMARK_LLM     = 4;
+     public static final int BENCHMARK_STABLEDIFFUSION= 5;
+     public static final int BENCHMARK_QNN_SAMPLE     = 6;
+     public static final int BENCHMARK_QNN_SAVER      = 7;
+     public static final int BENCHMARK_QNN_MATRIX     = 8;
+     public static final int BENCHMARK_QNN_GGML       = 9;
+     public static final int BENCHMARK_QNN_COMPLEX    = 10;
+     public static final int BENCHMARK_QNN_GGML_OP    = 11;
+     public static final int BENCHMARK_QNN_AUTO_UT    = 12;
+
+     //keep sync with ggml-qnn.h
+     public static final int QNN_BACKEND_CPU           = 0;
+     public static final int QNN_BACKEND_GPU           = 1;
+     public static final int QNN_BACKEND_HTP           = 2;
+     public static final int QNN_BACKEND_GGML          = 3; //"fake" QNN backend, just for compare performance between QNN and original GGML
 
 
      private static int       mASRMode = ASR_MODE_NORMAL;
@@ -3909,20 +3916,17 @@ public void onClick(DialogInterface dialog, int which) {
 
      public static String getBenchmarkDesc(int benchmarkIndex) {
          switch (benchmarkIndex) {
-             case BECHMARK_FULL:
-                 return "GGML whisper_encode";
+             case BENCHMARK_FULL:
+                 return "GGML whisper full";
 
-             case BECHMARK_MEMCPY:
-                 return "GGML memcopy";
+             case BENCHMARK_MEMCPY:
+                 return "GGML memcpy";
 
-             case BECHMARK_MULMAT:
+             case BENCHMARK_MULMAT:
                  return "GGML matrix multiply";
 
-             case BECHMARK_ASR:
-                 return "GGML ASR inference";
-
-             case BENCHMARK_MATRIX:
-                 return "GGML matrix";
+             case BENCHMARK_ASR:
+                 return "GGML whisper ASR";
 
              case BENCHMARK_LLM:
                  return "GGML LLAMA";
@@ -3936,20 +3940,53 @@ public static String getBenchmarkDesc(int benchmarkIndex) {
              case BENCHMARK_QNN_SAVER:
                  return "GGML QNN saver";
 
+
              case BENCHMARK_QNN_MATRIX:
-                 return "GGML QNN matrix manipulate";
+                 return "GGML QNN matrix addition";
 
              case BENCHMARK_QNN_GGML:
-                 return "GGML QNN ggml";
+                 return "GGML QNN mapping ggml tensor";
 
-             case BENCHMAKR_QNN_COMPLEX:
+             case BENCHMARK_QNN_COMPLEX:
                  return "GGML QNN complex graph";
+
+             case BENCHMARK_QNN_GGML_OP:
+                 return "GGML QNN OP UT"; //UT for PoC-S49: implementation of GGML OPs using QNN API
+
+             case BENCHMARK_QNN_AUTO_UT:
+                 return "GGML QNN OP UT automation"; //automation UT for PoC-S49: implementation of GGML OPs using QNN API
          }
 
          return "unknown";
      }
 
 
+     //keep sync with ggml-qnn.cpp
+     //QNN cDSP and HTA backend would not be used currently, just focus on QNN CPU/GPU/HTP(aka DSP) backend currently
+     public static String getBackendDesc(int n_backend_type) {
+         switch (n_backend_type) {
+             case 0:
+                 return "QNN-CPU";
+             case 1:
+                 return "QNN-GPU";
+             case 2:
+                 return "QNN-HTP(DSP)";
+             case 3:
+                 return "ggml";      //fake QNN backend, just used to compare performance between QNN and original GGML
+
+/*
+             case 3:
+                 return "QNN-cDSP";
+             case 4:
+                 return "QNN-HTA";
+*/
+
+             default:
+                 return "unknown";
+         }
+     }
+
+
      public static String getGGMLModeString(int ggmlModeType) {
          switch (ggmlModeType) {
              case 0:

cdeosplayer/cdeosplayer-lib/src/main/java/org/ggml/ggmljava.java

@@ -98,15 +98,16 @@ public enum ggml_op {
         GGML_OP_COUNT,
     };
 
-    public static native int  asr_init(String strModelPath, int nThreadCounts, int nASRMode);
+
+    public static native int  asr_init(String strModelPath, int nThreadCounts, int nASRMode, int nBackendType);
 
     public static native void asr_finalize();
 
     public static native void asr_start();
 
     public static native void asr_stop();
 
-    public static native int  asr_reset(String strModelPath, int nThreadCounts, int nASRMode);
+    public static native int  asr_reset(String strModelPath, int nThreadCounts, int nASRMode, int nBackendType);
 
     public static native String asr_get_systeminfo();
 
@@ -118,7 +119,7 @@ public enum ggml_op {
     /**
      * @param modelPath     /sdcard/kantv/ggml-xxxxxx.bin or  /sdcard/kantv/xxxxxx.gguf or qualcomm's prebuilt dedicated model.so or ""
      * @param audioPath     /sdcard/kantv/jfk.wav
-     * @param nBenchType    0: asr(transcription) 1: memcpy 2: mulmat  3: full/whisper_encode 4: matrix  5: LLAMA  6: stable diffusion 7: QNN sample 8: QNN saver 9: QNN matrix 10: QNN GGML 11: QNN complex
+     * @param nBenchType    0: whisper asr 1: memcpy 2: mulmat  3: whisper full 4: LLAMA 5: stable diffusion 6: QNN sample 7: QNN saver 8: QNN matrix 9: QNN GGML 10: QNN complex 11: QNN GGML OP(QNN UT) 12: QNN UT automation
      * @param nThreadCounts 1 - 8
      * @param nBackendType  0: CPU  1: GPU  2: DSP 3: ggml("fake" QNN backend, just for compare performance)
      * @param nOpType       type of matrix manipulate / GGML OP / type of various complex/complicated computation graph
@@ -130,5 +131,5 @@ public enum ggml_op {
     public static native String llm_get_systeminfo();
 
 
-    public static native String llm_inference(String modelPath, String prompt, int nBenchType, int nThreadCounts);
+    public static native String llm_inference(String modelPath, String prompt, int nBenchType, int nThreadCounts, int nBackendType);
 }

cdeosplayer/kantv/src/main/java/com/cdeos/kantv/app/IApplication.java

@@ -243,6 +243,28 @@ public void initGlobal() {
         CDEAssetLoader.copyAssetFile(mContext, ggmlModelFileName, CDEUtils.getDataPath() + ggmlModelFileName);
         CDEAssetLoader.copyAssetFile(mContext, ggmlSampleFileName, CDEUtils.getDataPath() + ggmlSampleFileName);
 
+
+        //for PoC:Add Qualcomm mobile SoC native backend for GGML, https://github.com/zhouwg/kantv/issues/121
+        CDEAssetLoader.copyAssetFile(mContext, "libInception_v3.so", CDEUtils.getDataPath(mContext) + "libInception_v3.so");
+        //qualcomm's prebuilt QNN userspace library
+        CDEAssetLoader.copyAssetFile(mContext, "libQnnCpu.so", CDEUtils.getDataPath(mContext) + "libQnnCpu.so");
+        CDEAssetLoader.copyAssetFile(mContext, "libQnnGpu.so", CDEUtils.getDataPath(mContext) + "libQnnGpu.so");
+        CDEAssetLoader.copyAssetFile(mContext, "libQnnDsp.so", CDEUtils.getDataPath(mContext) + "libQnnDsp.so");
+        CDEAssetLoader.copyAssetFile(mContext, "libQnnHtp.so", CDEUtils.getDataPath(mContext) + "libQnnHtp.so");
+        CDEAssetLoader.copyAssetFile(mContext, "libQnnHtpNetRunExtensions.so", CDEUtils.getDataPath(mContext) + "libQnnHtpNetRunExtensions.so");
+        CDEAssetLoader.copyAssetFile(mContext, "libQnnHtpPrepare.so", CDEUtils.getDataPath(mContext) + "libQnnHtpPrepare.so");
+        CDEAssetLoader.copyAssetFile(mContext, "libQnnHtpV75Stub.so", CDEUtils.getDataPath(mContext) + "libQnnHtpV75Stub.so");
+        CDEAssetLoader.copyAssetFile(mContext, "libQnnHtpV75Skel.so", CDEUtils.getDataPath(mContext) + "libQnnHtpV75Skel.so");
+        CDEAssetLoader.copyAssetFile(mContext, "libQnnSystem.so", CDEUtils.getDataPath(mContext) + "libQnnSystem.so");
+        CDEAssetLoader.copyAssetFile(mContext, "libQnnSaver.so", CDEUtils.getDataPath(mContext) + "libQnnSaver.so");
+        CDEAssetLoader.copyAssetFile(mContext, "params.bin", CDEUtils.getDataPath() + "params.bin");
+        //qualcomm's prebuilt binary file
+        CDEAssetLoader.copyAssetFile(mContext, "raw_list.txt", CDEUtils.getDataPath() + "raw_list.txt");
+        CDEAssetLoader.copyAssetDir(mContext, "data", CDEUtils.getDataPath() + "data");
+        //prebuilt data from https://github.com/karpathy/llm.c/blob/master/doc/layernorm/layernorm.md
+        CDEAssetLoader.copyAssetFile(mContext, "ln.bin", CDEUtils.getDataPath() + "ln.bin");
+
+
         CDEAssetLoader.copyAssetFile(mContext, "config.json", CDEAssetLoader.getDataPath(mContext) + "config.json");
         String configString = CDEAssetLoader.readTextFromFile(CDEAssetLoader.getDataPath(mContext) + "config.json");
         JSONObject jsonObject = JSON.parseObject(configString);
@@ -373,9 +395,9 @@ public void initGlobal() {
             CDELog.d(TAG, "cpu core counts:" + ggmljava.get_cpu_core_counts());
             CDELog.j(TAG, "asr mode: " + mSettings.getASRMode());
             if ((CDEUtils.ASR_MODE_NORMAL == mSettings.getASRMode()) || (CDEUtils.ASR_MODE_TRANSCRIPTION_RECORD == mSettings.getASRMode())) {
-                result = ggmljava.asr_init(modelPath, mSettings.getASRThreadCounts(), CDEUtils.ASR_MODE_NORMAL);
+                result = ggmljava.asr_init(modelPath, mSettings.getASRThreadCounts(), CDEUtils.ASR_MODE_NORMAL, CDEUtils.QNN_BACKEND_GGML);
             } else {
-                result = ggmljava.asr_init(modelPath, mSettings.getASRThreadCounts(), CDEUtils.ASR_MODE_PRESURETEST);
+                result = ggmljava.asr_init(modelPath, mSettings.getASRThreadCounts(), CDEUtils.ASR_MODE_PRESURETEST, CDEUtils.QNN_BACKEND_GGML);
             }
             CDEUtils.setASRConfig("whispercpp", modelPath, asrThreadCounts + 1, asrMode);
             CDEUtils.setTVASR(false);

cdeosplayer/kantv/src/main/java/com/cdeos/kantv/player/ffplayer/FFPlayerView.java

@@ -1577,9 +1577,9 @@ private void onASRStart(int asrMode) {
 
         if (CDEUtils.getASRSubsystemInit()) {
             if ((CDEUtils.ASR_MODE_NORMAL == mSettings.getASRMode()) || (CDEUtils.ASR_MODE_TRANSCRIPTION_RECORD == mSettings.getASRMode())) {
-                ggmljava.asr_reset(CDEUtils.getDataPath() + ggmlModelFileName, mSettings.getASRThreadCounts(), CDEUtils.ASR_MODE_NORMAL);
+                ggmljava.asr_reset(CDEUtils.getDataPath() + ggmlModelFileName, mSettings.getASRThreadCounts(), CDEUtils.ASR_MODE_NORMAL, CDEUtils.QNN_BACKEND_GGML);
             } else {
-                ggmljava.asr_reset(CDEUtils.getDataPath() + ggmlModelFileName, mSettings.getASRThreadCounts(), CDEUtils.ASR_MODE_PRESURETEST);
+                ggmljava.asr_reset(CDEUtils.getDataPath() + ggmlModelFileName, mSettings.getASRThreadCounts(), CDEUtils.ASR_MODE_PRESURETEST, CDEUtils.QNN_BACKEND_GGML);
             }
             ggmljava.asr_start();
         } else {