examples : support MiniCPM-V-2

examples/minicpmv2.0/CMakeLists.txt

@@ -0,0 +1,38 @@
+add_library(minicpmv OBJECT
+            minicpmv.cpp
+            minicpmv.h
+            clip.cpp
+            clip.h
+            )
+
+target_link_libraries(minicpmv PRIVATE ggml llama ${CMAKE_THREAD_LIBS_INIT})
+
+target_include_directories(minicpmv PUBLIC .)
+target_include_directories(minicpmv PUBLIC ../..)
+target_include_directories(minicpmv PUBLIC ../../common)
+
+target_compile_features(minicpmv PRIVATE cxx_std_17)
+
+add_library(minicpmv_static STATIC $<TARGET_OBJECTS:minicpmv>)
+if (BUILD_SHARED_LIBS)
+    set_target_properties(minicpmv PROPERTIES POSITION_INDEPENDENT_CODE ON)
+    target_compile_definitions(minicpmv PRIVATE LLAMA_SHARED LLAMA_BUILD)
+    add_library(minicpmv_shared SHARED $<TARGET_OBJECTS:minicpmv>)
+    target_link_libraries(minicpmv_shared PRIVATE ggml llama ${CMAKE_THREAD_LIBS_INIT})
+    install(TARGETS minicpmv_shared LIBRARY)
+endif()
+
+if (NOT MSVC)
+    target_compile_options(minicpmv PRIVATE -Wno-cast-qual) # stb_image.h
+endif()
+
+if(TARGET BUILD_INFO)
+    add_dependencies(minicpmv BUILD_INFO)
+endif()
+
+set(TARGET minicpmv-cli)
+add_executable(${TARGET} minicpmv-cli.cpp)
+set_target_properties(${TARGET} PROPERTIES OUTPUT_NAME minicpmv-cli)
+install(TARGETS ${TARGET} RUNTIME)
+target_link_libraries(${TARGET} PRIVATE common minicpmv ${CMAKE_THREAD_LIBS_INIT})
+target_compile_features(minicpmv PRIVATE cxx_std_17)

examples/minicpmv2.0/README.md

@@ -0,0 +1,16 @@
+# 所有命令在 llama.cpp 根目录执行，模型位于根目录上级目录处
+# All command should be executed under the root path of llama.cpp repo. We assume the MiniCPM-V-2 model are put in its parent folder.
+
+```bash
+make
+make minicpmv-cli
+
+python ./examples/minicpmv2.0/minicpm-surgery.py -m ../MiniCPM-V-2
+python ./examples/minicpmv2.0/convert-image-encoder-to-gguf.py -m ../MiniCPM-V-2 --llava-projector ../MiniCPM-V-2/llava.projector --output-dir ../MiniCPM-V-2 --image-mean 0.5 0.5 0.5 --image-std 0.5 0.5 0.5
+python ./convert-hf-to-gguf.py ../MiniCPM-V-2/model
+./minicpmv-cli -m ../MiniCPM-V-2/model/ggml-model-f16.gguf --mmproj ../MiniCPM-V-2/mmproj-model-f16.gguf -c 4096 --temp 0.6 --top-p 0.8 --top-k 100 --repeat-penalty 1.0 --image ../test.jpg -p "这张图里有什么?"
+
+# or run quantize int4 version
+./quantize ../MiniCPM-V-2/model/ggml-model-f16.gguf ../MiniCPM-V-2/model/ggml-model-Q4_K_M.gguf Q4_K_M
+./minicpmv-cli -m ../MiniCPM-V-2/model/ggml-model-Q4_K_M.gguf --mmproj ../MiniCPM-V-2/mmproj-model-f16.gguf -c 4096 --temp 0.6 --top-p 0.8 --top-k 100 --repeat-penalty 1.0 --image ../test.jpg -p "这张图里有什么?"
+```

examples/minicpmv2.0/android/adb_run.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+
+model_dir="/Users/cxt/model/llm/mobileVLM/MobileVLM-1.7B_processed"
+projector_name="mmproj-model-f16.gguf"
+llama_name="ggml-model-q4_k.gguf"
+img_dir="/Users/cxt/model/llm"
+img_name="demo.jpg"
+prompt="A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions. USER: <image>\nWho is the author of this book? \nAnswer the question using a single word or phrase. ASSISTANT:"
+# img_name="cat.jpeg"
+# prompt="A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions. USER: <image>\nWhat is in the image? ASSISTANT:"
+
+program_dir="build_64/bin"
+binName="minicpmv-cli"
+n_threads=4
+
+
+deviceDir="/data/local/tmp"
+saveDir="output"
+if [ ! -d ${saveDir} ]; then
+    mkdir ${saveDir}
+fi
+
+
+function android_run() {
+    # # copy resource into device
+    # adb push ${model_dir}/${projector_name} ${deviceDir}/${projector_name}
+    # adb push ${model_dir}/${llama_name} ${deviceDir}/${llama_name}
+    adb push ${img_dir}/${img_name} ${deviceDir}/${img_name}
+    # copy program into device
+    adb push ${program_dir}/${binName} ${deviceDir}/${binName}
+    adb shell "chmod 0777 ${deviceDir}/${binName}"
+
+    # run
+    adb shell "echo cd ${deviceDir} ${deviceDir}/${binName} \
+                                                 -m ${deviceDir}/${llama_name} \
+                                                 --mmproj ${deviceDir}/${projector_name} \
+                                                 -t ${n_threads} \
+                                                 --image ${deviceDir}/${img_name} \
+                                                 -p \"${prompt}\" \
+                                                 > ${deviceDir}/${modelName}_${projector_name}_${n_threads}_${img_name}.txt"
+    adb shell "cd ${deviceDir}; pwd; ${deviceDir}/${binName} \
+                                                 -m ${deviceDir}/${llama_name} \
+                                                 --mmproj ${deviceDir}/${projector_name} \
+                                                 -t ${n_threads} \
+                                                 --image ${deviceDir}/${img_name} \
+                                                 -p \"${prompt}\" \
+                                                 >> ${deviceDir}/${modelName}_${projector_name}_${n_threads}_${img_name}.txt 2>&1"
+    adb pull ${deviceDir}/${modelName}_${projector_name}_${n_threads}_${img_name}.txt ${saveDir}
+}
+
+android_run
+
+echo "android_run is Done!"