Make LLaVA fast again

Mozilla-Ocho · Nov 10, 2024 · 54d3c72 · 54d3c72
1 parent 9d6f89f
commit 54d3c72
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 10 deletions.
diff --git a/llama.cpp/llava/clip.cpp b/llama.cpp/llava/clip.cpp
@@ -1114,15 +1114,19 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) {
         }
     }
 
-#ifdef GGML_USE_CUDA
-    new_clip->backend = ggml_backend_cuda_init(0);
-    LOG_TEE("%s: CLIP using CUDA backend\n", __func__);
-#endif
+// #ifdef GGML_USE_CUDA
+    if (!new_clip->backend) { // [jart]
+        if ((new_clip->backend = ggml_backend_cuda_init(0)))
+            LOG_TEE("%s: CLIP using CUDA backend\n", __func__);
+    }
+// #endif
 
-#ifdef GGML_USE_METAL
-    new_clip->backend = ggml_backend_metal_init();
-    LOG_TEE("%s: CLIP using Metal backend\n", __func__);
-#endif
+// #ifdef GGML_USE_METAL
+    if (!new_clip->backend) { // [jart]
+        if ((new_clip->backend = ggml_backend_metal_init()))
+            LOG_TEE("%s: CLIP using Metal backend\n", __func__);
+    }
+// #endif
 
 #ifdef GGML_USE_CANN
     new_clip->backend = ggml_backend_cann_init(0);

diff --git a/llama.cpp/llava/llava-cli.cpp b/llama.cpp/llava/llava-cli.cpp
@@ -143,14 +143,14 @@ static struct llava_image_embed * load_image(llava_context * ctx_llava, gpt_para
         if (!params->image.empty()) {
             LOG_TEE("using base64 encoded image instead of command line image path\n");
         }
-        embed = llava_image_embed_make_with_prompt_base64(ctx_llava->ctx_clip, params->n_threads, prompt);
+        embed = llava_image_embed_make_with_prompt_base64(ctx_llava->ctx_clip, params->n_threads_batch, prompt); // [jart] batch
         if (!embed) {
             LOG_TEE("%s: can't load image from prompt\n", __func__);
             return NULL;
         }
         params->prompt = remove_image_from_prompt(prompt);
     } else {
-        embed = llava_image_embed_make_with_filename(ctx_llava->ctx_clip, params->n_threads, fname.c_str());
+        embed = llava_image_embed_make_with_filename(ctx_llava->ctx_clip, params->n_threads_batch, fname.c_str()); // [jart] batch
         if (!embed) {
             fprintf(stderr, "%s: is %s really an image file?\n", __func__, fname.c_str());
             return NULL;