Speed up KV in llamafile-bench

Mozilla-Ocho · Aug 29, 2024 · c7c4d65 · c7c4d65 · Djip007 · Aug 31, 2024
1 parent d2be732
commit c7c4d65
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/llama.cpp/llama-bench/llama-bench.cpp b/llama.cpp/llama-bench/llama-bench.cpp
@@ -265,8 +265,8 @@ static const cmd_params cmd_params_defaults = {
     /* n_pg          */ {},
     /* n_batch       */ {2048},
     /* n_ubatch      */ {512},
-    /* type_k        */ {GGML_TYPE_F16},
-    /* type_v        */ {GGML_TYPE_F16},
+    /* type_k        */ {X86_HAVE(AVX512_BF16) ? GGML_TYPE_BF16 : GGML_TYPE_F16},
+    /* type_v        */ {X86_HAVE(AVX512_BF16) ? GGML_TYPE_BF16 : GGML_TYPE_F16},
     /* n_threads     */ {cpu_get_num_math()},
     /* n_gpu_layers  */ {0},
     /* split_mode    */ {LLAMA_SPLIT_MODE_LAYER},