From fa4c5829d5cb230a12e21427eed9ac9d0da2bedc Mon Sep 17 00:00:00 2001
From: Ettore Di Giacinto <mudler@localai.io>
Date: Mon, 11 Mar 2024 19:44:18 +0100
Subject: [PATCH] fix(doc/examples): set defaults to mirostat

The default sampler on some models don't return enough candidates which
leads to a false sense of randomness. Tracing back the code it looks
that with the temperature sampler there might not be enough
candidates to pick from, and since the seed and "randomness" take effect
while picking a good candidate this yields to the same results over and
over.

Fixes https://github.com/mudler/LocalAI/issues/1723 by updating the
examples and documentation to use mirostat instead.
---
 docs/content/docs/advanced/advanced-usage.md  | 7 ++++---
 embedded/models/codellama-7b-gguf.yaml        | 6 +++++-
 embedded/models/dolphin-2.5-mixtral-8x7b.yaml | 5 ++++-
 embedded/models/llava.yaml                    | 3 +++
 embedded/models/mistral-openorca.yaml         | 4 ++++
 embedded/models/mixtral-instruct.yaml         | 4 ++++
 examples/configurations/phi-2.yaml            | 4 ++++
 7 files changed, 28 insertions(+), 5 deletions(-)

diff --git a/docs/content/docs/advanced/advanced-usage.md b/docs/content/docs/advanced/advanced-usage.md
index a892cc3690a3..c9926bab2761 100644
--- a/docs/content/docs/advanced/advanced-usage.md
+++ b/docs/content/docs/advanced/advanced-usage.md
@@ -130,13 +130,14 @@ parameters:
   typical_p:
   tfz:
   frequency_penalty:
-  mirostat_eta:
-  mirostat_tau:
-  mirostat: 
+
   rope_freq_base:
   rope_freq_scale:
   negative_prompt_scale:
 
+mirostat_eta:
+mirostat_tau:
+mirostat: 
 # Default context size
 context_size: 512
 # Default number of threads
diff --git a/embedded/models/codellama-7b-gguf.yaml b/embedded/models/codellama-7b-gguf.yaml
index d2a6b5184c83..413c838b7b00 100644
--- a/embedded/models/codellama-7b-gguf.yaml
+++ b/embedded/models/codellama-7b-gguf.yaml
@@ -2,10 +2,14 @@ name: codellama-7b-gguf
 backend: transformers
 parameters:
   model: huggingface://TheBloke/CodeLlama-7B-GGUF/codellama-7b.Q4_K_M.gguf
-  temperature: 0.2
+  temperature: 0.5
   top_k: 40
   seed: -1
   top_p: 0.95
+mirostat: 2
+mirostat_eta: 1.0
+mirostat_tau: 1.0
+
 context_size: 4096
 f16: true
 gpu_layers: 90
diff --git a/embedded/models/dolphin-2.5-mixtral-8x7b.yaml b/embedded/models/dolphin-2.5-mixtral-8x7b.yaml
index ba6020c0856d..12ee1efc2600 100644
--- a/embedded/models/dolphin-2.5-mixtral-8x7b.yaml
+++ b/embedded/models/dolphin-2.5-mixtral-8x7b.yaml
@@ -2,10 +2,13 @@ name: dolphin-mixtral-8x7b
 mmap: true
 parameters:
   model: huggingface://TheBloke/dolphin-2.5-mixtral-8x7b-GGUF/dolphin-2.5-mixtral-8x7b.Q2_K.gguf
-  temperature: 0.2
+  temperature: 0.5
   top_k: 40
   top_p: 0.95
   seed: -1
+mirostat: 2
+mirostat_eta: 1.0
+mirostat_tau: 1.0
 template:
   chat_message: |
     <|im_start|>{{if eq .RoleName "assistant"}}assistant{{else if eq .RoleName "system"}}system{{else if eq .RoleName "user"}}user{{end}}
diff --git a/embedded/models/llava.yaml b/embedded/models/llava.yaml
index 2e571f212e28..3d240681f175 100644
--- a/embedded/models/llava.yaml
+++ b/embedded/models/llava.yaml
@@ -18,6 +18,9 @@ parameters:
   top_k: 40
   top_p: 0.95
   seed: -1
+mirostat: 2
+mirostat_eta: 1.0
+mirostat_tau: 1.0
 
 template:
   chat: |
diff --git a/embedded/models/mistral-openorca.yaml b/embedded/models/mistral-openorca.yaml
index f40d854f72f1..0794a69b0a20 100644
--- a/embedded/models/mistral-openorca.yaml
+++ b/embedded/models/mistral-openorca.yaml
@@ -6,6 +6,10 @@ parameters:
   top_k: 40
   top_p: 0.95
   seed: -1
+mirostat: 2
+mirostat_eta: 1.0
+mirostat_tau: 1.0
+
 template:
   chat_message: |
     <|im_start|>{{if eq .RoleName "assistant"}}assistant{{else if eq .RoleName "system"}}system{{else if eq .RoleName "user"}}user{{end}}
diff --git a/embedded/models/mixtral-instruct.yaml b/embedded/models/mixtral-instruct.yaml
index 3272557a7177..246b23240f48 100644
--- a/embedded/models/mixtral-instruct.yaml
+++ b/embedded/models/mixtral-instruct.yaml
@@ -6,6 +6,10 @@ parameters:
   top_k: 40
   seed: -1
   top_p: 0.95
+mirostat: 2
+mirostat_eta: 1.0
+mirostat_tau: 1.0
+
 template:
   chat: &chat |
     [INST] {{.Input}} [/INST]    
diff --git a/examples/configurations/phi-2.yaml b/examples/configurations/phi-2.yaml
index cac1e9da9f5d..e5a13442d180 100644
--- a/examples/configurations/phi-2.yaml
+++ b/examples/configurations/phi-2.yaml
@@ -11,6 +11,10 @@ parameters:
   top_k: 40
   top_p: 0.95
   seed: -1
+  
+mirostat: 2
+mirostat_eta: 1.0
+mirostat_tau: 1.0
 template:
   chat: &template |-
     Instruct: {{.Input}}