dont add space when using special tokens

ggerganov · Oct 12, 2023 · 1c28116 · 1c28116
1 parent 5974d61
commit 1c28116
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/llama.cpp b/llama.cpp
@@ -6727,7 +6727,7 @@ static std::vector<llama_vocab::id> llama_tokenize_internal(const llama_vocab &
                         //  by modifying llm_tokenizer_x to operate with string offsets like pre-tokenizer
                         //  and passing 'add space prefix' as bool argument
                         //
-                        auto raw_text = " " + fragment.raw_text.substr(fragment.offset, fragment.length);
+                        auto raw_text = (special?"":" ") + fragment.raw_text.substr(fragment.offset, fragment.length);
 
 #ifdef PRETOKENIZERDEBUG
                         fprintf(stderr,"TT: (%ld %ld %ld) '%s'\n", raw_text.length(), fragment.offset, fragment.length, raw_text.c_str());