vocab : fix string init

ggerganov · slaren · web-flow · commit 3016faa8c7d8 · 2025-01-09T17:45:47.000+02:00
Co-authored-by: Diego Devesa &lt;slarengh@gmail.com&gt;
diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp
@@ -2456,7 +2456,7 @@ std::vector<llama_token> llama_vocab::tokenize(
 
                         // prefix with space if previous is special
                         if (tokenizer_add_space_prefix && is_prev_special) {
-                            text += ' ';
+                            text = ' ';
                         }
 
                         text += fragment.raw_text.substr(fragment.offset, fragment.length);

Original file line number	Diff line number	Diff line change
`@@ -2456,7 +2456,7 @@ std::vector<llama_token> llama_vocab::tokenize(`
`2456`	`2456`
`2457`	`2457`	`// prefix with space if previous is special`
`2458`	`2458`	`if (tokenizer_add_space_prefix && is_prev_special) {`
`2459`		`- text += ' ';`
	`2459`	`+ text = ' ';`
`2460`	`2460`	`}`
`2461`	`2461`
`2462`	`2462`	`text += fragment.raw_text.substr(fragment.offset, fragment.length);`