We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent 5ae5e2c commit c67e5efCopy full SHA for c67e5ef
utils.cpp
@@ -324,7 +324,7 @@ struct llama_tokenizer {
324
while (!text.empty()) {
325
llama_sp_symbol sym;
326
size_t char_len = utf8_len(text.data()[0]);
327
- sym.text = std::string_view(text.data(), char_len);
+ sym.text = std::string_view(text.data(), std::min(text.size(), char_len));
328
sym.prev = index - 1;
329
text.remove_prefix(char_len);
330
sym.next = text.empty() ? -1 : index + 1;
0 commit comments