From 2146a253e85ef4c27270909e0de34ea40c22b14d Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Mon, 22 Apr 2024 19:59:45 +0530 Subject: [PATCH 001/217] ChatOn: Capture the idea --- common/CMakeLists.txt | 1 + common/chaton.hpp | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 common/chaton.hpp diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index 0ec8d6d8d03b5..fe865abab708d 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -65,6 +65,7 @@ add_library(${TARGET} STATIC train.cpp ngram-cache.h ngram-cache.cpp + chaton.hpp ) if (BUILD_SHARED_LIBS) diff --git a/common/chaton.hpp b/common/chaton.hpp new file mode 100644 index 0000000000000..cd3679409aea3 --- /dev/null +++ b/common/chaton.hpp @@ -0,0 +1,22 @@ +#pragma once + +/*** + * Keep chatting with model and needed role tagging using special tokens simple and flexible, while building on existing interactive flow + * 1. Use a json file to configure the needed tags for each of the supported chat-handshake-template-standard + * a. system-prefix, system-suffix, + * b. user-prefix, user-suffix, + * c. reverse-prompt + * d. global-begin-marker, global-end-marker + * e. per-msg-begin-marker, per-msg-end-marker + * f. is per-msg-begin-marker used for system+user combo + * 2. Give the below option to user wrt system prompt, this should give the flexibility to either keep system prompt simple or complex in a flexible yet simple way. + * a. the system prompt they specify using -f, is used as is with parse_special when tokenising or + * b. whether the system prefix and suffix is added, but without parse_special tokenisation of system-prompt provided by user. + * 3. chat-apply-template uses the json file, which was loaded, to decide on how to generate the tagged messages for tokenisation + * a. input: [ { role: message }, { role: message}, ....] + * b. output: [ {flag: data}, { flag: data}, {flag: data}, ....] + * * flag is whether to do parse_special for this data, during tokenization or not + * + */ + + From 35f25196a00479df76c712e33be1644c3b35eaa5 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Mon, 22 Apr 2024 20:24:10 +0530 Subject: [PATCH 002/217] ChatOn:Common: Add the needed cmdline arg params and its parsing --- common/common.cpp | 24 ++++++++++++++++++++++++ common/common.h | 3 +++ 2 files changed, 27 insertions(+) diff --git a/common/common.cpp b/common/common.cpp index 467fb014eedb0..a065478a7bdd7 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -915,6 +915,28 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_pa params.chatml = true; return true; } + if (arg == "--chaton-json") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.chaton_json = argv[i]; + return true; + } + if (arg == "--chaton-template-id") { + if (++i >= argc) { + invalid_param = true; + return true; + } + std::string got = argv[i]; + std::regex whitespaces(R"(\s+)"); + std::string trimmed = std::regex_replace(got, whitespaces, ""); + if (!trimmed.empty()) { + params.chaton_template_id = trimmed; + params.chaton = true; + } + return true; + } if (arg == "--infill") { params.infill = true; return true; @@ -1419,6 +1441,8 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) { printf(" --interactive-first run in interactive mode and wait for input right away\n"); printf(" -ins, --instruct run in instruction mode (use with Alpaca models)\n"); printf(" -cml, --chatml run in chatml mode (use with ChatML-compatible models)\n"); + printf(" --chaton-json specify the json file containing chat-handshake-template-standard(s)"); + printf(" --chaton-template-id specify the specific template standard to use from loaded json file"); printf(" --multiline-input allows you to write or paste multiple lines without ending each in '\\'\n"); printf(" -r PROMPT, --reverse-prompt PROMPT\n"); printf(" halt generation at PROMPT, return control in interactive mode\n"); diff --git a/common/common.h b/common/common.h index 9252a4b63889b..f0709d38ab721 100644 --- a/common/common.h +++ b/common/common.h @@ -141,6 +141,9 @@ struct gpt_params { bool use_color = false; // use color to distinguish generations and inputs bool interactive = false; // interactive mode bool chatml = false; // chatml mode (used for models trained on chatml syntax) + bool chaton = false; // whether chaton is enabled or disabled + std::string chaton_json = ""; // name of the json file containing the chaton templates + std::string chaton_template_id = ""; // the specific chat-handshake-template-standard to use bool prompt_cache_all = false; // save user input and generations to prompt cache bool prompt_cache_ro = false; // open the prompt cache read-only and do not update it From dc56be951d63156d0dce91bc9171cd0a3fcece51 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Mon, 22 Apr 2024 20:50:17 +0530 Subject: [PATCH 003/217] ChatOn:Main: Load and dump any specified chaton meta file --- common/chaton.hpp | 30 ++++++++++++++++++++++++++++ examples/chaton_meta.json | 42 +++++++++++++++++++++++++++++++++++++++ examples/main/main.cpp | 6 ++++++ 3 files changed, 78 insertions(+) create mode 100644 examples/chaton_meta.json diff --git a/common/chaton.hpp b/common/chaton.hpp index cd3679409aea3..a9af7a565eb64 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -19,4 +19,34 @@ * */ +#include +#include +#include +#include +#include "log.h" + +using json = nlohmann::json; + +json conMeta; + +inline bool chaton_meta_load(std::string &fname) { + std::ifstream f(fname); + conMeta = json::parse(f); + return true; +} + +inline bool chaton_meta_ok() { + if (conMeta == nullptr) { + return false; + } + return true; +} + +inline void chaton_meta_dump() { + if (!chaton_meta_ok()) { + LOG_TEELN("ERRR:%s:ChatOn Meta: Not loaded yet...", __func__); + return; + } + LOG_TEELN("\n\nINFO:%s:ChatOn Meta\n%s", __func__, conMeta.dump(4).c_str()); +} diff --git a/examples/chaton_meta.json b/examples/chaton_meta.json new file mode 100644 index 0000000000000..77e4595d6ce45 --- /dev/null +++ b/examples/chaton_meta.json @@ -0,0 +1,42 @@ + +{ + "llama2": { + "global": { + "prefix": "[INST] ", + "suffix": " [/INST]", + }, + "system": { + "prefix": " <>\n", + "suffix": "\n<>\n\n", + }, + "user": { + "prefix": "", + "suffix": "", + }, + "assistant": { + "prefix": "", + "suffix": "", + }, + "reverse-prompt": "", + }, + "llama3": { + "global": { + "prefix": "", + "suffix": "", + }, + "system": { + "prefix": "<|start_header_id|>system<|end_header_id|>\n\n", + "suffix": "<|eot_id|>\n\n", + }, + "user": { + "prefix": "<|start_header_id|>user<|end_header_id|>\n\n", + "suffix": "<|eot_id|>\n\n", + }, + "assistant": { + "prefix": "<|start_header_id|>assistant<|end_header_id|>\n\n", + "suffix": "", + }, + "reverse-prompt": "<|eot_id|>", + }, +} + diff --git a/examples/main/main.cpp b/examples/main/main.cpp index eabbc2db38286..3432cb9f1f1a5 100644 --- a/examples/main/main.cpp +++ b/examples/main/main.cpp @@ -1,4 +1,5 @@ #include "common.h" +#include "chaton.hpp" #include "console.h" #include "llama.h" @@ -141,6 +142,11 @@ int main(int argc, char ** argv) { console::init(params.simple_io, params.use_color); atexit([]() { console::cleanup(); }); + if (params.chaton) { + chaton_meta_load(params.chaton_json); + chaton_meta_dump(); + } + if (params.logits_all) { printf("\n************\n"); printf("%s: please use the 'perplexity' tool for perplexity calculations\n", __func__); From 093abc29a2cc0508ef0d145a6fb0a3d0ad6eb562 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Mon, 22 Apr 2024 21:32:09 +0530 Subject: [PATCH 004/217] ChatOn: Update sample meta json to be a valid json --- examples/chaton_meta.json | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/examples/chaton_meta.json b/examples/chaton_meta.json index 77e4595d6ce45..e61f68319971e 100644 --- a/examples/chaton_meta.json +++ b/examples/chaton_meta.json @@ -3,40 +3,40 @@ "llama2": { "global": { "prefix": "[INST] ", - "suffix": " [/INST]", + "suffix": " [/INST]" }, "system": { "prefix": " <>\n", - "suffix": "\n<>\n\n", + "suffix": "\n<>\n\n" }, "user": { "prefix": "", - "suffix": "", + "suffix": "" }, "assistant": { "prefix": "", - "suffix": "", + "suffix": "" }, - "reverse-prompt": "", + "reverse-prompt": "" }, "llama3": { "global": { "prefix": "", - "suffix": "", + "suffix": "" }, "system": { "prefix": "<|start_header_id|>system<|end_header_id|>\n\n", - "suffix": "<|eot_id|>\n\n", + "suffix": "<|eot_id|>\n\n" }, "user": { "prefix": "<|start_header_id|>user<|end_header_id|>\n\n", - "suffix": "<|eot_id|>\n\n", + "suffix": "<|eot_id|>\n\n" }, "assistant": { "prefix": "<|start_header_id|>assistant<|end_header_id|>\n\n", - "suffix": "", + "suffix": "" }, - "reverse-prompt": "<|eot_id|>", - }, + "reverse-prompt": "<|eot_id|>" + } } From 1374a64200d4b281c4c1a6bfbe423902d042b46c Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Mon, 22 Apr 2024 21:59:15 +0530 Subject: [PATCH 005/217] Chaton:Meta: Add chatml meta data to sample meta json file --- examples/chaton_meta.json | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/examples/chaton_meta.json b/examples/chaton_meta.json index e61f68319971e..7b2114f867992 100644 --- a/examples/chaton_meta.json +++ b/examples/chaton_meta.json @@ -37,6 +37,25 @@ "suffix": "" }, "reverse-prompt": "<|eot_id|>" + }, + "chatml": { + "global": { + "prefix": "", + "suffix": "" + }, + "system": { + "prefix": "<|im_start|>system\n", + "suffix": "<|im_end|>\n" + }, + "user": { + "prefix": "\n<|im_start|>user\n", + "suffix": "<|im_end|>\n" + }, + "assistant": { + "prefix": "<|im_start|>assistant\n" + "suffix": "" + }, + "reverse-prompt": "<|im_start|>user\n" } } From 050d329e7e35e3a6b79fa14dff892556ad769d28 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Mon, 22 Apr 2024 22:20:16 +0530 Subject: [PATCH 006/217] ChatOn+Main: Initial go at chaton in main interactive flow --- common/chaton.hpp | 17 +++++++++++++++++ examples/main/main.cpp | 24 +++++++++++++++++++++--- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index a9af7a565eb64..c62ab146ebfc3 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -50,3 +50,20 @@ inline void chaton_meta_dump() { } LOG_TEELN("\n\nINFO:%s:ChatOn Meta\n%s", __func__, conMeta.dump(4).c_str()); } + +inline std::string chaton_tmpl_apply(const std::string &tmpl, const std::string &role, const std::string &content) { + std::stringstream ss; + ss << conMeta[tmpl]["global"]["prefix"]; + ss << conMeta[tmpl][role]["prefix"] << content << conMeta[tmpl][role]["suffix"]; + ss << conMeta[tmpl]["global"]["suffix"]; + std::string taggedStr = ss.str(); + return taggedStr; +} + +inline std::string chaton_tmpl_role_part(const std::string &tmpl, const std::string &role, const std::string &part) { + return conMeta[tmpl][role][part]; +} + +inline std::string chaton_tmpl_part(const std::string &tmpl, const std::string &part) { + return conMeta[tmpl][part]; +} diff --git a/examples/main/main.cpp b/examples/main/main.cpp index 3432cb9f1f1a5..f58888e99bb8f 100644 --- a/examples/main/main.cpp +++ b/examples/main/main.cpp @@ -256,11 +256,14 @@ int main(int argc, char ** argv) { std::vector embd_inp; - if (params.interactive_first || params.instruct || params.chatml || !params.prompt.empty() || session_tokens.empty()) { + if (params.interactive_first || params.instruct || params.chatml || params.chaton || !params.prompt.empty() || session_tokens.empty()) { LOG("tokenize the prompt\n"); if (params.chatml) { params.prompt = "<|im_start|>system\n" + params.prompt + "<|im_end|>"; } + if (params.chaton) { + params.prompt = chaton_tmpl_apply(params.chaton_template_id, "system", params.prompt); + } embd_inp = ::llama_tokenize(ctx, params.prompt, true, true); } else { LOG("use session tokens\n"); @@ -338,7 +341,7 @@ int main(int argc, char ** argv) { } // number of tokens to keep when resetting context - if (params.n_keep < 0 || params.n_keep > (int) embd_inp.size() || params.instruct || params.chatml) { + if (params.n_keep < 0 || params.n_keep > (int) embd_inp.size() || params.instruct || params.chatml || params.chaton) { params.n_keep = (int)embd_inp.size(); } else { params.n_keep += add_bos; // always keep the BOS token @@ -369,6 +372,16 @@ int main(int argc, char ** argv) { params.antiprompt.emplace_back("<|im_start|>user\n"); } + // chaton mode + const auto chaton_assitant_prefix = ::llama_tokenize(ctx, chaton_tmpl_role_part(params.chaton_template_id, "assistant", "prefix"), false, true); + if (params.chaton) { + params.interactive = true; // may remove later, by requiring user to explicitly request interactive mode + params.interactive_first = true; + params.input_prefix = chaton_tmpl_role_part(params.chaton_template_id, "user", "prefix"); + params.input_suffix = chaton_tmpl_role_part(params.chaton_template_id, "user", "suffix"); + params.antiprompt.emplace_back(chaton_tmpl_part(params.chaton_template_id, "reverse-prompt")); + } + // enable interactive mode if interactive start is specified if (params.interactive_first) { params.interactive = true; @@ -822,7 +835,7 @@ int main(int argc, char ** argv) { if (n_past > 0 && is_interacting) { LOG("waiting for user input\n"); - if (params.instruct || params.chatml) { + if (params.instruct || params.chatml || params.chaton) { printf("\n> "); } @@ -901,6 +914,11 @@ int main(int argc, char ** argv) { LOG("inserting chatml suffix\n"); embd_inp.insert(embd_inp.end(), cml_sfx.begin(), cml_sfx.end()); } + // chaton mode: insert assistant prefix + if (params.chaton) { + LOG("inserting chaton assistant prefix\n"); + embd_inp.insert(embd_inp.end(), chaton_assitant_prefix.begin(), chaton_assitant_prefix.end()); + } for (size_t i = original_size; i < embd_inp.size(); ++i) { const llama_token token = embd_inp[i]; From cdbe4f06cea79625fe466c225d810948fbd0a17b Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Mon, 22 Apr 2024 22:47:20 +0530 Subject: [PATCH 007/217] Chaton:Sample Meta JSON cleanup --- examples/chaton_meta.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/chaton_meta.json b/examples/chaton_meta.json index 7b2114f867992..4520b2966f12c 100644 --- a/examples/chaton_meta.json +++ b/examples/chaton_meta.json @@ -52,7 +52,7 @@ "suffix": "<|im_end|>\n" }, "assistant": { - "prefix": "<|im_start|>assistant\n" + "prefix": "<|im_start|>assistant\n", "suffix": "" }, "reverse-prompt": "<|im_start|>user\n" From d87d27512ec3625e885f14d4a5c606e0e646134d Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Mon, 22 Apr 2024 23:29:28 +0530 Subject: [PATCH 008/217] ChatOn: update sample meta json a bit Move [inst] [/inst] wrt llama2 from global to individual role specific parts. Avoid an extra \n wrt prefixes of llama3 --- examples/chaton_meta.json | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/examples/chaton_meta.json b/examples/chaton_meta.json index 4520b2966f12c..8301f42215123 100644 --- a/examples/chaton_meta.json +++ b/examples/chaton_meta.json @@ -2,16 +2,16 @@ { "llama2": { "global": { - "prefix": "[INST] ", - "suffix": " [/INST]" + "prefix": "", + "suffix": "" }, "system": { - "prefix": " <>\n", - "suffix": "\n<>\n\n" + "prefix": "[INST] <>\n", + "suffix": "\n<> [/INST]\n\n" }, "user": { - "prefix": "", - "suffix": "" + "prefix": "[INST] ", + "suffix": " [/INST]\n\n" }, "assistant": { "prefix": "", @@ -25,15 +25,15 @@ "suffix": "" }, "system": { - "prefix": "<|start_header_id|>system<|end_header_id|>\n\n", + "prefix": "<|start_header_id|>system<|end_header_id|>\n", "suffix": "<|eot_id|>\n\n" }, "user": { - "prefix": "<|start_header_id|>user<|end_header_id|>\n\n", + "prefix": "<|start_header_id|>user<|end_header_id|>\n", "suffix": "<|eot_id|>\n\n" }, "assistant": { - "prefix": "<|start_header_id|>assistant<|end_header_id|>\n\n", + "prefix": "<|start_header_id|>assistant<|end_header_id|>\n", "suffix": "" }, "reverse-prompt": "<|eot_id|>" From c4cf0e9075f7c3815de03395efe19ff2110f83e4 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Mon, 22 Apr 2024 23:48:24 +0530 Subject: [PATCH 009/217] ChatON:Cleanup: BeginEnd, Debug log Update the note Rename global-prefix|suffix to global-begin|end. Rename chat-apply-template to chat-apply-template-single, cas it handles only a single message. Add some debug log messages to the helper functions --- common/chaton.hpp | 29 +++++++++++++++++++---------- examples/chaton_meta.json | 12 ++++++------ examples/main/main.cpp | 2 +- 3 files changed, 26 insertions(+), 17 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index c62ab146ebfc3..5db64c631076f 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -1,14 +1,18 @@ #pragma once /*** - * Keep chatting with model and needed role tagging using special tokens simple and flexible, while building on existing interactive flow + * Keep chatting with model and needed role tagging using special tokens simple and flexible, + * while building on existing interactive flow and its in-prefix, in-suffix and antiprompt/reverse-promot + * * 1. Use a json file to configure the needed tags for each of the supported chat-handshake-template-standard * a. system-prefix, system-suffix, - * b. user-prefix, user-suffix, + * b. user-prefix, user-suffix, assistant-prefix + * * these override the in-prefix and in-suffix * c. reverse-prompt - * d. global-begin-marker, global-end-marker - * e. per-msg-begin-marker, per-msg-end-marker - * f. is per-msg-begin-marker used for system+user combo + * d. Later if required look at adding + * * global-begin-marker, global-end-marker + * * per-msg-begin-marker, per-msg-end-marker + * * is system-per-msg-end-marker and user-per-msg-begin-marker used for system+user combo * 2. Give the below option to user wrt system prompt, this should give the flexibility to either keep system prompt simple or complex in a flexible yet simple way. * a. the system prompt they specify using -f, is used as is with parse_special when tokenising or * b. whether the system prefix and suffix is added, but without parse_special tokenisation of system-prompt provided by user. @@ -51,19 +55,24 @@ inline void chaton_meta_dump() { LOG_TEELN("\n\nINFO:%s:ChatOn Meta\n%s", __func__, conMeta.dump(4).c_str()); } -inline std::string chaton_tmpl_apply(const std::string &tmpl, const std::string &role, const std::string &content) { +inline std::string chaton_tmpl_apply_single(const std::string &tmpl, const std::string &role, const std::string &content) { std::stringstream ss; - ss << conMeta[tmpl]["global"]["prefix"]; + ss << conMeta[tmpl]["global"]["begin"]; ss << conMeta[tmpl][role]["prefix"] << content << conMeta[tmpl][role]["suffix"]; - ss << conMeta[tmpl]["global"]["suffix"]; + ss << conMeta[tmpl]["global"]["end"]; std::string taggedStr = ss.str(); + LOG_TEELN("DBUG:%s:%s:%s:%s", __func__, tmpl.c_str(), role.c_str(), taggedStr.c_str()); return taggedStr; } inline std::string chaton_tmpl_role_part(const std::string &tmpl, const std::string &role, const std::string &part) { - return conMeta[tmpl][role][part]; + std::string got = conMeta[tmpl][role][part]; + LOG_TEELN("DBUG:%s:%s:%s:%s:%s", __func__, tmpl.c_str(), role.c_str(), part.c_str(), got.c_str()); + return got; } inline std::string chaton_tmpl_part(const std::string &tmpl, const std::string &part) { - return conMeta[tmpl][part]; + std::string got = conMeta[tmpl][part]; + LOG_TEELN("DBUG:%s:%s:%s:%s", __func__, tmpl.c_str(), part.c_str(), got.c_str()); + return got; } diff --git a/examples/chaton_meta.json b/examples/chaton_meta.json index 8301f42215123..4711ae7cddd04 100644 --- a/examples/chaton_meta.json +++ b/examples/chaton_meta.json @@ -2,8 +2,8 @@ { "llama2": { "global": { - "prefix": "", - "suffix": "" + "begin": "", + "end": "" }, "system": { "prefix": "[INST] <>\n", @@ -21,8 +21,8 @@ }, "llama3": { "global": { - "prefix": "", - "suffix": "" + "begin": "", + "end": "" }, "system": { "prefix": "<|start_header_id|>system<|end_header_id|>\n", @@ -40,8 +40,8 @@ }, "chatml": { "global": { - "prefix": "", - "suffix": "" + "begin": "", + "end": "" }, "system": { "prefix": "<|im_start|>system\n", diff --git a/examples/main/main.cpp b/examples/main/main.cpp index f58888e99bb8f..d16312a5e02bd 100644 --- a/examples/main/main.cpp +++ b/examples/main/main.cpp @@ -262,7 +262,7 @@ int main(int argc, char ** argv) { params.prompt = "<|im_start|>system\n" + params.prompt + "<|im_end|>"; } if (params.chaton) { - params.prompt = chaton_tmpl_apply(params.chaton_template_id, "system", params.prompt); + params.prompt = chaton_tmpl_apply_single(params.chaton_template_id, "system", params.prompt); } embd_inp = ::llama_tokenize(ctx, params.prompt, true, true); } else { From f03dd2439f2f22f122045816bc6ca9c1729cd437 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Mon, 22 Apr 2024 23:58:19 +0530 Subject: [PATCH 010/217] ChatOn:No global-begin/end in ChatApplyTmplSingle, ChatApplyTmpl Avoid adding global begin/end markers wrt ChatApplyTmplSingle. Add ChatApplyTmpl which goes through a vector of messages. --- common/chaton.hpp | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index 5db64c631076f..8de5ceabcf900 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -17,8 +17,8 @@ * a. the system prompt they specify using -f, is used as is with parse_special when tokenising or * b. whether the system prefix and suffix is added, but without parse_special tokenisation of system-prompt provided by user. * 3. chat-apply-template uses the json file, which was loaded, to decide on how to generate the tagged messages for tokenisation - * a. input: [ { role: message }, { role: message}, ....] - * b. output: [ {flag: data}, { flag: data}, {flag: data}, ....] + * a. input: [ { role, message }, { role, message}, ....] + * b. output: [ {flag, data}, { flag, data}, {flag, data}, ....] * * flag is whether to do parse_special for this data, during tokenization or not * */ @@ -55,16 +55,30 @@ inline void chaton_meta_dump() { LOG_TEELN("\n\nINFO:%s:ChatOn Meta\n%s", __func__, conMeta.dump(4).c_str()); } +// NOTE: This currently doesnt return about which parts of the tagged message contain tags and which parts the user message inline std::string chaton_tmpl_apply_single(const std::string &tmpl, const std::string &role, const std::string &content) { std::stringstream ss; - ss << conMeta[tmpl]["global"]["begin"]; ss << conMeta[tmpl][role]["prefix"] << content << conMeta[tmpl][role]["suffix"]; - ss << conMeta[tmpl]["global"]["end"]; std::string taggedStr = ss.str(); LOG_TEELN("DBUG:%s:%s:%s:%s", __func__, tmpl.c_str(), role.c_str(), taggedStr.c_str()); return taggedStr; } +// NOTE: This currently doesnt return about which parts of the tagged message contain tags and which parts the user message +inline std::string chaton_tmpl_apply(const std::string &tmpl, const std::vector &msgs) { + std::stringstream ss; + ss << conMeta[tmpl]["global"]["begin"]; + for(auto msg: msgs) { + auto role = msg.role; + auto content = msg.content; + ss << conMeta[tmpl][role]["prefix"] << content << conMeta[tmpl][role]["suffix"]; + } + ss << conMeta[tmpl]["global"]["end"]; + std::string taggedMsgs = ss.str(); + LOG_TEELN("DBUG:%s:%s:%s", __func__, tmpl.c_str(), taggedMsgs.c_str()); + return taggedMsgs; +} + inline std::string chaton_tmpl_role_part(const std::string &tmpl, const std::string &role, const std::string &part) { std::string got = conMeta[tmpl][role][part]; LOG_TEELN("DBUG:%s:%s:%s:%s:%s", __func__, tmpl.c_str(), role.c_str(), part.c_str(), got.c_str()); From 221ccd6462319ca94dd207fc5a98b9888a4e278b Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Tue, 23 Apr 2024 01:06:56 +0530 Subject: [PATCH 011/217] ChatOn: Add SystemUser-1st-User-Has-Prefix flag support Llama2 seems to need it, so chaton-meta-json sample file updated to use same. --- common/chaton.hpp | 44 +++++++++++++++++++++++++++++++++------ examples/chaton_meta.json | 9 +++++--- 2 files changed, 44 insertions(+), 9 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index 8de5ceabcf900..588c647ee3985 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -9,10 +9,10 @@ * b. user-prefix, user-suffix, assistant-prefix * * these override the in-prefix and in-suffix * c. reverse-prompt - * d. Later if required look at adding - * * global-begin-marker, global-end-marker - * * per-msg-begin-marker, per-msg-end-marker - * * is system-per-msg-end-marker and user-per-msg-begin-marker used for system+user combo + * d. global-begin, global-end + * d. systemuser-1st-user-has-prefix + * * if a combination of 1 system message followed by 1 or more user messages is seen, + * then include user prefix only if this flag is set. * 2. Give the below option to user wrt system prompt, this should give the flexibility to either keep system prompt simple or complex in a flexible yet simple way. * a. the system prompt they specify using -f, is used as is with parse_special when tokenising or * b. whether the system prefix and suffix is added, but without parse_special tokenisation of system-prompt provided by user. @@ -30,6 +30,14 @@ #include "log.h" + +const auto K_SYSTEM = "system"; +const auto K_USER = "user"; +const auto K_PREFIX = "prefix"; +const auto K_SUFFIX = "suffix"; +const auto K_SYSTEMUSER_1ST_USER_HAS_PREFIX = "systemuser-1st-user-has-prefix"; + + using json = nlohmann::json; json conMeta; @@ -55,6 +63,7 @@ inline void chaton_meta_dump() { LOG_TEELN("\n\nINFO:%s:ChatOn Meta\n%s", __func__, conMeta.dump(4).c_str()); } +// Return user-prefix + msg + user-suffix // NOTE: This currently doesnt return about which parts of the tagged message contain tags and which parts the user message inline std::string chaton_tmpl_apply_single(const std::string &tmpl, const std::string &role, const std::string &content) { std::stringstream ss; @@ -64,14 +73,37 @@ inline std::string chaton_tmpl_apply_single(const std::string &tmpl, const std:: return taggedStr; } +// global-begin + [role-prefix + msg + role-suffix] + global-end +// if there is a combination of system-user messages, +// then 1st user message will have user-prefix only if systemuser-1st-user-has-prefix is true // NOTE: This currently doesnt return about which parts of the tagged message contain tags and which parts the user message inline std::string chaton_tmpl_apply(const std::string &tmpl, const std::vector &msgs) { std::stringstream ss; ss << conMeta[tmpl]["global"]["begin"]; - for(auto msg: msgs) { + int cntSystem = 0; + int cntUser = 0; + int cntOthers = 0; + for(const auto msg: msgs) { auto role = msg.role; auto content = msg.content; - ss << conMeta[tmpl][role]["prefix"] << content << conMeta[tmpl][role]["suffix"]; + auto prefix = conMeta[tmpl][role][K_PREFIX]; + if (role == K_SYSTEM) { + cntSystem += 1; + ss << prefix; + } else if (role == K_USER) { + cntUser += 1; + if ((cntSystem == 1) && (cntUser == 1)) { + if (conMeta[tmpl][K_SYSTEMUSER_1ST_USER_HAS_PREFIX]) { + ss << prefix; + } + } else { + ss << prefix; + } + } else { + cntOthers += 1; + ss << prefix; + } + ss << content << conMeta[tmpl][role]["suffix"]; } ss << conMeta[tmpl]["global"]["end"]; std::string taggedMsgs = ss.str(); diff --git a/examples/chaton_meta.json b/examples/chaton_meta.json index 4711ae7cddd04..548d7cc8fd046 100644 --- a/examples/chaton_meta.json +++ b/examples/chaton_meta.json @@ -17,7 +17,8 @@ "prefix": "", "suffix": "" }, - "reverse-prompt": "" + "reverse-prompt": "", + "systemuser-1st-user-has-prefix": false }, "llama3": { "global": { @@ -36,7 +37,8 @@ "prefix": "<|start_header_id|>assistant<|end_header_id|>\n", "suffix": "" }, - "reverse-prompt": "<|eot_id|>" + "reverse-prompt": "<|eot_id|>", + "systemuser-1st-user-has-prefix": true }, "chatml": { "global": { @@ -55,7 +57,8 @@ "prefix": "<|im_start|>assistant\n", "suffix": "" }, - "reverse-prompt": "<|im_start|>user\n" + "reverse-prompt": "<|im_start|>user\n", + "systemuser-1st-user-has-prefix": true } } From 11b47fbcfcb97f7bb95b44739e33d062ed4bcfa7 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Tue, 23 Apr 2024 01:13:50 +0530 Subject: [PATCH 012/217] ChatON:MetaJson: Add key constants, check metaJson loaded ifNeeded --- common/chaton.hpp | 28 +++++++++++++++++----------- common/common.cpp | 4 ++-- common/common.h | 2 +- examples/main/main.cpp | 6 ++++-- 4 files changed, 24 insertions(+), 16 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index 588c647ee3985..e5f08e5868b92 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -13,6 +13,10 @@ * d. systemuser-1st-user-has-prefix * * if a combination of 1 system message followed by 1 or more user messages is seen, * then include user prefix only if this flag is set. + * * one or two models which I looked at seem to require not just BoS, but also the user-role-tag-prefix + * to also be controlled wrt this case. So not differentiating between BoS and any user-role-tag-prefix + * However if this needs to be decoupled, then maybe will add begin and end keys to role blocks in the json. + * then depending on what model needs, one can setup role-begin and role-prefix suitably. * 2. Give the below option to user wrt system prompt, this should give the flexibility to either keep system prompt simple or complex in a flexible yet simple way. * a. the system prompt they specify using -f, is used as is with parse_special when tokenising or * b. whether the system prefix and suffix is added, but without parse_special tokenisation of system-prompt provided by user. @@ -35,6 +39,9 @@ const auto K_SYSTEM = "system"; const auto K_USER = "user"; const auto K_PREFIX = "prefix"; const auto K_SUFFIX = "suffix"; +const auto K_BEGIN = "begin"; +const auto K_END = "end"; +const auto K_GLOBAL = "global"; const auto K_SYSTEMUSER_1ST_USER_HAS_PREFIX = "systemuser-1st-user-has-prefix"; @@ -48,26 +55,25 @@ inline bool chaton_meta_load(std::string &fname) { return true; } +inline void _chaton_meta_dump() { + LOG_TEELN("\n\nINFO:%s:ChatOn Meta\n%s", __func__, conMeta.dump(4).c_str()); +} + inline bool chaton_meta_ok() { if (conMeta == nullptr) { + LOG_TEELN("ERRR:%s:ChatOn Meta: Not loaded yet...", __func__); return false; } + _chaton_meta_dump(); return true; } -inline void chaton_meta_dump() { - if (!chaton_meta_ok()) { - LOG_TEELN("ERRR:%s:ChatOn Meta: Not loaded yet...", __func__); - return; - } - LOG_TEELN("\n\nINFO:%s:ChatOn Meta\n%s", __func__, conMeta.dump(4).c_str()); -} // Return user-prefix + msg + user-suffix // NOTE: This currently doesnt return about which parts of the tagged message contain tags and which parts the user message inline std::string chaton_tmpl_apply_single(const std::string &tmpl, const std::string &role, const std::string &content) { std::stringstream ss; - ss << conMeta[tmpl][role]["prefix"] << content << conMeta[tmpl][role]["suffix"]; + ss << conMeta[tmpl][role][K_PREFIX] << content << conMeta[tmpl][role][K_SUFFIX]; std::string taggedStr = ss.str(); LOG_TEELN("DBUG:%s:%s:%s:%s", __func__, tmpl.c_str(), role.c_str(), taggedStr.c_str()); return taggedStr; @@ -79,7 +85,7 @@ inline std::string chaton_tmpl_apply_single(const std::string &tmpl, const std:: // NOTE: This currently doesnt return about which parts of the tagged message contain tags and which parts the user message inline std::string chaton_tmpl_apply(const std::string &tmpl, const std::vector &msgs) { std::stringstream ss; - ss << conMeta[tmpl]["global"]["begin"]; + ss << conMeta[tmpl][K_GLOBAL][K_BEGIN]; int cntSystem = 0; int cntUser = 0; int cntOthers = 0; @@ -103,9 +109,9 @@ inline std::string chaton_tmpl_apply(const std::string &tmpl, const std::vector< cntOthers += 1; ss << prefix; } - ss << content << conMeta[tmpl][role]["suffix"]; + ss << content << conMeta[tmpl][role][K_SUFFIX]; } - ss << conMeta[tmpl]["global"]["end"]; + ss << conMeta[tmpl][K_GLOBAL][K_END]; std::string taggedMsgs = ss.str(); LOG_TEELN("DBUG:%s:%s:%s", __func__, tmpl.c_str(), taggedMsgs.c_str()); return taggedMsgs; diff --git a/common/common.cpp b/common/common.cpp index a065478a7bdd7..46789da416cc6 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -915,12 +915,12 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_pa params.chatml = true; return true; } - if (arg == "--chaton-json") { + if (arg == "--chaton-meta-json") { if (++i >= argc) { invalid_param = true; return true; } - params.chaton_json = argv[i]; + params.chaton_meta_json = argv[i]; return true; } if (arg == "--chaton-template-id") { diff --git a/common/common.h b/common/common.h index f0709d38ab721..302a2a2e52f25 100644 --- a/common/common.h +++ b/common/common.h @@ -142,7 +142,7 @@ struct gpt_params { bool interactive = false; // interactive mode bool chatml = false; // chatml mode (used for models trained on chatml syntax) bool chaton = false; // whether chaton is enabled or disabled - std::string chaton_json = ""; // name of the json file containing the chaton templates + std::string chaton_meta_json = ""; // name of the json file containing the chaton templates std::string chaton_template_id = ""; // the specific chat-handshake-template-standard to use bool prompt_cache_all = false; // save user input and generations to prompt cache bool prompt_cache_ro = false; // open the prompt cache read-only and do not update it diff --git a/examples/main/main.cpp b/examples/main/main.cpp index d16312a5e02bd..8eed62e657a69 100644 --- a/examples/main/main.cpp +++ b/examples/main/main.cpp @@ -143,8 +143,10 @@ int main(int argc, char ** argv) { atexit([]() { console::cleanup(); }); if (params.chaton) { - chaton_meta_load(params.chaton_json); - chaton_meta_dump(); + chaton_meta_load(params.chaton_meta_json); + if (!chaton_meta_ok()) { + exit(1); + } } if (params.logits_all) { From b1055641e98f91d08769f58bb9cdabf654ed0641 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Tue, 23 Apr 2024 08:29:05 +0530 Subject: [PATCH 013/217] ChatON: Update the notes a bit --- common/chaton.hpp | 61 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 43 insertions(+), 18 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index e5f08e5868b92..083f583a58d34 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -1,29 +1,53 @@ #pragma once /*** - * Keep chatting with model and needed role tagging using special tokens simple and flexible, - * while building on existing interactive flow and its in-prefix, in-suffix and antiprompt/reverse-promot + * + * ## Overview + * + * Helps chat with a model, by allowing role based special token tagging, based on the specified chat-handshake-template-standard. + * This is used by main, to build on existing interactive flow and its in-prefix, in-suffix and antiprompt/reverse-promot * * 1. Use a json file to configure the needed tags for each of the supported chat-handshake-template-standard - * a. system-prefix, system-suffix, - * b. user-prefix, user-suffix, assistant-prefix - * * these override the in-prefix and in-suffix + * a. system -> prefix & suffix, + * b. user -> prefix & suffix, assistant -> prefix + * * [main] these override the in-prefix and in-suffix * c. reverse-prompt - * d. global-begin, global-end + * * [main] this adds to any reverese-prompt specified using cmdline + * d. global -> begin & end * d. systemuser-1st-user-has-prefix - * * if a combination of 1 system message followed by 1 or more user messages is seen, - * then include user prefix only if this flag is set. - * * one or two models which I looked at seem to require not just BoS, but also the user-role-tag-prefix - * to also be controlled wrt this case. So not differentiating between BoS and any user-role-tag-prefix - * However if this needs to be decoupled, then maybe will add begin and end keys to role blocks in the json. - * then depending on what model needs, one can setup role-begin and role-prefix suitably. - * 2. Give the below option to user wrt system prompt, this should give the flexibility to either keep system prompt simple or complex in a flexible yet simple way. - * a. the system prompt they specify using -f, is used as is with parse_special when tokenising or - * b. whether the system prefix and suffix is added, but without parse_special tokenisation of system-prompt provided by user. - * 3. chat-apply-template uses the json file, which was loaded, to decide on how to generate the tagged messages for tokenisation + * * if a combination of system and user messages/prompts is passed, + * then for the 1st user message following the 1st system message, + * include user prefix only if this flag is set. [chaton-tmpl-apply] + * * [later] one or two models which I looked at seem to require not just BoS, but also the user-role-prefix-tag + * to also be controlled wrt this case. So not differentiating between BoS and any user-role-prefix-tag. + * However if bos and user-role-prefix-tag need to be decoupled, where only bos needs this treatment, + * then maybe add begin and end keys (to specify the BoS) in addition to prefix and suffix keys (to specify user-role-prefix-tag), to role blocks in the json. + * and inturn control only begin and not prefix, wrt whether to add or not. + * 2. [main] currently the user specified system prompt (-p + -f) is tagged using system role tags, + * and inturn this tagged message is tokenized with parse_special flag. + * So any special token related tags in the user specified system prompt will get parsed as special. + * 3. chaton-tmpl-apply uses the json file, which was loaded, to decide on how to generate the tagged messages for tokenisation. * a. input: [ { role, message }, { role, message}, ....] - * b. output: [ {flag, data}, { flag, data}, {flag, data}, ....] - * * flag is whether to do parse_special for this data, during tokenization or not + * b. output: currently a single string is returned which contains the tagged message(s). + * [later] if it is needed to differentiate between the special tags added by this from user specified prompts/messages, + * then return [ {flag, data}, { flag, data}, {flag, data}, ....], + * where the flag specifies whether parse_special should be used or not for the corresponding data, during tokenization. + * + * ## Adding support for new model / chat-handshake-template-standard + * + * 1. Add suitable entries in json for that model/standard + * 2. Update the flow in chaton-tmpl-apply, as needed. + * Try to update and or reuse the generic flow in chaton-tmpl-apply, as much as possible, + * before trying to add a custom logic. + * If you update the generic flow, cross check if existing json files will need to be updated or not. + * + * ## Notes + * + * Currently Main doesnt use chaton-tmpl-apply, but only + * * chaton-tmpl-apply-single (for system prompt) and + * * chaton-tmpl-role-part which maps the user prefix, suffix and reverse-prompt to + * in-prefix, in-suffix and antiprompt of main. + * These always adds any role specific prefix and suffix around the passed message. * */ @@ -33,6 +57,7 @@ #include #include "log.h" +#include "llama.h" const auto K_SYSTEM = "system"; From e8c24c0767574692662ad0d664dab44afda5f7de Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Tue, 23 Apr 2024 08:38:23 +0530 Subject: [PATCH 014/217] ChatOn:MetaOk: Allows template-id based cross check For a given template-id, cross check, all needed entries are there in the json. --- common/chaton.hpp | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index 083f583a58d34..bb1ae21389d07 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -62,6 +62,7 @@ const auto K_SYSTEM = "system"; const auto K_USER = "user"; +const auto K_ASSISTANT = "assistant"; const auto K_PREFIX = "prefix"; const auto K_SUFFIX = "suffix"; const auto K_BEGIN = "begin"; @@ -80,16 +81,37 @@ inline bool chaton_meta_load(std::string &fname) { return true; } -inline void _chaton_meta_dump() { - LOG_TEELN("\n\nINFO:%s:ChatOn Meta\n%s", __func__, conMeta.dump(4).c_str()); +inline void _chaton_meta_dump(std::string &tmpl) { + json theJson; + if (tmpl.empty()) { + theJson = conMeta; + } else { + theJson = conMeta[tmpl]; + } + LOG_TEELN("\n\nINFO:%s:ChatOn Meta\n%s", __func__, theJson.dump(4).c_str()); + if (!tmpl.empty()) { + LOG("INFO:%s:%s:%s", __func__, "global->begin", chaton_tmpl_role_part(tmpl, K_GLOBAL, K_BEGIN)); + LOG("INFO:%s:%s:%s", __func__, "global->end", chaton_tmpl_role_part(tmpl, K_GLOBAL, K_END)); + LOG("INFO:%s:%s:%s", __func__, "system->prefix", chaton_tmpl_role_part(tmpl, K_SYSTEM, K_PREFIX)); + LOG("INFO:%s:%s:%s", __func__, "system->suffix", chaton_tmpl_role_part(tmpl, K_SYSTEM, K_SUFFIX)); + LOG("INFO:%s:%s:%s", __func__, "user->prefix", chaton_tmpl_role_part(tmpl, K_USER, K_PREFIX)); + LOG("INFO:%s:%s:%s", __func__, "user->suffix", chaton_tmpl_role_part(tmpl, K_USER, K_SUFFIX)); + LOG("INFO:%s:%s:%s", __func__, "assistant->prefix", chaton_tmpl_role_part(tmpl, K_ASSISTANT, K_PREFIX)); + LOG("INFO:%s:%s:%s", __func__, "assistant->suffix", chaton_tmpl_role_part(tmpl, K_ASSISTANT, K_SUFFIX)); + } } -inline bool chaton_meta_ok() { +/** + * if tmpl is + * * empty string, then dump the full loaded chaton-meta + * * chaton-template-id, then dump contents related to that specific chat-handshake-template-standard + */ +inline bool chaton_meta_ok(std::string &tmpl) { if (conMeta == nullptr) { LOG_TEELN("ERRR:%s:ChatOn Meta: Not loaded yet...", __func__); return false; } - _chaton_meta_dump(); + _chaton_meta_dump(tmpl); return true; } From efb758ba7df96cf8f087573a8e241fa0c6a6e810 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Tue, 23 Apr 2024 09:15:25 +0530 Subject: [PATCH 015/217] ChatON: Rename helpers to kv suffix, updated wrt metaok rename because they return value of specified key. [main] update metaok to take template-id, so that one can cross check that all needed entries are there wrt that template-id in the chaton-meta-json file --- common/chaton.hpp | 28 ++++++++++++++-------------- examples/main/main.cpp | 10 +++++----- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index bb1ae21389d07..770c1cf4a4636 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -90,14 +90,14 @@ inline void _chaton_meta_dump(std::string &tmpl) { } LOG_TEELN("\n\nINFO:%s:ChatOn Meta\n%s", __func__, theJson.dump(4).c_str()); if (!tmpl.empty()) { - LOG("INFO:%s:%s:%s", __func__, "global->begin", chaton_tmpl_role_part(tmpl, K_GLOBAL, K_BEGIN)); - LOG("INFO:%s:%s:%s", __func__, "global->end", chaton_tmpl_role_part(tmpl, K_GLOBAL, K_END)); - LOG("INFO:%s:%s:%s", __func__, "system->prefix", chaton_tmpl_role_part(tmpl, K_SYSTEM, K_PREFIX)); - LOG("INFO:%s:%s:%s", __func__, "system->suffix", chaton_tmpl_role_part(tmpl, K_SYSTEM, K_SUFFIX)); - LOG("INFO:%s:%s:%s", __func__, "user->prefix", chaton_tmpl_role_part(tmpl, K_USER, K_PREFIX)); - LOG("INFO:%s:%s:%s", __func__, "user->suffix", chaton_tmpl_role_part(tmpl, K_USER, K_SUFFIX)); - LOG("INFO:%s:%s:%s", __func__, "assistant->prefix", chaton_tmpl_role_part(tmpl, K_ASSISTANT, K_PREFIX)); - LOG("INFO:%s:%s:%s", __func__, "assistant->suffix", chaton_tmpl_role_part(tmpl, K_ASSISTANT, K_SUFFIX)); + LOG("INFO:%s:%s:%s", __func__, "global->begin", chaton_tmpl_role_kv(tmpl, K_GLOBAL, K_BEGIN)); + LOG("INFO:%s:%s:%s", __func__, "global->end", chaton_tmpl_role_kv(tmpl, K_GLOBAL, K_END)); + LOG("INFO:%s:%s:%s", __func__, "system->prefix", chaton_tmpl_role_kv(tmpl, K_SYSTEM, K_PREFIX)); + LOG("INFO:%s:%s:%s", __func__, "system->suffix", chaton_tmpl_role_kv(tmpl, K_SYSTEM, K_SUFFIX)); + LOG("INFO:%s:%s:%s", __func__, "user->prefix", chaton_tmpl_role_kv(tmpl, K_USER, K_PREFIX)); + LOG("INFO:%s:%s:%s", __func__, "user->suffix", chaton_tmpl_role_kv(tmpl, K_USER, K_SUFFIX)); + LOG("INFO:%s:%s:%s", __func__, "assistant->prefix", chaton_tmpl_role_kv(tmpl, K_ASSISTANT, K_PREFIX)); + LOG("INFO:%s:%s:%s", __func__, "assistant->suffix", chaton_tmpl_role_kv(tmpl, K_ASSISTANT, K_SUFFIX)); } } @@ -164,14 +164,14 @@ inline std::string chaton_tmpl_apply(const std::string &tmpl, const std::vector< return taggedMsgs; } -inline std::string chaton_tmpl_role_part(const std::string &tmpl, const std::string &role, const std::string &part) { - std::string got = conMeta[tmpl][role][part]; - LOG_TEELN("DBUG:%s:%s:%s:%s:%s", __func__, tmpl.c_str(), role.c_str(), part.c_str(), got.c_str()); +inline std::string chaton_tmpl_role_kv(const std::string &tmpl, const std::string &role, const std::string &key) { + std::string got = conMeta[tmpl][role][key]; + LOG_TEELN("DBUG:%s:%s:%s:%s:%s", __func__, tmpl.c_str(), role.c_str(), key.c_str(), got.c_str()); return got; } -inline std::string chaton_tmpl_part(const std::string &tmpl, const std::string &part) { - std::string got = conMeta[tmpl][part]; - LOG_TEELN("DBUG:%s:%s:%s:%s", __func__, tmpl.c_str(), part.c_str(), got.c_str()); +inline std::string chaton_tmpl_kv(const std::string &tmpl, const std::string &key) { + std::string got = conMeta[tmpl][key]; + LOG_TEELN("DBUG:%s:%s:%s:%s", __func__, tmpl.c_str(), key.c_str(), got.c_str()); return got; } diff --git a/examples/main/main.cpp b/examples/main/main.cpp index 8eed62e657a69..be3daecdcb826 100644 --- a/examples/main/main.cpp +++ b/examples/main/main.cpp @@ -144,7 +144,7 @@ int main(int argc, char ** argv) { if (params.chaton) { chaton_meta_load(params.chaton_meta_json); - if (!chaton_meta_ok()) { + if (!chaton_meta_ok(params.chaton_template_id)) { exit(1); } } @@ -375,13 +375,13 @@ int main(int argc, char ** argv) { } // chaton mode - const auto chaton_assitant_prefix = ::llama_tokenize(ctx, chaton_tmpl_role_part(params.chaton_template_id, "assistant", "prefix"), false, true); + const auto chaton_assitant_prefix = ::llama_tokenize(ctx, chaton_tmpl_role_kv(params.chaton_template_id, "assistant", "prefix"), false, true); if (params.chaton) { params.interactive = true; // may remove later, by requiring user to explicitly request interactive mode params.interactive_first = true; - params.input_prefix = chaton_tmpl_role_part(params.chaton_template_id, "user", "prefix"); - params.input_suffix = chaton_tmpl_role_part(params.chaton_template_id, "user", "suffix"); - params.antiprompt.emplace_back(chaton_tmpl_part(params.chaton_template_id, "reverse-prompt")); + params.input_prefix = chaton_tmpl_role_kv(params.chaton_template_id, "user", "prefix"); + params.input_suffix = chaton_tmpl_role_kv(params.chaton_template_id, "user", "suffix"); + params.antiprompt.emplace_back(chaton_tmpl_kv(params.chaton_template_id, "reverse-prompt")); } // enable interactive mode if interactive start is specified From 42f6b455476c1ff44bc17685d0756eb156900049 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Tue, 23 Apr 2024 09:26:46 +0530 Subject: [PATCH 016/217] ChatON: Use the constants defined for the keys --- common/chaton.hpp | 1 + examples/main/main.cpp | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index 770c1cf4a4636..eb2309e5c0926 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -69,6 +69,7 @@ const auto K_BEGIN = "begin"; const auto K_END = "end"; const auto K_GLOBAL = "global"; const auto K_SYSTEMUSER_1ST_USER_HAS_PREFIX = "systemuser-1st-user-has-prefix"; +const auto K_REVERSE_PROMPT = "reverse-prompt"; using json = nlohmann::json; diff --git a/examples/main/main.cpp b/examples/main/main.cpp index be3daecdcb826..f75244800a0c0 100644 --- a/examples/main/main.cpp +++ b/examples/main/main.cpp @@ -264,7 +264,7 @@ int main(int argc, char ** argv) { params.prompt = "<|im_start|>system\n" + params.prompt + "<|im_end|>"; } if (params.chaton) { - params.prompt = chaton_tmpl_apply_single(params.chaton_template_id, "system", params.prompt); + params.prompt = chaton_tmpl_apply_single(params.chaton_template_id, K_SYSTEM, params.prompt); } embd_inp = ::llama_tokenize(ctx, params.prompt, true, true); } else { @@ -375,13 +375,13 @@ int main(int argc, char ** argv) { } // chaton mode - const auto chaton_assitant_prefix = ::llama_tokenize(ctx, chaton_tmpl_role_kv(params.chaton_template_id, "assistant", "prefix"), false, true); + const auto chaton_assitant_prefix = ::llama_tokenize(ctx, chaton_tmpl_role_kv(params.chaton_template_id, K_ASSISTANT, K_PREFIX), false, true); if (params.chaton) { params.interactive = true; // may remove later, by requiring user to explicitly request interactive mode params.interactive_first = true; - params.input_prefix = chaton_tmpl_role_kv(params.chaton_template_id, "user", "prefix"); - params.input_suffix = chaton_tmpl_role_kv(params.chaton_template_id, "user", "suffix"); - params.antiprompt.emplace_back(chaton_tmpl_kv(params.chaton_template_id, "reverse-prompt")); + params.input_prefix = chaton_tmpl_role_kv(params.chaton_template_id, K_USER, K_PREFIX); + params.input_suffix = chaton_tmpl_role_kv(params.chaton_template_id, K_USER, K_SUFFIX); + params.antiprompt.emplace_back(chaton_tmpl_kv(params.chaton_template_id, K_REVERSE_PROMPT)); } // enable interactive mode if interactive start is specified From 3f9dfc240c9153742d81d0eec4c414d37f6180a3 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Tue, 23 Apr 2024 09:45:55 +0530 Subject: [PATCH 017/217] ChatON: Check for the boolean entries in meta-json --- common/chaton.hpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/common/chaton.hpp b/common/chaton.hpp index eb2309e5c0926..53d7b28520102 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -99,6 +99,8 @@ inline void _chaton_meta_dump(std::string &tmpl) { LOG("INFO:%s:%s:%s", __func__, "user->suffix", chaton_tmpl_role_kv(tmpl, K_USER, K_SUFFIX)); LOG("INFO:%s:%s:%s", __func__, "assistant->prefix", chaton_tmpl_role_kv(tmpl, K_ASSISTANT, K_PREFIX)); LOG("INFO:%s:%s:%s", __func__, "assistant->suffix", chaton_tmpl_role_kv(tmpl, K_ASSISTANT, K_SUFFIX)); + LOG("INFO:%s:%s:%d", __func__, K_REVERSE_PROMPT, chaton_tmpl_kv(tmpl, K_REVERSE_PROMPT)); + LOG("INFO:%s:%s:%d", __func__, K_SYSTEMUSER_1ST_USER_HAS_PREFIX, chaton_tmpl_kv(tmpl, K_SYSTEMUSER_1ST_USER_HAS_PREFIX)); } } @@ -176,3 +178,9 @@ inline std::string chaton_tmpl_kv(const std::string &tmpl, const std::string &ke LOG_TEELN("DBUG:%s:%s:%s:%s", __func__, tmpl.c_str(), key.c_str(), got.c_str()); return got; } + +inline bool chaton_tmpl_kv_bool(const std::string &tmpl, const std::string &key) { + bool got = conMeta[tmpl][key]; + LOG_TEELN("DBUG:%s:%s:%s:%d", __func__, tmpl.c_str(), key.c_str(), got); + return got; +} From 217544e5ff870803dfea6cd9469fd027a9fa01ab Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Tue, 23 Apr 2024 09:53:05 +0530 Subject: [PATCH 018/217] ChatON: Keep compiler happy Order the functions so that no need for seperate prototypes Also use kv_bool wrt boolean entries. Convert string to c char * --- common/chaton.hpp | 77 +++++++++++++++++++++++++---------------------- 1 file changed, 41 insertions(+), 36 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index 53d7b28520102..b3a7222c0e62c 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -82,42 +82,6 @@ inline bool chaton_meta_load(std::string &fname) { return true; } -inline void _chaton_meta_dump(std::string &tmpl) { - json theJson; - if (tmpl.empty()) { - theJson = conMeta; - } else { - theJson = conMeta[tmpl]; - } - LOG_TEELN("\n\nINFO:%s:ChatOn Meta\n%s", __func__, theJson.dump(4).c_str()); - if (!tmpl.empty()) { - LOG("INFO:%s:%s:%s", __func__, "global->begin", chaton_tmpl_role_kv(tmpl, K_GLOBAL, K_BEGIN)); - LOG("INFO:%s:%s:%s", __func__, "global->end", chaton_tmpl_role_kv(tmpl, K_GLOBAL, K_END)); - LOG("INFO:%s:%s:%s", __func__, "system->prefix", chaton_tmpl_role_kv(tmpl, K_SYSTEM, K_PREFIX)); - LOG("INFO:%s:%s:%s", __func__, "system->suffix", chaton_tmpl_role_kv(tmpl, K_SYSTEM, K_SUFFIX)); - LOG("INFO:%s:%s:%s", __func__, "user->prefix", chaton_tmpl_role_kv(tmpl, K_USER, K_PREFIX)); - LOG("INFO:%s:%s:%s", __func__, "user->suffix", chaton_tmpl_role_kv(tmpl, K_USER, K_SUFFIX)); - LOG("INFO:%s:%s:%s", __func__, "assistant->prefix", chaton_tmpl_role_kv(tmpl, K_ASSISTANT, K_PREFIX)); - LOG("INFO:%s:%s:%s", __func__, "assistant->suffix", chaton_tmpl_role_kv(tmpl, K_ASSISTANT, K_SUFFIX)); - LOG("INFO:%s:%s:%d", __func__, K_REVERSE_PROMPT, chaton_tmpl_kv(tmpl, K_REVERSE_PROMPT)); - LOG("INFO:%s:%s:%d", __func__, K_SYSTEMUSER_1ST_USER_HAS_PREFIX, chaton_tmpl_kv(tmpl, K_SYSTEMUSER_1ST_USER_HAS_PREFIX)); - } -} - -/** - * if tmpl is - * * empty string, then dump the full loaded chaton-meta - * * chaton-template-id, then dump contents related to that specific chat-handshake-template-standard - */ -inline bool chaton_meta_ok(std::string &tmpl) { - if (conMeta == nullptr) { - LOG_TEELN("ERRR:%s:ChatOn Meta: Not loaded yet...", __func__); - return false; - } - _chaton_meta_dump(tmpl); - return true; -} - // Return user-prefix + msg + user-suffix // NOTE: This currently doesnt return about which parts of the tagged message contain tags and which parts the user message @@ -184,3 +148,44 @@ inline bool chaton_tmpl_kv_bool(const std::string &tmpl, const std::string &key) LOG_TEELN("DBUG:%s:%s:%s:%d", __func__, tmpl.c_str(), key.c_str(), got); return got; } + + +/** + * if tmpl is + * * empty string, then dump the full loaded chaton-meta + * * chaton-template-id, then dump contents related to that specific chat-handshake-template-standard + */ +inline void _chaton_meta_dump(std::string &tmpl) { + json theJson; + if (tmpl.empty()) { + theJson = conMeta; + } else { + theJson = conMeta[tmpl]; + } + LOG_TEELN("\n\nINFO:%s:ChatOn Meta\n%s", __func__, theJson.dump(4).c_str()); + if (!tmpl.empty()) { + LOG("INFO:%s:%s:%s", __func__, "global->begin", chaton_tmpl_role_kv(tmpl, K_GLOBAL, K_BEGIN).c_str()); + LOG("INFO:%s:%s:%s", __func__, "global->end", chaton_tmpl_role_kv(tmpl, K_GLOBAL, K_END).c_str()); + LOG("INFO:%s:%s:%s", __func__, "system->prefix", chaton_tmpl_role_kv(tmpl, K_SYSTEM, K_PREFIX).c_str()); + LOG("INFO:%s:%s:%s", __func__, "system->suffix", chaton_tmpl_role_kv(tmpl, K_SYSTEM, K_SUFFIX).c_str()); + LOG("INFO:%s:%s:%s", __func__, "user->prefix", chaton_tmpl_role_kv(tmpl, K_USER, K_PREFIX).c_str()); + LOG("INFO:%s:%s:%s", __func__, "user->suffix", chaton_tmpl_role_kv(tmpl, K_USER, K_SUFFIX).c_str()); + LOG("INFO:%s:%s:%s", __func__, "assistant->prefix", chaton_tmpl_role_kv(tmpl, K_ASSISTANT, K_PREFIX).c_str()); + LOG("INFO:%s:%s:%s", __func__, "assistant->suffix", chaton_tmpl_role_kv(tmpl, K_ASSISTANT, K_SUFFIX).c_str()); + LOG("INFO:%s:%s:%s", __func__, K_REVERSE_PROMPT, chaton_tmpl_kv(tmpl, K_REVERSE_PROMPT).c_str()); + LOG("INFO:%s:%s:%d", __func__, K_SYSTEMUSER_1ST_USER_HAS_PREFIX, chaton_tmpl_kv_bool(tmpl, K_SYSTEMUSER_1ST_USER_HAS_PREFIX)); + } +} + +/** + * Check that a meta-json file has been loaded. + * Verify that specified chaton-template-id contains required fields in meta-json, using meta-dump + */ +inline bool chaton_meta_ok(std::string &tmpl) { + if (conMeta == nullptr) { + LOG_TEELN("ERRR:%s:ChatOn Meta: Not loaded yet...", __func__); + return false; + } + _chaton_meta_dump(tmpl); + return true; +} From 57bd772bfd40eac13f0fa759f1ac1f2751481daa Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Tue, 23 Apr 2024 10:11:36 +0530 Subject: [PATCH 019/217] ChatON: Cleanup logging Avoid showing on screen the debug messages. meta-dump can either show on screen or not, based on how LOGXLN is defined. --- common/chaton.hpp | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index b3a7222c0e62c..be9957ff94e83 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -59,6 +59,7 @@ #include "log.h" #include "llama.h" +#define LOGXLN LOG_TEELN const auto K_SYSTEM = "system"; const auto K_USER = "user"; @@ -89,7 +90,7 @@ inline std::string chaton_tmpl_apply_single(const std::string &tmpl, const std:: std::stringstream ss; ss << conMeta[tmpl][role][K_PREFIX] << content << conMeta[tmpl][role][K_SUFFIX]; std::string taggedStr = ss.str(); - LOG_TEELN("DBUG:%s:%s:%s:%s", __func__, tmpl.c_str(), role.c_str(), taggedStr.c_str()); + LOGLN("DBUG:%s:%s:%s:%s", __func__, tmpl.c_str(), role.c_str(), taggedStr.c_str()); return taggedStr; } @@ -127,25 +128,26 @@ inline std::string chaton_tmpl_apply(const std::string &tmpl, const std::vector< } ss << conMeta[tmpl][K_GLOBAL][K_END]; std::string taggedMsgs = ss.str(); - LOG_TEELN("DBUG:%s:%s:%s", __func__, tmpl.c_str(), taggedMsgs.c_str()); + LOGLN("DBUG:%s:%s:%s", __func__, tmpl.c_str(), taggedMsgs.c_str()); + LOGLN("DBUG:%s:%s:CntSys[%d]:CntUsr[%d]:CntOthers[%d]", __func__, tmpl.c_str(), cntSystem, cntUser, cntOthers); return taggedMsgs; } inline std::string chaton_tmpl_role_kv(const std::string &tmpl, const std::string &role, const std::string &key) { std::string got = conMeta[tmpl][role][key]; - LOG_TEELN("DBUG:%s:%s:%s:%s:%s", __func__, tmpl.c_str(), role.c_str(), key.c_str(), got.c_str()); + LOGLN("DBUG:%s:%s:%s:%s:%s", __func__, tmpl.c_str(), role.c_str(), key.c_str(), got.c_str()); return got; } inline std::string chaton_tmpl_kv(const std::string &tmpl, const std::string &key) { std::string got = conMeta[tmpl][key]; - LOG_TEELN("DBUG:%s:%s:%s:%s", __func__, tmpl.c_str(), key.c_str(), got.c_str()); + LOGLN("DBUG:%s:%s:%s:%s", __func__, tmpl.c_str(), key.c_str(), got.c_str()); return got; } inline bool chaton_tmpl_kv_bool(const std::string &tmpl, const std::string &key) { bool got = conMeta[tmpl][key]; - LOG_TEELN("DBUG:%s:%s:%s:%d", __func__, tmpl.c_str(), key.c_str(), got); + LOGLN("DBUG:%s:%s:%s:%d", __func__, tmpl.c_str(), key.c_str(), got); return got; } @@ -162,18 +164,18 @@ inline void _chaton_meta_dump(std::string &tmpl) { } else { theJson = conMeta[tmpl]; } - LOG_TEELN("\n\nINFO:%s:ChatOn Meta\n%s", __func__, theJson.dump(4).c_str()); + LOGXLN("\n\nINFO:%s:ChatOn Meta\n%s", __func__, theJson.dump(4).c_str()); if (!tmpl.empty()) { - LOG("INFO:%s:%s:%s", __func__, "global->begin", chaton_tmpl_role_kv(tmpl, K_GLOBAL, K_BEGIN).c_str()); - LOG("INFO:%s:%s:%s", __func__, "global->end", chaton_tmpl_role_kv(tmpl, K_GLOBAL, K_END).c_str()); - LOG("INFO:%s:%s:%s", __func__, "system->prefix", chaton_tmpl_role_kv(tmpl, K_SYSTEM, K_PREFIX).c_str()); - LOG("INFO:%s:%s:%s", __func__, "system->suffix", chaton_tmpl_role_kv(tmpl, K_SYSTEM, K_SUFFIX).c_str()); - LOG("INFO:%s:%s:%s", __func__, "user->prefix", chaton_tmpl_role_kv(tmpl, K_USER, K_PREFIX).c_str()); - LOG("INFO:%s:%s:%s", __func__, "user->suffix", chaton_tmpl_role_kv(tmpl, K_USER, K_SUFFIX).c_str()); - LOG("INFO:%s:%s:%s", __func__, "assistant->prefix", chaton_tmpl_role_kv(tmpl, K_ASSISTANT, K_PREFIX).c_str()); - LOG("INFO:%s:%s:%s", __func__, "assistant->suffix", chaton_tmpl_role_kv(tmpl, K_ASSISTANT, K_SUFFIX).c_str()); - LOG("INFO:%s:%s:%s", __func__, K_REVERSE_PROMPT, chaton_tmpl_kv(tmpl, K_REVERSE_PROMPT).c_str()); - LOG("INFO:%s:%s:%d", __func__, K_SYSTEMUSER_1ST_USER_HAS_PREFIX, chaton_tmpl_kv_bool(tmpl, K_SYSTEMUSER_1ST_USER_HAS_PREFIX)); + LOGXLN("INFO:%s:%s:%s", __func__, "global->begin", chaton_tmpl_role_kv(tmpl, K_GLOBAL, K_BEGIN).c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "global->end", chaton_tmpl_role_kv(tmpl, K_GLOBAL, K_END).c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "system->prefix", chaton_tmpl_role_kv(tmpl, K_SYSTEM, K_PREFIX).c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "system->suffix", chaton_tmpl_role_kv(tmpl, K_SYSTEM, K_SUFFIX).c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "user->prefix", chaton_tmpl_role_kv(tmpl, K_USER, K_PREFIX).c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "user->suffix", chaton_tmpl_role_kv(tmpl, K_USER, K_SUFFIX).c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "assistant->prefix", chaton_tmpl_role_kv(tmpl, K_ASSISTANT, K_PREFIX).c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "assistant->suffix", chaton_tmpl_role_kv(tmpl, K_ASSISTANT, K_SUFFIX).c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, K_REVERSE_PROMPT, chaton_tmpl_kv(tmpl, K_REVERSE_PROMPT).c_str()); + LOGXLN("INFO:%s:%s:%d", __func__, K_SYSTEMUSER_1ST_USER_HAS_PREFIX, chaton_tmpl_kv_bool(tmpl, K_SYSTEMUSER_1ST_USER_HAS_PREFIX)); } } From 2a8028fba86136589eebab36208a16533cd2f2e9 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Tue, 23 Apr 2024 10:51:26 +0530 Subject: [PATCH 020/217] ChatON: Add Zephyr template to meta-json file --- examples/chaton_meta.json | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/examples/chaton_meta.json b/examples/chaton_meta.json index 548d7cc8fd046..52b977ad5b634 100644 --- a/examples/chaton_meta.json +++ b/examples/chaton_meta.json @@ -59,6 +59,26 @@ }, "reverse-prompt": "<|im_start|>user\n", "systemuser-1st-user-has-prefix": true + }, + "zephyr": { + "global": { + "begin": "", + "end": "" + }, + "system": { + "prefix": "<|system|>\n", + "suffix": "<|endoftext|>\n" + }, + "user": { + "prefix": "<|user|>\n", + "suffix": "<|endoftext|>\n" + }, + "assistant": { + "prefix": "<|assistant|>\n", + "suffix": "" + }, + "reverse-prompt": "", + "systemuser-1st-user-has-prefix": true } } From f4b54069f674e9ae2d020e4b1498e835ce5da46e Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Tue, 23 Apr 2024 11:23:20 +0530 Subject: [PATCH 021/217] ChatON: Add template for Gemma --- examples/chaton_meta.json | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/examples/chaton_meta.json b/examples/chaton_meta.json index 52b977ad5b634..05ffcc8fb52c5 100644 --- a/examples/chaton_meta.json +++ b/examples/chaton_meta.json @@ -79,6 +79,26 @@ }, "reverse-prompt": "", "systemuser-1st-user-has-prefix": true + }, + "gemma": { + "global": { + "begin": "", + "end": "" + }, + "system": { + "prefix": "", + "suffix": "" + }, + "user": { + "prefix": "user\n", + "suffix": "\n" + }, + "assistant": { + "prefix": "model\n", + "suffix": "" + }, + "reverse-prompt": "", + "systemuser-1st-user-has-prefix": true } } From 84367b9fd1f0aa5db1a1b80d14febe18e1e19a82 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Tue, 23 Apr 2024 12:46:40 +0530 Subject: [PATCH 022/217] ChatON: Add template for DeepSeek Was looking at the tokenized vector, and noticed that the EOS mentioned by existing chat_apply_template of llama.cpp, is different from what I noticed in tokenizer_config.json of deepseek llm, so I have added two entries * "deepseek-alt" which matches llama.cpp's chat_apply_template and * "deepseek" which matches that in tokenizer_config.json. This impacts the assistant suffix and reverse prompt entries. CasOfThis: Need to look into other entries which I added previously at a later time. However as the default logic should be picking the EOS from model file, so I assume reverse-prompt being outofsync, may not matter beyond a limit, potentially. --- common/chaton.hpp | 3 +++ examples/chaton_meta.json | 40 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/common/chaton.hpp b/common/chaton.hpp index be9957ff94e83..222c8657f9b08 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -49,6 +49,9 @@ * in-prefix, in-suffix and antiprompt of main. * These always adds any role specific prefix and suffix around the passed message. * + * Sample chaton_meta.json includes template info for + * * llama2, llama3, gemma, chatml, zephyr, deepseek + * */ #include diff --git a/examples/chaton_meta.json b/examples/chaton_meta.json index 05ffcc8fb52c5..4f6bcfc30550d 100644 --- a/examples/chaton_meta.json +++ b/examples/chaton_meta.json @@ -99,6 +99,46 @@ }, "reverse-prompt": "", "systemuser-1st-user-has-prefix": true + }, + "deepseek-alt": { + "global": { + "begin": "", + "end": "" + }, + "system": { + "prefix": "", + "suffix": "\n" + }, + "user": { + "prefix": "### Instruction:\n", + "suffix": "\n" + }, + "assistant": { + "prefix": "### Response:\n", + "suffix": "\n<|EOT|>\n" + }, + "reverse-prompt": "<|EOT|>", + "systemuser-1st-user-has-prefix": true + }, + "deepseek": { + "global": { + "begin": "", + "end": "" + }, + "system": { + "prefix": "", + "suffix": "\n\n" + }, + "user": { + "prefix": "User: ", + "suffix": "\n\n" + }, + "assistant": { + "prefix": "Assistant: ", + "suffix": " <｜end▁of▁sentence｜>\n" + }, + "reverse-prompt": "<｜end▁of▁sentence｜>", + "systemuser-1st-user-has-prefix": true } } From bdd279c0c9aa9174915d9a5febb949f4a1515f0a Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Tue, 23 Apr 2024 14:14:43 +0530 Subject: [PATCH 023/217] ChatOn:User Begin+Prefix note update, keep things simple consistent --- common/chaton.hpp | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index 222c8657f9b08..52722f04086a3 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -9,23 +9,23 @@ * * 1. Use a json file to configure the needed tags for each of the supported chat-handshake-template-standard * a. system -> prefix & suffix, - * b. user -> prefix & suffix, assistant -> prefix - * * [main] these override the in-prefix and in-suffix + * b. user -> begin, prefix & suffix; assistant -> prefix + * * [main] these override the in-prefix (begin+prefix) and in-suffix * c. reverse-prompt * * [main] this adds to any reverese-prompt specified using cmdline * d. global -> begin & end - * d. systemuser-1st-user-has-prefix - * * if a combination of system and user messages/prompts is passed, + * e. systemuser-1st-user-has-begin and systemuser-1st-user-has-prefix + * * [chaton-tmpl-apply] if a combination of system and user messages/prompts is passed, * then for the 1st user message following the 1st system message, - * include user prefix only if this flag is set. [chaton-tmpl-apply] - * * [later] one or two models which I looked at seem to require not just BoS, but also the user-role-prefix-tag - * to also be controlled wrt this case. So not differentiating between BoS and any user-role-prefix-tag. - * However if bos and user-role-prefix-tag need to be decoupled, where only bos needs this treatment, - * then maybe add begin and end keys (to specify the BoS) in addition to prefix and suffix keys (to specify user-role-prefix-tag), to role blocks in the json. - * and inturn control only begin and not prefix, wrt whether to add or not. + * include user begin and prefix only if corresponding flags is set. + * * begin should normally relate to BoS while prefix should relate to Role Identifier tag. + * If there is no need for seperate handling of BoS and RoleIdTag, then one could even + * set both BoS and RoleIdTag to one of these entries itself. + * * 2. [main] currently the user specified system prompt (-p + -f) is tagged using system role tags, * and inturn this tagged message is tokenized with parse_special flag. * So any special token related tags in the user specified system prompt will get parsed as special. + * * 3. chaton-tmpl-apply uses the json file, which was loaded, to decide on how to generate the tagged messages for tokenisation. * a. input: [ { role, message }, { role, message}, ....] * b. output: currently a single string is returned which contains the tagged message(s). From 0f713d4c4f3c5bc290fdc29274428dafb9ca4a64 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Tue, 23 Apr 2024 14:21:54 +0530 Subject: [PATCH 024/217] ChatOn: meta json update wrt the new begin related fields --- examples/chaton_meta.json | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/examples/chaton_meta.json b/examples/chaton_meta.json index 4f6bcfc30550d..5799cd3080a43 100644 --- a/examples/chaton_meta.json +++ b/examples/chaton_meta.json @@ -10,6 +10,7 @@ "suffix": "\n<> [/INST]\n\n" }, "user": { + "begin": "", "prefix": "[INST] ", "suffix": " [/INST]\n\n" }, @@ -18,6 +19,7 @@ "suffix": "" }, "reverse-prompt": "", + "systemuser-1st-user-has-begin": false, "systemuser-1st-user-has-prefix": false }, "llama3": { @@ -30,6 +32,7 @@ "suffix": "<|eot_id|>\n\n" }, "user": { + "begin": "", "prefix": "<|start_header_id|>user<|end_header_id|>\n", "suffix": "<|eot_id|>\n\n" }, @@ -38,6 +41,7 @@ "suffix": "" }, "reverse-prompt": "<|eot_id|>", + "systemuser-1st-user-has-begin": true, "systemuser-1st-user-has-prefix": true }, "chatml": { @@ -50,6 +54,7 @@ "suffix": "<|im_end|>\n" }, "user": { + "begin": "", "prefix": "\n<|im_start|>user\n", "suffix": "<|im_end|>\n" }, @@ -58,6 +63,7 @@ "suffix": "" }, "reverse-prompt": "<|im_start|>user\n", + "systemuser-1st-user-has-begin": true, "systemuser-1st-user-has-prefix": true }, "zephyr": { @@ -70,6 +76,7 @@ "suffix": "<|endoftext|>\n" }, "user": { + "begin": "", "prefix": "<|user|>\n", "suffix": "<|endoftext|>\n" }, @@ -78,6 +85,7 @@ "suffix": "" }, "reverse-prompt": "", + "systemuser-1st-user-has-begin": true, "systemuser-1st-user-has-prefix": true }, "gemma": { @@ -90,6 +98,7 @@ "suffix": "" }, "user": { + "begin": "", "prefix": "user\n", "suffix": "\n" }, @@ -98,6 +107,7 @@ "suffix": "" }, "reverse-prompt": "", + "systemuser-1st-user-has-begin": true, "systemuser-1st-user-has-prefix": true }, "deepseek-alt": { @@ -110,6 +120,7 @@ "suffix": "\n" }, "user": { + "begin": "", "prefix": "### Instruction:\n", "suffix": "\n" }, @@ -118,6 +129,7 @@ "suffix": "\n<|EOT|>\n" }, "reverse-prompt": "<|EOT|>", + "systemuser-1st-user-has-begin": true, "systemuser-1st-user-has-prefix": true }, "deepseek": { @@ -130,6 +142,7 @@ "suffix": "\n\n" }, "user": { + "begin": "", "prefix": "User: ", "suffix": "\n\n" }, @@ -138,6 +151,7 @@ "suffix": " <｜end▁of▁sentence｜>\n" }, "reverse-prompt": "<｜end▁of▁sentence｜>", + "systemuser-1st-user-has-begin": true, "systemuser-1st-user-has-prefix": true } } From d70fca7a45a4f9c986e278ca195201795f64a9ec Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Tue, 23 Apr 2024 14:38:43 +0530 Subject: [PATCH 025/217] ChatOn: Add begin to the mix along with prefix Dump shows user->begin. chat-template-apply[-single] updated to work with begin and prefix TODO: need to wrap begin in a try-catch, so that irrespective of role, begin+prefix will work, irrespoective of whether that role has a begin entry or not. --- common/chaton.hpp | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index 52722f04086a3..2214c7fe6d493 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -72,6 +72,7 @@ const auto K_SUFFIX = "suffix"; const auto K_BEGIN = "begin"; const auto K_END = "end"; const auto K_GLOBAL = "global"; +const auto K_SYSTEMUSER_1ST_USER_HAS_BEGIN = "systemuser-1st-user-has-begin"; const auto K_SYSTEMUSER_1ST_USER_HAS_PREFIX = "systemuser-1st-user-has-prefix"; const auto K_REVERSE_PROMPT = "reverse-prompt"; @@ -91,7 +92,10 @@ inline bool chaton_meta_load(std::string &fname) { // NOTE: This currently doesnt return about which parts of the tagged message contain tags and which parts the user message inline std::string chaton_tmpl_apply_single(const std::string &tmpl, const std::string &role, const std::string &content) { std::stringstream ss; - ss << conMeta[tmpl][role][K_PREFIX] << content << conMeta[tmpl][role][K_SUFFIX]; + std::string begin = conMeta[tmpl][role][K_BEGIN]; + std::string prefix = conMeta[tmpl][role][K_PREFIX]; + std::string suffix = conMeta[tmpl][role][K_SUFFIX]; + ss << begin << prefix << content << suffix; std::string taggedStr = ss.str(); LOGLN("DBUG:%s:%s:%s:%s", __func__, tmpl.c_str(), role.c_str(), taggedStr.c_str()); return taggedStr; @@ -110,22 +114,26 @@ inline std::string chaton_tmpl_apply(const std::string &tmpl, const std::vector< for(const auto msg: msgs) { auto role = msg.role; auto content = msg.content; + auto begin = conMeta[tmpl][role][K_BEGIN]; auto prefix = conMeta[tmpl][role][K_PREFIX]; if (role == K_SYSTEM) { cntSystem += 1; - ss << prefix; + ss << begin << prefix; } else if (role == K_USER) { cntUser += 1; if ((cntSystem == 1) && (cntUser == 1)) { + if (conMeta[tmpl][K_SYSTEMUSER_1ST_USER_HAS_BEGIN]) { + ss << begin; + } if (conMeta[tmpl][K_SYSTEMUSER_1ST_USER_HAS_PREFIX]) { ss << prefix; } } else { - ss << prefix; + ss << begin << prefix; } } else { cntOthers += 1; - ss << prefix; + ss << begin << prefix; } ss << content << conMeta[tmpl][role][K_SUFFIX]; } @@ -173,6 +181,7 @@ inline void _chaton_meta_dump(std::string &tmpl) { LOGXLN("INFO:%s:%s:%s", __func__, "global->end", chaton_tmpl_role_kv(tmpl, K_GLOBAL, K_END).c_str()); LOGXLN("INFO:%s:%s:%s", __func__, "system->prefix", chaton_tmpl_role_kv(tmpl, K_SYSTEM, K_PREFIX).c_str()); LOGXLN("INFO:%s:%s:%s", __func__, "system->suffix", chaton_tmpl_role_kv(tmpl, K_SYSTEM, K_SUFFIX).c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "user->begin", chaton_tmpl_role_kv(tmpl, K_USER, K_BEGIN).c_str()); LOGXLN("INFO:%s:%s:%s", __func__, "user->prefix", chaton_tmpl_role_kv(tmpl, K_USER, K_PREFIX).c_str()); LOGXLN("INFO:%s:%s:%s", __func__, "user->suffix", chaton_tmpl_role_kv(tmpl, K_USER, K_SUFFIX).c_str()); LOGXLN("INFO:%s:%s:%s", __func__, "assistant->prefix", chaton_tmpl_role_kv(tmpl, K_ASSISTANT, K_PREFIX).c_str()); From 724ff38345b0edc9d4b00003046c15ea6d58c34f Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Tue, 23 Apr 2024 15:01:41 +0530 Subject: [PATCH 026/217] ChatOn: Wrap getting begin in try-catch, so that even if a role doesnt contain begin, the logic will work fine. --- common/chaton.hpp | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index 2214c7fe6d493..149de8de5b48d 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -92,7 +92,12 @@ inline bool chaton_meta_load(std::string &fname) { // NOTE: This currently doesnt return about which parts of the tagged message contain tags and which parts the user message inline std::string chaton_tmpl_apply_single(const std::string &tmpl, const std::string &role, const std::string &content) { std::stringstream ss; - std::string begin = conMeta[tmpl][role][K_BEGIN]; + std::string begin = ""; + try { + begin = conMeta[tmpl][role][K_BEGIN]; + } catch (json::exception &err) { + + } std::string prefix = conMeta[tmpl][role][K_PREFIX]; std::string suffix = conMeta[tmpl][role][K_SUFFIX]; ss << begin << prefix << content << suffix; @@ -114,7 +119,12 @@ inline std::string chaton_tmpl_apply(const std::string &tmpl, const std::vector< for(const auto msg: msgs) { auto role = msg.role; auto content = msg.content; - auto begin = conMeta[tmpl][role][K_BEGIN]; + std::string begin = ""; + try { + begin = conMeta[tmpl][role][K_BEGIN]; + } catch (json::exception &err) { + + } auto prefix = conMeta[tmpl][role][K_PREFIX]; if (role == K_SYSTEM) { cntSystem += 1; From f1f39c525647c54be419daf77ccfcdc5baf59261 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Tue, 23 Apr 2024 15:15:25 +0530 Subject: [PATCH 027/217] ChatON:Add Monarch model template, which uses Begin + Prefix Inturn Begin/BoS is added only for non 1st user messages in a system+user prompts chain. --- common/chaton.hpp | 3 ++- examples/chaton_meta.json | 22 ++++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index 149de8de5b48d..a907f7483c274 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -50,7 +50,7 @@ * These always adds any role specific prefix and suffix around the passed message. * * Sample chaton_meta.json includes template info for - * * llama2, llama3, gemma, chatml, zephyr, deepseek + * * llama2, llama3, gemma, chatml, zephyr, deepseek, monarch * */ @@ -197,6 +197,7 @@ inline void _chaton_meta_dump(std::string &tmpl) { LOGXLN("INFO:%s:%s:%s", __func__, "assistant->prefix", chaton_tmpl_role_kv(tmpl, K_ASSISTANT, K_PREFIX).c_str()); LOGXLN("INFO:%s:%s:%s", __func__, "assistant->suffix", chaton_tmpl_role_kv(tmpl, K_ASSISTANT, K_SUFFIX).c_str()); LOGXLN("INFO:%s:%s:%s", __func__, K_REVERSE_PROMPT, chaton_tmpl_kv(tmpl, K_REVERSE_PROMPT).c_str()); + LOGXLN("INFO:%s:%s:%d", __func__, K_SYSTEMUSER_1ST_USER_HAS_BEGIN, chaton_tmpl_kv_bool(tmpl, K_SYSTEMUSER_1ST_USER_HAS_BEGIN)); LOGXLN("INFO:%s:%s:%d", __func__, K_SYSTEMUSER_1ST_USER_HAS_PREFIX, chaton_tmpl_kv_bool(tmpl, K_SYSTEMUSER_1ST_USER_HAS_PREFIX)); } } diff --git a/examples/chaton_meta.json b/examples/chaton_meta.json index 5799cd3080a43..1073884472cf0 100644 --- a/examples/chaton_meta.json +++ b/examples/chaton_meta.json @@ -153,6 +153,28 @@ "reverse-prompt": "<｜end▁of▁sentence｜>", "systemuser-1st-user-has-begin": true, "systemuser-1st-user-has-prefix": true + }, + "monarch": { + "global": { + "begin": "", + "end": "" + }, + "system": { + "prefix": "system\n", + "suffix": "\n" + }, + "user": { + "begin": "", + "prefix": "user\n", + "suffix": "\n" + }, + "assistant": { + "prefix": "assistant\n", + "suffix": " \n" + }, + "reverse-prompt": "", + "systemuser-1st-user-has-begin": false, + "systemuser-1st-user-has-prefix": true } } From 3064a36e7445f499970ebd3ab8e472edf0fb694b Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Tue, 23 Apr 2024 15:27:00 +0530 Subject: [PATCH 028/217] ChatON+:Update tmpl_role_kv to retrieve wrt multiple keys Use the same for user role's begin and prefix entries. --- common/chaton.hpp | 32 ++++++++++++++++++++------------ examples/main/main.cpp | 6 +++--- 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index a907f7483c274..690aa52d3c4ee 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -51,6 +51,8 @@ * * Sample chaton_meta.json includes template info for * * llama2, llama3, gemma, chatml, zephyr, deepseek, monarch + * * llama2 doesnt apply begin+prefix to 1st user msg following system msg + * * monarch doesnt apply begin to 1st user msg following system msg * */ @@ -154,9 +156,15 @@ inline std::string chaton_tmpl_apply(const std::string &tmpl, const std::vector< return taggedMsgs; } -inline std::string chaton_tmpl_role_kv(const std::string &tmpl, const std::string &role, const std::string &key) { - std::string got = conMeta[tmpl][role][key]; - LOGLN("DBUG:%s:%s:%s:%s:%s", __func__, tmpl.c_str(), role.c_str(), key.c_str(), got.c_str()); +inline std::string chaton_tmpl_role_kv(const std::string &tmpl, const std::string &role, const std::vector &keys) { + std::string got = ""; + std::string sKeys = ""; + for(auto key: keys) { + got += conMeta[tmpl][role][key]; + sKeys += "+"; + sKeys += key; + } + LOGLN("DBUG:%s:%s:%s:%s:%s", __func__, tmpl.c_str(), role.c_str(), sKeys.c_str(), got.c_str()); return got; } @@ -187,15 +195,15 @@ inline void _chaton_meta_dump(std::string &tmpl) { } LOGXLN("\n\nINFO:%s:ChatOn Meta\n%s", __func__, theJson.dump(4).c_str()); if (!tmpl.empty()) { - LOGXLN("INFO:%s:%s:%s", __func__, "global->begin", chaton_tmpl_role_kv(tmpl, K_GLOBAL, K_BEGIN).c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "global->end", chaton_tmpl_role_kv(tmpl, K_GLOBAL, K_END).c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "system->prefix", chaton_tmpl_role_kv(tmpl, K_SYSTEM, K_PREFIX).c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "system->suffix", chaton_tmpl_role_kv(tmpl, K_SYSTEM, K_SUFFIX).c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "user->begin", chaton_tmpl_role_kv(tmpl, K_USER, K_BEGIN).c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "user->prefix", chaton_tmpl_role_kv(tmpl, K_USER, K_PREFIX).c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "user->suffix", chaton_tmpl_role_kv(tmpl, K_USER, K_SUFFIX).c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "assistant->prefix", chaton_tmpl_role_kv(tmpl, K_ASSISTANT, K_PREFIX).c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "assistant->suffix", chaton_tmpl_role_kv(tmpl, K_ASSISTANT, K_SUFFIX).c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "global->begin", chaton_tmpl_role_kv(tmpl, K_GLOBAL, {K_BEGIN}).c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "global->end", chaton_tmpl_role_kv(tmpl, K_GLOBAL, {K_END}).c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "system->prefix", chaton_tmpl_role_kv(tmpl, K_SYSTEM, {K_PREFIX}).c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "system->suffix", chaton_tmpl_role_kv(tmpl, K_SYSTEM, {K_SUFFIX}).c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "user->begin", chaton_tmpl_role_kv(tmpl, K_USER, {K_BEGIN}).c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "user->prefix", chaton_tmpl_role_kv(tmpl, K_USER, {K_PREFIX}).c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "user->suffix", chaton_tmpl_role_kv(tmpl, K_USER, {K_SUFFIX}).c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "assistant->prefix", chaton_tmpl_role_kv(tmpl, K_ASSISTANT, {K_PREFIX}).c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "assistant->suffix", chaton_tmpl_role_kv(tmpl, K_ASSISTANT, {K_SUFFIX}).c_str()); LOGXLN("INFO:%s:%s:%s", __func__, K_REVERSE_PROMPT, chaton_tmpl_kv(tmpl, K_REVERSE_PROMPT).c_str()); LOGXLN("INFO:%s:%s:%d", __func__, K_SYSTEMUSER_1ST_USER_HAS_BEGIN, chaton_tmpl_kv_bool(tmpl, K_SYSTEMUSER_1ST_USER_HAS_BEGIN)); LOGXLN("INFO:%s:%s:%d", __func__, K_SYSTEMUSER_1ST_USER_HAS_PREFIX, chaton_tmpl_kv_bool(tmpl, K_SYSTEMUSER_1ST_USER_HAS_PREFIX)); diff --git a/examples/main/main.cpp b/examples/main/main.cpp index f75244800a0c0..a505c900b1258 100644 --- a/examples/main/main.cpp +++ b/examples/main/main.cpp @@ -375,12 +375,12 @@ int main(int argc, char ** argv) { } // chaton mode - const auto chaton_assitant_prefix = ::llama_tokenize(ctx, chaton_tmpl_role_kv(params.chaton_template_id, K_ASSISTANT, K_PREFIX), false, true); + const auto chaton_assitant_prefix = ::llama_tokenize(ctx, chaton_tmpl_role_kv(params.chaton_template_id, K_ASSISTANT, {K_PREFIX}), false, true); if (params.chaton) { params.interactive = true; // may remove later, by requiring user to explicitly request interactive mode params.interactive_first = true; - params.input_prefix = chaton_tmpl_role_kv(params.chaton_template_id, K_USER, K_PREFIX); - params.input_suffix = chaton_tmpl_role_kv(params.chaton_template_id, K_USER, K_SUFFIX); + params.input_prefix = chaton_tmpl_role_kv(params.chaton_template_id, K_USER, {K_BEGIN, K_PREFIX}); + params.input_suffix = chaton_tmpl_role_kv(params.chaton_template_id, K_USER, {K_SUFFIX}); params.antiprompt.emplace_back(chaton_tmpl_kv(params.chaton_template_id, K_REVERSE_PROMPT)); } From 9de1d6017fbfa28be44f0d1e7fa67809d17cf002 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Tue, 23 Apr 2024 22:01:30 +0530 Subject: [PATCH 029/217] ChatON:ChatParts class initial go Helps keep user prompt and chat-hs-template tag parts seperate, but in sequence --- common/chaton.hpp | 75 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/common/chaton.hpp b/common/chaton.hpp index 690aa52d3c4ee..a83c967e692bf 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -83,6 +83,70 @@ using json = nlohmann::json; json conMeta; + +/** + * Helps keep user prompt and chat-hs-template tag parts seperate, but in sequence + */ +class ChatParts { + + std::vector parts = {}; + std::string types = {""}; + +public: + // Identify string with special tokens that need to be processed. + static const auto S = 's'; + // Identify string which shouldnt have special token processing done. + static const auto N = 'n'; + // Identify no string condition and or ignore string. + static const auto X = '?'; + + ChatParts() :parts{}, types{""} {} + + char last_type() { + if (types.length() == 0) { + return ChatParts::X; + } + return types[types.length()-1]; + } + + void add_part(char type, const std::string &part) { + if (last_type() == type) { + parts[parts.size()-1] += part; + } else { + parts.emplace_back(part); + types += type; + } + } + + std::string str() { + std::string allin = ""; + for(auto part: parts) { + allin += part; + } + return allin; + } + + std::string name() { + return typeid(*this).name(); + } + + void dump() { + std::string me = name() + ":" + __func__; + LOGXLN("INFO:%s:NumTypes:%zu", me.c_str(), types.length()); + LOGXLN("INFO:%s:NumParts:%zu", me.c_str(), parts.size()); + LOGXLN("INFO:%s:StrLength:%zu", me.c_str(), str().length()); + if (parts.size() != types.length()) { + LOG_TEELN("DBUG:%s:Mismatch between parts and types", me.c_str()); + } + int i = 0; + for(auto part: parts) { + LOGXLN("INFO:%s:%c:%s", me.c_str(), types[i], part.c_str()); + i += 1; + } + } + +}; + inline bool chaton_meta_load(std::string &fname) { std::ifstream f(fname); conMeta = json::parse(f); @@ -93,6 +157,7 @@ inline bool chaton_meta_load(std::string &fname) { // Return user-prefix + msg + user-suffix // NOTE: This currently doesnt return about which parts of the tagged message contain tags and which parts the user message inline std::string chaton_tmpl_apply_single(const std::string &tmpl, const std::string &role, const std::string &content) { + ChatParts cp = {}; std::stringstream ss; std::string begin = ""; try { @@ -102,8 +167,18 @@ inline std::string chaton_tmpl_apply_single(const std::string &tmpl, const std:: } std::string prefix = conMeta[tmpl][role][K_PREFIX]; std::string suffix = conMeta[tmpl][role][K_SUFFIX]; + cp.add_part(ChatParts::S, begin); + cp.add_part(ChatParts::S, prefix); + cp.add_part(ChatParts::N, content); + cp.add_part(ChatParts::S, suffix); + cp.dump(); ss << begin << prefix << content << suffix; std::string taggedStr = ss.str(); + std::string cpStr = cp.str(); + if (taggedStr != cpStr) { + LOG_TEELN("DBUG:%s:Mismatch between CP[%s] and SS[%s]", __func__, cpStr.c_str(), taggedStr.c_str()); + exit(2); + } LOGLN("DBUG:%s:%s:%s:%s", __func__, tmpl.c_str(), role.c_str(), taggedStr.c_str()); return taggedStr; } From d1899728aa0d3e7299e4810b4df3354155851ad8 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Tue, 23 Apr 2024 22:22:06 +0530 Subject: [PATCH 030/217] ChatON: Test ChatParts in chat-template-apply --- common/chaton.hpp | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/common/chaton.hpp b/common/chaton.hpp index a83c967e692bf..c914cde5dcba2 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -188,8 +188,10 @@ inline std::string chaton_tmpl_apply_single(const std::string &tmpl, const std:: // then 1st user message will have user-prefix only if systemuser-1st-user-has-prefix is true // NOTE: This currently doesnt return about which parts of the tagged message contain tags and which parts the user message inline std::string chaton_tmpl_apply(const std::string &tmpl, const std::vector &msgs) { + ChatParts cp = {}; std::stringstream ss; ss << conMeta[tmpl][K_GLOBAL][K_BEGIN]; + cp.add_part(ChatParts::S, conMeta[tmpl][K_GLOBAL][K_BEGIN]); int cntSystem = 0; int cntUser = 0; int cntOthers = 0; @@ -199,6 +201,7 @@ inline std::string chaton_tmpl_apply(const std::string &tmpl, const std::vector< std::string begin = ""; try { begin = conMeta[tmpl][role][K_BEGIN]; + cp.add_part(ChatParts::S, begin); } catch (json::exception &err) { } @@ -206,26 +209,43 @@ inline std::string chaton_tmpl_apply(const std::string &tmpl, const std::vector< if (role == K_SYSTEM) { cntSystem += 1; ss << begin << prefix; + cp.add_part(ChatParts::S, begin); + cp.add_part(ChatParts::S, prefix); } else if (role == K_USER) { cntUser += 1; if ((cntSystem == 1) && (cntUser == 1)) { if (conMeta[tmpl][K_SYSTEMUSER_1ST_USER_HAS_BEGIN]) { ss << begin; + cp.add_part(ChatParts::S, begin); } if (conMeta[tmpl][K_SYSTEMUSER_1ST_USER_HAS_PREFIX]) { ss << prefix; + cp.add_part(ChatParts::S, prefix); } } else { ss << begin << prefix; + cp.add_part(ChatParts::S, begin); + cp.add_part(ChatParts::S, prefix); } } else { cntOthers += 1; ss << begin << prefix; + cp.add_part(ChatParts::S, begin); + cp.add_part(ChatParts::S, prefix); } ss << content << conMeta[tmpl][role][K_SUFFIX]; + cp.add_part(ChatParts::N, content); + cp.add_part(ChatParts::S, conMeta[tmpl][role][K_SUFFIX]); } ss << conMeta[tmpl][K_GLOBAL][K_END]; + cp.add_part(ChatParts::S, conMeta[tmpl][K_GLOBAL][K_END]); + cp.dump(); std::string taggedMsgs = ss.str(); + std::string cpStr = cp.str(); + if (taggedMsgs != cpStr) { + LOG_TEELN("DBUG:%s:Mismatch between CP[%s] and SS[%s]", __func__, cpStr.c_str(), taggedMsgs.c_str()); + exit(2); + } LOGLN("DBUG:%s:%s:%s", __func__, tmpl.c_str(), taggedMsgs.c_str()); LOGLN("DBUG:%s:%s:CntSys[%d]:CntUsr[%d]:CntOthers[%d]", __func__, tmpl.c_str(), cntSystem, cntUser, cntOthers); return taggedMsgs; From 6b23f15ffe97fa78ab83e91b88c5e0e29770f32d Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Wed, 24 Apr 2024 20:17:28 +0530 Subject: [PATCH 031/217] ChatON:ChatOnMetaJSon: Add suffix wrt assistant messages --- examples/chaton_meta.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/chaton_meta.json b/examples/chaton_meta.json index 1073884472cf0..25f05a17076e3 100644 --- a/examples/chaton_meta.json +++ b/examples/chaton_meta.json @@ -16,7 +16,7 @@ }, "assistant": { "prefix": "", - "suffix": "" + "suffix": "" }, "reverse-prompt": "", "systemuser-1st-user-has-begin": false, @@ -38,7 +38,7 @@ }, "assistant": { "prefix": "<|start_header_id|>assistant<|end_header_id|>\n", - "suffix": "" + "suffix": "<|eot_id|>\n\n" }, "reverse-prompt": "<|eot_id|>", "systemuser-1st-user-has-begin": true, @@ -60,7 +60,7 @@ }, "assistant": { "prefix": "<|im_start|>assistant\n", - "suffix": "" + "suffix": "<|im_end|>\n" }, "reverse-prompt": "<|im_start|>user\n", "systemuser-1st-user-has-begin": true, @@ -82,7 +82,7 @@ }, "assistant": { "prefix": "<|assistant|>\n", - "suffix": "" + "suffix": "<|endoftext|>\n" }, "reverse-prompt": "", "systemuser-1st-user-has-begin": true, @@ -104,7 +104,7 @@ }, "assistant": { "prefix": "model\n", - "suffix": "" + "suffix": "\n" }, "reverse-prompt": "", "systemuser-1st-user-has-begin": true, From 92e780fb1a600785700fcd3cb9656dacd938685a Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Wed, 24 Apr 2024 20:23:44 +0530 Subject: [PATCH 032/217] ChatON:ChatParts: Allow flexibility for more refined tokenization --- common/chaton.hpp | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index c914cde5dcba2..1cccee95b424b 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -85,7 +85,9 @@ json conMeta; /** - * Helps keep user prompt and chat-hs-template tag parts seperate, but in sequence + * Helps keep user prompt and chat-hs-template tag parts seperate, but in sequence. + * Inturn gives the flexibility to tokenize with or without parse_special flag, wrt the different parts of the chat msg(s). + * One could use the triplet of str, get_types and get_partslens to achieve the above mentioned flexibility. */ class ChatParts { @@ -100,7 +102,7 @@ class ChatParts { // Identify no string condition and or ignore string. static const auto X = '?'; - ChatParts() :parts{}, types{""} {} + ChatParts() : parts{}, types{""} {} char last_type() { if (types.length() == 0) { @@ -126,6 +128,18 @@ class ChatParts { return allin; } + std::string get_types() { + return types; + } + + std::vector get_partslens() { + std::vector lens = {}; + for(auto part: parts) { + lens.push_back(part.length()); + } + return lens; + } + std::string name() { return typeid(*this).name(); } From 825a78abaa972262e2fe408e862335283e033120 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Wed, 24 Apr 2024 20:54:19 +0530 Subject: [PATCH 033/217] ChatOn: ChatTemplateApplySingle[Ex] return parts detail Now there is a simple and extended version of returning tagged message wrt a single role and its content. The extended version returns the tagged string, as well as the details of the parts that make up that tagged message interms of the type of parts and the lengths of the parts. --- common/chaton.hpp | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index 1cccee95b424b..431aa0980b2d6 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -169,8 +169,14 @@ inline bool chaton_meta_load(std::string &fname) { // Return user-prefix + msg + user-suffix -// NOTE: This currently doesnt return about which parts of the tagged message contain tags and which parts the user message -inline std::string chaton_tmpl_apply_single(const std::string &tmpl, const std::string &role, const std::string &content) { +inline bool chaton_tmpl_apply_single_ex( + const std::string &tmpl, + const std::string &role, + const std::string &content, + std::string &tagged, + std::string &types, + std::vector &lens + ) { ChatParts cp = {}; std::stringstream ss; std::string begin = ""; @@ -187,14 +193,25 @@ inline std::string chaton_tmpl_apply_single(const std::string &tmpl, const std:: cp.add_part(ChatParts::S, suffix); cp.dump(); ss << begin << prefix << content << suffix; - std::string taggedStr = ss.str(); + tagged = ss.str(); std::string cpStr = cp.str(); - if (taggedStr != cpStr) { - LOG_TEELN("DBUG:%s:Mismatch between CP[%s] and SS[%s]", __func__, cpStr.c_str(), taggedStr.c_str()); + if (tagged != cpStr) { + LOG_TEELN("DBUG:%s:Mismatch between CP[%s] and SS[%s]", __func__, cpStr.c_str(), tagged.c_str()); exit(2); } - LOGLN("DBUG:%s:%s:%s:%s", __func__, tmpl.c_str(), role.c_str(), taggedStr.c_str()); - return taggedStr; + LOGLN("DBUG:%s:%s:%s:%s", __func__, tmpl.c_str(), role.c_str(), tagged.c_str()); + types = cp.get_types(); + lens = cp.get_partslens(); + return true; +} + +// Return user-prefix + msg + user-suffix, types string and lens vector wrt the parts that make up the returned string +inline std::string chaton_tmpl_apply_single(const std::string &tmpl, const std::string &role, const std::string &content) { + std::string tagged; + std::string types; + std::vector lens; + chaton_tmpl_apply_single_ex(tmpl, role, content, tagged, types, lens); + return tagged; } // global-begin + [role-prefix + msg + role-suffix] + global-end From 3f09eb5dea5a1792c988a8945435b04ece159931 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Wed, 24 Apr 2024 21:12:42 +0530 Subject: [PATCH 034/217] ChatOn: ChatTemplateApply[Ex] return tagged msgs parts detail Now there is a simple and extended version of returning tagged messages. The extended version returns the tagged string, as well as the details of the parts that make up that tagged message interms of the type of parts and the lengths of the parts. --- common/chaton.hpp | 35 +++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index 431aa0980b2d6..1cf94b562d961 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -214,11 +214,17 @@ inline std::string chaton_tmpl_apply_single(const std::string &tmpl, const std:: return tagged; } -// global-begin + [role-prefix + msg + role-suffix] + global-end +// global-begin + [[role-begin] + [role-prefix] + msg + role-suffix] + global-end // if there is a combination of system-user messages, // then 1st user message will have user-prefix only if systemuser-1st-user-has-prefix is true -// NOTE: This currently doesnt return about which parts of the tagged message contain tags and which parts the user message -inline std::string chaton_tmpl_apply(const std::string &tmpl, const std::vector &msgs) { +// NOTE: returns types and lens to help identify the parts of the tagged msg, which relate to passed and added tags +inline bool chaton_tmpl_apply_ex( + const std::string &tmpl, + const std::vector &msgs, + std::string &tagged, + std::string &types, + std::vector &lens + ) { ChatParts cp = {}; std::stringstream ss; ss << conMeta[tmpl][K_GLOBAL][K_BEGIN]; @@ -271,15 +277,28 @@ inline std::string chaton_tmpl_apply(const std::string &tmpl, const std::vector< ss << conMeta[tmpl][K_GLOBAL][K_END]; cp.add_part(ChatParts::S, conMeta[tmpl][K_GLOBAL][K_END]); cp.dump(); - std::string taggedMsgs = ss.str(); + tagged = ss.str(); std::string cpStr = cp.str(); - if (taggedMsgs != cpStr) { - LOG_TEELN("DBUG:%s:Mismatch between CP[%s] and SS[%s]", __func__, cpStr.c_str(), taggedMsgs.c_str()); + if (tagged != cpStr) { + LOG_TEELN("DBUG:%s:Mismatch between CP[%s] and SS[%s]", __func__, cpStr.c_str(), tagged.c_str()); exit(2); } - LOGLN("DBUG:%s:%s:%s", __func__, tmpl.c_str(), taggedMsgs.c_str()); + LOGLN("DBUG:%s:%s:%s", __func__, tmpl.c_str(), tagged.c_str()); LOGLN("DBUG:%s:%s:CntSys[%d]:CntUsr[%d]:CntOthers[%d]", __func__, tmpl.c_str(), cntSystem, cntUser, cntOthers); - return taggedMsgs; + types = cp.get_types(); + lens = cp.get_partslens(); + return true; +} + +// global-begin + [[role-begin] + [role-prefix] + msg + role-suffix] + global-end +// if there is a combination of system-user messages, +// then 1st user message will have user-prefix only if systemuser-1st-user-has-prefix is true +inline std::string chaton_tmpl_apply(const std::string &tmpl, const std::vector &msgs) { + std::string tagged; + std::string types; + std::vector lens; + chaton_tmpl_apply_ex(tmpl, msgs, tagged, types, lens); + return tagged; } inline std::string chaton_tmpl_role_kv(const std::string &tmpl, const std::string &role, const std::vector &keys) { From adab5775bf5df9d4aa4e614f0505d1aecdddcf04 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Wed, 24 Apr 2024 21:30:59 +0530 Subject: [PATCH 035/217] ChatON: more detailed/spreadout json fields --- common/chaton.hpp | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index 1cf94b562d961..c8ec25c7af75d 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -7,17 +7,18 @@ * Helps chat with a model, by allowing role based special token tagging, based on the specified chat-handshake-template-standard. * This is used by main, to build on existing interactive flow and its in-prefix, in-suffix and antiprompt/reverse-promot * - * 1. Use a json file to configure the needed tags for each of the supported chat-handshake-template-standard - * a. system -> prefix & suffix, - * b. user -> begin, prefix & suffix; assistant -> prefix - * * [main] these override the in-prefix (begin+prefix) and in-suffix + * 1. [ToDo] Use a json file to configure the needed tags for each of the supported chat-handshake-template-standard + * * global-> begin & end + * * system -> begin, prefix, suffix & end + * * user -> begin, prefix, suffix & end; assistant -> begin, prefix, suffix & end + * * [main] these override the in-prefix (begin+prefix) and in-suffix * c. reverse-prompt - * * [main] this adds to any reverese-prompt specified using cmdline - * d. global -> begin & end - * e. systemuser-1st-user-has-begin and systemuser-1st-user-has-prefix + * * [main] this adds to any reverese-prompt specified using cmdline + * e. systemuser-sys-has-suffix, systemuser-sys-has-end, systemuser-1st-user-has-begin and systemuser-1st-user-has-prefix * * [chaton-tmpl-apply] if a combination of system and user messages/prompts is passed, - * then for the 1st user message following the 1st system message, - * include user begin and prefix only if corresponding flags is set. + * then for system messages suffix and end, as well as + * for the 1st user message following the 1st system message, + * include system suffix and end and user begin and prefix only if corresponding flags is set. * * begin should normally relate to BoS while prefix should relate to Role Identifier tag. * If there is no need for seperate handling of BoS and RoleIdTag, then one could even * set both BoS and RoleIdTag to one of these entries itself. @@ -74,6 +75,8 @@ const auto K_SUFFIX = "suffix"; const auto K_BEGIN = "begin"; const auto K_END = "end"; const auto K_GLOBAL = "global"; +const auto K_SYSTEMUSER_SYSTEM_HAS_SUFFIX = "systemuser-systemp-has-suffix"; +const auto K_SYSTEMUSER_SYSTEM_HAS_END = "systemuser-system-has-end"; const auto K_SYSTEMUSER_1ST_USER_HAS_BEGIN = "systemuser-1st-user-has-begin"; const auto K_SYSTEMUSER_1ST_USER_HAS_PREFIX = "systemuser-1st-user-has-prefix"; const auto K_REVERSE_PROMPT = "reverse-prompt"; @@ -342,14 +345,21 @@ inline void _chaton_meta_dump(std::string &tmpl) { if (!tmpl.empty()) { LOGXLN("INFO:%s:%s:%s", __func__, "global->begin", chaton_tmpl_role_kv(tmpl, K_GLOBAL, {K_BEGIN}).c_str()); LOGXLN("INFO:%s:%s:%s", __func__, "global->end", chaton_tmpl_role_kv(tmpl, K_GLOBAL, {K_END}).c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "system->begin", chaton_tmpl_role_kv(tmpl, K_SYSTEM, {K_BEGIN}).c_str()); LOGXLN("INFO:%s:%s:%s", __func__, "system->prefix", chaton_tmpl_role_kv(tmpl, K_SYSTEM, {K_PREFIX}).c_str()); LOGXLN("INFO:%s:%s:%s", __func__, "system->suffix", chaton_tmpl_role_kv(tmpl, K_SYSTEM, {K_SUFFIX}).c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "system->end", chaton_tmpl_role_kv(tmpl, K_SYSTEM, {K_END}).c_str()); LOGXLN("INFO:%s:%s:%s", __func__, "user->begin", chaton_tmpl_role_kv(tmpl, K_USER, {K_BEGIN}).c_str()); LOGXLN("INFO:%s:%s:%s", __func__, "user->prefix", chaton_tmpl_role_kv(tmpl, K_USER, {K_PREFIX}).c_str()); LOGXLN("INFO:%s:%s:%s", __func__, "user->suffix", chaton_tmpl_role_kv(tmpl, K_USER, {K_SUFFIX}).c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "user->end", chaton_tmpl_role_kv(tmpl, K_USER, {K_END}).c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "assistant->begin", chaton_tmpl_role_kv(tmpl, K_ASSISTANT, {K_BEGIN}).c_str()); LOGXLN("INFO:%s:%s:%s", __func__, "assistant->prefix", chaton_tmpl_role_kv(tmpl, K_ASSISTANT, {K_PREFIX}).c_str()); LOGXLN("INFO:%s:%s:%s", __func__, "assistant->suffix", chaton_tmpl_role_kv(tmpl, K_ASSISTANT, {K_SUFFIX}).c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "assistant->end", chaton_tmpl_role_kv(tmpl, K_ASSISTANT, {K_END}).c_str()); LOGXLN("INFO:%s:%s:%s", __func__, K_REVERSE_PROMPT, chaton_tmpl_kv(tmpl, K_REVERSE_PROMPT).c_str()); + LOGXLN("INFO:%s:%s:%d", __func__, K_SYSTEMUSER_SYSTEM_HAS_SUFFIX, chaton_tmpl_kv_bool(tmpl, K_SYSTEMUSER_SYSTEM_HAS_SUFFIX)); + LOGXLN("INFO:%s:%s:%d", __func__, K_SYSTEMUSER_SYSTEM_HAS_END, chaton_tmpl_kv_bool(tmpl, K_SYSTEMUSER_SYSTEM_HAS_END)); LOGXLN("INFO:%s:%s:%d", __func__, K_SYSTEMUSER_1ST_USER_HAS_BEGIN, chaton_tmpl_kv_bool(tmpl, K_SYSTEMUSER_1ST_USER_HAS_BEGIN)); LOGXLN("INFO:%s:%s:%d", __func__, K_SYSTEMUSER_1ST_USER_HAS_PREFIX, chaton_tmpl_kv_bool(tmpl, K_SYSTEMUSER_1ST_USER_HAS_PREFIX)); } From 7ba0144e42d57cec424d72f1fe9df50a21466cb2 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Wed, 24 Apr 2024 21:50:59 +0530 Subject: [PATCH 036/217] ChatOn:chaton_tmpl_role_kv: try except to ignore missing ifany Cas of above reason, switch to directly accessing the keys in dump helper, which is inturn used by meta_ok check --- common/chaton.hpp | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index c8ec25c7af75d..c8ff21c06d865 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -308,7 +308,10 @@ inline std::string chaton_tmpl_role_kv(const std::string &tmpl, const std::strin std::string got = ""; std::string sKeys = ""; for(auto key: keys) { - got += conMeta[tmpl][role][key]; + try { + got += conMeta[tmpl][role][key]; + } catch (json::exception &err) { + } sKeys += "+"; sKeys += key; } @@ -343,20 +346,20 @@ inline void _chaton_meta_dump(std::string &tmpl) { } LOGXLN("\n\nINFO:%s:ChatOn Meta\n%s", __func__, theJson.dump(4).c_str()); if (!tmpl.empty()) { - LOGXLN("INFO:%s:%s:%s", __func__, "global->begin", chaton_tmpl_role_kv(tmpl, K_GLOBAL, {K_BEGIN}).c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "global->end", chaton_tmpl_role_kv(tmpl, K_GLOBAL, {K_END}).c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "system->begin", chaton_tmpl_role_kv(tmpl, K_SYSTEM, {K_BEGIN}).c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "system->prefix", chaton_tmpl_role_kv(tmpl, K_SYSTEM, {K_PREFIX}).c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "system->suffix", chaton_tmpl_role_kv(tmpl, K_SYSTEM, {K_SUFFIX}).c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "system->end", chaton_tmpl_role_kv(tmpl, K_SYSTEM, {K_END}).c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "user->begin", chaton_tmpl_role_kv(tmpl, K_USER, {K_BEGIN}).c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "user->prefix", chaton_tmpl_role_kv(tmpl, K_USER, {K_PREFIX}).c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "user->suffix", chaton_tmpl_role_kv(tmpl, K_USER, {K_SUFFIX}).c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "user->end", chaton_tmpl_role_kv(tmpl, K_USER, {K_END}).c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "assistant->begin", chaton_tmpl_role_kv(tmpl, K_ASSISTANT, {K_BEGIN}).c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "assistant->prefix", chaton_tmpl_role_kv(tmpl, K_ASSISTANT, {K_PREFIX}).c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "assistant->suffix", chaton_tmpl_role_kv(tmpl, K_ASSISTANT, {K_SUFFIX}).c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "assistant->end", chaton_tmpl_role_kv(tmpl, K_ASSISTANT, {K_END}).c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "global->begin", conMeta[tmpl][K_GLOBAL][K_BEGIN].c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "global->end", conMeta[tmpl][K_GLOBAL][K_END].c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "system->begin", conMeta[tmpl][K_SYSTEM][K_BEGIN].c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "system->prefix", conMeta[tmpl][K_SYSTEM][K_PREFIX].c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "system->suffix", conMeta[tmpl][K_SYSTEM][K_SUFFIX].c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "system->end", conMeta[tmpl][K_SYSTEM][K_END].c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "user->begin", conMeta[tmpl][K_USER][K_BEGIN].c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "user->prefix", conMeta[tmpl][K_USER][K_PREFIX].c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "user->suffix", conMeta[tmpl][K_USER][K_SUFFIX].c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "user->end", conMeta[tmpl][K_USER][K_END].c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "assistant->begin", conMeta[tmpl][K_ASSISTANT][K_BEGIN].c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "assistant->prefix", conMeta[tmpl][K_ASSISTANT][K_PREFIX].c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "assistant->suffix", conMeta[tmpl][K_ASSISTANT][K_SUFFIX].c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "assistant->end", conMeta[tmpl][K_ASSISTANT][K_END].c_str()); LOGXLN("INFO:%s:%s:%s", __func__, K_REVERSE_PROMPT, chaton_tmpl_kv(tmpl, K_REVERSE_PROMPT).c_str()); LOGXLN("INFO:%s:%s:%d", __func__, K_SYSTEMUSER_SYSTEM_HAS_SUFFIX, chaton_tmpl_kv_bool(tmpl, K_SYSTEMUSER_SYSTEM_HAS_SUFFIX)); LOGXLN("INFO:%s:%s:%d", __func__, K_SYSTEMUSER_SYSTEM_HAS_END, chaton_tmpl_kv_bool(tmpl, K_SYSTEMUSER_SYSTEM_HAS_END)); From 5d76f08d37ea2c4c586715e0199b8bd56114f7d9 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Wed, 24 Apr 2024 22:50:29 +0530 Subject: [PATCH 037/217] ChatON: Need to explicitly specify string to use c_str --- common/chaton.hpp | 43 +++++++++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index c8ff21c06d865..9c5a9b6e755f0 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -346,20 +346,35 @@ inline void _chaton_meta_dump(std::string &tmpl) { } LOGXLN("\n\nINFO:%s:ChatOn Meta\n%s", __func__, theJson.dump(4).c_str()); if (!tmpl.empty()) { - LOGXLN("INFO:%s:%s:%s", __func__, "global->begin", conMeta[tmpl][K_GLOBAL][K_BEGIN].c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "global->end", conMeta[tmpl][K_GLOBAL][K_END].c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "system->begin", conMeta[tmpl][K_SYSTEM][K_BEGIN].c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "system->prefix", conMeta[tmpl][K_SYSTEM][K_PREFIX].c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "system->suffix", conMeta[tmpl][K_SYSTEM][K_SUFFIX].c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "system->end", conMeta[tmpl][K_SYSTEM][K_END].c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "user->begin", conMeta[tmpl][K_USER][K_BEGIN].c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "user->prefix", conMeta[tmpl][K_USER][K_PREFIX].c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "user->suffix", conMeta[tmpl][K_USER][K_SUFFIX].c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "user->end", conMeta[tmpl][K_USER][K_END].c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "assistant->begin", conMeta[tmpl][K_ASSISTANT][K_BEGIN].c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "assistant->prefix", conMeta[tmpl][K_ASSISTANT][K_PREFIX].c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "assistant->suffix", conMeta[tmpl][K_ASSISTANT][K_SUFFIX].c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "assistant->end", conMeta[tmpl][K_ASSISTANT][K_END].c_str()); + std::string globalBegin = conMeta[tmpl][K_GLOBAL][K_BEGIN]; + std::string globalEnd = conMeta[tmpl][K_GLOBAL][K_END]; + std::string systemBegin = conMeta[tmpl][K_SYSTEM][K_BEGIN]; + std::string systemPrefix = conMeta[tmpl][K_SYSTEM][K_PREFIX]; + std::string systemSuffix = conMeta[tmpl][K_SYSTEM][K_SUFFIX]; + std::string systemEnd = conMeta[tmpl][K_SYSTEM][K_END]; + std::string userBegin = conMeta[tmpl][K_USER][K_BEGIN]; + std::string userPrefix = conMeta[tmpl][K_USER][K_PREFIX]; + std::string userSuffix = conMeta[tmpl][K_USER][K_SUFFIX]; + std::string userEnd = conMeta[tmpl][K_USER][K_END]; + std::string assistantBegin = conMeta[tmpl][K_ASSISTANT][K_BEGIN]; + std::string assistantPrefix = conMeta[tmpl][K_ASSISTANT][K_PREFIX]; + std::string assistantSuffix = conMeta[tmpl][K_ASSISTANT][K_SUFFIX]; + std::string assistantEnd = conMeta[tmpl][K_ASSISTANT][K_END]; + + LOGXLN("INFO:%s:%s:%s", __func__, "global->begin", globalBegin.c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "global->end", globalEnd.c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "system->begin", systemBegin.c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "system->prefix", systemPrefix.c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "system->suffix", systemSuffix.c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "system->end", systemEnd.c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "user->begin", userBegin.c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "user->prefix", userPrefix.c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "user->suffix", userSuffix.c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "user->end", userEnd.c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "assistant->begin", assistantBegin.c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "assistant->prefix", assistantPrefix.c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "assistant->suffix", assistantSuffix.c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "assistant->end", assistantEnd.c_str()); LOGXLN("INFO:%s:%s:%s", __func__, K_REVERSE_PROMPT, chaton_tmpl_kv(tmpl, K_REVERSE_PROMPT).c_str()); LOGXLN("INFO:%s:%s:%d", __func__, K_SYSTEMUSER_SYSTEM_HAS_SUFFIX, chaton_tmpl_kv_bool(tmpl, K_SYSTEMUSER_SYSTEM_HAS_SUFFIX)); LOGXLN("INFO:%s:%s:%d", __func__, K_SYSTEMUSER_SYSTEM_HAS_END, chaton_tmpl_kv_bool(tmpl, K_SYSTEMUSER_SYSTEM_HAS_END)); From f8ae21cec71f1ed5739eed322d01a954e25a7b0b Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Thu, 25 Apr 2024 06:32:24 +0530 Subject: [PATCH 038/217] ChatON:ChatTemplateApplySingle: update begin+prefix, suffix+end --- common/chaton.hpp | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index 9c5a9b6e755f0..d2db60ba84343 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -182,20 +182,13 @@ inline bool chaton_tmpl_apply_single_ex( ) { ChatParts cp = {}; std::stringstream ss; - std::string begin = ""; - try { - begin = conMeta[tmpl][role][K_BEGIN]; - } catch (json::exception &err) { - - } - std::string prefix = conMeta[tmpl][role][K_PREFIX]; - std::string suffix = conMeta[tmpl][role][K_SUFFIX]; - cp.add_part(ChatParts::S, begin); - cp.add_part(ChatParts::S, prefix); + std::string beginPrefix = chaton_tmpl_role_kv(tmpl, role, {K_BEGIN, K_PREFIX}); + std::string suffixEnd = chaton_tmpl_role_kv(tmpl, role, {K_SUFFIX, K_END}); + cp.add_part(ChatParts::S, beginPrefix); cp.add_part(ChatParts::N, content); - cp.add_part(ChatParts::S, suffix); + cp.add_part(ChatParts::S, suffixEnd); cp.dump(); - ss << begin << prefix << content << suffix; + ss << beginPrefix << content << suffixEnd; tagged = ss.str(); std::string cpStr = cp.str(); if (tagged != cpStr) { From 344857b6cbd4c681f8c84c740a54367754fcbccd Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Thu, 25 Apr 2024 06:49:30 +0530 Subject: [PATCH 039/217] ChatOn:ChatOnTemplateApply: suffix,end flag based control Also fix a oversight wrt begin, when flag based begin adding control was introduced. NOTE: Currently system role suffix/end conditional adding always triggered, if 1st system prompt seen or additional system prompt is seen. --- common/chaton.hpp | 39 +++++++++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index d2db60ba84343..ea42c41446990 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -223,22 +223,19 @@ inline bool chaton_tmpl_apply_ex( ) { ChatParts cp = {}; std::stringstream ss; - ss << conMeta[tmpl][K_GLOBAL][K_BEGIN]; - cp.add_part(ChatParts::S, conMeta[tmpl][K_GLOBAL][K_BEGIN]); + std::string globalBegin = chaton_tmpl_role_kv(tmpl, K_GLOBAL, {K_BEGIN}); + ss << globalBegin; + cp.add_part(ChatParts::S, globalBegin); int cntSystem = 0; int cntUser = 0; int cntOthers = 0; for(const auto msg: msgs) { auto role = msg.role; auto content = msg.content; - std::string begin = ""; - try { - begin = conMeta[tmpl][role][K_BEGIN]; - cp.add_part(ChatParts::S, begin); - } catch (json::exception &err) { - - } - auto prefix = conMeta[tmpl][role][K_PREFIX]; + std::string begin = chaton_tmpl_role_kv(tmpl, role, {K_BEGIN}); + auto prefix = chaton_tmpl_role_kv(tmpl, role, {K_PREFIX}); + auto suffix = chaton_tmpl_role_kv(tmpl, role, {K_SUFFIX}); + auto end = chaton_tmpl_role_kv(tmpl, role, {K_END}); if (role == K_SYSTEM) { cntSystem += 1; ss << begin << prefix; @@ -266,12 +263,26 @@ inline bool chaton_tmpl_apply_ex( cp.add_part(ChatParts::S, begin); cp.add_part(ChatParts::S, prefix); } - ss << content << conMeta[tmpl][role][K_SUFFIX]; + ss << content; cp.add_part(ChatParts::N, content); - cp.add_part(ChatParts::S, conMeta[tmpl][role][K_SUFFIX]); + if (role == K_SYSTEM) { + if (chaton_tmpl_kv_bool(tmpl, K_SYSTEMUSER_SYSTEM_HAS_SUFFIX)) { + ss << suffix; + cp.add_part(ChatParts::S, suffix); + } + if (chaton_tmpl_kv_bool(tmpl, K_SYSTEMUSER_SYSTEM_HAS_END)) { + ss << end; + cp.add_part(ChatParts::S, end); + } + } else { + ss << suffix << end; + cp.add_part(ChatParts::S, suffix); + cp.add_part(ChatParts::S, end); + } } - ss << conMeta[tmpl][K_GLOBAL][K_END]; - cp.add_part(ChatParts::S, conMeta[tmpl][K_GLOBAL][K_END]); + auto globalEnd = chaton_tmpl_role_kv(tmpl, K_GLOBAL, {K_END}); + ss << globalEnd; + cp.add_part(ChatParts::S, globalEnd); cp.dump(); tagged = ss.str(); std::string cpStr = cp.str(); From 6a0214c0675be3f96211822ca30e5ab133f44482 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Thu, 25 Apr 2024 06:55:30 +0530 Subject: [PATCH 040/217] ChatON:MetaOK->MetaDump: Alert if user->end is needed or not Because user messages dont normally need a EoS token. --- common/chaton.hpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/common/chaton.hpp b/common/chaton.hpp index ea42c41446990..e6d946029f5be 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -384,6 +384,10 @@ inline void _chaton_meta_dump(std::string &tmpl) { LOGXLN("INFO:%s:%s:%d", __func__, K_SYSTEMUSER_SYSTEM_HAS_END, chaton_tmpl_kv_bool(tmpl, K_SYSTEMUSER_SYSTEM_HAS_END)); LOGXLN("INFO:%s:%s:%d", __func__, K_SYSTEMUSER_1ST_USER_HAS_BEGIN, chaton_tmpl_kv_bool(tmpl, K_SYSTEMUSER_1ST_USER_HAS_BEGIN)); LOGXLN("INFO:%s:%s:%d", __func__, K_SYSTEMUSER_1ST_USER_HAS_PREFIX, chaton_tmpl_kv_bool(tmpl, K_SYSTEMUSER_1ST_USER_HAS_PREFIX)); + + if (!userEnd.empty()) { + LOG_TEELN("WARN:%s:User->End seems to be set to [%s], do cross check if this is proper and needed", __func__, userEnd); + } } } From 0cd7c627064a5de6042f3ac4a9d35c350163de2a Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Thu, 25 Apr 2024 17:41:24 +0530 Subject: [PATCH 041/217] ChatON: Keep compiler happy Move helpers to the begining, so can avoid adding prototype declerations/function signatures to the begining Get the char * wrt string data in the c++ string. --- common/chaton.hpp | 57 ++++++++++++++++++++++++----------------------- 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index e6d946029f5be..d3a8bfba584b4 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -171,6 +171,34 @@ inline bool chaton_meta_load(std::string &fname) { } +inline std::string chaton_tmpl_role_kv(const std::string &tmpl, const std::string &role, const std::vector &keys) { + std::string got = ""; + std::string sKeys = ""; + for(auto key: keys) { + try { + got += conMeta[tmpl][role][key]; + } catch (json::exception &err) { + } + sKeys += "+"; + sKeys += key; + } + LOGLN("DBUG:%s:%s:%s:%s:%s", __func__, tmpl.c_str(), role.c_str(), sKeys.c_str(), got.c_str()); + return got; +} + +inline std::string chaton_tmpl_kv(const std::string &tmpl, const std::string &key) { + std::string got = conMeta[tmpl][key]; + LOGLN("DBUG:%s:%s:%s:%s", __func__, tmpl.c_str(), key.c_str(), got.c_str()); + return got; +} + +inline bool chaton_tmpl_kv_bool(const std::string &tmpl, const std::string &key) { + bool got = conMeta[tmpl][key]; + LOGLN("DBUG:%s:%s:%s:%d", __func__, tmpl.c_str(), key.c_str(), got); + return got; +} + + // Return user-prefix + msg + user-suffix inline bool chaton_tmpl_apply_single_ex( const std::string &tmpl, @@ -308,33 +336,6 @@ inline std::string chaton_tmpl_apply(const std::string &tmpl, const std::vector< return tagged; } -inline std::string chaton_tmpl_role_kv(const std::string &tmpl, const std::string &role, const std::vector &keys) { - std::string got = ""; - std::string sKeys = ""; - for(auto key: keys) { - try { - got += conMeta[tmpl][role][key]; - } catch (json::exception &err) { - } - sKeys += "+"; - sKeys += key; - } - LOGLN("DBUG:%s:%s:%s:%s:%s", __func__, tmpl.c_str(), role.c_str(), sKeys.c_str(), got.c_str()); - return got; -} - -inline std::string chaton_tmpl_kv(const std::string &tmpl, const std::string &key) { - std::string got = conMeta[tmpl][key]; - LOGLN("DBUG:%s:%s:%s:%s", __func__, tmpl.c_str(), key.c_str(), got.c_str()); - return got; -} - -inline bool chaton_tmpl_kv_bool(const std::string &tmpl, const std::string &key) { - bool got = conMeta[tmpl][key]; - LOGLN("DBUG:%s:%s:%s:%d", __func__, tmpl.c_str(), key.c_str(), got); - return got; -} - /** * if tmpl is @@ -386,7 +387,7 @@ inline void _chaton_meta_dump(std::string &tmpl) { LOGXLN("INFO:%s:%s:%d", __func__, K_SYSTEMUSER_1ST_USER_HAS_PREFIX, chaton_tmpl_kv_bool(tmpl, K_SYSTEMUSER_1ST_USER_HAS_PREFIX)); if (!userEnd.empty()) { - LOG_TEELN("WARN:%s:User->End seems to be set to [%s], do cross check if this is proper and needed", __func__, userEnd); + LOG_TEELN("WARN:%s:User->End seems to be set to [%s], do cross check if this is proper and needed", __func__, userEnd.c_str()); } } } From bf1167bfdb638c5f9694e2163950612e5cecab43 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Thu, 25 Apr 2024 17:56:32 +0530 Subject: [PATCH 042/217] ChatON: Backup the current simple meta json file --- examples/chaton_meta.old_simple.json | 180 +++++++++++++++++++++++++++ 1 file changed, 180 insertions(+) create mode 100644 examples/chaton_meta.old_simple.json diff --git a/examples/chaton_meta.old_simple.json b/examples/chaton_meta.old_simple.json new file mode 100644 index 0000000000000..25f05a17076e3 --- /dev/null +++ b/examples/chaton_meta.old_simple.json @@ -0,0 +1,180 @@ + +{ + "llama2": { + "global": { + "begin": "", + "end": "" + }, + "system": { + "prefix": "[INST] <>\n", + "suffix": "\n<> [/INST]\n\n" + }, + "user": { + "begin": "", + "prefix": "[INST] ", + "suffix": " [/INST]\n\n" + }, + "assistant": { + "prefix": "", + "suffix": "" + }, + "reverse-prompt": "", + "systemuser-1st-user-has-begin": false, + "systemuser-1st-user-has-prefix": false + }, + "llama3": { + "global": { + "begin": "", + "end": "" + }, + "system": { + "prefix": "<|start_header_id|>system<|end_header_id|>\n", + "suffix": "<|eot_id|>\n\n" + }, + "user": { + "begin": "", + "prefix": "<|start_header_id|>user<|end_header_id|>\n", + "suffix": "<|eot_id|>\n\n" + }, + "assistant": { + "prefix": "<|start_header_id|>assistant<|end_header_id|>\n", + "suffix": "<|eot_id|>\n\n" + }, + "reverse-prompt": "<|eot_id|>", + "systemuser-1st-user-has-begin": true, + "systemuser-1st-user-has-prefix": true + }, + "chatml": { + "global": { + "begin": "", + "end": "" + }, + "system": { + "prefix": "<|im_start|>system\n", + "suffix": "<|im_end|>\n" + }, + "user": { + "begin": "", + "prefix": "\n<|im_start|>user\n", + "suffix": "<|im_end|>\n" + }, + "assistant": { + "prefix": "<|im_start|>assistant\n", + "suffix": "<|im_end|>\n" + }, + "reverse-prompt": "<|im_start|>user\n", + "systemuser-1st-user-has-begin": true, + "systemuser-1st-user-has-prefix": true + }, + "zephyr": { + "global": { + "begin": "", + "end": "" + }, + "system": { + "prefix": "<|system|>\n", + "suffix": "<|endoftext|>\n" + }, + "user": { + "begin": "", + "prefix": "<|user|>\n", + "suffix": "<|endoftext|>\n" + }, + "assistant": { + "prefix": "<|assistant|>\n", + "suffix": "<|endoftext|>\n" + }, + "reverse-prompt": "", + "systemuser-1st-user-has-begin": true, + "systemuser-1st-user-has-prefix": true + }, + "gemma": { + "global": { + "begin": "", + "end": "" + }, + "system": { + "prefix": "", + "suffix": "" + }, + "user": { + "begin": "", + "prefix": "user\n", + "suffix": "\n" + }, + "assistant": { + "prefix": "model\n", + "suffix": "\n" + }, + "reverse-prompt": "", + "systemuser-1st-user-has-begin": true, + "systemuser-1st-user-has-prefix": true + }, + "deepseek-alt": { + "global": { + "begin": "", + "end": "" + }, + "system": { + "prefix": "", + "suffix": "\n" + }, + "user": { + "begin": "", + "prefix": "### Instruction:\n", + "suffix": "\n" + }, + "assistant": { + "prefix": "### Response:\n", + "suffix": "\n<|EOT|>\n" + }, + "reverse-prompt": "<|EOT|>", + "systemuser-1st-user-has-begin": true, + "systemuser-1st-user-has-prefix": true + }, + "deepseek": { + "global": { + "begin": "", + "end": "" + }, + "system": { + "prefix": "", + "suffix": "\n\n" + }, + "user": { + "begin": "", + "prefix": "User: ", + "suffix": "\n\n" + }, + "assistant": { + "prefix": "Assistant: ", + "suffix": " <｜end▁of▁sentence｜>\n" + }, + "reverse-prompt": "<｜end▁of▁sentence｜>", + "systemuser-1st-user-has-begin": true, + "systemuser-1st-user-has-prefix": true + }, + "monarch": { + "global": { + "begin": "", + "end": "" + }, + "system": { + "prefix": "system\n", + "suffix": "\n" + }, + "user": { + "begin": "", + "prefix": "user\n", + "suffix": "\n" + }, + "assistant": { + "prefix": "assistant\n", + "suffix": " \n" + }, + "reverse-prompt": "", + "systemuser-1st-user-has-begin": false, + "systemuser-1st-user-has-prefix": true + } +} + From b9e31304a529d63e8ffad3422044d47a8d5f4744 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Thu, 25 Apr 2024 18:00:01 +0530 Subject: [PATCH 043/217] ChatON: Update to new detailed format wrt llama2 and llama3 Wrt llama2 * add bos wrt llama2 system and user begins, but not assistant * split system suffix into suffix and end, and add systemuser-system flags so that end can be avoided wrt system+user message combo * add eos wrt assistant end * With these potentially this should work with main and server flows Wrt llama3 * add empty begin, end fields and systemuser-system flags * This should potentially work with main and server flows --- examples/chaton_meta.json | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/examples/chaton_meta.json b/examples/chaton_meta.json index 25f05a17076e3..333ad620e7b5f 100644 --- a/examples/chaton_meta.json +++ b/examples/chaton_meta.json @@ -6,19 +6,26 @@ "end": "" }, "system": { + "begin": "", "prefix": "[INST] <>\n", - "suffix": "\n<> [/INST]\n\n" + "suffix": "\n<> ", + "end": "[/INST]\n\n" }, "user": { - "begin": "", + "begin": "", "prefix": "[INST] ", - "suffix": " [/INST]\n\n" + "suffix": " [/INST]\n\n", + "end": "" }, "assistant": { + "begin": "", "prefix": "", - "suffix": "" + "suffix": "", + "end": "" }, "reverse-prompt": "", + "systemuser-system-has-suffix": true, + "systemuser-system-has-end": false, "systemuser-1st-user-has-begin": false, "systemuser-1st-user-has-prefix": false }, @@ -28,19 +35,26 @@ "end": "" }, "system": { + "begin": "", "prefix": "<|start_header_id|>system<|end_header_id|>\n", - "suffix": "<|eot_id|>\n\n" + "suffix": "<|eot_id|>\n\n", + "end": "" }, "user": { "begin": "", "prefix": "<|start_header_id|>user<|end_header_id|>\n", - "suffix": "<|eot_id|>\n\n" + "suffix": "<|eot_id|>\n\n", + "end": "" }, "assistant": { + "begin": "", "prefix": "<|start_header_id|>assistant<|end_header_id|>\n", - "suffix": "<|eot_id|>\n\n" + "suffix": "<|eot_id|>\n\n", + "end": "" }, "reverse-prompt": "<|eot_id|>", + "systemuser-system-has-suffix": true, + "systemuser-system-has-end": true, "systemuser-1st-user-has-begin": true, "systemuser-1st-user-has-prefix": true }, From 13857f29d69b00961fe2df56bde1415f189e27a4 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Thu, 25 Apr 2024 18:49:53 +0530 Subject: [PATCH 044/217] ChatON+Main: Updates wrt detailed meta json Fix a oversight wrt key name. Add a alert in case if passed meta json file contains begin(BoS) wrt assistant role, similar to check for end (EoS) wrt user role. Bcas normally both (ie EoS wrt User and BoS wrt Assistant) shouldnt be needed. Update main wrt begin & prefix and suffix & end addition. --- common/chaton.hpp | 5 ++++- examples/main/main.cpp | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index d3a8bfba584b4..7ea7b193279e3 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -75,7 +75,7 @@ const auto K_SUFFIX = "suffix"; const auto K_BEGIN = "begin"; const auto K_END = "end"; const auto K_GLOBAL = "global"; -const auto K_SYSTEMUSER_SYSTEM_HAS_SUFFIX = "systemuser-systemp-has-suffix"; +const auto K_SYSTEMUSER_SYSTEM_HAS_SUFFIX = "systemuser-system-has-suffix"; const auto K_SYSTEMUSER_SYSTEM_HAS_END = "systemuser-system-has-end"; const auto K_SYSTEMUSER_1ST_USER_HAS_BEGIN = "systemuser-1st-user-has-begin"; const auto K_SYSTEMUSER_1ST_USER_HAS_PREFIX = "systemuser-1st-user-has-prefix"; @@ -389,6 +389,9 @@ inline void _chaton_meta_dump(std::string &tmpl) { if (!userEnd.empty()) { LOG_TEELN("WARN:%s:User->End seems to be set to [%s], do cross check if this is proper and needed", __func__, userEnd.c_str()); } + if (!assistantBegin.empty()) { + LOG_TEELN("WARN:%s:Assistant->Begin seems to be set to [%s], do cross check if this is proper and needed", __func__, assistantBegin.c_str()); + } } } diff --git a/examples/main/main.cpp b/examples/main/main.cpp index a505c900b1258..074db4598dd5a 100644 --- a/examples/main/main.cpp +++ b/examples/main/main.cpp @@ -375,12 +375,12 @@ int main(int argc, char ** argv) { } // chaton mode - const auto chaton_assitant_prefix = ::llama_tokenize(ctx, chaton_tmpl_role_kv(params.chaton_template_id, K_ASSISTANT, {K_PREFIX}), false, true); + const auto chaton_assitant_prefix = ::llama_tokenize(ctx, chaton_tmpl_role_kv(params.chaton_template_id, K_ASSISTANT, {K_BEGIN, K_PREFIX}), false, true); if (params.chaton) { params.interactive = true; // may remove later, by requiring user to explicitly request interactive mode params.interactive_first = true; params.input_prefix = chaton_tmpl_role_kv(params.chaton_template_id, K_USER, {K_BEGIN, K_PREFIX}); - params.input_suffix = chaton_tmpl_role_kv(params.chaton_template_id, K_USER, {K_SUFFIX}); + params.input_suffix = chaton_tmpl_role_kv(params.chaton_template_id, K_USER, {K_SUFFIX, K_END}); params.antiprompt.emplace_back(chaton_tmpl_kv(params.chaton_template_id, K_REVERSE_PROMPT)); } From 01c8db70f706566fab1a5c0a0ab73c88525e2068 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Fri, 26 Apr 2024 06:12:55 +0530 Subject: [PATCH 045/217] ChatON+Main: Add C_API wrapper for single Add a c api wrapper for a single message tagging scenario. Inturn to match convention followed by existing chat_apply_template code, make it return the size expected of the tagged message string buffer. Update internal single logic to help with same. Explicitly check if tmpl specified is available in the loaded json or not and then return a error if not found. --- common/chaton.hpp | 60 +++++++++++++++++++++++++++++++++++++++--- examples/main/main.cpp | 2 +- 2 files changed, 58 insertions(+), 4 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index 7ea7b193279e3..d74dc08dea8f9 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -165,11 +165,27 @@ class ChatParts { }; inline bool chaton_meta_load(std::string &fname) { + if (conMeta != nullptr) { + LOGXLN("WARN:%s:ChatOn Meta: overwriting???", __func__); + } std::ifstream f(fname); conMeta = json::parse(f); return true; } +inline bool chaton_tmpl_exists(const std::string &tmpl) { + if (conMeta == nullptr) { + LOG_TEELN("ERRR:%s:ChatOnMeta: Not loaded yet...", __func__); + return false; + } + try { + auto tmplData = conMeta[tmpl]; + return true; + } catch (json::exception &err) { + LOG_TEELN("WARN:%s:ChatOnMeta: tmpl[%s] not found...", __func__, tmpl.c_str()); + return false; + } +} inline std::string chaton_tmpl_role_kv(const std::string &tmpl, const std::string &role, const std::vector &keys) { std::string got = ""; @@ -230,12 +246,50 @@ inline bool chaton_tmpl_apply_single_ex( } // Return user-prefix + msg + user-suffix, types string and lens vector wrt the parts that make up the returned string -inline std::string chaton_tmpl_apply_single(const std::string &tmpl, const std::string &role, const std::string &content) { - std::string tagged; +inline size_t chaton_tmpl_apply_single( + const std::string &tmpl, + const std::string &role, + const std::string &content, + std::string &tagged + ) { + if (!chaton_tmpl_exists(tmpl)) { + return -1; + } std::string types; std::vector lens; chaton_tmpl_apply_single_ex(tmpl, role, content, tagged, types, lens); - return tagged; + return tagged.size(); +} + +/** + * Apply chat-handshake-template for the specified template standard and role. + * If the passed char array is smaller that that required for the tagged message, + * * part of the tagged message which fits within dest buffer is copied + * * the returned value, indicates the size of the tagged message + * NOTE: + * * the passed char array should be able to fit the tagged message+0|null char. + * * if the return value from this function is larger than or equal to destLength, + * then you will have to increase the size of the dest buffer, and call this + * function a second time, to ensure that one gets the full tagged message. + */ +inline size_t chat_tmpl_apply_single_capi( + const char *tmpl, + const char *role, + const char *content, + char *dest, + const size_t destLength + ) { + std::string tagged; + std::string types; + std::vector lens; + auto taggedLength = chaton_tmpl_apply_single(tmpl, role, content, tagged); + if (taggedLength <= 0) { + return taggedLength; + } + if (dest && (destLength > 0)) { + strlcpy(dest, tagged.c_str(), destLength); + } + return taggedLength; } // global-begin + [[role-begin] + [role-prefix] + msg + role-suffix] + global-end diff --git a/examples/main/main.cpp b/examples/main/main.cpp index 074db4598dd5a..2a41c8e1e74e8 100644 --- a/examples/main/main.cpp +++ b/examples/main/main.cpp @@ -264,7 +264,7 @@ int main(int argc, char ** argv) { params.prompt = "<|im_start|>system\n" + params.prompt + "<|im_end|>"; } if (params.chaton) { - params.prompt = chaton_tmpl_apply_single(params.chaton_template_id, K_SYSTEM, params.prompt); + chaton_tmpl_apply_single(params.chaton_template_id, K_SYSTEM, params.prompt, params.prompt); } embd_inp = ::llama_tokenize(ctx, params.prompt, true, true); } else { From ea3a0f19cc9f99ecc859138c1409b9ad5971d564 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Fri, 26 Apr 2024 06:23:24 +0530 Subject: [PATCH 046/217] ChatON: Rather check for tmpl existance in single_ex --- common/chaton.hpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index d74dc08dea8f9..2f2a0ad62ebc0 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -224,6 +224,9 @@ inline bool chaton_tmpl_apply_single_ex( std::string &types, std::vector &lens ) { + if (!chaton_tmpl_exists(tmpl)) { + return false; + } ChatParts cp = {}; std::stringstream ss; std::string beginPrefix = chaton_tmpl_role_kv(tmpl, role, {K_BEGIN, K_PREFIX}); @@ -252,12 +255,11 @@ inline size_t chaton_tmpl_apply_single( const std::string &content, std::string &tagged ) { - if (!chaton_tmpl_exists(tmpl)) { - return -1; - } std::string types; std::vector lens; - chaton_tmpl_apply_single_ex(tmpl, role, content, tagged, types, lens); + if (!chaton_tmpl_apply_single_ex(tmpl, role, content, tagged, types, lens)) { + return -1; + } return tagged.size(); } From e62699f9233cfbfcd3fb0effde361caf1e036be4 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Fri, 26 Apr 2024 06:55:41 +0530 Subject: [PATCH 047/217] ChatON: Add alertAssistantAtEnd flag & logic wrt MultiMsgs Apply While sending the current chat session along with new user query to the model, many models expect that a tag be added at the end to indicate that user is expecting the model to respond, this flags allows for the same. --- common/chaton.hpp | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index 2f2a0ad62ebc0..7cfdd8f998bb8 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -303,7 +303,8 @@ inline bool chaton_tmpl_apply_ex( const std::vector &msgs, std::string &tagged, std::string &types, - std::vector &lens + std::vector &lens, + bool alertAssistantAtEnd ) { ChatParts cp = {}; std::stringstream ss; @@ -364,6 +365,11 @@ inline bool chaton_tmpl_apply_ex( cp.add_part(ChatParts::S, end); } } + if (alertAssistantAtEnd) { + auto assistantBeginPrefix = chaton_tmpl_role_kv(tmpl, K_ASSISTANT, {K_BEGIN, K_PREFIX}); + ss << assistantBeginPrefix; + cp.add_part(ChatParts::S, assistantBeginPrefix); + } auto globalEnd = chaton_tmpl_role_kv(tmpl, K_GLOBAL, {K_END}); ss << globalEnd; cp.add_part(ChatParts::S, globalEnd); @@ -384,11 +390,15 @@ inline bool chaton_tmpl_apply_ex( // global-begin + [[role-begin] + [role-prefix] + msg + role-suffix] + global-end // if there is a combination of system-user messages, // then 1st user message will have user-prefix only if systemuser-1st-user-has-prefix is true -inline std::string chaton_tmpl_apply(const std::string &tmpl, const std::vector &msgs) { +inline std::string chaton_tmpl_apply( + const std::string &tmpl, + const std::vector &msgs, + bool alertAssistantAtEnd + ) { std::string tagged; std::string types; std::vector lens; - chaton_tmpl_apply_ex(tmpl, msgs, tagged, types, lens); + chaton_tmpl_apply_ex(tmpl, msgs, tagged, types, lens, alertAssistantAtEnd); return tagged; } From 308d3bf3ffd8089fbcad4244672c31d6039a2849 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Fri, 26 Apr 2024 07:58:40 +0530 Subject: [PATCH 048/217] ChatON:WIP:Add c api wrapper for chat_template_apply Initial skeletons Update existing logics to help with same. Also the inbetween helper was having a bad signature wrt returning status and data, thats also fixed. --- common/chaton.hpp | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index 7cfdd8f998bb8..7c31ad4dc1a73 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -306,6 +306,9 @@ inline bool chaton_tmpl_apply_ex( std::vector &lens, bool alertAssistantAtEnd ) { + if (!chaton_tmpl_exists(tmpl)) { + return false; + } ChatParts cp = {}; std::stringstream ss; std::string globalBegin = chaton_tmpl_role_kv(tmpl, K_GLOBAL, {K_BEGIN}); @@ -390,16 +393,37 @@ inline bool chaton_tmpl_apply_ex( // global-begin + [[role-begin] + [role-prefix] + msg + role-suffix] + global-end // if there is a combination of system-user messages, // then 1st user message will have user-prefix only if systemuser-1st-user-has-prefix is true -inline std::string chaton_tmpl_apply( +inline int32_t chaton_tmpl_apply( const std::string &tmpl, const std::vector &msgs, - bool alertAssistantAtEnd + bool alertAssistantAtEnd, + std::string &tagged ) { - std::string tagged; std::string types; std::vector lens; - chaton_tmpl_apply_ex(tmpl, msgs, tagged, types, lens, alertAssistantAtEnd); - return tagged; + if (!chaton_tmpl_apply_ex(tmpl, msgs, tagged, types, lens, alertAssistantAtEnd)) { + return -1; + } + return tagged.size(); +} + +inline int32_t chaton_tmpl_apply_capi( + const char *tmpl, + const struct llama_chat_message *msgs, + const size_t numMsgs, + bool alertAssistantAtEnd, + char *dest, + int32_t destLength + ) { + if ((tmpl == nullptr) || (dest == nullptr)) { + return -1; + } + std::vector vMsgs; + for(size_t i=0; i Date: Fri, 26 Apr 2024 09:09:41 +0530 Subject: [PATCH 049/217] ChatON:ChatTemplateApplyCAPI remaining base logic As c doesnt have the concept of pass by reference, and inturn the existing c api uses pointers wrt llama chat message structure, so switching to same wrt chat_tmpl_apply logics. Also fix a oversight in previous commit and add the remaining logic. --- common/chaton.hpp | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index 7c31ad4dc1a73..e072289f851cc 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -300,7 +300,7 @@ inline size_t chat_tmpl_apply_single_capi( // NOTE: returns types and lens to help identify the parts of the tagged msg, which relate to passed and added tags inline bool chaton_tmpl_apply_ex( const std::string &tmpl, - const std::vector &msgs, + const std::vector &msgs, std::string &tagged, std::string &types, std::vector &lens, @@ -318,8 +318,8 @@ inline bool chaton_tmpl_apply_ex( int cntUser = 0; int cntOthers = 0; for(const auto msg: msgs) { - auto role = msg.role; - auto content = msg.content; + auto role = msg->role; + auto content = msg->content; std::string begin = chaton_tmpl_role_kv(tmpl, role, {K_BEGIN}); auto prefix = chaton_tmpl_role_kv(tmpl, role, {K_PREFIX}); auto suffix = chaton_tmpl_role_kv(tmpl, role, {K_SUFFIX}); @@ -395,7 +395,7 @@ inline bool chaton_tmpl_apply_ex( // then 1st user message will have user-prefix only if systemuser-1st-user-has-prefix is true inline int32_t chaton_tmpl_apply( const std::string &tmpl, - const std::vector &msgs, + const std::vector &msgs, bool alertAssistantAtEnd, std::string &tagged ) { @@ -420,10 +420,17 @@ inline int32_t chaton_tmpl_apply_capi( } std::vector vMsgs; for(size_t i=0; i 0) { + strlcpy(dest, taggedMsgs.c_str(), destLength); + } + return taggedLength; } From 58e1ff16bc0afa7ba8610fd39845fb0ceb49a499 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Fri, 26 Apr 2024 09:46:00 +0530 Subject: [PATCH 050/217] ChatON: switch to ordered_json from json library to be in sync with the json namespace in server. --- common/chaton.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index e072289f851cc..00ade584558d0 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -82,7 +82,7 @@ const auto K_SYSTEMUSER_1ST_USER_HAS_PREFIX = "systemuser-1st-user-has-prefix"; const auto K_REVERSE_PROMPT = "reverse-prompt"; -using json = nlohmann::json; +using json = nlohmann::ordered_json; json conMeta; From fee887fe31925f01946019f15af262d40cd0944a Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Fri, 26 Apr 2024 10:31:17 +0530 Subject: [PATCH 051/217] ChatON:Common:Update the cmdline argument name used Had forgotten to update it before --- common/common.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/common/common.cpp b/common/common.cpp index 46789da416cc6..3447714333eab 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -1441,8 +1441,10 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) { printf(" --interactive-first run in interactive mode and wait for input right away\n"); printf(" -ins, --instruct run in instruction mode (use with Alpaca models)\n"); printf(" -cml, --chatml run in chatml mode (use with ChatML-compatible models)\n"); - printf(" --chaton-json specify the json file containing chat-handshake-template-standard(s)"); - printf(" --chaton-template-id specify the specific template standard to use from loaded json file"); + printf(" --chaton-meta-json JsonFile\n"); + printf(" specify the json file containing chat-handshake-template-standard(s)"); + printf(" --chaton-template-id ChatHandshakeTemplateId\n"); + printf(" specify the specific template standard to use from loaded json file"); printf(" --multiline-input allows you to write or paste multiple lines without ending each in '\\'\n"); printf(" -r PROMPT, --reverse-prompt PROMPT\n"); printf(" halt generation at PROMPT, return control in interactive mode\n"); From d61b071b8d2dcba3d9114d5cf29f7fdda76a3d07 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Fri, 26 Apr 2024 10:47:03 +0530 Subject: [PATCH 052/217] Chaton:Common:Add missing newline wrt cmdline arg usage --- common/common.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/common.cpp b/common/common.cpp index 3447714333eab..c48758b52e63b 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -1442,9 +1442,9 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) { printf(" -ins, --instruct run in instruction mode (use with Alpaca models)\n"); printf(" -cml, --chatml run in chatml mode (use with ChatML-compatible models)\n"); printf(" --chaton-meta-json JsonFile\n"); - printf(" specify the json file containing chat-handshake-template-standard(s)"); + printf(" specify the json file containing chat-handshake-template-standard(s)\n"); printf(" --chaton-template-id ChatHandshakeTemplateId\n"); - printf(" specify the specific template standard to use from loaded json file"); + printf(" specify the specific template standard to use from loaded json file\n"); printf(" --multiline-input allows you to write or paste multiple lines without ending each in '\\'\n"); printf(" -r PROMPT, --reverse-prompt PROMPT\n"); printf(" halt generation at PROMPT, return control in interactive mode\n"); From a4b3285034b70138b4a2dda99b80a0e4cd977060 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sat, 27 Apr 2024 05:49:08 +0530 Subject: [PATCH 053/217] ChatON:Show Log on screen when template is applied --- common/chaton.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index 00ade584558d0..b50ea5708bb71 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -242,7 +242,7 @@ inline bool chaton_tmpl_apply_single_ex( LOG_TEELN("DBUG:%s:Mismatch between CP[%s] and SS[%s]", __func__, cpStr.c_str(), tagged.c_str()); exit(2); } - LOGLN("DBUG:%s:%s:%s:%s", __func__, tmpl.c_str(), role.c_str(), tagged.c_str()); + LOGXLN("DBUG:%s:%s:%s:%s", __func__, tmpl.c_str(), role.c_str(), tagged.c_str()); types = cp.get_types(); lens = cp.get_partslens(); return true; @@ -383,7 +383,7 @@ inline bool chaton_tmpl_apply_ex( LOG_TEELN("DBUG:%s:Mismatch between CP[%s] and SS[%s]", __func__, cpStr.c_str(), tagged.c_str()); exit(2); } - LOGLN("DBUG:%s:%s:%s", __func__, tmpl.c_str(), tagged.c_str()); + LOGXLN("DBUG:%s:%s:%s", __func__, tmpl.c_str(), tagged.c_str()); LOGLN("DBUG:%s:%s:CntSys[%d]:CntUsr[%d]:CntOthers[%d]", __func__, tmpl.c_str(), cntSystem, cntUser, cntOthers); types = cp.get_types(); lens = cp.get_partslens(); From 403a6c43231b4022641b6a3adb705fd95108917a Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sat, 27 Apr 2024 06:46:28 +0530 Subject: [PATCH 054/217] ChatON:Gemma: update for detailed meta json Also as part of same add user role entry for system role also. --- examples/chaton_meta.json | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/examples/chaton_meta.json b/examples/chaton_meta.json index 333ad620e7b5f..e9b3ed8daffaa 100644 --- a/examples/chaton_meta.json +++ b/examples/chaton_meta.json @@ -108,19 +108,26 @@ "end": "" }, "system": { - "prefix": "", - "suffix": "" + "begin": "", + "prefix": "user\n", + "suffix": "\n", + "end": "" }, "user": { "begin": "", "prefix": "user\n", - "suffix": "\n" + "suffix": "\n", + "end": "" }, "assistant": { + "begin": "", "prefix": "model\n", - "suffix": "\n" + "suffix": "\n", + "end": "" }, "reverse-prompt": "", + "systemuser-system-has-suffix": true, + "systemuser-system-has-end": false, "systemuser-1st-user-has-begin": true, "systemuser-1st-user-has-prefix": true }, From 1b2e9211865aff028ec092206144dd613f9971f6 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sat, 27 Apr 2024 06:57:56 +0530 Subject: [PATCH 055/217] ChatON:DeepSeek: Update support wrt detailed meta json --- examples/chaton_meta.json | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/examples/chaton_meta.json b/examples/chaton_meta.json index e9b3ed8daffaa..a6685b2b2ba4a 100644 --- a/examples/chaton_meta.json +++ b/examples/chaton_meta.json @@ -159,20 +159,27 @@ "end": "" }, "system": { + "begin": "<｜begin▁of▁sentence｜>", "prefix": "", - "suffix": "\n\n" + "suffix": "\n\n", + "end": "" }, "user": { - "begin": "", + "begin": "<｜begin▁of▁sentence｜>", "prefix": "User: ", - "suffix": "\n\n" + "suffix": "\n\n", + "end": "" }, "assistant": { + "begin": "", "prefix": "Assistant: ", - "suffix": " <｜end▁of▁sentence｜>\n" + "suffix": " <｜end▁of▁sentence｜>\n", + "end": "" }, "reverse-prompt": "<｜end▁of▁sentence｜>", - "systemuser-1st-user-has-begin": true, + "systemuser-system-has-suffix": true, + "systemuser-system-has-end": false, + "systemuser-1st-user-has-begin": false, "systemuser-1st-user-has-prefix": true }, "monarch": { From 006a398ebf9d6facd6e551355933b611e4e42e57 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sat, 27 Apr 2024 07:11:05 +0530 Subject: [PATCH 056/217] ChatON:DeepSeekCoder: Update tmplid and wrt detailed meta json --- examples/chaton_meta.json | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/examples/chaton_meta.json b/examples/chaton_meta.json index a6685b2b2ba4a..8d64479932da3 100644 --- a/examples/chaton_meta.json +++ b/examples/chaton_meta.json @@ -131,26 +131,31 @@ "systemuser-1st-user-has-begin": true, "systemuser-1st-user-has-prefix": true }, - "deepseek-alt": { + "deepseek-coder": { "global": { "begin": "", "end": "" }, "system": { + "begin": "\n<｜begin▁of▁sentence｜>", "prefix": "", - "suffix": "\n" + "suffix": "\n", + "end": "" }, "user": { "begin": "", "prefix": "### Instruction:\n", - "suffix": "\n" + "suffix": "\n", + "end": "" }, "assistant": { "prefix": "### Response:\n", "suffix": "\n<|EOT|>\n" }, "reverse-prompt": "<|EOT|>", - "systemuser-1st-user-has-begin": true, + "systemuser-system-has-suffix": true, + "systemuser-system-has-end": false, + "systemuser-1st-user-has-begin": false, "systemuser-1st-user-has-prefix": true }, "deepseek": { From 18cd12524f082304fb28a9ef91b09bf85c3a131a Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sat, 27 Apr 2024 07:24:38 +0530 Subject: [PATCH 057/217] ChatON:Monarch:Update wrt detailed meta json --- examples/chaton_meta.json | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/examples/chaton_meta.json b/examples/chaton_meta.json index 8d64479932da3..619050379ec4a 100644 --- a/examples/chaton_meta.json +++ b/examples/chaton_meta.json @@ -193,20 +193,27 @@ "end": "" }, "system": { - "prefix": "system\n", - "suffix": "\n" + "begin": "", + "prefix": "system\n", + "suffix": "", + "end": "\n" }, "user": { "begin": "", "prefix": "user\n", - "suffix": "\n" + "suffix": "", + "end": "\n" }, "assistant": { - "prefix": "assistant\n", - "suffix": " \n" + "begin": "", + "prefix": "assistant\n", + "suffix": "", + "end": "\n" }, "reverse-prompt": "", - "systemuser-1st-user-has-begin": false, + "systemuser-system-has-suffix": true, + "systemuser-system-has-end": true, + "systemuser-1st-user-has-begin": true, "systemuser-1st-user-has-prefix": true } } From a64dcd7796b0ba79f4df6891053bd9f4c9bfeb6c Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sat, 27 Apr 2024 07:39:32 +0530 Subject: [PATCH 058/217] ChatON:Zephyr: Update wrt detailed meta json, also update eos Pick eos from zephyr's tokenizer_config, which is different from what was hardcoded in the existing llama_chat_apply_template. --- examples/chaton_meta.json | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/examples/chaton_meta.json b/examples/chaton_meta.json index 619050379ec4a..da70b3cc734c7 100644 --- a/examples/chaton_meta.json +++ b/examples/chaton_meta.json @@ -82,23 +82,32 @@ }, "zephyr": { "global": { + "alt-end": "<|endoftext|>\n", "begin": "", "end": "" }, "system": { + "begin": "", "prefix": "<|system|>\n", - "suffix": "<|endoftext|>\n" + "suffix": "", + "end": "\n" }, "user": { "begin": "", "prefix": "<|user|>\n", - "suffix": "<|endoftext|>\n" + "suffix": "", + "end": "\n" }, "assistant": { + "end-alt": "<|endoftext|>\n", + "begin": "", "prefix": "<|assistant|>\n", - "suffix": "<|endoftext|>\n" + "suffix": "", + "end": "\n" }, "reverse-prompt": "", + "systemuser-system-has-suffix": true, + "systemuser-system-has-end": true, "systemuser-1st-user-has-begin": true, "systemuser-1st-user-has-prefix": true }, From 368fbf17a1452025204671010d67772860a9cab2 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sat, 27 Apr 2024 07:51:59 +0530 Subject: [PATCH 059/217] ChatON:ChatML: Update wrt detailed meta json --- examples/chaton_meta.json | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/examples/chaton_meta.json b/examples/chaton_meta.json index da70b3cc734c7..e3e6f9f91f83b 100644 --- a/examples/chaton_meta.json +++ b/examples/chaton_meta.json @@ -64,19 +64,26 @@ "end": "" }, "system": { + "begin": "", "prefix": "<|im_start|>system\n", - "suffix": "<|im_end|>\n" + "suffix": "<|im_end|>\n", + "end": "" }, "user": { "begin": "", - "prefix": "\n<|im_start|>user\n", - "suffix": "<|im_end|>\n" + "prefix": "<|im_start|>user\n", + "suffix": "<|im_end|>\n", + "end": "" }, "assistant": { + "begin": "", "prefix": "<|im_start|>assistant\n", - "suffix": "<|im_end|>\n" + "suffix": "<|im_end|>\n", + "end": "" }, "reverse-prompt": "<|im_start|>user\n", + "systemuser-system-has-suffix": true, + "systemuser-system-has-end": true, "systemuser-1st-user-has-begin": true, "systemuser-1st-user-has-prefix": true }, From ad5e5216ce0b00cd46cba864168f547b1480b48e Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sat, 27 Apr 2024 14:58:07 +0530 Subject: [PATCH 060/217] ChatON:Mistral: Add detailed meta json entries --- examples/chaton_meta.json | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/examples/chaton_meta.json b/examples/chaton_meta.json index e3e6f9f91f83b..8764bb18178ce 100644 --- a/examples/chaton_meta.json +++ b/examples/chaton_meta.json @@ -231,6 +231,35 @@ "systemuser-system-has-end": true, "systemuser-1st-user-has-begin": true, "systemuser-1st-user-has-prefix": true + }, + "mistral": { + "global": { + "begin": "", + "end": "" + }, + "system": { + "begin": "", + "prefix": "[INST] ", + "suffix": " [/INST]\n", + "end": "" + }, + "user": { + "begin": "", + "prefix": "[INST] ", + "suffix": " [/INST]\n", + "end": "" + }, + "assistant": { + "begin": "", + "prefix": "", + "suffix": "", + "end": " \n" + }, + "reverse-prompt": "", + "systemuser-system-has-suffix": true, + "systemuser-system-has-end": true, + "systemuser-1st-user-has-begin": true, + "systemuser-1st-user-has-prefix": true } } From 55e3d63f13f30d34bb41c4bce1ea123d34c5715b Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sat, 27 Apr 2024 18:39:52 +0530 Subject: [PATCH 061/217] ChatON:Mistral: Update to match jinja file --- examples/chaton_meta.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/chaton_meta.json b/examples/chaton_meta.json index 8764bb18178ce..dbc0cc5fad5dc 100644 --- a/examples/chaton_meta.json +++ b/examples/chaton_meta.json @@ -256,10 +256,10 @@ "end": " \n" }, "reverse-prompt": "", - "systemuser-system-has-suffix": true, - "systemuser-system-has-end": true, - "systemuser-1st-user-has-begin": true, - "systemuser-1st-user-has-prefix": true + "systemuser-system-has-suffix": false, + "systemuser-system-has-end": false, + "systemuser-1st-user-has-begin": false, + "systemuser-1st-user-has-prefix": false } } From cad50c527e5ef7b40a0baedc37fcd61b1649c987 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sat, 27 Apr 2024 21:12:31 +0530 Subject: [PATCH 062/217] ChatON: Update the note to match current logic --- common/chaton.hpp | 204 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 155 insertions(+), 49 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index b50ea5708bb71..4fa17cc984f06 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -1,60 +1,166 @@ #pragma once -/*** - * +/** + * * ## Overview + * + * Helps chat with models, by tagging chat messages based on the specified + * chat-handshake-template-standard. This uses a generic tagging code driven + * by a json meta data file, which specifies the handshake template details. + * + * This can be used by + * + * * main, to build on existing interactive flow and its in-prefix, in-suffix + * and antiprompt/reverse-prompt + * + * * server, by replacing its existing llama_chat_apply_template with the + * equivalent helper here. + * + * + * ## The common pattern + * + * As a convention, the tagging used by LLMs to differentiate between the + * different parts when chatting with them normally follows a general pattern of + * + * * + * + * * The Roles could include System, User and Assistant (ie the Model) + * + * * A chat normally consists of + * + * * a System message/prompt followed by + * + * * multiple user message/query - model message/response pairs + * + * The different models will normally have all or some subset of the tagging mentioned above. + * + * You may also notice some common patterns like + * + * * Because a user message is normally followed by model/assistant response, in most models + * + * * user messages wont have EndOfSentenceTag and + * + * * the following model response wont have BeginOfSentenceTag + * + * * Because a system message will normally be immidiately followed by a user query, + * + * * in many models, there wont be a EndOfSentenceTag following the system message and + * BeginOfSentenceTag wrt the 1st user message following the system message. + * + * * in some models there wont even be a RoleSuffixTag following system message + * and RolePrefixTag wrt the 1st user message following the system message. + * + * * however in many of these models, the subsequent user messages will have the + * BeginOfSentenceTag and or RolePrefixTag. + * + * + * ## The Strategy + * + * The template meta data json file allows the user to specify the above mentioned tags wrt + * each of the Role. Depending on whether a given model uses a given tag or not you either + * specify the required tag or else you specify a empty string. + * + * A tag could be a single word or multiple words, and may include newline char specified + * using \n and so on. The tag is always demarcated using double quotes and thus also allows + * spaces at the begining or end of the tag, if needed. + * + * In order to account for the conditionality of tags between the system message and the 1st + * user message, flags are provided to explicitly control whether each of these possible tags + * is used by a specific model or not, as part of its template info. + * + * The Roles are identified in the json file using "system", "user" and "assistant". However + * the model may use different words to identify these roles, in which case setup RolePrefix + * and or RoleSuffix appropriately. + * + * To identify that model is finished with generating response to user query, depending on + * the model's handshake template standard, one will need to set the reverse-prompt to either + * the assistant's suffix or end tag or to the user's begin or prefix tag, depending on what + * is generated by the model at the end of its response. + * + * + * ## The JSON File + * + * Can contain the template info wrt multiple models/handshake-standards. And inturn each + * unique template is identified by a unique template id string. + * + * The fields that make up a given chat-handshake-template-standard include + * + * * global-> begin & end + * + * * system -> begin, prefix, suffix & end + * + * * user -> begin, prefix, suffix & end + * + * * assistant -> begin, prefix, suffix & end + * + * * reverse-prompt + * + * * systemuser-system-has-suffix, systemuser-system-has-end, + * systemuser-1st-user-has-begin and systemuser-1st-user-has-prefix + * * - * Helps chat with a model, by allowing role based special token tagging, based on the specified chat-handshake-template-standard. - * This is used by main, to build on existing interactive flow and its in-prefix, in-suffix and antiprompt/reverse-promot - * - * 1. [ToDo] Use a json file to configure the needed tags for each of the supported chat-handshake-template-standard - * * global-> begin & end - * * system -> begin, prefix, suffix & end - * * user -> begin, prefix, suffix & end; assistant -> begin, prefix, suffix & end - * * [main] these override the in-prefix (begin+prefix) and in-suffix - * c. reverse-prompt - * * [main] this adds to any reverese-prompt specified using cmdline - * e. systemuser-sys-has-suffix, systemuser-sys-has-end, systemuser-1st-user-has-begin and systemuser-1st-user-has-prefix - * * [chaton-tmpl-apply] if a combination of system and user messages/prompts is passed, - * then for system messages suffix and end, as well as - * for the 1st user message following the 1st system message, - * include system suffix and end and user begin and prefix only if corresponding flags is set. - * * begin should normally relate to BoS while prefix should relate to Role Identifier tag. - * If there is no need for seperate handling of BoS and RoleIdTag, then one could even - * set both BoS and RoleIdTag to one of these entries itself. - * - * 2. [main] currently the user specified system prompt (-p + -f) is tagged using system role tags, - * and inturn this tagged message is tokenized with parse_special flag. - * So any special token related tags in the user specified system prompt will get parsed as special. - * - * 3. chaton-tmpl-apply uses the json file, which was loaded, to decide on how to generate the tagged messages for tokenisation. - * a. input: [ { role, message }, { role, message}, ....] - * b. output: currently a single string is returned which contains the tagged message(s). - * [later] if it is needed to differentiate between the special tags added by this from user specified prompts/messages, - * then return [ {flag, data}, { flag, data}, {flag, data}, ....], - * where the flag specifies whether parse_special should be used or not for the corresponding data, during tokenization. - * + * ## Usage + * + * One needs to load the json file containing the template meta data and inturn call the + * other helper functions as needed. + * + * Inturn one can use the helper functions to either extract a given tag or to apply all + * tags specified wrt a given role to the passed message or to apply tags as needed for + * a bunch of messages in one go. + * + * The individual message tagging helper, will apply all tags specified wrt that role. + * + * The multiple messages tagging helper chaton-tmpl-apply, will look at the boolean flags + * when tagging the passed messages. In this the system suffix, system end, user begin and + * user prefix get included only if corresponding flag is set. + * + * Both the single and multi messages tagging helpers provide two versions. + * * one which returns a single string which contains the tagged message(s) + * * one which returns + * * [tagged msg] the string containing the tagged message(s) + * * [parts lengths] an array of integers, which specifies the part lengths, + * which divides the returned string into parts. + * * [parts types] a string where each character indicates whether the corresponding + * part is a normal part which needs to be tokenized without parse_special + * or is a special part which needs to be tokenized with parse-special. + * + * + * ## example/main + * + * The interactive commandline program under example/main, uses + * + * * the system role related tags to tag the system prompt + * * the system prompt includes contents of -p if any + * * followed by contents of file specified using -f if any + * * the user begin+prefix to map to in-prefix + * * the user suffix+end to map to in-suffix + * * the reverse-prompt to map to antiprompt + * * wrt tokenization + * * the user specified system prompt is tokenized with parse_special flag. + * * however the user messages are tokenized without parse_special flag. + * + * Currently Main doesnt use chaton-tmpl-apply, but only + * * chaton-tmpl-apply-single (for system prompt) and + * * chaton-tmpl-role-kv which maps the user prefix, suffix and reverse-prompt + * to in-prefix, in-suffix and antiprompt of main. + * These always adds any role specific begin+prefix and suffix+end around + * the passed message. + * + * * ## Adding support for new model / chat-handshake-template-standard - * + * * 1. Add suitable entries in json for that model/standard - * 2. Update the flow in chaton-tmpl-apply, as needed. - * Try to update and or reuse the generic flow in chaton-tmpl-apply, as much as possible, + * 2. Try to reuse the generic flow in chaton-tmpl-apply, as much as possible, * before trying to add a custom logic. - * If you update the generic flow, cross check if existing json files will need to be updated or not. - * + * If you update the generic flow, cross check if existing json files will + * need to be updated or not. + * + * * ## Notes - * - * Currently Main doesnt use chaton-tmpl-apply, but only - * * chaton-tmpl-apply-single (for system prompt) and - * * chaton-tmpl-role-part which maps the user prefix, suffix and reverse-prompt to - * in-prefix, in-suffix and antiprompt of main. - * These always adds any role specific prefix and suffix around the passed message. - * - * Sample chaton_meta.json includes template info for - * * llama2, llama3, gemma, chatml, zephyr, deepseek, monarch - * * llama2 doesnt apply begin+prefix to 1st user msg following system msg - * * monarch doesnt apply begin to 1st user msg following system msg - * + * + * Look at the sample chaton_meta.json in examples folder for how the above may apply + * * llama2, llama3, gemma, chatml, zephyr, deepseek(normal and coder), monarch, mistral + * */ #include From 32e672c5dd627fadb3119a2ca02fef35b5b64129 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sat, 27 Apr 2024 21:58:11 +0530 Subject: [PATCH 063/217] ChatON: Dont log final tagged message string to screen --- common/chaton.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index 4fa17cc984f06..b2e0c88fea8fc 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -348,7 +348,7 @@ inline bool chaton_tmpl_apply_single_ex( LOG_TEELN("DBUG:%s:Mismatch between CP[%s] and SS[%s]", __func__, cpStr.c_str(), tagged.c_str()); exit(2); } - LOGXLN("DBUG:%s:%s:%s:%s", __func__, tmpl.c_str(), role.c_str(), tagged.c_str()); + LOGLN("DBUG:%s:%s:%s:%s", __func__, tmpl.c_str(), role.c_str(), tagged.c_str()); types = cp.get_types(); lens = cp.get_partslens(); return true; @@ -489,7 +489,7 @@ inline bool chaton_tmpl_apply_ex( LOG_TEELN("DBUG:%s:Mismatch between CP[%s] and SS[%s]", __func__, cpStr.c_str(), tagged.c_str()); exit(2); } - LOGXLN("DBUG:%s:%s:%s", __func__, tmpl.c_str(), tagged.c_str()); + LOGLN("DBUG:%s:%s:%s", __func__, tmpl.c_str(), tagged.c_str()); LOGLN("DBUG:%s:%s:CntSys[%d]:CntUsr[%d]:CntOthers[%d]", __func__, tmpl.c_str(), cntSystem, cntUser, cntOthers); types = cp.get_types(); lens = cp.get_partslens(); From a724fd90bd132d6941d9a14ea8013f694c39564a Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sat, 27 Apr 2024 21:59:31 +0530 Subject: [PATCH 064/217] ChatON:Tests: Add a test templates program for chaton --- tests/CMakeLists.txt | 1 + tests/test-chat-template-chaton.cpp | 141 ++++++++++++++++++++++++++++ 2 files changed, 142 insertions(+) create mode 100644 tests/test-chat-template-chaton.cpp diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 208f0cf7b54bc..fb6efe30e28fc 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -115,6 +115,7 @@ llama_target_and_test(test-quantize-fns.cpp) llama_target_and_test(test-quantize-perf.cpp) llama_target_and_test(test-sampling.cpp) llama_target_and_test(test-chat-template.cpp) +llama_target_and_test(test-chat-template-chaton.cpp) llama_target_and_test(test-grammar-parser.cpp) llama_target_and_test(test-llama-grammar.cpp) diff --git a/tests/test-chat-template-chaton.cpp b/tests/test-chat-template-chaton.cpp new file mode 100644 index 0000000000000..b99c460fde40e --- /dev/null +++ b/tests/test-chat-template-chaton.cpp @@ -0,0 +1,141 @@ +#include +#include +#include +#include + +#undef NDEBUG +#include + +#include "llama.h" +#include "chaton.hpp" + + +std::vector templateIds = { "llama2", "llama3", "chatml", + "zephyr", "gemma", "deepseek-coder", "deepseek", "monarch", "mistral" }; + +llama_chat_message conversation[] = { + {"system", "You are a helpful assistant"}, + {"user", "Hello"}, + {"assistant", "Hi there"}, + {"user", "Who are you"}, + {"assistant", " I am an assistant "}, + {"user", "Another question"}, +}; +size_t message_count = 6; + +std::vector templates = { + // teknium/OpenHermes-2.5-Mistral-7B + "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\\n' + message['content'] + '<|im_end|>' + '\\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\\n' }}{% endif %}", + // mistralai/Mistral-7B-Instruct-v0.2 + "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token}}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}", + // TheBloke/FusionNet_34Bx2_MoE-AWQ + "{%- for idx in range(0, messages|length) -%}\\n{%- if messages[idx]['role'] == 'user' -%}\\n{%- if idx > 1 -%}\\n{{- bos_token + '[INST] ' + messages[idx]['content'] + ' [/INST]' -}}\\n{%- else -%}\\n{{- messages[idx]['content'] + ' [/INST]' -}}\\n{%- endif -%}\\n{% elif messages[idx]['role'] == 'system' %}\\n{{- '[INST] <>\\\\n' + messages[idx]['content'] + '\\\\n<>\\\\n\\\\n' -}}\\n{%- elif messages[idx]['role'] == 'assistant' -%}\\n{{- ' ' + messages[idx]['content'] + ' ' + eos_token -}}\\n{% endif %}\\n{% endfor %}", + // bofenghuang/vigogne-2-70b-chat + "{{ bos_token }}{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% elif true == true and not '<>' in messages[0]['content'] %}{% set loop_messages = messages %}{% set system_message = 'Vous êtes Vigogne, un assistant IA créé par Zaion Lab. Vous suivez extrêmement bien les instructions. Aidez autant que vous le pouvez.' %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 and system_message != false %}{% set content = '<>\\\\n' + system_message + '\\\\n<>\\\\n\\\\n' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + content.strip() + ' [/INST]' }}{% elif message['role'] == 'system' %}{{ '<>\\\\n' + content.strip() + '\\\\n<>\\\\n\\\\n' }}{% elif message['role'] == 'assistant' %}{{ ' ' + content.strip() + ' ' + eos_token }}{% endif %}{% endfor %}", + // mlabonne/AlphaMonarch-7B + "{% for message in messages %}{{bos_token + message['role'] + '\\n' + message['content'] + eos_token + '\\n'}}{% endfor %}{% if add_generation_prompt %}{{ bos_token + 'assistant\\n' }}{% endif %}", + // google/gemma-7b-it + "{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '' + role + '\\n' + message['content'] | trim + '\\n' }}{% endfor %}{% if add_generation_prompt %}{{'model\\n'}}{% endif %}", + // OrionStarAI/Orion-14B-Chat + "{% for message in messages %}{% if loop.first %}{{ bos_token }}{% endif %}{% if message['role'] == 'user' %}{{ 'Human: ' + message['content'] + '\\n\\nAssistant: ' + eos_token }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token }}{% endif %}{% endfor %}", + // openchat/openchat-3.5-0106 + // The included chat_template differs from the author's suggestions here: https://huggingface.co/openchat/openchat_3.5/discussions/5#65448109b4a3f3a2f486fd9d + // So we match against the included template but implement the suggested version. + "{{ bos_token }}{% for message in messages %}{{ 'GPT4 Correct ' + message['role'].title() + ': ' + message['content'] + '<|end_of_turn|>'}}{% endfor %}{% if add_generation_prompt %}{{ 'GPT4 Correct Assistant:' }}{% endif %}", + // deepseek-ai/deepseek-coder-33b-instruct + "{% if not add_generation_prompt is defined %}\n{% set add_generation_prompt = false %}\n{% endif %}\n{%- set ns = namespace(found=false) -%}\n{%- for message in messages -%}\n {%- if message['role'] == 'system' -%}\n {%- set ns.found = true -%}\n {%- endif -%}\n{%- endfor -%}\n{{bos_token}}{%- if not ns.found -%}\n{{'You are an AI programming assistant, utilizing the Deepseek Coder model, developed by Deepseek Company, and you only answer questions related to computer science. For politically sensitive questions, security and privacy issues, and other non-computer science questions, you will refuse to answer\\n'}}\n{%- endif %}\n{%- for message in messages %}\n {%- if message['role'] == 'system' %}\n{{ message['content'] }}\n {%- else %}\n {%- if message['role'] == 'user' %}\n{{'### Instruction:\\n' + message['content'] + '\\n'}}\n {%- else %}\n{{'### Response:\\n' + message['content'] + '\\n<|EOT|>\\n'}}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{% if add_generation_prompt %}\n{{'### Response:'}}\n{% endif %}", + // eachadea/vicuna-13b-1.1 + // No template included in tokenizer_config.json, so this template likely needs to be manually set. + "{%- for message in messages %}{%- if message['role'] == 'system' -%}{{- '' + message['content'] + '\n\n' -}}{%- else -%}{%- if message['role'] == 'user' -%}{{-'USER: ' + message['content'] + '\n'-}}{%- else -%}{{-'ASSISTANT: ' + message['content'] + '\n' -}}{%- endif -%}{%- endif -%}{%- endfor -%}{%- if add_generation_prompt -%}{{-'ASSISTANT:'-}}{%- endif -%}", + // Orca-Vicuna + // No template included in tokenizer_config.json, so this template likely needs to be manually set. + "{%- for message in messages %}{%- if message['role'] == 'system' -%}{{-'SYSTEM: ' + message['content'] + '\n' -}}{%- else -%}{%- if message['role'] == 'user' -%}{{-'USER: ' + message['content'] + '\n'-}}{%- else -%}{{-'ASSISTANT: ' + message['content'] + '\n' -}}{%- endif -%}{%- endif -%}{%- endfor -%}{%- if add_generation_prompt -%}{{-'ASSISTANT:'-}}{%- endif -%}", + // CohereForAI/c4ai-command-r-plus + "{{ bos_token }}{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% elif false == true %}{% set loop_messages = messages %}{% set system_message = 'You are Command-R, a brilliant, sophisticated, AI-assistant trained to assist human users by providing thorough responses. You are trained by Cohere.' %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% if system_message != false %}{{ '<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>' + system_message + '<|END_OF_TURN_TOKEN|>' }}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ '<|START_OF_TURN_TOKEN|><|USER_TOKEN|>' + content.strip() + '<|END_OF_TURN_TOKEN|>' }}{% elif message['role'] == 'assistant' %}{{ '<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>' + content.strip() + '<|END_OF_TURN_TOKEN|>' }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>' }}{% endif %}", + // Llama-3 + "{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}", +}; +std::vector expected_output = { + // teknium/OpenHermes-2.5-Mistral-7B + "<|im_start|>system\nYou are a helpful assistant<|im_end|>\n<|im_start|>user\nHello<|im_end|>\n<|im_start|>assistant\nHi there<|im_end|>\n<|im_start|>user\nWho are you<|im_end|>\n<|im_start|>assistant\n I am an assistant <|im_end|>\n<|im_start|>user\nAnother question<|im_end|>\n<|im_start|>assistant\n", + // mistralai/Mistral-7B-Instruct-v0.2 + "[INST] You are a helpful assistant\nHello [/INST]Hi there[INST] Who are you [/INST] I am an assistant [INST] Another question [/INST]", + // TheBloke/FusionNet_34Bx2_MoE-AWQ + "[INST] <>\nYou are a helpful assistant\n<>\n\nHello [/INST] Hi there [INST] Who are you [/INST] I am an assistant [INST] Another question [/INST]", + // bofenghuang/vigogne-2-70b-chat + "[INST] <>\nYou are a helpful assistant\n<>\n\nHello [/INST] Hi there [INST] Who are you [/INST] I am an assistant [INST] Another question [/INST]", + // mlabonne/AlphaMonarch-7B + "system\nYou are a helpful assistant\nuser\nHello\nassistant\nHi there\nuser\nWho are you\nassistant\n I am an assistant \nuser\nAnother question\nassistant\n", + // google/gemma-7b-it + "user\nYou are a helpful assistant\n\nHello\nmodel\nHi there\nuser\nWho are you\nmodel\nI am an assistant\nuser\nAnother question\nmodel\n", + // OrionStarAI/Orion-14B-Chat + "Human: You are a helpful assistant\n\nHello\n\nAssistant: Hi thereHuman: Who are you\n\nAssistant: I am an assistant Human: Another question\n\nAssistant: ", + // openchat/openchat-3.5-0106 + "You are a helpful assistant<|end_of_turn|>GPT4 Correct User: Hello<|end_of_turn|>GPT4 Correct Assistant: Hi there<|end_of_turn|>GPT4 Correct User: Who are you<|end_of_turn|>GPT4 Correct Assistant: I am an assistant <|end_of_turn|>GPT4 Correct User: Another question<|end_of_turn|>GPT4 Correct Assistant:", + // deepseek-ai/deepseek-coder-33b-instruct + "You are a helpful assistant### Instruction:\nHello\n### Response:\nHi there\n<|EOT|>\n### Instruction:\nWho are you\n### Response:\n I am an assistant \n<|EOT|>\n### Instruction:\nAnother question\n### Response:\n", + // eachadea/vicuna-13b-1.1 + "You are a helpful assistant\n\nUSER: Hello\nASSISTANT: Hi there\nUSER: Who are you\nASSISTANT: I am an assistant \nUSER: Another question\nASSISTANT:", + // Orca-Vicuna + "SYSTEM: You are a helpful assistant\nUSER: Hello\nASSISTANT: Hi there\nUSER: Who are you\nASSISTANT: I am an assistant \nUSER: Another question\nASSISTANT:", + // CohereForAI/c4ai-command-r-plus + "<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>You are a helpful assistant<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|USER_TOKEN|>Hello<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>Hi there<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|USER_TOKEN|>Who are you<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>I am an assistant<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|USER_TOKEN|>Another question<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>", + // Llama 3 + "<|start_header_id|>system<|end_header_id|>\n\nYou are a helpful assistant<|eot_id|><|start_header_id|>user<|end_header_id|>\n\nHello<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\nHi there<|eot_id|><|start_header_id|>user<|end_header_id|>\n\nWho are you<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\nI am an assistant<|eot_id|><|start_header_id|>user<|end_header_id|>\n\nAnother question<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n", +}; + +static void check_default() { + std::vector formatted_chat(1024); + int32_t res; + + // test invalid chat template + res = llama_chat_apply_template(nullptr, "INVALID TEMPLATE", conversation, message_count, true, formatted_chat.data(), formatted_chat.size()); + assert(res < 0); + + for (size_t i = 0; i < templates.size(); i++) { + std::string custom_template = templates[i]; + std::string expected = expected_output[i]; + formatted_chat.resize(1024); + res = llama_chat_apply_template( + nullptr, + custom_template.c_str(), + conversation, + message_count, + true, + formatted_chat.data(), + formatted_chat.size() + ); + formatted_chat.resize(res); + std::string output(formatted_chat.data(), formatted_chat.size()); + std::cout << output << "\n-------------------------\n"; + assert(output == expected); + } +} + +static void check_chaton(std::string &metaJson) { + std::vector formatted_chat(1024); + int32_t res; + + chaton_meta_load(metaJson); + for(auto tmplId: templateIds) { + formatted_chat.resize(1024); + std::cout << "\n----------" << tmplId << "---------------\n"; + res = chaton_tmpl_apply_capi(tmplId.c_str(), conversation, message_count, true, formatted_chat.data(), formatted_chat.size()); + assert(res > 0); + formatted_chat.resize(res); + std::string output(formatted_chat.data(), formatted_chat.size()); + std::cout << "*** TAGGEDMSG ***\n" << output; + std::cout << "\n----------" << tmplId << "---------------\n"; + } + +} + +int main(int argc, char **argv) { + if (argc != 2) { + std::cout << argv[0] << " meta.json" << std::endl; + exit(1); + } + std::string metaJson(argv[1]); + check_chaton(metaJson); + return 0; +} From c4e829d49280c4a6b4c3737b723f16a0b2ac0a3e Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sat, 27 Apr 2024 22:14:00 +0530 Subject: [PATCH 065/217] ChatON:Mistral: Decouple \n from suffix, use wrt sys msg --- examples/chaton_meta.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/chaton_meta.json b/examples/chaton_meta.json index dbc0cc5fad5dc..5e3c6f8e880f3 100644 --- a/examples/chaton_meta.json +++ b/examples/chaton_meta.json @@ -240,14 +240,14 @@ "system": { "begin": "", "prefix": "[INST] ", - "suffix": " [/INST]\n", - "end": "" + "suffix": " [/INST]", + "end": "\n" }, "user": { "begin": "", "prefix": "[INST] ", - "suffix": " [/INST]\n", - "end": "" + "suffix": " [/INST]", + "end": "\n" }, "assistant": { "begin": "", @@ -257,7 +257,7 @@ }, "reverse-prompt": "", "systemuser-system-has-suffix": false, - "systemuser-system-has-end": false, + "systemuser-system-has-end": true, "systemuser-1st-user-has-begin": false, "systemuser-1st-user-has-prefix": false } From ff5f68826b2f7a7b774c6be795d6d74755c666c1 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sun, 28 Apr 2024 08:59:28 +0530 Subject: [PATCH 066/217] ChatON:ChatTmplApplySingle: Avoid streamstring, update func notes --- common/chaton.hpp | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index b2e0c88fea8fc..b6ef6e1f9ce91 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -321,7 +321,14 @@ inline bool chaton_tmpl_kv_bool(const std::string &tmpl, const std::string &key) } -// Return user-prefix + msg + user-suffix +// Given the template standard, role and a message, this returns +// a tagged message, types string and lens vector wrt the parts that make up the returned string +// +// * a string containing the tagged message +// * user-(begin+prefix) + msg + user-(suffix+end) +// * a string where the chars contain info about +// type of sub-strings/parts that make up the tagged message. +// * a vector of ints, which give the length of each part in the tagged message. inline bool chaton_tmpl_apply_single_ex( const std::string &tmpl, const std::string &role, @@ -334,27 +341,23 @@ inline bool chaton_tmpl_apply_single_ex( return false; } ChatParts cp = {}; - std::stringstream ss; std::string beginPrefix = chaton_tmpl_role_kv(tmpl, role, {K_BEGIN, K_PREFIX}); std::string suffixEnd = chaton_tmpl_role_kv(tmpl, role, {K_SUFFIX, K_END}); cp.add_part(ChatParts::S, beginPrefix); cp.add_part(ChatParts::N, content); cp.add_part(ChatParts::S, suffixEnd); cp.dump(); - ss << beginPrefix << content << suffixEnd; - tagged = ss.str(); - std::string cpStr = cp.str(); - if (tagged != cpStr) { - LOG_TEELN("DBUG:%s:Mismatch between CP[%s] and SS[%s]", __func__, cpStr.c_str(), tagged.c_str()); - exit(2); - } + tagged = cp.str(); LOGLN("DBUG:%s:%s:%s:%s", __func__, tmpl.c_str(), role.c_str(), tagged.c_str()); types = cp.get_types(); lens = cp.get_partslens(); return true; } -// Return user-prefix + msg + user-suffix, types string and lens vector wrt the parts that make up the returned string +// Given the template standard, role and a message, this returns the tagged message. +// +// * a string containing the tagged message +// * user-(begin+prefix) + msg + user-(suffix+end) inline size_t chaton_tmpl_apply_single( const std::string &tmpl, const std::string &role, @@ -371,11 +374,11 @@ inline size_t chaton_tmpl_apply_single( /** * Apply chat-handshake-template for the specified template standard and role. - * If the passed char array is smaller that that required for the tagged message, + * If the passed char array is smaller than that required for the tagged message, * * part of the tagged message which fits within dest buffer is copied - * * the returned value, indicates the size of the tagged message + * * the returned value, indicates the size of the actual tagged message * NOTE: - * * the passed char array should be able to fit the tagged message+0|null char. + * * ideally the passed char array should be able to fit the tagged message+0|null char. * * if the return value from this function is larger than or equal to destLength, * then you will have to increase the size of the dest buffer, and call this * function a second time, to ensure that one gets the full tagged message. From 889a45ff2805b1bf21108fb05b049945f101a8ad Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sun, 28 Apr 2024 09:22:07 +0530 Subject: [PATCH 067/217] ChatON:ChatTmplApply:Update the function notes --- common/chaton.hpp | 39 ++++++++++++++++++++++++++++++--------- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index b6ef6e1f9ce91..c9a879e339fb0 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -325,7 +325,7 @@ inline bool chaton_tmpl_kv_bool(const std::string &tmpl, const std::string &key) // a tagged message, types string and lens vector wrt the parts that make up the returned string // // * a string containing the tagged message -// * user-(begin+prefix) + msg + user-(suffix+end) +// * role-(begin+prefix) + msg + role-(suffix+end) // * a string where the chars contain info about // type of sub-strings/parts that make up the tagged message. // * a vector of ints, which give the length of each part in the tagged message. @@ -357,7 +357,7 @@ inline bool chaton_tmpl_apply_single_ex( // Given the template standard, role and a message, this returns the tagged message. // // * a string containing the tagged message -// * user-(begin+prefix) + msg + user-(suffix+end) +// * role-(begin+prefix) + msg + role-(suffix+end) inline size_t chaton_tmpl_apply_single( const std::string &tmpl, const std::string &role, @@ -403,10 +403,19 @@ inline size_t chat_tmpl_apply_single_capi( return taggedLength; } -// global-begin + [[role-begin] + [role-prefix] + msg + role-suffix] + global-end -// if there is a combination of system-user messages, -// then 1st user message will have user-prefix only if systemuser-1st-user-has-prefix is true -// NOTE: returns types and lens to help identify the parts of the tagged msg, which relate to passed and added tags + +// Given the template standard and a bunch of messages including their roles, this returns +// tagged messages, types string and lens vector. Returned types string and lens vector help +// identify the parts of the tagged msgs string, which relate to passed msgs and added tags. +// +// * a string containing the tagged messages +// * global-begin + 1 or more [[role-begin] + [role-prefix] + msg + [role-suffix] +[role-end]] + global-end +// * a string where the chars contain info about +// type of sub-strings/parts that make up the tagged messages string. +// * a vector of ints, which give the length of each part in the tagged messages string. +// +// if a combination of system-user messages is passed, then tags between the system +// and the 1st user message, is based on the flags set wrt the corresponding template standard. inline bool chaton_tmpl_apply_ex( const std::string &tmpl, const std::vector &msgs, @@ -499,9 +508,9 @@ inline bool chaton_tmpl_apply_ex( return true; } -// global-begin + [[role-begin] + [role-prefix] + msg + role-suffix] + global-end -// if there is a combination of system-user messages, -// then 1st user message will have user-prefix only if systemuser-1st-user-has-prefix is true +// Given the template standard and a bunch of messages including their roles, this returns +// the tagged messages as a string. +// global-begin + 1 or more [[role-begin] + [role-prefix] + msg + [role-suffix] +[role-end]] + global-end inline int32_t chaton_tmpl_apply( const std::string &tmpl, const std::vector &msgs, @@ -516,6 +525,18 @@ inline int32_t chaton_tmpl_apply( return tagged.size(); } +// Given the template standard and a bunch of messages including their roles, this returns +// the tagged messages as a string. +// global-begin + 1 or more [[role-begin] + [role-prefix] + msg + [role-suffix] +[role-end]] + global-end +// +// If the passed char array is smaller than that required for the tagged messages string, +// * part of the tagged messages string which fits within dest buffer is copied +// * the returned value, indicates the size of the actual tagged message +// NOTE: +// * ideally the passed char array should be able to fit the tagged messages string + 0|null char. +// * if the return value from this function is larger than or equal to destLength, +// then you will have to increase the size of the dest buffer, and call this +// function a second time, to ensure that one gets the full tagged messages string. inline int32_t chaton_tmpl_apply_capi( const char *tmpl, const struct llama_chat_message *msgs, From af9a0a211b8624c1203942151f96cebf35945274 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sun, 28 Apr 2024 09:31:15 +0530 Subject: [PATCH 068/217] ChatON:ChatTmplApply: Avoid the stringstream --- common/chaton.hpp | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index c9a879e339fb0..1140ba409e8a6 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -428,9 +428,7 @@ inline bool chaton_tmpl_apply_ex( return false; } ChatParts cp = {}; - std::stringstream ss; std::string globalBegin = chaton_tmpl_role_kv(tmpl, K_GLOBAL, {K_BEGIN}); - ss << globalBegin; cp.add_part(ChatParts::S, globalBegin); int cntSystem = 0; int cntUser = 0; @@ -444,63 +442,47 @@ inline bool chaton_tmpl_apply_ex( auto end = chaton_tmpl_role_kv(tmpl, role, {K_END}); if (role == K_SYSTEM) { cntSystem += 1; - ss << begin << prefix; cp.add_part(ChatParts::S, begin); cp.add_part(ChatParts::S, prefix); } else if (role == K_USER) { cntUser += 1; if ((cntSystem == 1) && (cntUser == 1)) { if (conMeta[tmpl][K_SYSTEMUSER_1ST_USER_HAS_BEGIN]) { - ss << begin; cp.add_part(ChatParts::S, begin); } if (conMeta[tmpl][K_SYSTEMUSER_1ST_USER_HAS_PREFIX]) { - ss << prefix; cp.add_part(ChatParts::S, prefix); } } else { - ss << begin << prefix; cp.add_part(ChatParts::S, begin); cp.add_part(ChatParts::S, prefix); } } else { cntOthers += 1; - ss << begin << prefix; cp.add_part(ChatParts::S, begin); cp.add_part(ChatParts::S, prefix); } - ss << content; cp.add_part(ChatParts::N, content); if (role == K_SYSTEM) { if (chaton_tmpl_kv_bool(tmpl, K_SYSTEMUSER_SYSTEM_HAS_SUFFIX)) { - ss << suffix; cp.add_part(ChatParts::S, suffix); } if (chaton_tmpl_kv_bool(tmpl, K_SYSTEMUSER_SYSTEM_HAS_END)) { - ss << end; cp.add_part(ChatParts::S, end); } } else { - ss << suffix << end; cp.add_part(ChatParts::S, suffix); cp.add_part(ChatParts::S, end); } } if (alertAssistantAtEnd) { auto assistantBeginPrefix = chaton_tmpl_role_kv(tmpl, K_ASSISTANT, {K_BEGIN, K_PREFIX}); - ss << assistantBeginPrefix; cp.add_part(ChatParts::S, assistantBeginPrefix); } auto globalEnd = chaton_tmpl_role_kv(tmpl, K_GLOBAL, {K_END}); - ss << globalEnd; cp.add_part(ChatParts::S, globalEnd); cp.dump(); - tagged = ss.str(); - std::string cpStr = cp.str(); - if (tagged != cpStr) { - LOG_TEELN("DBUG:%s:Mismatch between CP[%s] and SS[%s]", __func__, cpStr.c_str(), tagged.c_str()); - exit(2); - } + tagged = cp.str(); LOGLN("DBUG:%s:%s:%s", __func__, tmpl.c_str(), tagged.c_str()); LOGLN("DBUG:%s:%s:CntSys[%d]:CntUsr[%d]:CntOthers[%d]", __func__, tmpl.c_str(), cntSystem, cntUser, cntOthers); types = cp.get_types(); From ce75d434dc248201787a11795d31f59fac7df7b9 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sun, 28 Apr 2024 10:57:51 +0530 Subject: [PATCH 069/217] SimpCfg: Initial skeleton : get and set string and bool values --- common/simpcfg.hpp | 62 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 common/simpcfg.hpp diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp new file mode 100644 index 0000000000000..d62c80564f3e8 --- /dev/null +++ b/common/simpcfg.hpp @@ -0,0 +1,62 @@ +#pragma once + +/** + * Provide a simple config file logic + * It can consist of multiple config groups. + * * the group name needs to start at the begining of the line. + * Each group can inturn contain multiple config fields wrt that group. + * * the group fields need to have 1 or more space at the begining of line. + * + */ + +#include +#include +#include +#include + +#include "log.h" + + +std::map> mapStrings = {}; +std::map> mapBools = {}; + +void sc_set_string(std::string &group, std::string &key, std::string &value) { + auto gm = mapStrings[group]; + gm[key] = value; +} + +void sc_set_bool(std::string &group, std::string &key, bool value) { + auto gm = mapBools[group]; + gm[key] = value; +} + +std::string sc_get_string(std::string &group, std::string &key) { + auto gm = mapStrings[group]; + return gm[key]; +} + +bool sc_get_bool(std::string &group, std::string &key) { + auto gm = mapBools[group]; + return gm[key]; +} + +std::string str_trim(std::string &sin) { + std::string sout = sin; + sout.erase(sout.find_last); +} + +bool sc_load(std::string &fname) { + std::ifstream f {fname}; + if (!f) { + LOG_TEELN("ERRR:%s:%s:failed to load...", __func__, fname); + exit(2); + } + while(!f.eof()) { + std::string curl; + getline(f, curl); + if (curl.empty()) { + continue; + } + + } +} From f4687fa5d43dd07f9fc9aa838c71a5a91f597a1c Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sun, 28 Apr 2024 12:10:00 +0530 Subject: [PATCH 070/217] SimpCfg:Parse config file and load string key-value fields --- common/simpcfg.hpp | 62 +++++++++++++++++++++++++++++++++++++++------- 1 file changed, 53 insertions(+), 9 deletions(-) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index d62c80564f3e8..83b88c4c92e1c 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -14,8 +14,12 @@ #include #include +#define TEST_LOGIC +#ifdef TEST_LOGIC +#define LOG_TEELN(FMT, ...) printf(FMT, __VA_ARGS__) +#else #include "log.h" - +#endif std::map> mapStrings = {}; std::map> mapBools = {}; @@ -40,23 +44,63 @@ bool sc_get_bool(std::string &group, std::string &key) { return gm[key]; } -std::string str_trim(std::string &sin) { - std::string sout = sin; - sout.erase(sout.find_last); +std::string str_trim(std::string sin) { + sin.erase(sin.find_last_not_of(" \t\n")+1); + sin.erase(0, sin.find_first_not_of(" \t\n")); + return sin; } bool sc_load(std::string &fname) { std::ifstream f {fname}; if (!f) { LOG_TEELN("ERRR:%s:%s:failed to load...", __func__, fname); - exit(2); + throw std::runtime_error { "ERRR:SimpCfg:File not found" }; + } else { + LOG_TEELN("DBUG:%s:%s", __func__, fname); } + std::string group; + int iLine = 0; while(!f.eof()) { - std::string curl; - getline(f, curl); - if (curl.empty()) { + iLine += 1; + std::string curL; + getline(f, curL); + if (curL.empty()) { continue; } - + if (curL[0] == '#') { + continue; + } + bool bGroup = !isspace(curL[0]); + curL = str_trim(curL); + if (bGroup) { + group = curL; + LOG_TEELN("DBUG:%s:%s:%s:%s", __func__, group); + continue; + } + auto dPos = curL.find(':'); + if (dPos == std::string::npos) { + LOG_TEELN("ERRR:%s:%d:invalid key value line:%s", __func__, iLine, curL); + throw std::runtime_error { "ERRR:SimpCfg:Invalid key value line" }; + } + auto dEnd = curL.length() - dPos; + if ((dPos == 0) || (dEnd < 2)) { + LOG_TEELN("ERRR:%s:%d:invalid key value line:%s", __func__, iLine, curL); + throw std::runtime_error { "ERRR:SimpCfg:Invalid key value line" }; + } + std::string key = curL.substr(0, dPos); + key = str_trim(key); + std::string value = curL.substr(dPos+1); + value = str_trim(value); + LOG_TEELN("DBUG:%s:%s:%s:%s", __func__, group, key, value); + sc_set_string(group, key, value); } } + + +#ifdef TEST_LOGIC +int main(int argc, char **argv) { + std::string fname {argv[1]}; + sc_load(fname); + return 0; +} +#endif From aea685013102ec226b37e52779d54759b7e9b397 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sun, 28 Apr 2024 12:34:57 +0530 Subject: [PATCH 071/217] SimpCfg: Keep compiler happy, also add newline wrt alt logging def --- common/simpcfg.hpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index 83b88c4c92e1c..026c1beaa14a7 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -16,7 +16,7 @@ #define TEST_LOGIC #ifdef TEST_LOGIC -#define LOG_TEELN(FMT, ...) printf(FMT, __VA_ARGS__) +#define LOG_TEELN(FMT, ...) printf(FMT"\n", __VA_ARGS__) #else #include "log.h" #endif @@ -50,13 +50,13 @@ std::string str_trim(std::string sin) { return sin; } -bool sc_load(std::string &fname) { +void sc_load(std::string &fname) { std::ifstream f {fname}; if (!f) { - LOG_TEELN("ERRR:%s:%s:failed to load...", __func__, fname); + LOG_TEELN("ERRR:%s:%s:failed to load...", __func__, fname.c_str()); throw std::runtime_error { "ERRR:SimpCfg:File not found" }; } else { - LOG_TEELN("DBUG:%s:%s", __func__, fname); + LOG_TEELN("DBUG:%s:%s", __func__, fname.c_str()); } std::string group; int iLine = 0; @@ -74,24 +74,24 @@ bool sc_load(std::string &fname) { curL = str_trim(curL); if (bGroup) { group = curL; - LOG_TEELN("DBUG:%s:%s:%s:%s", __func__, group); + LOG_TEELN("DBUG:%s:%s", __func__, group.c_str()); continue; } auto dPos = curL.find(':'); if (dPos == std::string::npos) { - LOG_TEELN("ERRR:%s:%d:invalid key value line:%s", __func__, iLine, curL); + LOG_TEELN("ERRR:%s:%d:invalid key value line:%s", __func__, iLine, curL.c_str()); throw std::runtime_error { "ERRR:SimpCfg:Invalid key value line" }; } auto dEnd = curL.length() - dPos; if ((dPos == 0) || (dEnd < 2)) { - LOG_TEELN("ERRR:%s:%d:invalid key value line:%s", __func__, iLine, curL); + LOG_TEELN("ERRR:%s:%d:invalid key value line:%s", __func__, iLine, curL.c_str()); throw std::runtime_error { "ERRR:SimpCfg:Invalid key value line" }; } std::string key = curL.substr(0, dPos); key = str_trim(key); std::string value = curL.substr(dPos+1); value = str_trim(value); - LOG_TEELN("DBUG:%s:%s:%s:%s", __func__, group, key, value); + LOG_TEELN("DBUG:%s:%s:%s:%s", __func__, group.c_str(), key.c_str(), value.c_str()); sc_set_string(group, key, value); } } From f728dbddd0737f42173eba65aa0b423335218bd5 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sun, 28 Apr 2024 13:07:43 +0530 Subject: [PATCH 072/217] ChatON: Add simpcfg based config file matching chaton_meta.json Add missing begin and end fields wrt deepseek-coder assistant in chaton_meta.json. Idea is to avoid json library dependency by adding a simple text based config file support. --- examples/chaton_meta.json | 4 +- examples/chaton_meta.simpcfg | 225 +++++++++++++++++++++++++++++++++++ 2 files changed, 228 insertions(+), 1 deletion(-) create mode 100644 examples/chaton_meta.simpcfg diff --git a/examples/chaton_meta.json b/examples/chaton_meta.json index 5e3c6f8e880f3..994301386e383 100644 --- a/examples/chaton_meta.json +++ b/examples/chaton_meta.json @@ -165,8 +165,10 @@ "end": "" }, "assistant": { + "begin": "", "prefix": "### Response:\n", - "suffix": "\n<|EOT|>\n" + "suffix": "\n<|EOT|>\n", + "end": "" }, "reverse-prompt": "<|EOT|>", "systemuser-system-has-suffix": true, diff --git a/examples/chaton_meta.simpcfg b/examples/chaton_meta.simpcfg new file mode 100644 index 0000000000000..2a209fd21c7df --- /dev/null +++ b/examples/chaton_meta.simpcfg @@ -0,0 +1,225 @@ +"llama2" + "global-begin": "" + "global-end": "" + + "system-begin": "" + "system-prefix": "[INST] <>\n" + "system-suffix": "\n<> " + "system-end": "[/INST]\n\n" + + "user-begin": "" + "user-prefix": "[INST] " + "user-suffix": " [/INST]\n\n" + "user-end": "" + + "assistant-begin": "" + "assistant-prefix": "" + "assistant-suffix": "" + "assistant-end": "" + + "reverse-prompt": "", + "systemuser-system-has-suffix": true, + "systemuser-system-has-end": false, + "systemuser-1st-user-has-begin": false, + "systemuser-1st-user-has-prefix": false + +"llama3" + "global-begin": "" + "global-end": "" + + "system-begin": "" + "system-prefix": "<|start_header_id|>system<|end_header_id|>\n" + "system-suffix": "<|eot_id|>\n\n" + "system-end": "" + + "user-begin": "" + "user-prefix": "<|start_header_id|>user<|end_header_id|>\n" + "user-suffix": "<|eot_id|>\n\n" + "user-end": "" + + "assistant-begin": "" + "assistant-prefix": "<|start_header_id|>assistant<|end_header_id|>\n" + "assistant-suffix": "<|eot_id|>\n\n" + "assistant-end": "" + + "reverse-prompt": "<|eot_id|>", + "systemuser-system-has-suffix": true, + "systemuser-system-has-end": true, + "systemuser-1st-user-has-begin": true, + "systemuser-1st-user-has-prefix": true + +"chatml" + "global-begin": "" + "global-end": "" + + "system-begin": "" + "system-prefix": "<|im_start|>system\n" + "system-suffix": "<|im_end|>\n" + "system-end": "" + + "user-begin": "" + "user-prefix": "<|im_start|>user\n" + "user-suffix": "<|im_end|>\n" + "user-end": "" + + "assistant-begin": "" + "assistant-prefix": "<|im_start|>assistant\n" + "assistant-suffix": "<|im_end|>\n" + "assistant-end": "" + + "reverse-prompt": "<|im_start|>user\n", + "systemuser-system-has-suffix": true, + "systemuser-system-has-end": true, + "systemuser-1st-user-has-begin": true, + "systemuser-1st-user-has-prefix": true + +"zephyr" + "global-alt-end": "<|endoftext|>\n" + "global-begin": "" + "global-end": "" + + "system-begin": "" + "system-prefix": "<|system|>\n" + "system-suffix": "" + "system-end": "\n" + + "user-begin": "" + "user-prefix": "<|user|>\n" + "user-suffix": "" + "user-end": "\n" + + "assistant-end-alt": "<|endoftext|>\n" + "assistant-begin": "" + "assistant-prefix": "<|assistant|>\n" + "assistant-suffix": "" + "assistant-end": "\n" + + "reverse-prompt": "", + "systemuser-system-has-suffix": true, + "systemuser-system-has-end": true, + "systemuser-1st-user-has-begin": true, + "systemuser-1st-user-has-prefix": true + +"gemma" + "global-begin": "" + "global-end": "" + + "system-begin": "" + "system-prefix": "user\n" + "system-suffix": "\n" + "system-end": "" + + "user-begin": "" + "user-prefix": "user\n" + "user-suffix": "\n" + "user-end": "" + + "assistant-begin": "" + "assistant-prefix": "model\n" + "assistant-suffix": "\n" + "assistant-end": "" + + "reverse-prompt": "", + "systemuser-system-has-suffix": true, + "systemuser-system-has-end": false, + "systemuser-1st-user-has-begin": true, + "systemuser-1st-user-has-prefix": true + +"deepseek-coder" + "global-begin": "" + "global-end": "" + + "system-begin": "\n<｜begin▁of▁sentence｜>" + "system-prefix": "" + "system-suffix": "\n" + "system-end": "" + + "user-begin": "" + "user-prefix": "### Instruction:\n" + "user-suffix": "\n" + "user-end": "" + + "assistant-prefix": "### Response:\n" + "assistant-suffix": "\n<|EOT|>\n + + "reverse-prompt": "<|EOT|>" + "systemuser-system-has-suffix": true, + "systemuser-system-has-end": false, + "systemuser-1st-user-has-begin": false, + "systemuser-1st-user-has-prefix": true + +"deepseek" + "global-begin": "" + "global-end": "" + + "system-begin": "<｜begin▁of▁sentence｜>" + "system-prefix": "" + "system-suffix": "\n\n" + "system-end": "" + + "user-begin": "<｜begin▁of▁sentence｜>" + "user-prefix": "User: " + "user-suffix": "\n\n" + "user-end": "" + + "assistant-begin": "" + "assistant-prefix": "Assistant: " + "assistant-suffix": " <｜end▁of▁sentence｜>\n" + "assistant-end": "" + + "reverse-prompt": "<｜end▁of▁sentence｜>", + "systemuser-system-has-suffix": true, + "systemuser-system-has-end": false, + "systemuser-1st-user-has-begin": false, + "systemuser-1st-user-has-prefix": true + +"monarch" + "global-begin": "" + "global-end": "" + + "system-begin": "" + "system-prefix": "system\n" + "system-suffix": "" + "system-end": "\n" + + "user-begin": "" + "user-prefix": "user\n" + "user-suffix": "" + "user-end": "\n" + + "assistant-begin": "" + "assistant-prefix": "assistant\n" + "assistant-suffix": "" + "assistant-end": "\n" + + "reverse-prompt": "", + "systemuser-system-has-suffix": true, + "systemuser-system-has-end": true, + "systemuser-1st-user-has-begin": true, + "systemuser-1st-user-has-prefix": true + +"mistral" + "global-begin": "" + "global-end": "" + + "system-begin": "" + "system-prefix": "[INST] " + "system-suffix": " [/INST]" + "system-end": "\n" + + "user-begin": "" + "user-prefix": "[INST] " + "user-suffix": " [/INST]" + "user-end": "\n" + + "assistant-begin": "" + "assistant-prefix": "" + "assistant-suffix": "" + "assistant-end": " \n" + + "reverse-prompt": "", + "systemuser-system-has-suffix": false, + "systemuser-system-has-end": true, + "systemuser-1st-user-has-begin": false, + "systemuser-1st-user-has-prefix": false + From 2cbb00c340d2a63544360e2f9f0a77566c5783ff Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sun, 28 Apr 2024 13:10:59 +0530 Subject: [PATCH 073/217] SimpCfg: Add support for boolean fields wrt key-value --- common/simpcfg.hpp | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index 026c1beaa14a7..ff68f85ce89c8 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -74,7 +74,7 @@ void sc_load(std::string &fname) { curL = str_trim(curL); if (bGroup) { group = curL; - LOG_TEELN("DBUG:%s:%s", __func__, group.c_str()); + LOG_TEELN("DBUG:%s:group:%s", __func__, group.c_str()); continue; } auto dPos = curL.find(':'); @@ -91,8 +91,12 @@ void sc_load(std::string &fname) { key = str_trim(key); std::string value = curL.substr(dPos+1); value = str_trim(value); - LOG_TEELN("DBUG:%s:%s:%s:%s", __func__, group.c_str(), key.c_str(), value.c_str()); - sc_set_string(group, key, value); + LOG_TEELN("DBUG:%s:kv:%s:%s:%s", __func__, group.c_str(), key.c_str(), value.c_str()); + if ((value == "true") || (value == "false")) { + sc_set_bool(group, key, value == "true" ? true : false); + } else { + sc_set_string(group, key, value); + } } } From 28ae0c5b02d6f0fac03928dbe95657262d061464 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sun, 28 Apr 2024 13:20:39 +0530 Subject: [PATCH 074/217] SimpCfg:Make str_trim flexible, use to trim , wrt value Now one can pass the list/string of chars to trim at either end. --- common/simpcfg.hpp | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index ff68f85ce89c8..287662eae561a 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -44,9 +44,9 @@ bool sc_get_bool(std::string &group, std::string &key) { return gm[key]; } -std::string str_trim(std::string sin) { - sin.erase(sin.find_last_not_of(" \t\n")+1); - sin.erase(0, sin.find_first_not_of(" \t\n")); +std::string str_trim(std::string sin, std::string trimChars=" \t\n") { + sin.erase(sin.find_last_not_of(trimChars)+1); + sin.erase(0, sin.find_first_not_of(trimChars)); return sin; } @@ -91,12 +91,15 @@ void sc_load(std::string &fname) { key = str_trim(key); std::string value = curL.substr(dPos+1); value = str_trim(value); - LOG_TEELN("DBUG:%s:kv:%s:%s:%s", __func__, group.c_str(), key.c_str(), value.c_str()); + value = str_trim(value, ","); + std::string vtype = "bool"; if ((value == "true") || (value == "false")) { sc_set_bool(group, key, value == "true" ? true : false); } else { + vtype = "string"; sc_set_string(group, key, value); } + LOG_TEELN("DBUG:%s:kv:%s:%s:%s:%s", __func__, group.c_str(), key.c_str(), vtype.c_str(), value.c_str()); } } From 1ecca5a7ec0a85540978faa30f0564db22619a68 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sun, 28 Apr 2024 13:32:00 +0530 Subject: [PATCH 075/217] SimpCfg: Convert to a class --- common/simpcfg.hpp | 149 ++++++++++++++++++++++++--------------------- 1 file changed, 78 insertions(+), 71 deletions(-) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index 287662eae561a..4d441296e499b 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -21,28 +21,6 @@ #include "log.h" #endif -std::map> mapStrings = {}; -std::map> mapBools = {}; - -void sc_set_string(std::string &group, std::string &key, std::string &value) { - auto gm = mapStrings[group]; - gm[key] = value; -} - -void sc_set_bool(std::string &group, std::string &key, bool value) { - auto gm = mapBools[group]; - gm[key] = value; -} - -std::string sc_get_string(std::string &group, std::string &key) { - auto gm = mapStrings[group]; - return gm[key]; -} - -bool sc_get_bool(std::string &group, std::string &key) { - auto gm = mapBools[group]; - return gm[key]; -} std::string str_trim(std::string sin, std::string trimChars=" \t\n") { sin.erase(sin.find_last_not_of(trimChars)+1); @@ -50,64 +28,93 @@ std::string str_trim(std::string sin, std::string trimChars=" \t\n") { return sin; } -void sc_load(std::string &fname) { - std::ifstream f {fname}; - if (!f) { - LOG_TEELN("ERRR:%s:%s:failed to load...", __func__, fname.c_str()); - throw std::runtime_error { "ERRR:SimpCfg:File not found" }; - } else { - LOG_TEELN("DBUG:%s:%s", __func__, fname.c_str()); + +class SimpCfg { +private: + std::map> mapStrings = {}; + std::map> mapBools = {}; +public: + void sc_set_string(std::string &group, std::string &key, std::string &value) { + auto gm = mapStrings[group]; + gm[key] = value; } - std::string group; - int iLine = 0; - while(!f.eof()) { - iLine += 1; - std::string curL; - getline(f, curL); - if (curL.empty()) { - continue; - } - if (curL[0] == '#') { - continue; - } - bool bGroup = !isspace(curL[0]); - curL = str_trim(curL); - if (bGroup) { - group = curL; - LOG_TEELN("DBUG:%s:group:%s", __func__, group.c_str()); - continue; - } - auto dPos = curL.find(':'); - if (dPos == std::string::npos) { - LOG_TEELN("ERRR:%s:%d:invalid key value line:%s", __func__, iLine, curL.c_str()); - throw std::runtime_error { "ERRR:SimpCfg:Invalid key value line" }; - } - auto dEnd = curL.length() - dPos; - if ((dPos == 0) || (dEnd < 2)) { - LOG_TEELN("ERRR:%s:%d:invalid key value line:%s", __func__, iLine, curL.c_str()); - throw std::runtime_error { "ERRR:SimpCfg:Invalid key value line" }; - } - std::string key = curL.substr(0, dPos); - key = str_trim(key); - std::string value = curL.substr(dPos+1); - value = str_trim(value); - value = str_trim(value, ","); - std::string vtype = "bool"; - if ((value == "true") || (value == "false")) { - sc_set_bool(group, key, value == "true" ? true : false); + + void sc_set_bool(std::string &group, std::string &key, bool value) { + auto gm = mapBools[group]; + gm[key] = value; + } + + std::string sc_get_string(std::string &group, std::string &key) { + auto gm = mapStrings[group]; + return gm[key]; + } + + bool sc_get_bool(std::string &group, std::string &key) { + auto gm = mapBools[group]; + return gm[key]; + } + + void sc_load(std::string &fname) { + std::ifstream f {fname}; + if (!f) { + LOG_TEELN("ERRR:%s:%s:failed to load...", __func__, fname.c_str()); + throw std::runtime_error { "ERRR:SimpCfg:File not found" }; } else { - vtype = "string"; - sc_set_string(group, key, value); + LOG_TEELN("DBUG:%s:%s", __func__, fname.c_str()); + } + std::string group; + int iLine = 0; + while(!f.eof()) { + iLine += 1; + std::string curL; + getline(f, curL); + if (curL.empty()) { + continue; + } + if (curL[0] == '#') { + continue; + } + bool bGroup = !isspace(curL[0]); + curL = str_trim(curL); + if (bGroup) { + group = curL; + LOG_TEELN("DBUG:%s:group:%s", __func__, group.c_str()); + continue; + } + auto dPos = curL.find(':'); + if (dPos == std::string::npos) { + LOG_TEELN("ERRR:%s:%d:invalid key value line:%s", __func__, iLine, curL.c_str()); + throw std::runtime_error { "ERRR:SimpCfg:Invalid key value line" }; + } + auto dEnd = curL.length() - dPos; + if ((dPos == 0) || (dEnd < 2)) { + LOG_TEELN("ERRR:%s:%d:invalid key value line:%s", __func__, iLine, curL.c_str()); + throw std::runtime_error { "ERRR:SimpCfg:Invalid key value line" }; + } + std::string key = curL.substr(0, dPos); + key = str_trim(key); + std::string value = curL.substr(dPos+1); + value = str_trim(value); + value = str_trim(value, ","); + std::string vtype = "bool"; + if ((value == "true") || (value == "false")) { + sc_set_bool(group, key, value == "true" ? true : false); + } else { + vtype = "string"; + sc_set_string(group, key, value); + } + LOG_TEELN("DBUG:%s:kv:%s:%s:%s:%s", __func__, group.c_str(), key.c_str(), vtype.c_str(), value.c_str()); } - LOG_TEELN("DBUG:%s:kv:%s:%s:%s:%s", __func__, group.c_str(), key.c_str(), vtype.c_str(), value.c_str()); } -} + +}; #ifdef TEST_LOGIC int main(int argc, char **argv) { std::string fname {argv[1]}; - sc_load(fname); + SimpCfg sc; + sc.sc_load(fname); return 0; } #endif From 6de8a14f3239ac759b203fe56ceac9af69f35f09 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sun, 28 Apr 2024 13:33:53 +0530 Subject: [PATCH 076/217] SimpCfg: Rename member functions to avoid sc_ prefix now that logic has been converted into a class, no need for this prefix --- common/simpcfg.hpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index 4d441296e499b..fc435a73a02be 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -34,27 +34,27 @@ class SimpCfg { std::map> mapStrings = {}; std::map> mapBools = {}; public: - void sc_set_string(std::string &group, std::string &key, std::string &value) { + void set_string(std::string &group, std::string &key, std::string &value) { auto gm = mapStrings[group]; gm[key] = value; } - void sc_set_bool(std::string &group, std::string &key, bool value) { + void set_bool(std::string &group, std::string &key, bool value) { auto gm = mapBools[group]; gm[key] = value; } - std::string sc_get_string(std::string &group, std::string &key) { + std::string get_string(std::string &group, std::string &key) { auto gm = mapStrings[group]; return gm[key]; } - bool sc_get_bool(std::string &group, std::string &key) { + bool get_bool(std::string &group, std::string &key) { auto gm = mapBools[group]; return gm[key]; } - void sc_load(std::string &fname) { + void load(std::string &fname) { std::ifstream f {fname}; if (!f) { LOG_TEELN("ERRR:%s:%s:failed to load...", __func__, fname.c_str()); @@ -98,10 +98,10 @@ class SimpCfg { value = str_trim(value, ","); std::string vtype = "bool"; if ((value == "true") || (value == "false")) { - sc_set_bool(group, key, value == "true" ? true : false); + set_bool(group, key, value == "true" ? true : false); } else { vtype = "string"; - sc_set_string(group, key, value); + set_string(group, key, value); } LOG_TEELN("DBUG:%s:kv:%s:%s:%s:%s", __func__, group.c_str(), key.c_str(), vtype.c_str(), value.c_str()); } @@ -114,7 +114,7 @@ class SimpCfg { int main(int argc, char **argv) { std::string fname {argv[1]}; SimpCfg sc; - sc.sc_load(fname); + sc.load(fname); return 0; } #endif From d514c818290881b2fca793c22ee49fca09d57688 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sun, 28 Apr 2024 15:41:43 +0530 Subject: [PATCH 077/217] SimpCfg: Add the const which I had forgotten wrt args --- common/simpcfg.hpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index fc435a73a02be..6435262484495 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -34,27 +34,27 @@ class SimpCfg { std::map> mapStrings = {}; std::map> mapBools = {}; public: - void set_string(std::string &group, std::string &key, std::string &value) { + void set_string(const std::string &group, const std::string &key, const std::string &value) { auto gm = mapStrings[group]; gm[key] = value; } - void set_bool(std::string &group, std::string &key, bool value) { + void set_bool(const std::string &group, const std::string &key, bool value) { auto gm = mapBools[group]; gm[key] = value; } - std::string get_string(std::string &group, std::string &key) { + std::string get_string(const std::string &group, const std::string &key) { auto gm = mapStrings[group]; return gm[key]; } - bool get_bool(std::string &group, std::string &key) { + bool get_bool(const std::string &group, const std::string &key) { auto gm = mapBools[group]; return gm[key]; } - void load(std::string &fname) { + void load(const std::string &fname) { std::ifstream f {fname}; if (!f) { LOG_TEELN("ERRR:%s:%s:failed to load...", __func__, fname.c_str()); @@ -115,6 +115,7 @@ int main(int argc, char **argv) { std::string fname {argv[1]}; SimpCfg sc; sc.load(fname); + sc.get_bool("testme", "key101"); return 0; } #endif From 951fbc33968ca1fe47cb013c97bccd10b6c8ad8f Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sun, 28 Apr 2024 15:59:05 +0530 Subject: [PATCH 078/217] SimpCfg: Change logging to LDBUG and LERRR helpers --- common/simpcfg.hpp | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index 6435262484495..73a4cc395b1e3 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -16,9 +16,12 @@ #define TEST_LOGIC #ifdef TEST_LOGIC -#define LOG_TEELN(FMT, ...) printf(FMT"\n", __VA_ARGS__) +#define LDBUG_LN(FMT, ...) printf(FMT"\n", __VA_ARGS__) +#define LERRR_LN(FMT, ...) printf(FMT"\n", __VA_ARGS__) #else #include "log.h" +#define LDBUG_LN LOGLN +#define LERRR_LN LOG_TEELN #endif @@ -37,30 +40,36 @@ class SimpCfg { void set_string(const std::string &group, const std::string &key, const std::string &value) { auto gm = mapStrings[group]; gm[key] = value; + LDBUG_LN("DBUG:SC:%s:%s:%s:%s", __func__, group.c_str(), key.c_str(), value.c_str()); } void set_bool(const std::string &group, const std::string &key, bool value) { auto gm = mapBools[group]; gm[key] = value; + LDBUG_LN("DBUG:SC:%s:%s:%s:%d", __func__, group.c_str(), key.c_str(), value); } std::string get_string(const std::string &group, const std::string &key) { auto gm = mapStrings[group]; - return gm[key]; + auto value = gm[key]; + LDBUG_LN("DBUG:SC:%s:%s:%s:%s", __func__, group.c_str(), key.c_str(), value.c_str()); + return value; } bool get_bool(const std::string &group, const std::string &key) { auto gm = mapBools[group]; - return gm[key]; + auto value = gm[key]; + LDBUG_LN("DBUG:SC:%s:%s:%s:%d", __func__, group.c_str(), key.c_str(), value); + return value; } void load(const std::string &fname) { std::ifstream f {fname}; if (!f) { - LOG_TEELN("ERRR:%s:%s:failed to load...", __func__, fname.c_str()); + LERRR_LN("ERRR:%s:%s:failed to load...", __func__, fname.c_str()); throw std::runtime_error { "ERRR:SimpCfg:File not found" }; } else { - LOG_TEELN("DBUG:%s:%s", __func__, fname.c_str()); + LDBUG_LN("DBUG:%s:%s", __func__, fname.c_str()); } std::string group; int iLine = 0; @@ -78,17 +87,17 @@ class SimpCfg { curL = str_trim(curL); if (bGroup) { group = curL; - LOG_TEELN("DBUG:%s:group:%s", __func__, group.c_str()); + LDBUG_LN("DBUG:%s:group:%s", __func__, group.c_str()); continue; } auto dPos = curL.find(':'); if (dPos == std::string::npos) { - LOG_TEELN("ERRR:%s:%d:invalid key value line:%s", __func__, iLine, curL.c_str()); + LERRR_LN("ERRR:%s:%d:invalid key value line:%s", __func__, iLine, curL.c_str()); throw std::runtime_error { "ERRR:SimpCfg:Invalid key value line" }; } auto dEnd = curL.length() - dPos; if ((dPos == 0) || (dEnd < 2)) { - LOG_TEELN("ERRR:%s:%d:invalid key value line:%s", __func__, iLine, curL.c_str()); + LERRR_LN("ERRR:%s:%d:invalid key value line:%s", __func__, iLine, curL.c_str()); throw std::runtime_error { "ERRR:SimpCfg:Invalid key value line" }; } std::string key = curL.substr(0, dPos); @@ -103,7 +112,7 @@ class SimpCfg { vtype = "string"; set_string(group, key, value); } - LOG_TEELN("DBUG:%s:kv:%s:%s:%s:%s", __func__, group.c_str(), key.c_str(), vtype.c_str(), value.c_str()); + //LDBUG_LN("DBUG:%s:kv:%s:%s:%s:%s", __func__, group.c_str(), key.c_str(), vtype.c_str(), value.c_str()); } } @@ -115,7 +124,8 @@ int main(int argc, char **argv) { std::string fname {argv[1]}; SimpCfg sc; sc.load(fname); - sc.get_bool("testme", "key101"); + sc.get_bool("testme", "key101b"); + sc.get_string("testme", "key101s"); return 0; } #endif From 9940bd8ed7810b64f061cc227fb3f69fe72de040 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sun, 28 Apr 2024 16:29:49 +0530 Subject: [PATCH 079/217] SimpCfg: Allow default values wrt set string and set bool --- common/simpcfg.hpp | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index 73a4cc395b1e3..77542b8f1cc90 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -18,10 +18,12 @@ #ifdef TEST_LOGIC #define LDBUG_LN(FMT, ...) printf(FMT"\n", __VA_ARGS__) #define LERRR_LN(FMT, ...) printf(FMT"\n", __VA_ARGS__) +#define LWARN_LN(FMT, ...) printf(FMT"\n", __VA_ARGS__) #else #include "log.h" #define LDBUG_LN LOGLN #define LERRR_LN LOG_TEELN +#define LWARN_LN LOG_TEELN #endif @@ -49,15 +51,23 @@ class SimpCfg { LDBUG_LN("DBUG:SC:%s:%s:%s:%d", __func__, group.c_str(), key.c_str(), value); } - std::string get_string(const std::string &group, const std::string &key) { + std::string get_string(const std::string &group, const std::string &key, const std::string &defaultValue) { auto gm = mapStrings[group]; + if (gm.find(key) == gm.end()) { + LWARN_LN("DBUG:SC:%s:%s:%s:%s[default]", __func__, group.c_str(), key.c_str(), defaultValue.c_str()); + return defaultValue; + } auto value = gm[key]; LDBUG_LN("DBUG:SC:%s:%s:%s:%s", __func__, group.c_str(), key.c_str(), value.c_str()); return value; } - bool get_bool(const std::string &group, const std::string &key) { + bool get_bool(const std::string &group, const std::string &key, bool defaultValue) { auto gm = mapBools[group]; + if (gm.find(key) == gm.end()) { + LWARN_LN("DBUG:SC:%s:%s:%s:%d[default]", __func__, group.c_str(), key.c_str(), defaultValue); + return defaultValue; + } auto value = gm[key]; LDBUG_LN("DBUG:SC:%s:%s:%s:%d", __func__, group.c_str(), key.c_str(), value); return value; @@ -124,8 +134,12 @@ int main(int argc, char **argv) { std::string fname {argv[1]}; SimpCfg sc; sc.load(fname); - sc.get_bool("testme", "key101b"); - sc.get_string("testme", "key101s"); + sc.get_bool("testme", "key101b", false); + sc.get_string("testme", "key101s", "Not found"); + sc.set_bool("testme", "key201b", true); + sc.set_string("testme", "key201s", "hello world"); + sc.get_bool("testme", "key201b", false); + sc.get_string("testme", "key201s", "Not found"); return 0; } #endif From 82348e284053eef9e5a2c8e17a046ae4da98ab13 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sun, 28 Apr 2024 16:45:38 +0530 Subject: [PATCH 080/217] SimpCfg: Put GroupMap back into Map, Iterate during get if DBUG TODO: Have to look into C++ a bit later including these default container types. Not sure my current flow is efficient. --- common/simpcfg.hpp | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index 77542b8f1cc90..aca6bd2a69be5 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -14,6 +14,7 @@ #include #include +#define SC_DEBUG #define TEST_LOGIC #ifdef TEST_LOGIC #define LDBUG_LN(FMT, ...) printf(FMT"\n", __VA_ARGS__) @@ -42,17 +43,24 @@ class SimpCfg { void set_string(const std::string &group, const std::string &key, const std::string &value) { auto gm = mapStrings[group]; gm[key] = value; + mapStrings[group] = gm; LDBUG_LN("DBUG:SC:%s:%s:%s:%s", __func__, group.c_str(), key.c_str(), value.c_str()); } void set_bool(const std::string &group, const std::string &key, bool value) { auto gm = mapBools[group]; gm[key] = value; + mapBools[group] = gm; LDBUG_LN("DBUG:SC:%s:%s:%s:%d", __func__, group.c_str(), key.c_str(), value); } std::string get_string(const std::string &group, const std::string &key, const std::string &defaultValue) { auto gm = mapStrings[group]; +#ifdef SC_DEBUG + for(auto k: gm) { + LDBUG_LN("DBUG:SC:%s:Iterate:%s:%s", __func__, k.first.c_str(), k.second.c_str()); + } +#endif if (gm.find(key) == gm.end()) { LWARN_LN("DBUG:SC:%s:%s:%s:%s[default]", __func__, group.c_str(), key.c_str(), defaultValue.c_str()); return defaultValue; @@ -64,6 +72,11 @@ class SimpCfg { bool get_bool(const std::string &group, const std::string &key, bool defaultValue) { auto gm = mapBools[group]; +#ifdef SC_DEBUG + for(auto k: gm) { + LDBUG_LN("DBUG:SC:%s:Iterate:%s:%d", __func__, k.first.c_str(), k.second); + } +#endif if (gm.find(key) == gm.end()) { LWARN_LN("DBUG:SC:%s:%s:%s:%d[default]", __func__, group.c_str(), key.c_str(), defaultValue); return defaultValue; @@ -140,6 +153,8 @@ int main(int argc, char **argv) { sc.set_string("testme", "key201s", "hello world"); sc.get_bool("testme", "key201b", false); sc.get_string("testme", "key201s", "Not found"); + sc.get_string("mistral", "system-prefix", "Not found"); + sc.get_string("\"mistral\"", "\"system-prefix\"", "Not found"); return 0; } #endif From ca5a04d60798ae823aa194afd812f329cc5cab05 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sun, 28 Apr 2024 17:21:12 +0530 Subject: [PATCH 081/217] SimpCfg: Remove double quotes around group, key or string value --- common/simpcfg.hpp | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index aca6bd2a69be5..e17a38f12b34c 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -34,6 +34,28 @@ std::string str_trim(std::string sin, std::string trimChars=" \t\n") { return sin; } +// Remove atmost 1 char at the begin and 1 char at the end of the passed string, +// provided the char belongs to one of the chars in trimChars. +// NOTE: Not sure this handles non english utf8 multibyte chars properly, +// need to cross check. +std::string str_trim_single(std::string sin, std::string trimChars=" \t\n") { + if (sin.empty()) return sin; + for(auto c: trimChars) { + if (c == sin.front()) { + sin = sin.substr(1, std::string::npos); + break; + } + } + if (sin.empty()) return sin; + for(auto c: trimChars) { + if (c == sin.back()) { + sin = sin.substr(0, sin.length()-1); + break; + } + } + return sin; +} + class SimpCfg { private: @@ -109,6 +131,7 @@ class SimpCfg { bool bGroup = !isspace(curL[0]); curL = str_trim(curL); if (bGroup) { + curL = str_trim_single(curL, "\""); group = curL; LDBUG_LN("DBUG:%s:group:%s", __func__, group.c_str()); continue; @@ -125,6 +148,7 @@ class SimpCfg { } std::string key = curL.substr(0, dPos); key = str_trim(key); + key = str_trim_single(key, "\""); std::string value = curL.substr(dPos+1); value = str_trim(value); value = str_trim(value, ","); @@ -133,6 +157,7 @@ class SimpCfg { set_bool(group, key, value == "true" ? true : false); } else { vtype = "string"; + value = str_trim_single(value, "\""); set_string(group, key, value); } //LDBUG_LN("DBUG:%s:kv:%s:%s:%s:%s", __func__, group.c_str(), key.c_str(), vtype.c_str(), value.c_str()); From 0a534e6897c05c0a526b499f68d81dc0e9db71c9 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sun, 28 Apr 2024 17:27:21 +0530 Subject: [PATCH 082/217] SimpCfg: Rename test program related #define --- common/simpcfg.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index e17a38f12b34c..b28839ccee8de 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -15,8 +15,8 @@ #include #define SC_DEBUG -#define TEST_LOGIC -#ifdef TEST_LOGIC +#define SC_TEST_PRG +#ifdef SC_TEST_PRG #define LDBUG_LN(FMT, ...) printf(FMT"\n", __VA_ARGS__) #define LERRR_LN(FMT, ...) printf(FMT"\n", __VA_ARGS__) #define LWARN_LN(FMT, ...) printf(FMT"\n", __VA_ARGS__) @@ -167,7 +167,7 @@ class SimpCfg { }; -#ifdef TEST_LOGIC +#ifdef SC_TEST_PRG int main(int argc, char **argv) { std::string fname {argv[1]}; SimpCfg sc; From d0b3ebf32e86a865c965f6265385e69188f0a3e1 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sun, 28 Apr 2024 18:41:26 +0530 Subject: [PATCH 083/217] SimpCfg: Use & wrt destination of [] operation so that one can update the value-item's content, without needing to explicitly update/store the value-item back into map after the content has been updated. This should make these setting operations/helpers more efficient. --- common/simpcfg.hpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index b28839ccee8de..313341721b528 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -63,16 +63,14 @@ class SimpCfg { std::map> mapBools = {}; public: void set_string(const std::string &group, const std::string &key, const std::string &value) { - auto gm = mapStrings[group]; + auto &gm = mapStrings[group]; gm[key] = value; - mapStrings[group] = gm; LDBUG_LN("DBUG:SC:%s:%s:%s:%s", __func__, group.c_str(), key.c_str(), value.c_str()); } void set_bool(const std::string &group, const std::string &key, bool value) { - auto gm = mapBools[group]; + auto &gm = mapBools[group]; gm[key] = value; - mapBools[group] = gm; LDBUG_LN("DBUG:SC:%s:%s:%s:%d", __func__, group.c_str(), key.c_str(), value); } From fb9a7dc7febb2e4fce212073ecc16610a8dbd154 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sun, 28 Apr 2024 23:03:13 +0530 Subject: [PATCH 084/217] SimpCfg:Initial skeleton towards supporting int and floating point --- common/simpcfg.hpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index 313341721b528..abecf1352eff8 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -13,6 +13,8 @@ #include #include #include +#include + #define SC_DEBUG #define SC_TEST_PRG @@ -61,6 +63,10 @@ class SimpCfg { private: std::map> mapStrings = {}; std::map> mapBools = {}; + std::map> mapInts = {}; + std::map> mapDoubles = {}; + std::regex rInt {R"(^[-+]?\d+$)"}; + std::regex rFloat {R"(^[-+]?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?$)"}; public: void set_string(const std::string &group, const std::string &key, const std::string &value) { auto &gm = mapStrings[group]; @@ -153,6 +159,12 @@ class SimpCfg { std::string vtype = "bool"; if ((value == "true") || (value == "false")) { set_bool(group, key, value == "true" ? true : false); + } else if (std::regex_match(value, rInt)) { + vtype = "int"; + set_int(group, key, value); + } else if (std::regex_match(value, rFloat)) { + vtype = "float"; + set_float(group, key, value); } else { vtype = "string"; value = str_trim_single(value, "\""); From 41811642177c2159e9ce572218def5b1525e71c2 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sun, 28 Apr 2024 23:17:53 +0530 Subject: [PATCH 085/217] SimpCfg:Implement set_int64 and set_double Also update the sample simpcfg file, to test for int and float values. --- common/simpcfg.hpp | 28 +++++++++++++++++++++++++--- examples/chaton_meta.simpcfg | 7 +++++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index abecf1352eff8..587b9dbb03d63 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -63,7 +63,7 @@ class SimpCfg { private: std::map> mapStrings = {}; std::map> mapBools = {}; - std::map> mapInts = {}; + std::map> mapInt64s = {}; std::map> mapDoubles = {}; std::regex rInt {R"(^[-+]?\d+$)"}; std::regex rFloat {R"(^[-+]?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?$)"}; @@ -80,6 +80,28 @@ class SimpCfg { LDBUG_LN("DBUG:SC:%s:%s:%s:%d", __func__, group.c_str(), key.c_str(), value); } + void set_int64(const std::string &group, const std::string &key, int64_t value) { + auto &gm = mapInt64s[group]; + gm[key] = value; + LDBUG_LN("DBUG:SC:%s:%s:%s:%lld", __func__, group.c_str(), key.c_str(), value); + } + + void set_int64(const std::string &group, const std::string &key, std::string &value) { + auto ivalue = strtoll(value.c_str(), nullptr, 0); + set_int64(group, key, ivalue); + } + + void set_double(const std::string &group, const std::string &key, double value) { + auto &gm = mapDoubles[group]; + gm[key] = value; + LDBUG_LN("DBUG:SC:%s:%s:%s:%f", __func__, group.c_str(), key.c_str(), value); + } + + void set_double(const std::string &group, const std::string &key, std::string &value) { + auto dvalue = strtod(value.c_str(), nullptr); + set_double(group, key, dvalue); + } + std::string get_string(const std::string &group, const std::string &key, const std::string &defaultValue) { auto gm = mapStrings[group]; #ifdef SC_DEBUG @@ -161,10 +183,10 @@ class SimpCfg { set_bool(group, key, value == "true" ? true : false); } else if (std::regex_match(value, rInt)) { vtype = "int"; - set_int(group, key, value); + set_int64(group, key, value); } else if (std::regex_match(value, rFloat)) { vtype = "float"; - set_float(group, key, value); + set_double(group, key, value); } else { vtype = "string"; value = str_trim_single(value, "\""); diff --git a/examples/chaton_meta.simpcfg b/examples/chaton_meta.simpcfg index 2a209fd21c7df..1a3b2dcaac4eb 100644 --- a/examples/chaton_meta.simpcfg +++ b/examples/chaton_meta.simpcfg @@ -223,3 +223,10 @@ "systemuser-1st-user-has-begin": false, "systemuser-1st-user-has-prefix": false +"testme" + "int": 1234 + "sint": -9876543210 + "float": 12.34 + "double1": +12.0e-123 + "double2": -12.0e-123 + From a6648b02f2711e5d5fc069f58657b268ad31b828 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sun, 28 Apr 2024 23:44:33 +0530 Subject: [PATCH 086/217] SimpCfg:Show floating point values in normal and exponential form --- common/simpcfg.hpp | 2 +- examples/chaton_meta.simpcfg | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index 587b9dbb03d63..7346087bb782f 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -94,7 +94,7 @@ class SimpCfg { void set_double(const std::string &group, const std::string &key, double value) { auto &gm = mapDoubles[group]; gm[key] = value; - LDBUG_LN("DBUG:SC:%s:%s:%s:%f", __func__, group.c_str(), key.c_str(), value); + LDBUG_LN("DBUG:SC:%s:%s:%s:%f[%e]", __func__, group.c_str(), key.c_str(), value, value); } void set_double(const std::string &group, const std::string &key, std::string &value) { diff --git a/examples/chaton_meta.simpcfg b/examples/chaton_meta.simpcfg index 1a3b2dcaac4eb..c8c0cb49bbe87 100644 --- a/examples/chaton_meta.simpcfg +++ b/examples/chaton_meta.simpcfg @@ -228,5 +228,6 @@ "sint": -9876543210 "float": 12.34 "double1": +12.0e-123 - "double2": -12.0e-123 + "double2": -12.0e-123678 + "double3": -12.0e300 From 000245b8e8779e7adb27d462448a8a16a3881f1b Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Mon, 29 Apr 2024 00:00:11 +0530 Subject: [PATCH 087/217] SimpCfg:Warn possible nonstring strings, some invalid floats Warn if something not starting with double quote is being treated as a string. Show some examples of invalid floating point values wrt this logics floating point determination code --- common/simpcfg.hpp | 15 +++++++++------ examples/chaton_meta.simpcfg | 6 ++++++ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index 7346087bb782f..33623312ee84f 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -137,10 +137,10 @@ class SimpCfg { void load(const std::string &fname) { std::ifstream f {fname}; if (!f) { - LERRR_LN("ERRR:%s:%s:failed to load...", __func__, fname.c_str()); + LERRR_LN("ERRR:SC:%s:%s:failed to load...", __func__, fname.c_str()); throw std::runtime_error { "ERRR:SimpCfg:File not found" }; } else { - LDBUG_LN("DBUG:%s:%s", __func__, fname.c_str()); + LDBUG_LN("DBUG:SC:%s:%s", __func__, fname.c_str()); } std::string group; int iLine = 0; @@ -159,17 +159,17 @@ class SimpCfg { if (bGroup) { curL = str_trim_single(curL, "\""); group = curL; - LDBUG_LN("DBUG:%s:group:%s", __func__, group.c_str()); + LDBUG_LN("DBUG:SC:%s:group:%s", __func__, group.c_str()); continue; } auto dPos = curL.find(':'); if (dPos == std::string::npos) { - LERRR_LN("ERRR:%s:%d:invalid key value line:%s", __func__, iLine, curL.c_str()); + LERRR_LN("ERRR:SC:%s:%d:invalid key value line:%s", __func__, iLine, curL.c_str()); throw std::runtime_error { "ERRR:SimpCfg:Invalid key value line" }; } auto dEnd = curL.length() - dPos; if ((dPos == 0) || (dEnd < 2)) { - LERRR_LN("ERRR:%s:%d:invalid key value line:%s", __func__, iLine, curL.c_str()); + LERRR_LN("ERRR:SC:%s:%d:invalid key value line:%s", __func__, iLine, curL.c_str()); throw std::runtime_error { "ERRR:SimpCfg:Invalid key value line" }; } std::string key = curL.substr(0, dPos); @@ -189,10 +189,13 @@ class SimpCfg { set_double(group, key, value); } else { vtype = "string"; + if (!value.empty() && (value.front() != '"')) { + LWARN_LN("WARN:SC:%s:kv:is this string:%s", __func__, value.c_str()); + } value = str_trim_single(value, "\""); set_string(group, key, value); } - //LDBUG_LN("DBUG:%s:kv:%s:%s:%s:%s", __func__, group.c_str(), key.c_str(), vtype.c_str(), value.c_str()); + //LDBUG_LN("DBUG:SC:%s:kv:%s:%s:%s:%s", __func__, group.c_str(), key.c_str(), vtype.c_str(), value.c_str()); } } diff --git a/examples/chaton_meta.simpcfg b/examples/chaton_meta.simpcfg index c8c0cb49bbe87..283cc238e9a3d 100644 --- a/examples/chaton_meta.simpcfg +++ b/examples/chaton_meta.simpcfg @@ -226,8 +226,14 @@ "testme" "int": 1234 "sint": -9876543210 + "another int": 0xff "float": 12.34 "double1": +12.0e-123 "double2": -12.0e-123678 "double3": -12.0e300 + "isit double1": -12.e300 + "isit double2": -12. + "yes double2": -12.0 + "isit double3": .0123 + "yes double3": 0.0123 From 8ad2c17e5d755e0d61c64d50f2eef008187539a1 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Mon, 29 Apr 2024 00:10:56 +0530 Subject: [PATCH 088/217] SimpCfg: get_int64 logic --- common/simpcfg.hpp | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index 33623312ee84f..c734feedf8380 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -134,6 +134,22 @@ class SimpCfg { return value; } + int64_t get_int64(const std::string &group, const std::string &key, int64_t defaultValue) { + auto gm = mapInt64s[group]; +#ifdef SC_DEBUG + for(auto k: gm) { + LDBUG_LN("DBUG:SC:%s:Iterate:%s:%lld", __func__, k.first.c_str(), k.second); + } +#endif + if (gm.find(key) == gm.end()) { + LWARN_LN("DBUG:SC:%s:%s:%s:%lld[default]", __func__, group.c_str(), key.c_str(), defaultValue); + return defaultValue; + } + auto value = gm[key]; + LDBUG_LN("DBUG:SC:%s:%s:%s:%lld", __func__, group.c_str(), key.c_str(), value); + return value; + } + void load(const std::string &fname) { std::ifstream f {fname}; if (!f) { @@ -207,12 +223,19 @@ int main(int argc, char **argv) { std::string fname {argv[1]}; SimpCfg sc; sc.load(fname); + sc.get_bool("testme", "key101b", false); sc.get_string("testme", "key101s", "Not found"); + sc.get_int64("testme", "key101i", 123456); + sc.set_bool("testme", "key201b", true); sc.set_string("testme", "key201s", "hello world"); + sc.set_int64("testme", "key201i", 987654); + sc.get_bool("testme", "key201b", false); sc.get_string("testme", "key201s", "Not found"); + sc.get_int64("testme", "key201i", 123456); + sc.get_string("mistral", "system-prefix", "Not found"); sc.get_string("\"mistral\"", "\"system-prefix\"", "Not found"); return 0; From 44c05305d0bbcc900ed0768379746994cb6176d9 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Mon, 29 Apr 2024 00:20:17 +0530 Subject: [PATCH 089/217] SimpCfg: Add support for get_double --- common/simpcfg.hpp | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index c734feedf8380..883dbb0d5e01e 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -150,6 +150,22 @@ class SimpCfg { return value; } + double get_double(const std::string &group, const std::string &key, double defaultValue) { + auto gm = mapDoubles[group]; +#ifdef SC_DEBUG + for(auto k: gm) { + LDBUG_LN("DBUG:SC:%s:Iterate:%s:%f[%e]", __func__, k.first.c_str(), k.second, k.second); + } +#endif + if (gm.find(key) == gm.end()) { + LWARN_LN("DBUG:SC:%s:%s:%s:%f[%e][default]", __func__, group.c_str(), key.c_str(), defaultValue, defaultValue); + return defaultValue; + } + auto value = gm[key]; + LDBUG_LN("DBUG:SC:%s:%s:%s:%f[%e]", __func__, group.c_str(), key.c_str(), value, value); + return value; + } + void load(const std::string &fname) { std::ifstream f {fname}; if (!f) { @@ -227,14 +243,17 @@ int main(int argc, char **argv) { sc.get_bool("testme", "key101b", false); sc.get_string("testme", "key101s", "Not found"); sc.get_int64("testme", "key101i", 123456); + sc.get_double("testme", "key101d", 123456.789); sc.set_bool("testme", "key201b", true); sc.set_string("testme", "key201s", "hello world"); sc.set_int64("testme", "key201i", 987654); + sc.set_double("testme", "key201d", 9988.7766); sc.get_bool("testme", "key201b", false); sc.get_string("testme", "key201s", "Not found"); sc.get_int64("testme", "key201i", 123456); + sc.get_double("testme", "key201d", 123456.789); sc.get_string("mistral", "system-prefix", "Not found"); sc.get_string("\"mistral\"", "\"system-prefix\"", "Not found"); From a108000cfd9dc5e02d9e5ac279e6e2e3ccac1c14 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Mon, 29 Apr 2024 11:25:14 +0530 Subject: [PATCH 090/217] ChatON:Phi3: Add template details to detailed meta.json --- examples/chaton_meta.json | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/examples/chaton_meta.json b/examples/chaton_meta.json index 994301386e383..eda31fee62b16 100644 --- a/examples/chaton_meta.json +++ b/examples/chaton_meta.json @@ -262,6 +262,35 @@ "systemuser-system-has-end": true, "systemuser-1st-user-has-begin": false, "systemuser-1st-user-has-prefix": false + }, + "phi3": { + "global": { + "begin": "", + "end": "" + }, + "system": { + "begin": "", + "prefix": "<|system|>\n", + "suffix": "<|end|>\n", + "end": "" + }, + "user": { + "begin": "", + "prefix": "<|user|>\n", + "suffix": "<|end|>\n", + "end": "" + }, + "assistant": { + "begin": "", + "prefix": "<|assistant|>\n", + "suffix": "<|end|>\n", + "end": "" + }, + "reverse-prompt": "<|end|>", + "systemuser-system-has-suffix": true, + "systemuser-system-has-end": true, + "systemuser-1st-user-has-begin": true, + "systemuser-1st-user-has-prefix": true } } From a09571318a86856683fdaa118ac55fe720e12d3e Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Mon, 29 Apr 2024 12:02:03 +0530 Subject: [PATCH 091/217] ChatON: meta-dump returns flag inturn returned by meta-ok test-chat-template-chaton now tries to check if meta-ok is ok wrt the template-id being looked into. Log template-id info also, where it was previously missed out. --- common/chaton.hpp | 14 +++++++++----- tests/test-chat-template-chaton.cpp | 3 +++ 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index 1140ba409e8a6..c608845a3bb35 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -551,14 +551,18 @@ inline int32_t chaton_tmpl_apply_capi( * * empty string, then dump the full loaded chaton-meta * * chaton-template-id, then dump contents related to that specific chat-handshake-template-standard */ -inline void _chaton_meta_dump(std::string &tmpl) { +inline bool _chaton_meta_dump(std::string &tmpl) { json theJson; if (tmpl.empty()) { theJson = conMeta; } else { theJson = conMeta[tmpl]; + if (theJson.empty()) { + LOGXLN("ERRR:%s:Specified template-id [%s] not found", __func__, tmpl.c_str()); + return false; + } } - LOGXLN("\n\nINFO:%s:ChatOn Meta\n%s", __func__, theJson.dump(4).c_str()); + LOGXLN("\n\nINFO:%s:ChatOn Meta:%s:\n%s", __func__, tmpl.c_str(), theJson.dump(4).c_str()); if (!tmpl.empty()) { std::string globalBegin = conMeta[tmpl][K_GLOBAL][K_BEGIN]; std::string globalEnd = conMeta[tmpl][K_GLOBAL][K_END]; @@ -602,6 +606,7 @@ inline void _chaton_meta_dump(std::string &tmpl) { LOG_TEELN("WARN:%s:Assistant->Begin seems to be set to [%s], do cross check if this is proper and needed", __func__, assistantBegin.c_str()); } } + return true; } /** @@ -610,9 +615,8 @@ inline void _chaton_meta_dump(std::string &tmpl) { */ inline bool chaton_meta_ok(std::string &tmpl) { if (conMeta == nullptr) { - LOG_TEELN("ERRR:%s:ChatOn Meta: Not loaded yet...", __func__); + LOG_TEELN("ERRR:%s:%s:ChatOn Meta: Not loaded yet...", __func__, tmpl.c_str()); return false; } - _chaton_meta_dump(tmpl); - return true; + return _chaton_meta_dump(tmpl); } diff --git a/tests/test-chat-template-chaton.cpp b/tests/test-chat-template-chaton.cpp index b99c460fde40e..8cf60edf54da5 100644 --- a/tests/test-chat-template-chaton.cpp +++ b/tests/test-chat-template-chaton.cpp @@ -120,6 +120,9 @@ static void check_chaton(std::string &metaJson) { for(auto tmplId: templateIds) { formatted_chat.resize(1024); std::cout << "\n----------" << tmplId << "---------------\n"; + if (!chaton_meta_ok(tmplId)) { + exit(1); + } res = chaton_tmpl_apply_capi(tmplId.c_str(), conversation, message_count, true, formatted_chat.data(), formatted_chat.size()); assert(res > 0); formatted_chat.resize(res); From 7302b3ab36a60facf025e89ff342bf64bca3c83a Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Tue, 30 Apr 2024 11:58:29 +0530 Subject: [PATCH 092/217] SimpCfg: Use stderr wrt internal Log messaging helpers --- common/simpcfg.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index 883dbb0d5e01e..2b41b6dc7b743 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -19,9 +19,9 @@ #define SC_DEBUG #define SC_TEST_PRG #ifdef SC_TEST_PRG -#define LDBUG_LN(FMT, ...) printf(FMT"\n", __VA_ARGS__) -#define LERRR_LN(FMT, ...) printf(FMT"\n", __VA_ARGS__) -#define LWARN_LN(FMT, ...) printf(FMT"\n", __VA_ARGS__) +#define LDBUG_LN(FMT, ...) fprintf(stderr, FMT"\n", __VA_ARGS__) +#define LERRR_LN(FMT, ...) fprintf(stderr, FMT"\n", __VA_ARGS__) +#define LWARN_LN(FMT, ...) fprintf(stderr, FMT"\n", __VA_ARGS__) #else #include "log.h" #define LDBUG_LN LOGLN From ee1a62c876aebb215ed40f9bbd9ec0b1accd832d Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Tue, 30 Apr 2024 16:35:37 +0530 Subject: [PATCH 093/217] SimpCfg:WIP:Switch to C++ Variant type - initial skeleton --- common/simpcfg.hpp | 95 +++++++++++++++++----------------------------- 1 file changed, 35 insertions(+), 60 deletions(-) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index 2b41b6dc7b743..abe75c99a230c 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -14,6 +14,8 @@ #include #include #include +#include +#include #define SC_DEBUG @@ -60,30 +62,33 @@ std::string str_trim_single(std::string sin, std::string trimChars=" \t\n") { class SimpCfg { + private: - std::map> mapStrings = {}; - std::map> mapBools = {}; - std::map> mapInt64s = {}; - std::map> mapDoubles = {}; + std::map>> mapV = {}; std::regex rInt {R"(^[-+]?\d+$)"}; std::regex rFloat {R"(^[-+]?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?$)"}; + public: - void set_string(const std::string &group, const std::string &key, const std::string &value) { - auto &gm = mapStrings[group]; + + template + void set_value(const std::string &group, const std::string &key, const SupportedDataType &value) { + auto &gm = mapV[group]; gm[key] = value; - LDBUG_LN("DBUG:SC:%s:%s:%s:%s", __func__, group.c_str(), key.c_str(), value.c_str()); + std::stringstream ss; + ss << value; + LDBUG_LN("DBUG:SC:%s:%s:%s:%s", __func__, group.c_str(), key.c_str(), ss.str().c_str()); + } + + void set_string(const std::string &group, const std::string &key, const std::string &value) { + set_value(group, key, value); } void set_bool(const std::string &group, const std::string &key, bool value) { - auto &gm = mapBools[group]; - gm[key] = value; - LDBUG_LN("DBUG:SC:%s:%s:%s:%d", __func__, group.c_str(), key.c_str(), value); + set_value(group, key, value); } void set_int64(const std::string &group, const std::string &key, int64_t value) { - auto &gm = mapInt64s[group]; - gm[key] = value; - LDBUG_LN("DBUG:SC:%s:%s:%s:%lld", __func__, group.c_str(), key.c_str(), value); + set_value(group, key, value); } void set_int64(const std::string &group, const std::string &key, std::string &value) { @@ -92,9 +97,7 @@ class SimpCfg { } void set_double(const std::string &group, const std::string &key, double value) { - auto &gm = mapDoubles[group]; - gm[key] = value; - LDBUG_LN("DBUG:SC:%s:%s:%s:%f[%e]", __func__, group.c_str(), key.c_str(), value, value); + set_value(group, key, value); } void set_double(const std::string &group, const std::string &key, std::string &value) { @@ -102,68 +105,40 @@ class SimpCfg { set_double(group, key, dvalue); } - std::string get_string(const std::string &group, const std::string &key, const std::string &defaultValue) { - auto gm = mapStrings[group]; + template + SupportedDataType get_value(const std::string &group, const std::string &key, const SupportedDataType &defaultValue) { + auto gm = mapV[group]; #ifdef SC_DEBUG for(auto k: gm) { - LDBUG_LN("DBUG:SC:%s:Iterate:%s:%s", __func__, k.first.c_str(), k.second.c_str()); + std::stringstream ss << k.second; + LDBUG_LN("DBUG:SC:%s:Iterate:%s:%s", __func__, k.first.c_str(), ss.str().c_str()); } #endif if (gm.find(key) == gm.end()) { - LWARN_LN("DBUG:SC:%s:%s:%s:%s[default]", __func__, group.c_str(), key.c_str(), defaultValue.c_str()); + std::stringstream ss << defaultValue; + LWARN_LN("DBUG:SC:%s:%s:%s:%s[default]", __func__, group.c_str(), key.c_str(), ss.str().c_str()); return defaultValue; } auto value = gm[key]; - LDBUG_LN("DBUG:SC:%s:%s:%s:%s", __func__, group.c_str(), key.c_str(), value.c_str()); + std::stringstream ss << defaultValue; + LDBUG_LN("DBUG:SC:%s:%s:%s:%s", __func__, group.c_str(), key.c_str(), ss.str().c_str()); return value; } + std::string get_string(const std::string &group, const std::string &key, const std::string &defaultValue) { + return get_value(group, key, defaultValue); + } + bool get_bool(const std::string &group, const std::string &key, bool defaultValue) { - auto gm = mapBools[group]; -#ifdef SC_DEBUG - for(auto k: gm) { - LDBUG_LN("DBUG:SC:%s:Iterate:%s:%d", __func__, k.first.c_str(), k.second); - } -#endif - if (gm.find(key) == gm.end()) { - LWARN_LN("DBUG:SC:%s:%s:%s:%d[default]", __func__, group.c_str(), key.c_str(), defaultValue); - return defaultValue; - } - auto value = gm[key]; - LDBUG_LN("DBUG:SC:%s:%s:%s:%d", __func__, group.c_str(), key.c_str(), value); - return value; + return get_value(group, key, defaultValue); } int64_t get_int64(const std::string &group, const std::string &key, int64_t defaultValue) { - auto gm = mapInt64s[group]; -#ifdef SC_DEBUG - for(auto k: gm) { - LDBUG_LN("DBUG:SC:%s:Iterate:%s:%lld", __func__, k.first.c_str(), k.second); - } -#endif - if (gm.find(key) == gm.end()) { - LWARN_LN("DBUG:SC:%s:%s:%s:%lld[default]", __func__, group.c_str(), key.c_str(), defaultValue); - return defaultValue; - } - auto value = gm[key]; - LDBUG_LN("DBUG:SC:%s:%s:%s:%lld", __func__, group.c_str(), key.c_str(), value); - return value; + return get_value(group, key, defaultValue); } double get_double(const std::string &group, const std::string &key, double defaultValue) { - auto gm = mapDoubles[group]; -#ifdef SC_DEBUG - for(auto k: gm) { - LDBUG_LN("DBUG:SC:%s:Iterate:%s:%f[%e]", __func__, k.first.c_str(), k.second, k.second); - } -#endif - if (gm.find(key) == gm.end()) { - LWARN_LN("DBUG:SC:%s:%s:%s:%f[%e][default]", __func__, group.c_str(), key.c_str(), defaultValue, defaultValue); - return defaultValue; - } - auto value = gm[key]; - LDBUG_LN("DBUG:SC:%s:%s:%s:%f[%e]", __func__, group.c_str(), key.c_str(), value, value); - return value; + return get_value(group, key, defaultValue); } void load(const std::string &fname) { From 1dc7fd0e85ee8ee2a6a96125f5f52313f7a5c492 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Tue, 30 Apr 2024 17:03:52 +0530 Subject: [PATCH 094/217] SimpCfg:WIP:Variant TypeDef, to_str and std::get Cleanup the use of the variant Initialize and << op stringstream seperately. --- common/simpcfg.hpp | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index abe75c99a230c..07985ba86d2e3 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -61,15 +61,26 @@ std::string str_trim_single(std::string sin, std::string trimChars=" \t\n") { } +typedef std::variant SimpCfgData; + class SimpCfg { private: - std::map>> mapV = {}; + std::map> mapV = {}; std::regex rInt {R"(^[-+]?\d+$)"}; std::regex rFloat {R"(^[-+]?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?$)"}; public: + std::string to_str(const SimpCfgData &value) { + auto visitor = [](auto value) { + std::stringstream ss; + ss << value; + return ss.str(); + }; + return std::visit(visitor, value); + } + template void set_value(const std::string &group, const std::string &key, const SupportedDataType &value) { auto &gm = mapV[group]; @@ -110,19 +121,18 @@ class SimpCfg { auto gm = mapV[group]; #ifdef SC_DEBUG for(auto k: gm) { - std::stringstream ss << k.second; - LDBUG_LN("DBUG:SC:%s:Iterate:%s:%s", __func__, k.first.c_str(), ss.str().c_str()); + LDBUG_LN("DBUG:SC:%s:Iterate:%s:%s", __func__, k.first.c_str(), to_str(k.second).c_str()); } #endif if (gm.find(key) == gm.end()) { - std::stringstream ss << defaultValue; + std::stringstream ss; + ss << defaultValue; LWARN_LN("DBUG:SC:%s:%s:%s:%s[default]", __func__, group.c_str(), key.c_str(), ss.str().c_str()); return defaultValue; } auto value = gm[key]; - std::stringstream ss << defaultValue; - LDBUG_LN("DBUG:SC:%s:%s:%s:%s", __func__, group.c_str(), key.c_str(), ss.str().c_str()); - return value; + LDBUG_LN("DBUG:SC:%s:%s:%s:%s", __func__, group.c_str(), key.c_str(), to_str(value).c_str()); + return std::get(value); } std::string get_string(const std::string &group, const std::string &key, const std::string &defaultValue) { From f05f71bdc408162e3971b9c8e623f7dda3266303 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Tue, 30 Apr 2024 19:36:30 +0530 Subject: [PATCH 095/217] SimpCfg:SetBool string value, str_tolower, SetValueTypeLogging --- common/simpcfg.hpp | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index 07985ba86d2e3..b49ed090b0556 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -60,6 +60,12 @@ std::string str_trim_single(std::string sin, std::string trimChars=" \t\n") { return sin; } +std::string str_tolower(const std::string &sin) { + std::string sout; + std::transform(sin.begin(), sin.end(),sout.begin(), [](char c)->char {return std::tolower(c);}); + return sout; +} + typedef std::variant SimpCfgData; @@ -73,7 +79,7 @@ class SimpCfg { public: std::string to_str(const SimpCfgData &value) { - auto visitor = [](auto value) { + auto visitor = [](auto value) -> auto { std::stringstream ss; ss << value; return ss.str(); @@ -87,7 +93,7 @@ class SimpCfg { gm[key] = value; std::stringstream ss; ss << value; - LDBUG_LN("DBUG:SC:%s:%s:%s:%s", __func__, group.c_str(), key.c_str(), ss.str().c_str()); + LDBUG_LN("DBUG:SC:%s_%s:%s:%s:%s", __func__, typeid(value).name(), group.c_str(), key.c_str(), ss.str().c_str()); } void set_string(const std::string &group, const std::string &key, const std::string &value) { @@ -98,6 +104,10 @@ class SimpCfg { set_value(group, key, value); } + void set_bool(const std::string &group, const std::string &key, const std::string &value) { + set_bool(group, key, str_tolower(value) == "true" ? true : false); + } + void set_int64(const std::string &group, const std::string &key, int64_t value) { set_value(group, key, value); } @@ -197,7 +207,7 @@ class SimpCfg { value = str_trim(value, ","); std::string vtype = "bool"; if ((value == "true") || (value == "false")) { - set_bool(group, key, value == "true" ? true : false); + set_bool(group, key, value); } else if (std::regex_match(value, rInt)) { vtype = "int"; set_int64(group, key, value); From 6b475e444fe2dd1b586c662714538b8c68a805e7 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Tue, 30 Apr 2024 20:04:20 +0530 Subject: [PATCH 096/217] SimpCfg: Log Caller of Set/GetValue --- common/simpcfg.hpp | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index b49ed090b0556..c5c23d639b91f 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -88,20 +88,20 @@ class SimpCfg { } template - void set_value(const std::string &group, const std::string &key, const SupportedDataType &value) { + void set_value(const std::string &group, const std::string &key, const SupportedDataType &value, const std::string &callerName="") { auto &gm = mapV[group]; gm[key] = value; std::stringstream ss; ss << value; - LDBUG_LN("DBUG:SC:%s_%s:%s:%s:%s", __func__, typeid(value).name(), group.c_str(), key.c_str(), ss.str().c_str()); + LDBUG_LN("DBUG:SC:%s_%s:%s:%s:%s", __func__, callerName.c_str(), group.c_str(), key.c_str(), ss.str().c_str()); } void set_string(const std::string &group, const std::string &key, const std::string &value) { - set_value(group, key, value); + set_value(group, key, value, __func__); } void set_bool(const std::string &group, const std::string &key, bool value) { - set_value(group, key, value); + set_value(group, key, value, __func__); } void set_bool(const std::string &group, const std::string &key, const std::string &value) { @@ -109,7 +109,7 @@ class SimpCfg { } void set_int64(const std::string &group, const std::string &key, int64_t value) { - set_value(group, key, value); + set_value(group, key, value, __func__); } void set_int64(const std::string &group, const std::string &key, std::string &value) { @@ -118,7 +118,7 @@ class SimpCfg { } void set_double(const std::string &group, const std::string &key, double value) { - set_value(group, key, value); + set_value(group, key, value, __func__); } void set_double(const std::string &group, const std::string &key, std::string &value) { @@ -127,7 +127,7 @@ class SimpCfg { } template - SupportedDataType get_value(const std::string &group, const std::string &key, const SupportedDataType &defaultValue) { + SupportedDataType get_value(const std::string &group, const std::string &key, const SupportedDataType &defaultValue, const std::string &callerName="") { auto gm = mapV[group]; #ifdef SC_DEBUG for(auto k: gm) { @@ -137,7 +137,7 @@ class SimpCfg { if (gm.find(key) == gm.end()) { std::stringstream ss; ss << defaultValue; - LWARN_LN("DBUG:SC:%s:%s:%s:%s[default]", __func__, group.c_str(), key.c_str(), ss.str().c_str()); + LWARN_LN("DBUG:SC:%s_%s:%s:%s:%s[default]", __func__, callerName.c_str(), group.c_str(), key.c_str(), ss.str().c_str()); return defaultValue; } auto value = gm[key]; @@ -146,19 +146,19 @@ class SimpCfg { } std::string get_string(const std::string &group, const std::string &key, const std::string &defaultValue) { - return get_value(group, key, defaultValue); + return get_value(group, key, defaultValue, __func__); } bool get_bool(const std::string &group, const std::string &key, bool defaultValue) { - return get_value(group, key, defaultValue); + return get_value(group, key, defaultValue, __func__); } int64_t get_int64(const std::string &group, const std::string &key, int64_t defaultValue) { - return get_value(group, key, defaultValue); + return get_value(group, key, defaultValue, __func__); } double get_double(const std::string &group, const std::string &key, double defaultValue) { - return get_value(group, key, defaultValue); + return get_value(group, key, defaultValue, __func__); } void load(const std::string &fname) { From 5aa1072aac969e7c2d0fb951dac44c404f59d85a Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Tue, 30 Apr 2024 21:32:26 +0530 Subject: [PATCH 097/217] SimpCfg: Move dump into its own func, Avoid KV iter wrt Get --- common/simpcfg.hpp | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index c5c23d639b91f..8084150a1162c 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -21,11 +21,13 @@ #define SC_DEBUG #define SC_TEST_PRG #ifdef SC_TEST_PRG +#define LINFO_LN(FMT, ...) fprintf(stdout, FMT"\n", __VA_ARGS__) #define LDBUG_LN(FMT, ...) fprintf(stderr, FMT"\n", __VA_ARGS__) #define LERRR_LN(FMT, ...) fprintf(stderr, FMT"\n", __VA_ARGS__) #define LWARN_LN(FMT, ...) fprintf(stderr, FMT"\n", __VA_ARGS__) #else #include "log.h" +#define LINFO_LN LOG_TEELN #define LDBUG_LN LOGLN #define LERRR_LN LOG_TEELN #define LWARN_LN LOG_TEELN @@ -126,14 +128,21 @@ class SimpCfg { set_double(group, key, dvalue); } + void dump(const std::string &group) { + for (auto gm: mapV) { + if (!group.empty() && (gm.first != group)) { + LINFO_LN("INFO:SC:%s:%s:Skipping...", __func__, gm.first.c_str()); + continue; + } + for(auto k: gm.second) { + LINFO_LN("DBUG:SC:%s:%s:Iterate:%s:%s", __func__, gm.first.c_str(), k.first.c_str(), to_str(k.second).c_str()); + } + } + } + template SupportedDataType get_value(const std::string &group, const std::string &key, const SupportedDataType &defaultValue, const std::string &callerName="") { auto gm = mapV[group]; -#ifdef SC_DEBUG - for(auto k: gm) { - LDBUG_LN("DBUG:SC:%s:Iterate:%s:%s", __func__, k.first.c_str(), to_str(k.second).c_str()); - } -#endif if (gm.find(key) == gm.end()) { std::stringstream ss; ss << defaultValue; @@ -234,6 +243,7 @@ int main(int argc, char **argv) { std::string fname {argv[1]}; SimpCfg sc; sc.load(fname); + sc.dump(""); sc.get_bool("testme", "key101b", false); sc.get_string("testme", "key101s", "Not found"); @@ -245,6 +255,7 @@ int main(int argc, char **argv) { sc.set_int64("testme", "key201i", 987654); sc.set_double("testme", "key201d", 9988.7766); + sc.dump("testme"); sc.get_bool("testme", "key201b", false); sc.get_string("testme", "key201s", "Not found"); sc.get_int64("testme", "key201i", 123456); From ef5a2cf391dece8667c9704de318913a4cdc00b3 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Tue, 30 Apr 2024 22:36:22 +0530 Subject: [PATCH 098/217] SimpCfg:Dbug why bool is not setting properly --- common/simpcfg.hpp | 6 +++++- examples/chaton_meta.simpcfg | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index 8084150a1162c..abb7ec8b3469c 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -65,6 +65,7 @@ std::string str_trim_single(std::string sin, std::string trimChars=" \t\n") { std::string str_tolower(const std::string &sin) { std::string sout; std::transform(sin.begin(), sin.end(),sout.begin(), [](char c)->char {return std::tolower(c);}); + LDBUG_LN("DBUG:%s:%s:%s", __func__, sin.c_str(), sout.c_str()); return sout; } @@ -107,7 +108,10 @@ class SimpCfg { } void set_bool(const std::string &group, const std::string &key, const std::string &value) { - set_bool(group, key, str_tolower(value) == "true" ? true : false); + std::string sValue = str_tolower(value); + bool bValue = sValue == "true" ? true : false; + LDBUG_LN("DBUG:%s:%s:%s:%d", __func__, value.c_str(), sValue.c_str(), bValue); + set_bool(group, key, bValue); } void set_int64(const std::string &group, const std::string &key, int64_t value) { diff --git a/examples/chaton_meta.simpcfg b/examples/chaton_meta.simpcfg index 283cc238e9a3d..8bf12d8990384 100644 --- a/examples/chaton_meta.simpcfg +++ b/examples/chaton_meta.simpcfg @@ -18,8 +18,8 @@ "assistant-end": "" "reverse-prompt": "", - "systemuser-system-has-suffix": true, - "systemuser-system-has-end": false, + "systemuser-system-has-suffix": tRUe, + "systemuser-system-has-end": faLSe, "systemuser-1st-user-has-begin": false, "systemuser-1st-user-has-prefix": false From eb56517f20e97402c0319782df8bc56aa0aa53a5 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Tue, 30 Apr 2024 22:59:38 +0530 Subject: [PATCH 099/217] SimpCfg:Bools:Make lowercase b4 checking true/false for bool path TODO: string check wrt true/false, doesnt seem to be working after str_tolower was introduced. I seem to be doing some silly mistake not able to make out, moving in and out of sleep, need to check tomorrow. string == string-literal failed string == string-view failed string.compare(string-literal) failed Bit strange --- common/simpcfg.hpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index abb7ec8b3469c..fda3ceef5bd34 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -65,7 +65,7 @@ std::string str_trim_single(std::string sin, std::string trimChars=" \t\n") { std::string str_tolower(const std::string &sin) { std::string sout; std::transform(sin.begin(), sin.end(),sout.begin(), [](char c)->char {return std::tolower(c);}); - LDBUG_LN("DBUG:%s:%s:%s", __func__, sin.c_str(), sout.c_str()); + //LDBUG_LN("DBUG:%s:%s:%s", __func__, sin.c_str(), sout.c_str()); return sout; } @@ -219,7 +219,9 @@ class SimpCfg { value = str_trim(value); value = str_trim(value, ","); std::string vtype = "bool"; - if ((value == "true") || (value == "false")) { + auto valueLower = str_tolower(value); + LDBUG_LN("DBUG:%s:BoolCheck:%s:%s", __func__, value.c_str(), valueLower.c_str()); + if ((valueLower.compare("true") == 0) || (valueLower.compare("false") == 0)) { set_bool(group, key, value); } else if (std::regex_match(value, rInt)) { vtype = "int"; From 0e0d7da18f2f20a5c44bdc52e6a56caa45c6115b Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Wed, 1 May 2024 00:41:22 +0530 Subject: [PATCH 100/217] SimpCfg:Found issue with str_tolower, transform doesnt resize dst --- common/simpcfg.hpp | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index fda3ceef5bd34..b3b53db569a83 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -64,11 +64,20 @@ std::string str_trim_single(std::string sin, std::string trimChars=" \t\n") { std::string str_tolower(const std::string &sin) { std::string sout; - std::transform(sin.begin(), sin.end(),sout.begin(), [](char c)->char {return std::tolower(c);}); + sout.resize(sin.size()); + std::transform(sin.begin(), sin.end(), sout.begin(), [](char c)->char {return std::tolower(c);}); //LDBUG_LN("DBUG:%s:%s:%s", __func__, sin.c_str(), sout.c_str()); return sout; } +void str_compare_dump(const std::string &s1, const std::string &s2) { + LDBUG_LN("DBUG:%s:%s:Len:%zu", __func__, s1.c_str(), s1.length()); + LDBUG_LN("DBUG:%s:%s:Len:%zu", __func__, s2.c_str(), s2.length()); + int minLen = s1.length() < s2.length() ? s1.length() : s2.length(); + for(int i=0; i SimpCfgData; @@ -221,6 +230,8 @@ class SimpCfg { std::string vtype = "bool"; auto valueLower = str_tolower(value); LDBUG_LN("DBUG:%s:BoolCheck:%s:%s", __func__, value.c_str(), valueLower.c_str()); + str_compare_dump(value, valueLower); + str_compare_dump(valueLower, "true"); if ((valueLower.compare("true") == 0) || (valueLower.compare("false") == 0)) { set_bool(group, key, value); } else if (std::regex_match(value, rInt)) { From 08b9711d68bb7d324d7e8c321ae726a86830d19a Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Wed, 1 May 2024 00:50:52 +0530 Subject: [PATCH 101/217] SimpCfg:Remove dbug logs wrt str_tolower and set_bool Also the warning wrt is it string, now also logs the line number, group and key, to help user identify the line better. Misc: pass time last week Another life, Anchakkallakokkan, Deadloch --- common/simpcfg.hpp | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index b3b53db569a83..673e62f55d89e 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -119,7 +119,7 @@ class SimpCfg { void set_bool(const std::string &group, const std::string &key, const std::string &value) { std::string sValue = str_tolower(value); bool bValue = sValue == "true" ? true : false; - LDBUG_LN("DBUG:%s:%s:%s:%d", __func__, value.c_str(), sValue.c_str(), bValue); + //LDBUG_LN("DBUG:%s:%s:%s:%d", __func__, value.c_str(), sValue.c_str(), bValue); set_bool(group, key, bValue); } @@ -229,9 +229,6 @@ class SimpCfg { value = str_trim(value, ","); std::string vtype = "bool"; auto valueLower = str_tolower(value); - LDBUG_LN("DBUG:%s:BoolCheck:%s:%s", __func__, value.c_str(), valueLower.c_str()); - str_compare_dump(value, valueLower); - str_compare_dump(valueLower, "true"); if ((valueLower.compare("true") == 0) || (valueLower.compare("false") == 0)) { set_bool(group, key, value); } else if (std::regex_match(value, rInt)) { @@ -243,12 +240,12 @@ class SimpCfg { } else { vtype = "string"; if (!value.empty() && (value.front() != '"')) { - LWARN_LN("WARN:SC:%s:kv:is this string:%s", __func__, value.c_str()); + LWARN_LN("WARN:SC:%s:%d:%s:k:%s:v:%s:is this string?", __func__, iLine, group.c_str(), key.c_str(), value.c_str()); } value = str_trim_single(value, "\""); set_string(group, key, value); } - //LDBUG_LN("DBUG:SC:%s:kv:%s:%s:%s:%s", __func__, group.c_str(), key.c_str(), vtype.c_str(), value.c_str()); + //LDBUG_LN("DBUG:SC:%s:%d:kv:%s:%s:%s:%s", __func__, iLine, group.c_str(), key.c_str(), vtype.c_str(), value.c_str()); } } From 8fdc80533f59989514d9fba001f46f01cf23ec26 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Wed, 1 May 2024 16:40:24 +0530 Subject: [PATCH 102/217] SimpCfg:Cleanup:Cmdline Arg, GetValueCallerLogging, StringCmp Bring the direct string comparison also back, as the issue with cmp was more to do with string.transform. --- common/simpcfg.hpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index 673e62f55d89e..4461dd1bae033 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -163,7 +163,7 @@ class SimpCfg { return defaultValue; } auto value = gm[key]; - LDBUG_LN("DBUG:SC:%s:%s:%s:%s", __func__, group.c_str(), key.c_str(), to_str(value).c_str()); + LDBUG_LN("DBUG:SC:%s_%s:%s:%s:%s", __func__, callerName.c_str(), group.c_str(), key.c_str(), to_str(value).c_str()); return std::get(value); } @@ -229,7 +229,7 @@ class SimpCfg { value = str_trim(value, ","); std::string vtype = "bool"; auto valueLower = str_tolower(value); - if ((valueLower.compare("true") == 0) || (valueLower.compare("false") == 0)) { + if ((valueLower.compare("true") == 0) || (valueLower == "false")) { set_bool(group, key, value); } else if (std::regex_match(value, rInt)) { vtype = "int"; @@ -254,6 +254,10 @@ class SimpCfg { #ifdef SC_TEST_PRG int main(int argc, char **argv) { + if (argc != 2) { + LERRR_LN("USAGE:%s simp.cfg", argv[0]); + exit(1); + } std::string fname {argv[1]}; SimpCfg sc; sc.load(fname); From 561f50930ec76aa04bba0d46ba881e009ad9d224 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Wed, 1 May 2024 17:00:05 +0530 Subject: [PATCH 103/217] SimpCfg: initial go at adding support for spreadout arrays By having a seperate entry for each element of the array, the existing logic itself can be repurposed with minimal additions. --- common/simpcfg.hpp | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index 4461dd1bae033..9028bf598b153 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -183,6 +183,32 @@ class SimpCfg { return get_value(group, key, defaultValue, __func__); } + + template + std::vector get_array(const std::string &group, const std::string &key, const std::vector &defaultValue, const std::string &callerName="") { + auto gm = mapV[group]; + std::vector array; + int i = 0; + while(true) { + std::stringstream ssArrayKey; + ssArrayKey << key << "-" << i; + auto arrayKey = ssArrayKey.str(); + if (gm.find(arrayKey) == gm.end()) { + break; + } + array.push_back(std::get(gm[arrayKey])); + } + if (array.empty()) { + std::stringstream ss; + ss << defaultValue; + LWARN_LN("DBUG:SC:%s_%s:%s:%s:%s[default]", __func__, callerName.c_str(), group.c_str(), key.c_str(), ss.str().c_str()); + return defaultValue; + } + LDBUG_LN("DBUG:SC:%s_%s:%s:%s:%s", __func__, callerName.c_str(), group.c_str(), key.c_str(), to_str(value).c_str()); + return array; + } + + void load(const std::string &fname) { std::ifstream f {fname}; if (!f) { @@ -281,6 +307,11 @@ int main(int argc, char **argv) { sc.get_string("mistral", "system-prefix", "Not found"); sc.get_string("\"mistral\"", "\"system-prefix\"", "Not found"); + + sc.set_int64("testme", "keyA300-0", 330); + sc.set_int64("testme", "keyA300-1", 331); + sc.set_int64("testme", "keyA300-2", 332); + sc.get_array("testme", "keyA300", {1, 2, 3}); return 0; } #endif From 1e1f54ec1d8b49959ca79875b2e327115d15fd00 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Wed, 1 May 2024 17:43:25 +0530 Subject: [PATCH 104/217] SimpCfg:GetArray flesh out, helpers to convert to string Good: Forgotten love, All the light we cant see, laapataa ladies --- common/simpcfg.hpp | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index 9028bf598b153..a0a80c29121e4 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -79,6 +79,29 @@ void str_compare_dump(const std::string &s1, const std::string &s2) { } } + +template +std::string str(TypeWithStrSupp value) { + std::stringstream ss; + ss << value; + return ss.str(); +} + +template +std::string str(std::vector values) { + std::stringstream ss; + ss << "[ "; + int cnt = 0; + for(auto value: values) { + cnt += 1; + if (cnt != 1) ss << ", "; + ss << value; + } + ss << " ]"; + return ss.str(); +} + + typedef std::variant SimpCfgData; class SimpCfg { @@ -197,14 +220,13 @@ class SimpCfg { break; } array.push_back(std::get(gm[arrayKey])); + i += 1; } if (array.empty()) { - std::stringstream ss; - ss << defaultValue; - LWARN_LN("DBUG:SC:%s_%s:%s:%s:%s[default]", __func__, callerName.c_str(), group.c_str(), key.c_str(), ss.str().c_str()); + LWARN_LN("DBUG:SC:%s_%s:%s:%s:%s[default]", __func__, callerName.c_str(), group.c_str(), key.c_str(), str(defaultValue).c_str()); return defaultValue; } - LDBUG_LN("DBUG:SC:%s_%s:%s:%s:%s", __func__, callerName.c_str(), group.c_str(), key.c_str(), to_str(value).c_str()); + LDBUG_LN("DBUG:SC:%s_%s:%s:%s:%s", __func__, callerName.c_str(), group.c_str(), key.c_str(), str(array).c_str()); return array; } @@ -308,10 +330,11 @@ int main(int argc, char **argv) { sc.get_string("mistral", "system-prefix", "Not found"); sc.get_string("\"mistral\"", "\"system-prefix\"", "Not found"); + sc.get_array("testme", "keyA100", {1, 2, 3}); sc.set_int64("testme", "keyA300-0", 330); sc.set_int64("testme", "keyA300-1", 331); sc.set_int64("testme", "keyA300-2", 332); - sc.get_array("testme", "keyA300", {1, 2, 3}); + sc.get_array("testme", "keyA300", {1, 2, 3}); return 0; } #endif From 86e776c857695253fb26d78d2d13fabf95a2a309 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Wed, 1 May 2024 18:27:39 +0530 Subject: [PATCH 105/217] SimpCfg: Rename to get_vector, add some test code --- common/simpcfg.hpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index a0a80c29121e4..3c07a777800e8 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -208,7 +208,7 @@ class SimpCfg { template - std::vector get_array(const std::string &group, const std::string &key, const std::vector &defaultValue, const std::string &callerName="") { + std::vector get_vector(const std::string &group, const std::string &key, const std::vector &defaultValue, const std::string &callerName="") { auto gm = mapV[group]; std::vector array; int i = 0; @@ -330,11 +330,16 @@ int main(int argc, char **argv) { sc.get_string("mistral", "system-prefix", "Not found"); sc.get_string("\"mistral\"", "\"system-prefix\"", "Not found"); - sc.get_array("testme", "keyA100", {1, 2, 3}); + sc.get_vector("testme", "keyA100", {1, 2, 3}); + sc.get_vector("testme", "keyA100", { "A", "അ", "अ", "ಅ" }); sc.set_int64("testme", "keyA300-0", 330); sc.set_int64("testme", "keyA300-1", 331); sc.set_int64("testme", "keyA300-2", 332); - sc.get_array("testme", "keyA300", {1, 2, 3}); + sc.set_string("testme", "keyA301-0", "India"); + sc.set_value("testme", "keyA301-1", "World"); + sc.set_string("testme", "keyA301-2", "AkashaGanga"); + sc.get_vector("testme", "keyA300", {1, 2, 3}); + sc.get_vector("testme", "keyA301", { "yes 1", "No 2", "very well 3" }); return 0; } #endif From 56f19c7a681ca83f1f8a4f2b41db8adbe2064e96 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Thu, 2 May 2024 06:11:40 +0530 Subject: [PATCH 106/217] SimpCfg: Test c++ string handling --- common/simpcfg.hpp | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index 3c07a777800e8..cc99e66210ea0 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -16,6 +16,8 @@ #include #include #include +#include +#include #define SC_DEBUG @@ -301,11 +303,35 @@ class SimpCfg { #ifdef SC_TEST_PRG + +#include + +void check_strings() { + + std::string sT11 = "123"; + std::string sT12 = "1अ3"; + std::cout << std::format("string: [{}] len[{}] size[{}]", sT11, sT11.length(), sT11.size()) << std::endl; + std::cout << std::format("string: [{}] len[{}] size[{}]", sT12, sT12.length(), sT12.size()) << std::endl; + + std::u8string sT21 = u8"123"; + std::u8string sT22 = u8"1अ3"; + std::wstring sT31 = L"1अ3"; + std::wstring sT32 (sT22.begin(), sT22.end()); + std::string sT21x (sT21.begin(), sT21.end()); + std::string sT22x (sT22.begin(), sT22.end()); + std::cout << std::format("u8string: [{}] len[{}] size[{}]", sT21x, sT21.length(), sT21.size()) << std::endl; + std::cout << std::format("u8string: [{}] len[{}] size[{}]", sT22x, sT22.length(), sT22.size()) << std::endl; + +} + int main(int argc, char **argv) { if (argc != 2) { LERRR_LN("USAGE:%s simp.cfg", argv[0]); exit(1); } + + check_strings(); + std::string fname {argv[1]}; SimpCfg sc; sc.load(fname); From 713520caac131a54d4ef7cc5037b51ecc718612e Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Thu, 2 May 2024 06:23:31 +0530 Subject: [PATCH 107/217] SimpCfg:CheckStrings: Organise and Probe - p1 std::string --- common/simpcfg.hpp | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index cc99e66210ea0..b3fdb7d7566f5 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -308,10 +308,15 @@ class SimpCfg { void check_strings() { - std::string sT11 = "123"; - std::string sT12 = "1अ3"; - std::cout << std::format("string: [{}] len[{}] size[{}]", sT11, sT11.length(), sT11.size()) << std::endl; - std::cout << std::format("string: [{}] len[{}] size[{}]", sT12, sT12.length(), sT12.size()) << std::endl; + std::vector vStandard = { "123", "1अ3" }; + for(auto sCur: vStandard) { + std::cout << std::format("string: [{}] len[{}] size[{}]", sCur, sCur.length(), sCur.size()) << std::endl; + int i = 0; + for(auto c: sCur) { + std::cout << std::format("string:{}:pos:{}:char:{}[{:x}]\n", sCur, i, c, (uint8_t)c); + i += 1; + } + } std::u8string sT21 = u8"123"; std::u8string sT22 = u8"1अ3"; From 691d0d43b5478f91566daa0fb6e4c3fd629c9b2d Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Thu, 2 May 2024 07:22:21 +0530 Subject: [PATCH 108/217] SimpCfg:CheckStrings: Organise and Probe - P2 - std::u8string --- common/simpcfg.hpp | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index b3fdb7d7566f5..2c6e54eadf770 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -313,19 +313,24 @@ void check_strings() { std::cout << std::format("string: [{}] len[{}] size[{}]", sCur, sCur.length(), sCur.size()) << std::endl; int i = 0; for(auto c: sCur) { - std::cout << std::format("string:{}:pos:{}:char:{}[{:x}]\n", sCur, i, c, (uint8_t)c); + std::cout << std::format("string:{}:pos:{}:char:{}[0x{:x}]\n", sCur, i, c, (uint8_t)c); + i += 1; + } + } + + std::vector vU8s = { u8"123", u8"1अ3" }; + for(auto sCur: vU8s) { + std::string sCurx (sCur.begin(), sCur.end()); + std::cout << std::format("u8string: [{}] len[{}] size[{}]", sCurx, sCur.length(), sCur.size()) << std::endl; + int i = 0; + for(auto c: sCur) { + //std::cout << c << std::endl; + std::cout << std::format("u8string:{}:pos:{}:char:{}[0x{:x}]\n", sCurx, i, (unsigned char)c, (unsigned char)c); i += 1; } } - std::u8string sT21 = u8"123"; - std::u8string sT22 = u8"1अ3"; std::wstring sT31 = L"1अ3"; - std::wstring sT32 (sT22.begin(), sT22.end()); - std::string sT21x (sT21.begin(), sT21.end()); - std::string sT22x (sT22.begin(), sT22.end()); - std::cout << std::format("u8string: [{}] len[{}] size[{}]", sT21x, sT21.length(), sT21.size()) << std::endl; - std::cout << std::format("u8string: [{}] len[{}] size[{}]", sT22x, sT22.length(), sT22.size()) << std::endl; } From a448fec486fde0c0d4fdde5d6a7ce70990626467 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Thu, 2 May 2024 07:38:37 +0530 Subject: [PATCH 109/217] SimpCfg:CheckString: organise and probe - p3 wstring wcouts' involving 2nd wstring with non english char in it not showing up? --- common/simpcfg.hpp | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index 2c6e54eadf770..f6e5b33176573 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -309,6 +309,7 @@ class SimpCfg { void check_strings() { std::vector vStandard = { "123", "1अ3" }; + std::cout << "**** string **** " << vStandard.size() << std::endl; for(auto sCur: vStandard) { std::cout << std::format("string: [{}] len[{}] size[{}]", sCur, sCur.length(), sCur.size()) << std::endl; int i = 0; @@ -319,6 +320,7 @@ void check_strings() { } std::vector vU8s = { u8"123", u8"1अ3" }; + std::cout << "**** u8string **** " << vU8s.size() << std::endl; for(auto sCur: vU8s) { std::string sCurx (sCur.begin(), sCur.end()); std::cout << std::format("u8string: [{}] len[{}] size[{}]", sCurx, sCur.length(), sCur.size()) << std::endl; @@ -330,7 +332,16 @@ void check_strings() { } } - std::wstring sT31 = L"1अ3"; + std::vector vWide = { L"123", L"1अ3" }; + std::cout << "**** wstring **** " << vWide.size() << std::endl; + for(auto sCur: vWide) { + std::wcout << std::format(L"wstring: [{}] len[{}] size[{}]", sCur, sCur.length(), sCur.size()) << std::endl; + int i = 0; + for(auto c: sCur) { + std::wcout << std::format(L"wstring:{}:pos:{}:char:{}[0x{:x}]\n", sCur, i, c, (uint8_t)c); + i += 1; + } + } } From 3ad5cec47eb6322da416a715f7ea960ca82b126e Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Thu, 2 May 2024 07:59:48 +0530 Subject: [PATCH 110/217] SimpCfg:CheckStrings:MacOS, wstring and wcout Without using imbue, I couldnt get non-english wstrings to print on mac. Need to check on linux also. Also avoid the uint8_t typecasting, given that wchar isnt 8bit --- common/simpcfg.hpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index f6e5b33176573..429fb02a67a12 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -332,13 +332,15 @@ void check_strings() { } } + std::wcout.imbue(std::locale("en_US.UTF-8")); std::vector vWide = { L"123", L"1अ3" }; std::cout << "**** wstring **** " << vWide.size() << std::endl; for(auto sCur: vWide) { + std::wcout << sCur << std::endl; std::wcout << std::format(L"wstring: [{}] len[{}] size[{}]", sCur, sCur.length(), sCur.size()) << std::endl; int i = 0; for(auto c: sCur) { - std::wcout << std::format(L"wstring:{}:pos:{}:char:{}[0x{:x}]\n", sCur, i, c, (uint8_t)c); + std::wcout << std::format(L"wstring:{}:pos:{}:char:{}[0x{:x}]\n", sCur, i, c, c); i += 1; } } From 66d6fa62b7931673f97f4b00d8d201481e0d25e2 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Thu, 2 May 2024 11:33:13 +0530 Subject: [PATCH 111/217] SimpCfg: C++ and strings is a mess even after decades Seperate out the checks wrt different string types. Add a wstring_basic, which verifies that wstring iterator handles non english chars propery or atleast better. --- common/simpcfg.hpp | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index 429fb02a67a12..e82d58b5bfdf2 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -306,8 +306,7 @@ class SimpCfg { #include -void check_strings() { - +void check_string() { std::vector vStandard = { "123", "1अ3" }; std::cout << "**** string **** " << vStandard.size() << std::endl; for(auto sCur: vStandard) { @@ -318,7 +317,9 @@ void check_strings() { i += 1; } } +} +void check_u8string() { std::vector vU8s = { u8"123", u8"1अ3" }; std::cout << "**** u8string **** " << vU8s.size() << std::endl; for(auto sCur: vU8s) { @@ -331,7 +332,9 @@ void check_strings() { i += 1; } } +} +void check_wstring() { std::wcout.imbue(std::locale("en_US.UTF-8")); std::vector vWide = { L"123", L"1अ3" }; std::cout << "**** wstring **** " << vWide.size() << std::endl; @@ -344,7 +347,27 @@ void check_strings() { i += 1; } } +} + +void check_wstring_basic() { + std::vector vWide = { L"123", L"1अ3" }; + std::cout << "**** wstring basic **** " << vWide.size() << std::endl; + for(auto sCur: vWide) { + std::string sCurx (sCur.begin(), sCur.end()); + std::cout << std::format("wstring: [{}] len[{}] size[{}]", sCurx, sCur.length(), sCur.size()) << std::endl; + int i = 0; + for(auto c: sCur) { + std::cout << std::format("wstring:{}:pos:{}:char:{}[0x{:x}]\n", sCurx, i, c, c); + i += 1; + } + } +} +void check_strings() { + check_string(); + check_u8string(); + //check_wstring(); + check_wstring_basic(); } int main(int argc, char **argv) { From 1a618a42f841d4be3d8c87bde83852ba6e03f431 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Thu, 2 May 2024 18:48:29 +0530 Subject: [PATCH 112/217] SimpCfg: Update the func notes with alert --- common/simpcfg.hpp | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index e82d58b5bfdf2..21ef372af64f2 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -36,6 +36,13 @@ #endif +// Remove chars from begin and end of the passed string, provided the char belongs +// to one of the chars in trimChars. +// NOTE: Chars being trimmed (ie trimChars) needs to be 1byte encoded chars. +// NOTE: This will work provided the string being trimmed as well the chars being +// trimmed are made up of 1byte encoded chars including in utf8 encoding space. +// If the string being trimmed includes multibyte encoded characters at the end, +// then trimming can mess things up. std::string str_trim(std::string sin, std::string trimChars=" \t\n") { sin.erase(sin.find_last_not_of(trimChars)+1); sin.erase(0, sin.find_first_not_of(trimChars)); @@ -44,8 +51,11 @@ std::string str_trim(std::string sin, std::string trimChars=" \t\n") { // Remove atmost 1 char at the begin and 1 char at the end of the passed string, // provided the char belongs to one of the chars in trimChars. -// NOTE: Not sure this handles non english utf8 multibyte chars properly, -// need to cross check. +// NOTE: Chars being trimmed (ie trimChars) needs to be 1byte encoded chars. +// NOTE: This will work provided the string being trimmed as well the chars being +// trimmed are made up of 1byte encoded chars including in utf8 encoding space. +// If the string being trimmed includes multibyte encoded characters at the end, +// then trimming can mess things up. std::string str_trim_single(std::string sin, std::string trimChars=" \t\n") { if (sin.empty()) return sin; for(auto c: trimChars) { @@ -64,6 +74,8 @@ std::string str_trim_single(std::string sin, std::string trimChars=" \t\n") { return sin; } +// This works for 1byte encoded chars, including in utf8 encoding space. +// This wont work for multibyte encoded chars. std::string str_tolower(const std::string &sin) { std::string sout; sout.resize(sin.size()); From 7607dbc8c7ff8b94ac3e54902b58eecb1989a26a Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Thu, 2 May 2024 19:20:32 +0530 Subject: [PATCH 113/217] SimpCfg:CheckStrings: Try fixup wstring handling --- common/simpcfg.hpp | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index 21ef372af64f2..fe9561317c34c 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -36,6 +36,18 @@ #endif +size_t wcs_to_mbs(std::string &sDest, const std::wstring &wSrc) { + auto reqLen = std::wcstombs(nullptr, wSrc.c_str(), 0); + sDest.resize(reqLen); + return std::wcstombs(sDest.data(), wSrc.c_str(), sDest.length()); +} + +size_t mbs_to_wcs(std::wstring &wDest, std::string &sSrc) { + auto reqLen = std::mbstowcs(nullptr, sSrc.c_str(), 0); + wDest.resize(reqLen); + return std::mbstowcs(wDest.data(), sSrc.c_str(), wDest.length()); +} + // Remove chars from begin and end of the passed string, provided the char belongs // to one of the chars in trimChars. // NOTE: Chars being trimmed (ie trimChars) needs to be 1byte encoded chars. @@ -366,10 +378,21 @@ void check_wstring_basic() { std::cout << "**** wstring basic **** " << vWide.size() << std::endl; for(auto sCur: vWide) { std::string sCurx (sCur.begin(), sCur.end()); + std::string sCury; + wcs_to_mbs(sCury, sCur); std::cout << std::format("wstring: [{}] len[{}] size[{}]", sCurx, sCur.length(), sCur.size()) << std::endl; + std::cout << std::format("wstring: [{}] len[{}] size[{}]", sCury, sCur.length(), sCur.size()) << std::endl; int i = 0; for(auto c: sCur) { - std::cout << std::format("wstring:{}:pos:{}:char:{}[0x{:x}]\n", sCurx, i, c, c); + std::wstringstream wsc; + wsc << c; + /* + std::string u8 (ws.str().begin(), ws.str().end()); + std::cout << std::format("wstring:{}:pos:{}:char:{}[0x{:x}]\n", sCurx, i, u8, (uint32_t)c); + */ + std::string ssc; + wcs_to_mbs(ssc, wsc.str()); + std::cout << std::format("wstring:{}:pos:{}:char:{}[0x{:x}]\n", sCurx, i, ssc, (uint32_t)c); i += 1; } } From 2cda78f1ad3664d2e0e5fec7daab8423b2e62b40 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Thu, 2 May 2024 21:45:08 +0530 Subject: [PATCH 114/217] SimpCfg:CheckStrings: WString2String finally The constructor method doesnt convert wstring to string, when it involves non-english chars which will encode to multibyte chars in utf8. even thou it does work for the already utf8 u8string. wcstombs doesnt seem to work for non english chars, when the locale is set to the default c, need to change to something like en_US.UTF-8, to allow it to do the conversion properly. --- common/simpcfg.hpp | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index fe9561317c34c..b0a48d8439da3 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -37,7 +37,10 @@ size_t wcs_to_mbs(std::string &sDest, const std::wstring &wSrc) { - auto reqLen = std::wcstombs(nullptr, wSrc.c_str(), 0); + std::mbstate_t mbState = std::mbstate_t(); + const wchar_t *wSrcP = wSrc.c_str(); + auto reqLen = std::wcsrtombs(nullptr, &wSrcP, 0, &mbState); + LDBUG_LN("DBUG:%s:%zu", __func__, reqLen); sDest.resize(reqLen); return std::wcstombs(sDest.data(), wSrc.c_str(), sDest.length()); } @@ -378,9 +381,9 @@ void check_wstring_basic() { std::cout << "**** wstring basic **** " << vWide.size() << std::endl; for(auto sCur: vWide) { std::string sCurx (sCur.begin(), sCur.end()); + std::cout << std::format("wstring: [{}] len[{}] size[{}]", sCurx, sCur.length(), sCur.size()) << std::endl; std::string sCury; wcs_to_mbs(sCury, sCur); - std::cout << std::format("wstring: [{}] len[{}] size[{}]", sCurx, sCur.length(), sCur.size()) << std::endl; std::cout << std::format("wstring: [{}] len[{}] size[{}]", sCury, sCur.length(), sCur.size()) << std::endl; int i = 0; for(auto c: sCur) { @@ -399,6 +402,10 @@ void check_wstring_basic() { } void check_strings() { + std::string prevLoc = std::setlocale(LC_ALL, nullptr); + LDBUG_LN("DBUG:%s:Locale:%s", __func__, prevLoc.c_str()); + prevLoc = std::setlocale(LC_ALL, "en_US.UTF-8"); + LDBUG_LN("DBUG:%s:Locale:%s", __func__, prevLoc.c_str()); check_string(); check_u8string(); //check_wstring(); From 23acf07bb2959820d46afab926c8b6ff2b9f088f Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Thu, 2 May 2024 22:07:51 +0530 Subject: [PATCH 115/217] SimpCfg:CheckStrings: Cleanup wstring flow to needed parts --- common/simpcfg.hpp | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index b0a48d8439da3..37c72f63acb5d 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -40,9 +40,8 @@ size_t wcs_to_mbs(std::string &sDest, const std::wstring &wSrc) { std::mbstate_t mbState = std::mbstate_t(); const wchar_t *wSrcP = wSrc.c_str(); auto reqLen = std::wcsrtombs(nullptr, &wSrcP, 0, &mbState); - LDBUG_LN("DBUG:%s:%zu", __func__, reqLen); sDest.resize(reqLen); - return std::wcstombs(sDest.data(), wSrc.c_str(), sDest.length()); + return std::wcsrtombs(sDest.data(), &wSrcP, sDest.length(), &mbState); } size_t mbs_to_wcs(std::wstring &wDest, std::string &sSrc) { @@ -361,10 +360,10 @@ void check_u8string() { } } -void check_wstring() { +void check_wstring_wcout() { std::wcout.imbue(std::locale("en_US.UTF-8")); std::vector vWide = { L"123", L"1अ3" }; - std::cout << "**** wstring **** " << vWide.size() << std::endl; + std::cout << "**** wstring wcout **** " << vWide.size() << std::endl; for(auto sCur: vWide) { std::wcout << sCur << std::endl; std::wcout << std::format(L"wstring: [{}] len[{}] size[{}]", sCur, sCur.length(), sCur.size()) << std::endl; @@ -376,12 +375,10 @@ void check_wstring() { } } -void check_wstring_basic() { +void check_wstring_cout() { std::vector vWide = { L"123", L"1अ3" }; - std::cout << "**** wstring basic **** " << vWide.size() << std::endl; + std::cout << "**** wstring cout **** " << vWide.size() << std::endl; for(auto sCur: vWide) { - std::string sCurx (sCur.begin(), sCur.end()); - std::cout << std::format("wstring: [{}] len[{}] size[{}]", sCurx, sCur.length(), sCur.size()) << std::endl; std::string sCury; wcs_to_mbs(sCury, sCur); std::cout << std::format("wstring: [{}] len[{}] size[{}]", sCury, sCur.length(), sCur.size()) << std::endl; @@ -389,13 +386,9 @@ void check_wstring_basic() { for(auto c: sCur) { std::wstringstream wsc; wsc << c; - /* - std::string u8 (ws.str().begin(), ws.str().end()); - std::cout << std::format("wstring:{}:pos:{}:char:{}[0x{:x}]\n", sCurx, i, u8, (uint32_t)c); - */ std::string ssc; wcs_to_mbs(ssc, wsc.str()); - std::cout << std::format("wstring:{}:pos:{}:char:{}[0x{:x}]\n", sCurx, i, ssc, (uint32_t)c); + std::cout << std::format("wstring:{}:pos:{}:char:{}[0x{:x}]\n", sCury, i, ssc, (uint32_t)c); i += 1; } } @@ -408,8 +401,8 @@ void check_strings() { LDBUG_LN("DBUG:%s:Locale:%s", __func__, prevLoc.c_str()); check_string(); check_u8string(); - //check_wstring(); - check_wstring_basic(); + //check_wstring_wcout(); + check_wstring_cout(); } int main(int argc, char **argv) { From 2325764180ab02ac0d99b730e9534efd689bee38 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Thu, 2 May 2024 22:41:09 +0530 Subject: [PATCH 116/217] SimpCfg:CheckStrings: Switch Mbs2Wcs to multithread safe calls --- common/simpcfg.hpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index 37c72f63acb5d..fd54cf0a4c2e3 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -45,9 +45,11 @@ size_t wcs_to_mbs(std::string &sDest, const std::wstring &wSrc) { } size_t mbs_to_wcs(std::wstring &wDest, std::string &sSrc) { - auto reqLen = std::mbstowcs(nullptr, sSrc.c_str(), 0); + std::mbstate_t mbState = std::mbstate_t(); + const char *sSrcP = sSrc.c_str(); + auto reqLen = std::mbsrtowcs(nullptr, &sSrcP, 0, &mbState); wDest.resize(reqLen); - return std::mbstowcs(wDest.data(), sSrc.c_str(), wDest.length()); + return std::mbsrtowcs(wDest.data(), &sSrcP, wDest.length(), &mbState); } // Remove chars from begin and end of the passed string, provided the char belongs From d1156cc05588a36854e14f28e225744870acecb1 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Fri, 3 May 2024 07:37:20 +0530 Subject: [PATCH 117/217] SimpCfg: As locale manipulation reqd for better processing --- common/simpcfg.hpp | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index fd54cf0a4c2e3..ac1ea689751f3 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -260,6 +260,16 @@ class SimpCfg { return array; } + static void locale_prepare(std::string &sSavedLocale) { + sSavedLocale = std::setlocale(LC_ALL, nullptr); + auto sUpdatedLocale = std::setlocale(LC_ALL, "en_US.UTF-8"); + LDBUG_LN("DBUG:%s:Locale:Prev:%s:Cur:%s", __func__, sSavedLocale.c_str(), sUpdatedLocale); + } + + static void locale_restore(const std::string &sSavedLocale) { + auto sCurLocale = std::setlocale(LC_ALL, sSavedLocale.c_str()); + LDBUG_LN("DBUG:%s:Locale:Requested:%s:Got:%s", __func__, sSavedLocale.c_str(), sCurLocale); + } void load(const std::string &fname) { std::ifstream f {fname}; @@ -397,14 +407,13 @@ void check_wstring_cout() { } void check_strings() { - std::string prevLoc = std::setlocale(LC_ALL, nullptr); - LDBUG_LN("DBUG:%s:Locale:%s", __func__, prevLoc.c_str()); - prevLoc = std::setlocale(LC_ALL, "en_US.UTF-8"); - LDBUG_LN("DBUG:%s:Locale:%s", __func__, prevLoc.c_str()); + std::string sSavedLocale; + SimpCfg::locale_prepare(sSavedLocale); check_string(); check_u8string(); //check_wstring_wcout(); check_wstring_cout(); + SimpCfg::locale_restore(sSavedLocale); } int main(int argc, char **argv) { From cae0fff715ddcb4a0037a0a18919055a3818f037 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Fri, 3 May 2024 08:10:25 +0530 Subject: [PATCH 118/217] SimpCfg: Update notes; Try add a better trimming logic --- common/simpcfg.hpp | 45 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index ac1ea689751f3..ade8fd2077787 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -1,12 +1,27 @@ #pragma once /** - * Provide a simple config file logic + * Provides a simple direct 1-level only config file logic + * + * ## File format + * * It can consist of multiple config groups. * * the group name needs to start at the begining of the line. - * Each group can inturn contain multiple config fields wrt that group. + * Each group can inturn contain multiple config fields (key:value pairs) wrt that group. * * the group fields need to have 1 or more space at the begining of line. * + * ## Supported data types + * + * The fields can have values belonging to ane one of the below types + * * strings - enclosed in double quotes + * this is also the fallback catch all type, but dont rely on this behaviour. + * * int - using decimal number system + * * float - needs to have a decimal point and or e/E + * if decimal point is used, there should be atleast one decimal number on its either side + * * bool - either true or false + * + * It tries to provide a crude expanded form of array wrt any of the above supported types. + * For this one needs to define keys using the pattern TheKeyName-0, TheKeyName-1, .... */ #include @@ -35,6 +50,13 @@ #define LWARN_LN LOG_TEELN #endif +#undef SC_STR_OVERSMART +#ifdef SC_STR_OVERSMART +#define str_trim str_trim_oversmart +#else +#define str_trim str_trim_dumb +#endif + size_t wcs_to_mbs(std::string &sDest, const std::wstring &wSrc) { std::mbstate_t mbState = std::mbstate_t(); @@ -59,12 +81,29 @@ size_t mbs_to_wcs(std::wstring &wDest, std::string &sSrc) { // trimmed are made up of 1byte encoded chars including in utf8 encoding space. // If the string being trimmed includes multibyte encoded characters at the end, // then trimming can mess things up. -std::string str_trim(std::string sin, std::string trimChars=" \t\n") { +std::string str_trim_dumb(std::string sin, std::string trimChars=" \t\n") { sin.erase(sin.find_last_not_of(trimChars)+1); sin.erase(0, sin.find_first_not_of(trimChars)); return sin; } +// Remove chars from begin and end of the passed string, provided the char belongs +// to one of the chars in trimChars. +// NOTE: Internally converts to wchar/wstring to try and support proper trimming, +// wrt possibly more languages, to some extent, ie even if the passed string +// contains multibyte encoded characters in it. +std::string str_trim_oversmart(std::string sIn, std::string trimChars=" \t\n") { + std::wstring wIn; + mbs_to_wcs(wIn, sIn); + std::wstring wTrimChars; + mbs_to_wcs(wTrimChars, trimChars); + wIn.erase(wIn.find_last_not_of(wTrimChars)+1); + wIn.erase(0, wIn.find_first_not_of(wTrimChars)); + std::string sOut; + wcs_to_mbs(sOut, wIn); + return sOut; +} + // Remove atmost 1 char at the begin and 1 char at the end of the passed string, // provided the char belongs to one of the chars in trimChars. // NOTE: Chars being trimmed (ie trimChars) needs to be 1byte encoded chars. From 554b00f0277bfe297f18bfd2290789427004e19c Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Fri, 3 May 2024 08:38:30 +0530 Subject: [PATCH 119/217] SimpCfg: Add some missing const refs --- common/simpcfg.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index ade8fd2077787..4358483a30239 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -66,7 +66,7 @@ size_t wcs_to_mbs(std::string &sDest, const std::wstring &wSrc) { return std::wcsrtombs(sDest.data(), &wSrcP, sDest.length(), &mbState); } -size_t mbs_to_wcs(std::wstring &wDest, std::string &sSrc) { +size_t mbs_to_wcs(std::wstring &wDest, const std::string &sSrc) { std::mbstate_t mbState = std::mbstate_t(); const char *sSrcP = sSrc.c_str(); auto reqLen = std::mbsrtowcs(nullptr, &sSrcP, 0, &mbState); @@ -81,7 +81,7 @@ size_t mbs_to_wcs(std::wstring &wDest, std::string &sSrc) { // trimmed are made up of 1byte encoded chars including in utf8 encoding space. // If the string being trimmed includes multibyte encoded characters at the end, // then trimming can mess things up. -std::string str_trim_dumb(std::string sin, std::string trimChars=" \t\n") { +std::string str_trim_dumb(std::string sin, const std::string &trimChars=" \t\n") { sin.erase(sin.find_last_not_of(trimChars)+1); sin.erase(0, sin.find_first_not_of(trimChars)); return sin; @@ -92,7 +92,7 @@ std::string str_trim_dumb(std::string sin, std::string trimChars=" \t\n") { // NOTE: Internally converts to wchar/wstring to try and support proper trimming, // wrt possibly more languages, to some extent, ie even if the passed string // contains multibyte encoded characters in it. -std::string str_trim_oversmart(std::string sIn, std::string trimChars=" \t\n") { +std::string str_trim_oversmart(std::string sIn, const std::string &trimChars=" \t\n") { std::wstring wIn; mbs_to_wcs(wIn, sIn); std::wstring wTrimChars; From bf111a83f112d0d336b6ad1275889cca8da54123 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Fri, 3 May 2024 11:11:07 +0530 Subject: [PATCH 120/217] SimpCfg:TemplatedDumbTrim; Test dumb and oversmart trim logics --- common/simpcfg.hpp | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index 4358483a30239..538c2f9b97dd2 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -33,6 +33,7 @@ #include #include #include +#include #define SC_DEBUG @@ -74,6 +75,14 @@ size_t mbs_to_wcs(std::wstring &wDest, const std::string &sSrc) { return std::mbsrtowcs(wDest.data(), &sSrcP, wDest.length(), &mbState); } +void dumphex_string(const std::string &sIn, const std::string &msgTag){ + std::cout << msgTag << "[ "; + for(auto c: sIn) { + std::cout << std::format("{:02x}, ", (uint8_t)c); + } + std::cout << " ]" << std::endl; +} + // Remove chars from begin and end of the passed string, provided the char belongs // to one of the chars in trimChars. // NOTE: Chars being trimmed (ie trimChars) needs to be 1byte encoded chars. @@ -81,7 +90,10 @@ size_t mbs_to_wcs(std::wstring &wDest, const std::string &sSrc) { // trimmed are made up of 1byte encoded chars including in utf8 encoding space. // If the string being trimmed includes multibyte encoded characters at the end, // then trimming can mess things up. -std::string str_trim_dumb(std::string sin, const std::string &trimChars=" \t\n") { +template +TString str_trim_dumb(TString sin, const TString &trimChars=" \t\n") { + dumphex_string(sin, "DBUG:TrimDumb:Str:"); + dumphex_string(trimChars, "DBUG:TrimDumb:Tim:"); sin.erase(sin.find_last_not_of(trimChars)+1); sin.erase(0, sin.find_first_not_of(trimChars)); return sin; @@ -353,7 +365,7 @@ class SimpCfg { key = str_trim_single(key, "\""); std::string value = curL.substr(dPos+1); value = str_trim(value); - value = str_trim(value, ","); + value = str_trim(value, {","}); std::string vtype = "bool"; auto valueLower = str_tolower(value); if ((valueLower.compare("true") == 0) || (valueLower == "false")) { @@ -381,8 +393,6 @@ class SimpCfg { #ifdef SC_TEST_PRG -#include - void check_string() { std::vector vStandard = { "123", "1अ3" }; std::cout << "**** string **** " << vStandard.size() << std::endl; @@ -445,6 +455,21 @@ void check_wstring_cout() { } } +void check_nonenglish() { + std::vector vTest1 = { "\n\tAഅअಅ\n\t", "\n\tAഅअಅ " }; + for (auto sTest: vTest1) { + std::string sGotDumb = str_trim_dumb(sTest, {" \n\t"}); + std::string sGotOSmart = str_trim_oversmart(sTest, {" \n\t"}); + std::cout << std::format("{}: Test1[{}] Dumb[{}] OverSmart[{}]", __func__, sTest, sGotDumb, sGotOSmart) << std::endl; + } + std::vector vTest2 = { "\n\t this र remove 0s at end 000 ", "\n\tthis र remove 0s and अs at end 000रअ0अ "}; + for (auto sTest: vTest2) { + std::string sGotDumb = str_trim_dumb(sTest, {" \n\t0अ"}); + std::string sGotOSmart = str_trim_oversmart(sTest, {" \n\t0अ"}); + std::cout << std::format("{}: Test2[{}] Dumb[{}] OverSmart[{}]", __func__, sTest, sGotDumb, sGotOSmart) << std::endl; + } +} + void check_strings() { std::string sSavedLocale; SimpCfg::locale_prepare(sSavedLocale); @@ -452,6 +477,7 @@ void check_strings() { check_u8string(); //check_wstring_wcout(); check_wstring_cout(); + check_nonenglish(); SimpCfg::locale_restore(sSavedLocale); } From 97ac443bba55311a1cf1d174686b2c8c4515155e Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Fri, 3 May 2024 23:31:24 +0530 Subject: [PATCH 121/217] SimpCfg:Cleanup, updated notes, templated code Update the notes to match the templated flow now and some of the nitty gritties involved. Update DumpHexString to be templated. Split check nonenglish flow wrt trim dumb and oversmart testing, so that things with work with one, but not the other can be differentiated in the flow. --- common/simpcfg.hpp | 74 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 59 insertions(+), 15 deletions(-) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index 538c2f9b97dd2..686c8de51d385 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -75,25 +75,59 @@ size_t mbs_to_wcs(std::wstring &wDest, const std::string &sSrc) { return std::mbsrtowcs(wDest.data(), &sSrcP, wDest.length(), &mbState); } -void dumphex_string(const std::string &sIn, const std::string &msgTag){ +template +void dumphex_string(const TString &sIn, const std::string &msgTag){ std::cout << msgTag << "[ "; for(auto c: sIn) { - std::cout << std::format("{:02x}, ", (uint8_t)c); + auto cSize = sizeof(c); + if (cSize == 1) { + std::cout << std::format("{:02x}, ", (uint8_t)c); + } else if (cSize == 2) { + std::cout << std::format("{:04x}, ", (uint16_t)c); + } else if (cSize == 4) { + std::cout << std::format("{:08x}, ", (uint32_t)c); + } else { + throw std::runtime_error( std::format("ERRR:{}:Unsupported char type with size [{}]", __func__, cSize) ); + } } std::cout << " ]" << std::endl; } -// Remove chars from begin and end of the passed string, provided the char belongs -// to one of the chars in trimChars. -// NOTE: Chars being trimmed (ie trimChars) needs to be 1byte encoded chars. -// NOTE: This will work provided the string being trimmed as well the chars being -// trimmed are made up of 1byte encoded chars including in utf8 encoding space. -// If the string being trimmed includes multibyte encoded characters at the end, -// then trimming can mess things up. +// Remove chars from begin and end of the passed string, provided the char +// belongs to one of the chars in trimChars. +// +// NOTE: This will work perfectly provided the string being trimmed as well as +// chars being trimmed are made up of FixedSize chars from the same encoded space. +// For utf-8, this means the ascii equivalent 1byteSized chars of utf8 and not +// variable length ones. +// NOTE: It will also work, if atleast either end of string have fixedSize chars +// from their encoding space, rather than variable length based chars if any. +// And the trimChars are also fixedSize encoded chars. +// +// NOTE: Given the way UTF-8 char encoding is designed, where fixedSize 1byte +// encoded chars are fully unique and dont overlap with any bytes from any of +// the variable length encoded chars in the utf-8 space, so as long as the +// trimChars belong to the fixedSize chars subset, the logic should work, even +// if the string has a mixture of fixed and variable length encoded chars. +// Chances are utf-16 and utf-32 also have similar characteristics wrt thier +// fixedSize encoded chars, and so equivalent semantic applies to them also. +// +// ALERT: Given that this simple minded logic, works at individual bytes level +// only, If trimChars involve variable length encoded chars, then +// * because different bytes from different trim chars when clubbed together +// can map to some other new char, if there is that new char at either end +// of the string, it may get trimmed, because of the possibility of mix up +// mentioned. +// * given that different variable length encoded chars may have some common +// bytes between them, if one of these chars is at either end of the string +// and another char is in trimChars, then string may get partially trimmed. +// template TString str_trim_dumb(TString sin, const TString &trimChars=" \t\n") { +#ifdef SC_DEBUG dumphex_string(sin, "DBUG:TrimDumb:Str:"); dumphex_string(trimChars, "DBUG:TrimDumb:Tim:"); +#endif sin.erase(sin.find_last_not_of(trimChars)+1); sin.erase(0, sin.find_first_not_of(trimChars)); return sin; @@ -102,8 +136,9 @@ TString str_trim_dumb(TString sin, const TString &trimChars=" \t\n") { // Remove chars from begin and end of the passed string, provided the char belongs // to one of the chars in trimChars. // NOTE: Internally converts to wchar/wstring to try and support proper trimming, -// wrt possibly more languages, to some extent, ie even if the passed string -// contains multibyte encoded characters in it. +// wrt possibly more languages, to some extent. IE even if the passed string +// contains multibyte encoded characters in it in utf-8 space, it may get converted +// to fixed size chars in the expanded wchar_t encoding space. std::string str_trim_oversmart(std::string sIn, const std::string &trimChars=" \t\n") { std::wstring wIn; mbs_to_wcs(wIn, sIn); @@ -118,11 +153,13 @@ std::string str_trim_oversmart(std::string sIn, const std::string &trimChars=" \ // Remove atmost 1 char at the begin and 1 char at the end of the passed string, // provided the char belongs to one of the chars in trimChars. -// NOTE: Chars being trimmed (ie trimChars) needs to be 1byte encoded chars. +// NOTE: Chars being trimmed (ie in trimChars) needs to be 1byte encoded chars, to +// avoid mix up when working utf-8/variable length encoded strings. // NOTE: This will work provided the string being trimmed as well the chars being // trimmed are made up of 1byte encoded chars including in utf8 encoding space. // If the string being trimmed includes multibyte encoded characters at the end, -// then trimming can mess things up. +// then trimming can mess things up, if you have multibyte encoded utf-8 chars +// in the trimChars set. std::string str_trim_single(std::string sin, std::string trimChars=" \t\n") { if (sin.empty()) return sin; for(auto c: trimChars) { @@ -462,11 +499,18 @@ void check_nonenglish() { std::string sGotOSmart = str_trim_oversmart(sTest, {" \n\t"}); std::cout << std::format("{}: Test1[{}] Dumb[{}] OverSmart[{}]", __func__, sTest, sGotDumb, sGotOSmart) << std::endl; } - std::vector vTest2 = { "\n\t this र remove 0s at end 000 ", "\n\tthis र remove 0s and अs at end 000रअ0अ "}; + std::vector vTest2 = { "\n\t this र remove 0s at end 000 ", "\n\tthis र remove 0s and अs at end 000रअ0अ ", "\n\tthis र remove 0s and अs at end 000रअ0\xa4अ "}; for (auto sTest: vTest2) { std::string sGotDumb = str_trim_dumb(sTest, {" \n\t0अ"}); + std::cout << std::format("{}: Test2[{}] Dumb[{}]", __func__, sTest, sGotDumb) << std::endl; + } + // This partly invalid utf8 string will mess up str_trim_dumb "\n\tthis र remove 0s and अs at end 000रअ0\xa4अ " + // but will trigger a exception with oversmart. + // std::vector vTest3 = { "\n\t this र remove 0s at end 000 ", "\n\tthis र remove 0s and अs at end 000रअ0अ ", "\n\tthis र remove 0s and अs at end 000रअ0\xa4अ "}; + std::vector vTest3 = { "\n\t this र remove 0s at end 000 ", "\n\tthis र remove 0s and अs at end 000रअ0अ ", "\n\tthis र remove 0s and अs at end 000रअ0\xe0\xa4\x30अ "}; // \xe0\xa4 + for (auto sTest: vTest3) { std::string sGotOSmart = str_trim_oversmart(sTest, {" \n\t0अ"}); - std::cout << std::format("{}: Test2[{}] Dumb[{}] OverSmart[{}]", __func__, sTest, sGotDumb, sGotOSmart) << std::endl; + std::cout << std::format("{}: Test3[{}] OverSmart[{}]", __func__, sTest, sGotOSmart) << std::endl; } } From d030a26f3cfad21bee2d1d606f5d5510ac78ef6a Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sat, 4 May 2024 00:51:52 +0530 Subject: [PATCH 122/217] SimpCfg:Update TrimOverSmart use templated TrimDumb after wstrconv --- common/simpcfg.hpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index 686c8de51d385..02404d6344c20 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -144,10 +144,9 @@ std::string str_trim_oversmart(std::string sIn, const std::string &trimChars=" \ mbs_to_wcs(wIn, sIn); std::wstring wTrimChars; mbs_to_wcs(wTrimChars, trimChars); - wIn.erase(wIn.find_last_not_of(wTrimChars)+1); - wIn.erase(0, wIn.find_first_not_of(wTrimChars)); + auto wOut = str_trim_dumb(wIn, wTrimChars); std::string sOut; - wcs_to_mbs(sOut, wIn); + wcs_to_mbs(sOut, wOut); return sOut; } From 5b8bf849c05a7d28c41a659d2980d71468a84798 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sat, 4 May 2024 10:24:28 +0530 Subject: [PATCH 123/217] SimpCfg: Fixed & ~Variable Length to Native & MultiNativeCharSize So as to make the notes, more generic. --- common/simpcfg.hpp | 52 +++++++++++++++++++++++++--------------------- 1 file changed, 28 insertions(+), 24 deletions(-) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index 02404d6344c20..831e463966436 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -97,30 +97,32 @@ void dumphex_string(const TString &sIn, const std::string &msgTag){ // belongs to one of the chars in trimChars. // // NOTE: This will work perfectly provided the string being trimmed as well as -// chars being trimmed are made up of FixedSize chars from the same encoded space. +// chars being trimmed are made up of NativeCharSize chars from same encoded space. // For utf-8, this means the ascii equivalent 1byteSized chars of utf8 and not -// variable length ones. -// NOTE: It will also work, if atleast either end of string have fixedSize chars -// from their encoding space, rather than variable length based chars if any. -// And the trimChars are also fixedSize encoded chars. +// variable length MultiNativeCharSize (ie multibye in case of utf-8) ones. +// NOTE: It will also work, if atleast either end of string as well as trimChars +// have NativeCharSize chars from their encoding space, rather than variable +// length MultiNativeCharSize based chars if any. // -// NOTE: Given the way UTF-8 char encoding is designed, where fixedSize 1byte -// encoded chars are fully unique and dont overlap with any bytes from any of -// the variable length encoded chars in the utf-8 space, so as long as the -// trimChars belong to the fixedSize chars subset, the logic should work, even -// if the string has a mixture of fixed and variable length encoded chars. +// NOTE: Given the way UTF-8 char encoding is designed, where NativeCharSize 1byte +// encoded chars are fully unique and dont overlap with any bytes from any of the +// variable length MultiNativeCharSize encoded chars in the utf-8 space, so as long as +// the trimChars belong to NativeCharSize chars subset, the logic should work, even +// if string has a mixture of NativeCharSize and MultiNativeCharSize encoded chars. // Chances are utf-16 and utf-32 also have similar characteristics wrt thier -// fixedSize encoded chars, and so equivalent semantic applies to them also. +// NativeCharSize encoded chars (ie fully encoded within single 16bit and 32bit value +// respectively), and so equivalent semantic applies to them also. // -// ALERT: Given that this simple minded logic, works at individual bytes level -// only, If trimChars involve variable length encoded chars, then -// * because different bytes from different trim chars when clubbed together -// can map to some other new char, if there is that new char at either end -// of the string, it may get trimmed, because of the possibility of mix up -// mentioned. -// * given that different variable length encoded chars may have some common -// bytes between them, if one of these chars is at either end of the string -// and another char is in trimChars, then string may get partially trimmed. +// ALERT: Given that this simple minded logic, works at individual NativeCharSize level +// only, If trimChars involve variable length MultiNativeCharSize encoded chars, then +// * because different NativeCharSize subparts (bytes in case of utf-8) from different +// MultiNativeCharSize trim chars when clubbed together can map to some other new char +// in a variable length encoded char space, if there is that new char at either end +// of the string, it may get trimmed, because of the possibility of mix up mentioned. +// * given that different variable length MultiNativeCharSize encoded chars may have +// some common NativeCharSize subparts (bytes in case of utf-8) between them, if one +// of these chars is at either end of the string and another char is in trimChars, +// then string may get partially trimmed. // template TString str_trim_dumb(TString sin, const TString &trimChars=" \t\n") { @@ -137,8 +139,10 @@ TString str_trim_dumb(TString sin, const TString &trimChars=" \t\n") { // to one of the chars in trimChars. // NOTE: Internally converts to wchar/wstring to try and support proper trimming, // wrt possibly more languages, to some extent. IE even if the passed string -// contains multibyte encoded characters in it in utf-8 space, it may get converted -// to fixed size chars in the expanded wchar_t encoding space. +// contains multibyte encoded characters in it in utf-8 space (ie MultiNativeCharSize), +// it may get converted to NativeCharSize chars in the expanded wchar_t encoding space, +// thus leading to fixed NativeCharSize driven logic itself handling things sufficiently. +// Look at str_trim_dumb comments for additional aspects. std::string str_trim_oversmart(std::string sIn, const std::string &trimChars=" \t\n") { std::wstring wIn; mbs_to_wcs(wIn, sIn); @@ -152,8 +156,8 @@ std::string str_trim_oversmart(std::string sIn, const std::string &trimChars=" \ // Remove atmost 1 char at the begin and 1 char at the end of the passed string, // provided the char belongs to one of the chars in trimChars. -// NOTE: Chars being trimmed (ie in trimChars) needs to be 1byte encoded chars, to -// avoid mix up when working utf-8/variable length encoded strings. +// NOTE: Chars being trimmed (ie in trimChars) needs to be FixedSize encoded chars, +// to avoid mix up when working with strings which can utf-8/variable length encoded strings. // NOTE: This will work provided the string being trimmed as well the chars being // trimmed are made up of 1byte encoded chars including in utf8 encoding space. // If the string being trimmed includes multibyte encoded characters at the end, From 32ba195a8395e347d003fccc63320bd0b33cf684 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sat, 4 May 2024 10:48:28 +0530 Subject: [PATCH 124/217] SimpCfg: Templatize str_trim_single Also use NativeCharSize and MultiNativeCharSize wording to make the note more generic --- common/simpcfg.hpp | 39 +++++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index 831e463966436..08140f77b9cec 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -59,6 +59,9 @@ #endif +// **** **** **** String related helpers **** **** **** // + + size_t wcs_to_mbs(std::string &sDest, const std::wstring &wSrc) { std::mbstate_t mbState = std::mbstate_t(); const wchar_t *wSrcP = wSrc.c_str(); @@ -156,18 +159,27 @@ std::string str_trim_oversmart(std::string sIn, const std::string &trimChars=" \ // Remove atmost 1 char at the begin and 1 char at the end of the passed string, // provided the char belongs to one of the chars in trimChars. -// NOTE: Chars being trimmed (ie in trimChars) needs to be FixedSize encoded chars, -// to avoid mix up when working with strings which can utf-8/variable length encoded strings. -// NOTE: This will work provided the string being trimmed as well the chars being -// trimmed are made up of 1byte encoded chars including in utf8 encoding space. -// If the string being trimmed includes multibyte encoded characters at the end, -// then trimming can mess things up, if you have multibyte encoded utf-8 chars -// in the trimChars set. -std::string str_trim_single(std::string sin, std::string trimChars=" \t\n") { +// +// NOTE: Chars being trimmed (ie in trimChars) needs to be part of NativeCharSize +// subset of the string's encoded char space, to avoid mix up when working with +// strings which can be utf-8/utf-16/utf-32/sane-variable-length encoded strings. +// NOTE:UTF8: This will work provided the string being trimmed as well the chars +// being trimmed are made up of 1byte encoded chars in case of utf8 encoding space. +// If the string being trimmed includes multibyte (ie MultiNativeCharSize) encoded +// characters at the end, then trimming can mess things up, if you have multibyte +// encoded utf-8 chars in the trimChars set. +// +// Currently given that SimpCfg only uses this with NativeCharSize chars in the +// trimChars and most of the platforms are likely to be using utf-8 based char +// space (which is a realtively sane variable length char encoding from this +// logics perspective), so not providing oversmart variant. +// +template +TString str_trim_single(TString sin, const TString& trimChars=" \t\n") { if (sin.empty()) return sin; for(auto c: trimChars) { if (c == sin.front()) { - sin = sin.substr(1, std::string::npos); + sin = sin.substr(1, TString::npos); break; } } @@ -223,6 +235,9 @@ std::string str(std::vector values) { } +// **** **** **** SimpCfg related helpers **** **** **** // + + typedef std::variant SimpCfgData; class SimpCfg { @@ -385,7 +400,7 @@ class SimpCfg { bool bGroup = !isspace(curL[0]); curL = str_trim(curL); if (bGroup) { - curL = str_trim_single(curL, "\""); + curL = str_trim_single(curL, {"\""}); group = curL; LDBUG_LN("DBUG:SC:%s:group:%s", __func__, group.c_str()); continue; @@ -402,7 +417,7 @@ class SimpCfg { } std::string key = curL.substr(0, dPos); key = str_trim(key); - key = str_trim_single(key, "\""); + key = str_trim_single(key, {"\""}); std::string value = curL.substr(dPos+1); value = str_trim(value); value = str_trim(value, {","}); @@ -421,7 +436,7 @@ class SimpCfg { if (!value.empty() && (value.front() != '"')) { LWARN_LN("WARN:SC:%s:%d:%s:k:%s:v:%s:is this string?", __func__, iLine, group.c_str(), key.c_str(), value.c_str()); } - value = str_trim_single(value, "\""); + value = str_trim_single(value, {"\""}); set_string(group, key, value); } //LDBUG_LN("DBUG:SC:%s:%d:kv:%s:%s:%s:%s", __func__, iLine, group.c_str(), key.c_str(), vtype.c_str(), value.c_str()); From 33619a3b92f421e2e7057583a3f2fea39197ed30 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sat, 4 May 2024 11:32:37 +0530 Subject: [PATCH 125/217] SimpCfg: Templatize str_lower --- common/simpcfg.hpp | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index 08140f77b9cec..2a7f96d6bd20d 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -22,6 +22,25 @@ * * It tries to provide a crude expanded form of array wrt any of the above supported types. * For this one needs to define keys using the pattern TheKeyName-0, TheKeyName-1, .... + * + * ## Additional notes + * + * NativeCharSize encoded char refers to chars which fit within the size of char type in a given + * type of c++ string or base bitsize of a encoding standard, like 1 byte in case of std::string, + * utf-8, ... + * * example english alphabets in utf-8 encoding space are 1byte chars, in its variable length + * encoding space. + * + * MultiNativeCharSize encoded char refers to chars which occupy multiple base-char-bit-size of + * a c++ string type or char encoding standard. + * * example indian scripts alphabets in utf-8 encoding space occupy multiple bytes in its variable + * length encoding space. + * + * Sane variable length encoding - refers to encoding where the values of NativeCharSized chars of + * a char encoding space cant overlap with values in NativeCharSize subparts of MultiNativeCharSized + * chars of the same char encoding standard. + * * utf-8 shows this behaviour + * * chances are utf-16 and utf-32 also show this behaviour (need to cross check once) */ #include @@ -163,6 +182,7 @@ std::string str_trim_oversmart(std::string sIn, const std::string &trimChars=" \ // NOTE: Chars being trimmed (ie in trimChars) needs to be part of NativeCharSize // subset of the string's encoded char space, to avoid mix up when working with // strings which can be utf-8/utf-16/utf-32/sane-variable-length encoded strings. +// // NOTE:UTF8: This will work provided the string being trimmed as well the chars // being trimmed are made up of 1byte encoded chars in case of utf8 encoding space. // If the string being trimmed includes multibyte (ie MultiNativeCharSize) encoded @@ -193,13 +213,17 @@ TString str_trim_single(TString sin, const TString& trimChars=" \t\n") { return sin; } -// This works for 1byte encoded chars, including in utf8 encoding space. +// This works for NativeCharSize encoded chars, including in utf8 encoding space. // This wont work for multibyte encoded chars. -std::string str_tolower(const std::string &sin) { - std::string sout; +template +TString str_tolower(const TString &sin) { + TString sout; sout.resize(sin.size()); - std::transform(sin.begin(), sin.end(), sout.begin(), [](char c)->char {return std::tolower(c);}); - //LDBUG_LN("DBUG:%s:%s:%s", __func__, sin.c_str(), sout.c_str()); + std::transform(sin.begin(), sin.end(), sout.begin(), [](auto c)->auto {return std::tolower(c);}); +#ifdef SC_DEBUG_VERBOSE + dumphex_string(sin, std::format("DBUG:{}:in:", __func__)); + dumphex_string(sout, std::format("DBUG:{}:out:", __func__)); +#endif return sout; } From 3287fdba28530671dfac3144983a790b4cc1eba4 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sat, 4 May 2024 11:43:15 +0530 Subject: [PATCH 126/217] SimpCfg:Fix/cleanup trim related test samples and flow Use the commonality between Indian languages to show mixup issue with the simple minded trim_dump logic and how trim_oversmart could potentially avoid that. Given that I am using valid strings to show the pitfalls of fixed native char size driven logic, so no need to keep the dump and oversmart flows seperate, so merge into a common loop. --- common/simpcfg.hpp | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index 2a7f96d6bd20d..27fb9c56dd912 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -541,18 +541,15 @@ void check_nonenglish() { std::string sGotOSmart = str_trim_oversmart(sTest, {" \n\t"}); std::cout << std::format("{}: Test1[{}] Dumb[{}] OverSmart[{}]", __func__, sTest, sGotDumb, sGotOSmart) << std::endl; } - std::vector vTest2 = { "\n\t this र remove 0s at end 000 ", "\n\tthis र remove 0s and अs at end 000रअ0अ ", "\n\tthis र remove 0s and अs at end 000रअ0\xa4अ "}; + // The string "\n\tthis र remove 0s and अs at end 000रअ0\xa4अ ", + // * will mess up str_trim_dumb, + // * but will rightly trigger a exception with oversmart. + std::vector vTest2 = { "\n\t this र remove 0s at end 000 ", "\n\tthis र remove 0s, अs, ഇs at end 000रअ0अ ", "\n\tthis र remove 0s, अs, ഇs at end 000रअ0इअ "}; + std::string trimChars = {" \n\tഇ0अ"}; for (auto sTest: vTest2) { - std::string sGotDumb = str_trim_dumb(sTest, {" \n\t0अ"}); - std::cout << std::format("{}: Test2[{}] Dumb[{}]", __func__, sTest, sGotDumb) << std::endl; - } - // This partly invalid utf8 string will mess up str_trim_dumb "\n\tthis र remove 0s and अs at end 000रअ0\xa4अ " - // but will trigger a exception with oversmart. - // std::vector vTest3 = { "\n\t this र remove 0s at end 000 ", "\n\tthis र remove 0s and अs at end 000रअ0अ ", "\n\tthis र remove 0s and अs at end 000रअ0\xa4अ "}; - std::vector vTest3 = { "\n\t this र remove 0s at end 000 ", "\n\tthis र remove 0s and अs at end 000रअ0अ ", "\n\tthis र remove 0s and अs at end 000रअ0\xe0\xa4\x30अ "}; // \xe0\xa4 - for (auto sTest: vTest3) { - std::string sGotOSmart = str_trim_oversmart(sTest, {" \n\t0अ"}); - std::cout << std::format("{}: Test3[{}] OverSmart[{}]", __func__, sTest, sGotOSmart) << std::endl; + std::string sGotDumb = str_trim_dumb(sTest, trimChars); + std::string sGotOSmart = str_trim_oversmart(sTest, trimChars); + std::cout << std::format("{}: Test2 [{}]\n\tDumb[{}]\n\tOverSmart[{}]", __func__, sTest, sGotDumb, sGotOSmart) << std::endl; } } From f53c19baac7c75de73f6672204944f609004be11 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sat, 4 May 2024 12:22:06 +0530 Subject: [PATCH 127/217] SimpCfg: Update the notes wrt tolower and add test code --- common/simpcfg.hpp | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index 27fb9c56dd912..a048607145fa2 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -213,8 +213,15 @@ TString str_trim_single(TString sin, const TString& trimChars=" \t\n") { return sin; } -// This works for NativeCharSize encoded chars, including in utf8 encoding space. -// This wont work for multibyte encoded chars. +// Convert to lower case, if language has upper and lower case semantic +// +// This works for fixed size encoded char spaces. +// +// For variable length encoded char spaces, it can work +// * if one is doing the conversion for languages which fit into NativeCharSized chars in it +// * AND if one is working with a sane variable length encoding standard +// * ex: this will work if trying to do the conversion for english language within utf-8 +// template TString str_tolower(const TString &sin) { TString sout; @@ -539,7 +546,8 @@ void check_nonenglish() { for (auto sTest: vTest1) { std::string sGotDumb = str_trim_dumb(sTest, {" \n\t"}); std::string sGotOSmart = str_trim_oversmart(sTest, {" \n\t"}); - std::cout << std::format("{}: Test1[{}] Dumb[{}] OverSmart[{}]", __func__, sTest, sGotDumb, sGotOSmart) << std::endl; + std::string sLower = str_tolower(sTest); + std::cout << std::format("{}: Test1 [{}]\n\tTrimDumb[{}]\n\tTrimOverSmart[{}]\n\tLowerDumb[{}]", __func__, sTest, sGotDumb, sGotOSmart, sLower) << std::endl; } // The string "\n\tthis र remove 0s and अs at end 000रअ0\xa4अ ", // * will mess up str_trim_dumb, From 20e5b383c5be4a9e5df747a38ef26f4e40b9d280 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sat, 4 May 2024 12:25:57 +0530 Subject: [PATCH 128/217] SimpCfg:Trim DumpHexString only if SC_DEBUG_VERBOSE --- common/simpcfg.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index a048607145fa2..abc573195f235 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -56,6 +56,7 @@ #define SC_DEBUG +#undef SC_DEBUG_VERBOSE #define SC_TEST_PRG #ifdef SC_TEST_PRG #define LINFO_LN(FMT, ...) fprintf(stdout, FMT"\n", __VA_ARGS__) @@ -148,7 +149,7 @@ void dumphex_string(const TString &sIn, const std::string &msgTag){ // template TString str_trim_dumb(TString sin, const TString &trimChars=" \t\n") { -#ifdef SC_DEBUG +#ifdef SC_DEBUG_VERBOSE dumphex_string(sin, "DBUG:TrimDumb:Str:"); dumphex_string(trimChars, "DBUG:TrimDumb:Tim:"); #endif From 2b14bcaddb23b28fe6e08e4b73feaa41a8535155 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sat, 4 May 2024 12:28:13 +0530 Subject: [PATCH 129/217] SimpCfg:ChatON: add by Humans for All note --- common/chaton.hpp | 3 +++ common/simpcfg.hpp | 1 + 2 files changed, 4 insertions(+) diff --git a/common/chaton.hpp b/common/chaton.hpp index c608845a3bb35..a14f3351faa67 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -1,6 +1,9 @@ #pragma once /** + * + * Generic tagging logic + text config file based chat templates handling + * by Humans for All * * ## Overview * diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index abc573195f235..af50d45d2f50d 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -2,6 +2,7 @@ /** * Provides a simple direct 1-level only config file logic + * by Humans for All * * ## File format * From 5380b1e86e783efea3c6001a53fa8bdcc275bb61 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sat, 4 May 2024 13:28:21 +0530 Subject: [PATCH 130/217] ChatON:Update meta.json wrt command-r models template info picked from tokenizer config's default entry, Verified that same is used in the existing hardcoded chat apply template flow. --- examples/chaton_meta.json | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/examples/chaton_meta.json b/examples/chaton_meta.json index eda31fee62b16..1214319a1f738 100644 --- a/examples/chaton_meta.json +++ b/examples/chaton_meta.json @@ -291,6 +291,35 @@ "systemuser-system-has-end": true, "systemuser-1st-user-has-begin": true, "systemuser-1st-user-has-prefix": true + }, + "command-r": { + "global": { + "begin": "", + "end": "" + }, + "system": { + "begin": "", + "prefix": "<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>", + "suffix": "<|END_OF_TURN_TOKEN|>", + "end": "" + }, + "user": { + "begin": "", + "prefix": "<|START_OF_TURN_TOKEN|><|USER_TOKEN|>", + "suffix": "<|END_OF_TURN_TOKEN|>", + "end": "" + }, + "assistant": { + "begin": "", + "prefix": "<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>", + "suffix": "<|END_OF_TURN_TOKEN|>", + "end": "" + }, + "reverse-prompt": "<|END_OF_TURN_TOKEN|>", + "systemuser-system-has-suffix": true, + "systemuser-system-has-end": true, + "systemuser-1st-user-has-begin": true, + "systemuser-1st-user-has-prefix": true } } From c6ecd9316e721a4800a83f01ecd746f1d0e6a64f Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sat, 4 May 2024 17:20:52 +0530 Subject: [PATCH 131/217] SimpCfg: Use to_str instead of using stringstream directly --- common/simpcfg.hpp | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index af50d45d2f50d..ac497e7d783b9 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -295,9 +295,9 @@ class SimpCfg { void set_value(const std::string &group, const std::string &key, const SupportedDataType &value, const std::string &callerName="") { auto &gm = mapV[group]; gm[key] = value; - std::stringstream ss; - ss << value; - LDBUG_LN("DBUG:SC:%s_%s:%s:%s:%s", __func__, callerName.c_str(), group.c_str(), key.c_str(), ss.str().c_str()); +#ifdef SC_DEBUG + LDBUG_LN("DBUG:SC:%s_%s:%s:%s:%s", __func__, callerName.c_str(), group.c_str(), key.c_str(), to_str(value).c_str()); +#endif } void set_string(const std::string &group, const std::string &key, const std::string &value) { @@ -349,13 +349,15 @@ class SimpCfg { SupportedDataType get_value(const std::string &group, const std::string &key, const SupportedDataType &defaultValue, const std::string &callerName="") { auto gm = mapV[group]; if (gm.find(key) == gm.end()) { - std::stringstream ss; - ss << defaultValue; - LWARN_LN("DBUG:SC:%s_%s:%s:%s:%s[default]", __func__, callerName.c_str(), group.c_str(), key.c_str(), ss.str().c_str()); +#ifdef SC_DEBUG + LWARN_LN("WARN:SC:%s_%s:%s:%s:%s[default]", __func__, callerName.c_str(), group.c_str(), key.c_str(), to_str(defaultValue).c_str()); +#endif return defaultValue; } auto value = gm[key]; +#ifdef SC_DEBUG LDBUG_LN("DBUG:SC:%s_%s:%s:%s:%s", __func__, callerName.c_str(), group.c_str(), key.c_str(), to_str(value).c_str()); +#endif return std::get(value); } From 623d0b60dabad7055e62daa2ac827a8f809f84cd Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sat, 4 May 2024 18:09:35 +0530 Subject: [PATCH 132/217] SimpCfg: General MultiPart support, KeyParts not Key wrt SetValue --- common/simpcfg.hpp | 73 ++++++++++++++++++++++++++++------------------ 1 file changed, 44 insertions(+), 29 deletions(-) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index ac497e7d783b9..33a89cde9dbef 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -272,6 +272,7 @@ std::string str(std::vector values) { typedef std::variant SimpCfgData; +typedef std::vector MultiPart; class SimpCfg { @@ -282,6 +283,19 @@ class SimpCfg { public: + static std::string joiner(const MultiPart& parts) { + std::stringstream joined; + int iCnt = 0; + for(auto part: parts) { + if (iCnt != 0) { + joined << "-"; + } + iCnt += 1; + joined << part; + } + return joined.str(); + } + std::string to_str(const SimpCfgData &value) { auto visitor = [](auto value) -> auto { std::stringstream ss; @@ -292,7 +306,8 @@ class SimpCfg { } template - void set_value(const std::string &group, const std::string &key, const SupportedDataType &value, const std::string &callerName="") { + void set_value(const std::string &group, const MultiPart &keyParts, const SupportedDataType &value, const std::string &callerName="") { + auto key = joiner(keyParts); auto &gm = mapV[group]; gm[key] = value; #ifdef SC_DEBUG @@ -300,37 +315,37 @@ class SimpCfg { #endif } - void set_string(const std::string &group, const std::string &key, const std::string &value) { - set_value(group, key, value, __func__); + void set_string(const std::string &group, const MultiPart &keyParts, const std::string &value) { + set_value(group, keyParts, value, __func__); } - void set_bool(const std::string &group, const std::string &key, bool value) { - set_value(group, key, value, __func__); + void set_bool(const std::string &group, const MultiPart &keyParts, bool value) { + set_value(group, keyParts, value, __func__); } - void set_bool(const std::string &group, const std::string &key, const std::string &value) { + void set_bool(const std::string &group, const MultiPart &keyParts, const std::string &value) { std::string sValue = str_tolower(value); bool bValue = sValue == "true" ? true : false; //LDBUG_LN("DBUG:%s:%s:%s:%d", __func__, value.c_str(), sValue.c_str(), bValue); - set_bool(group, key, bValue); + set_bool(group, keyParts, bValue); } - void set_int64(const std::string &group, const std::string &key, int64_t value) { - set_value(group, key, value, __func__); + void set_int64(const std::string &group, const MultiPart &keyParts, int64_t value) { + set_value(group, keyParts, value, __func__); } - void set_int64(const std::string &group, const std::string &key, std::string &value) { + void set_int64(const std::string &group, const MultiPart &keyParts, std::string &value) { auto ivalue = strtoll(value.c_str(), nullptr, 0); - set_int64(group, key, ivalue); + set_int64(group, keyParts, ivalue); } - void set_double(const std::string &group, const std::string &key, double value) { - set_value(group, key, value, __func__); + void set_double(const std::string &group, const MultiPart &keyParts, double value) { + set_value(group, keyParts, value, __func__); } - void set_double(const std::string &group, const std::string &key, std::string &value) { + void set_double(const std::string &group, const MultiPart &keyParts, std::string &value) { auto dvalue = strtod(value.c_str(), nullptr); - set_double(group, key, dvalue); + set_double(group, keyParts, dvalue); } void dump(const std::string &group) { @@ -459,20 +474,20 @@ class SimpCfg { std::string vtype = "bool"; auto valueLower = str_tolower(value); if ((valueLower.compare("true") == 0) || (valueLower == "false")) { - set_bool(group, key, value); + set_bool(group, {key}, value); } else if (std::regex_match(value, rInt)) { vtype = "int"; - set_int64(group, key, value); + set_int64(group, {key}, value); } else if (std::regex_match(value, rFloat)) { vtype = "float"; - set_double(group, key, value); + set_double(group, {key}, value); } else { vtype = "string"; if (!value.empty() && (value.front() != '"')) { LWARN_LN("WARN:SC:%s:%d:%s:k:%s:v:%s:is this string?", __func__, iLine, group.c_str(), key.c_str(), value.c_str()); } value = str_trim_single(value, {"\""}); - set_string(group, key, value); + set_string(group, {key}, value); } //LDBUG_LN("DBUG:SC:%s:%d:kv:%s:%s:%s:%s", __func__, iLine, group.c_str(), key.c_str(), vtype.c_str(), value.c_str()); } @@ -594,10 +609,10 @@ int main(int argc, char **argv) { sc.get_int64("testme", "key101i", 123456); sc.get_double("testme", "key101d", 123456.789); - sc.set_bool("testme", "key201b", true); - sc.set_string("testme", "key201s", "hello world"); - sc.set_int64("testme", "key201i", 987654); - sc.set_double("testme", "key201d", 9988.7766); + sc.set_bool("testme", {"key201b"}, true); + sc.set_string("testme", {"key201s"}, "hello world"); + sc.set_int64("testme", {"key201i"}, 987654); + sc.set_double("testme", {"key201d"}, 9988.7766); sc.dump("testme"); sc.get_bool("testme", "key201b", false); @@ -610,12 +625,12 @@ int main(int argc, char **argv) { sc.get_vector("testme", "keyA100", {1, 2, 3}); sc.get_vector("testme", "keyA100", { "A", "അ", "अ", "ಅ" }); - sc.set_int64("testme", "keyA300-0", 330); - sc.set_int64("testme", "keyA300-1", 331); - sc.set_int64("testme", "keyA300-2", 332); - sc.set_string("testme", "keyA301-0", "India"); - sc.set_value("testme", "keyA301-1", "World"); - sc.set_string("testme", "keyA301-2", "AkashaGanga"); + sc.set_int64("testme", {"keyA300-0"}, 330); + sc.set_int64("testme", {"keyA300-1"}, 331); + sc.set_int64("testme", {"keyA300-2"}, 332); + sc.set_string("testme", {"keyA301-0"}, "India"); + sc.set_value("testme", {"keyA301", "1"}, "World"); + sc.set_string("testme", {"keyA301", "2"}, "AkashaGanga"); sc.get_vector("testme", "keyA300", {1, 2, 3}); sc.get_vector("testme", "keyA301", { "yes 1", "No 2", "very well 3" }); return 0; From 19d3c88e8a9e9725ccc18e7a7a58cae463910bd0 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sat, 4 May 2024 18:21:59 +0530 Subject: [PATCH 133/217] SimpCfg:MultiPart keys wrt get_value etal --- common/simpcfg.hpp | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index 33a89cde9dbef..af7a9bdc6ff8f 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -361,7 +361,8 @@ class SimpCfg { } template - SupportedDataType get_value(const std::string &group, const std::string &key, const SupportedDataType &defaultValue, const std::string &callerName="") { + SupportedDataType get_value(const std::string &group, const MultiPart &keyParts, const SupportedDataType &defaultValue, const std::string &callerName="") { + auto key = joiner(keyParts); auto gm = mapV[group]; if (gm.find(key) == gm.end()) { #ifdef SC_DEBUG @@ -376,20 +377,20 @@ class SimpCfg { return std::get(value); } - std::string get_string(const std::string &group, const std::string &key, const std::string &defaultValue) { - return get_value(group, key, defaultValue, __func__); + std::string get_string(const std::string &group, const MultiPart &keyParts, const std::string &defaultValue) { + return get_value(group, keyParts, defaultValue, __func__); } - bool get_bool(const std::string &group, const std::string &key, bool defaultValue) { - return get_value(group, key, defaultValue, __func__); + bool get_bool(const std::string &group, const MultiPart &keyParts, bool defaultValue) { + return get_value(group, keyParts, defaultValue, __func__); } - int64_t get_int64(const std::string &group, const std::string &key, int64_t defaultValue) { - return get_value(group, key, defaultValue, __func__); + int64_t get_int64(const std::string &group, const MultiPart &keyParts, int64_t defaultValue) { + return get_value(group, keyParts, defaultValue, __func__); } - double get_double(const std::string &group, const std::string &key, double defaultValue) { - return get_value(group, key, defaultValue, __func__); + double get_double(const std::string &group, const MultiPart &keyParts, double defaultValue) { + return get_value(group, keyParts, defaultValue, __func__); } @@ -604,10 +605,10 @@ int main(int argc, char **argv) { sc.load(fname); sc.dump(""); - sc.get_bool("testme", "key101b", false); - sc.get_string("testme", "key101s", "Not found"); - sc.get_int64("testme", "key101i", 123456); - sc.get_double("testme", "key101d", 123456.789); + sc.get_bool("testme", {"key101b"}, false); + sc.get_string("testme", {"key101s"}, "Not found"); + sc.get_int64("testme", {"key101i"}, 123456); + sc.get_double("testme", {"key101d"}, 123456.789); sc.set_bool("testme", {"key201b"}, true); sc.set_string("testme", {"key201s"}, "hello world"); @@ -615,13 +616,13 @@ int main(int argc, char **argv) { sc.set_double("testme", {"key201d"}, 9988.7766); sc.dump("testme"); - sc.get_bool("testme", "key201b", false); - sc.get_string("testme", "key201s", "Not found"); - sc.get_int64("testme", "key201i", 123456); - sc.get_double("testme", "key201d", 123456.789); + sc.get_bool("testme", {"key201b"}, false); + sc.get_string("testme", {"key201s"}, "Not found"); + sc.get_int64("testme", {"key201i"}, 123456); + sc.get_double("testme", {"key201d"}, 123456.789); - sc.get_string("mistral", "system-prefix", "Not found"); - sc.get_string("\"mistral\"", "\"system-prefix\"", "Not found"); + sc.get_string("mistral", {"system-prefix"}, "Not found"); + sc.get_string("\"mistral\"", {"\"system-prefix\""}, "Not found"); sc.get_vector("testme", "keyA100", {1, 2, 3}); sc.get_vector("testme", "keyA100", { "A", "അ", "अ", "ಅ" }); From 344c068d7b9571574cf4b8d694fc7dba7090154f Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sat, 4 May 2024 18:29:25 +0530 Subject: [PATCH 134/217] SimpCfg:MultiPart keys wrt get_vector With this and past few commits, now there is simple yet sufficient support to help move multi-level-hierarchy config files into the SimpCfg's simple physically 1-level, but if reqd logically multi level hierarchy flow. B4 this series of commits also one could have still achieved this, but there would have been bit more effort needed. --- common/simpcfg.hpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index af7a9bdc6ff8f..46efb60738378 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -395,7 +395,8 @@ class SimpCfg { template - std::vector get_vector(const std::string &group, const std::string &key, const std::vector &defaultValue, const std::string &callerName="") { + std::vector get_vector(const std::string &group, const MultiPart &keyParts, const std::vector &defaultValue, const std::string &callerName="") { + auto key = joiner(keyParts); auto gm = mapV[group]; std::vector array; int i = 0; @@ -624,16 +625,16 @@ int main(int argc, char **argv) { sc.get_string("mistral", {"system-prefix"}, "Not found"); sc.get_string("\"mistral\"", {"\"system-prefix\""}, "Not found"); - sc.get_vector("testme", "keyA100", {1, 2, 3}); - sc.get_vector("testme", "keyA100", { "A", "അ", "अ", "ಅ" }); + sc.get_vector("testme", {"keyA100"}, {1, 2, 3}); + sc.get_vector("testme", {"keyA100"}, { "A", "അ", "अ", "ಅ" }); sc.set_int64("testme", {"keyA300-0"}, 330); sc.set_int64("testme", {"keyA300-1"}, 331); sc.set_int64("testme", {"keyA300-2"}, 332); sc.set_string("testme", {"keyA301-0"}, "India"); sc.set_value("testme", {"keyA301", "1"}, "World"); sc.set_string("testme", {"keyA301", "2"}, "AkashaGanga"); - sc.get_vector("testme", "keyA300", {1, 2, 3}); - sc.get_vector("testme", "keyA301", { "yes 1", "No 2", "very well 3" }); + sc.get_vector("testme", {"keyA300"}, {1, 2, 3}); + sc.get_vector("testme", {"keyA301"}, { "yes 1", "No 2", "very well 3" }); return 0; } #endif From 989c6c4125a7553ca730f22b3952f8696d299922 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sun, 5 May 2024 07:50:32 +0530 Subject: [PATCH 135/217] SimpCfg: Cleanup the Note a bit to avoid some ambiguities --- common/simpcfg.hpp | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index 46efb60738378..58725f930ab63 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -126,7 +126,8 @@ void dumphex_string(const TString &sIn, const std::string &msgTag){ // variable length MultiNativeCharSize (ie multibye in case of utf-8) ones. // NOTE: It will also work, if atleast either end of string as well as trimChars // have NativeCharSize chars from their encoding space, rather than variable -// length MultiNativeCharSize based chars if any. +// length MultiNativeCharSize based chars if any. There needs to be NativeCharSized +// chars beyond any chars that get trimmed, on either side. // // NOTE: Given the way UTF-8 char encoding is designed, where NativeCharSize 1byte // encoded chars are fully unique and dont overlap with any bytes from any of the @@ -134,8 +135,8 @@ void dumphex_string(const TString &sIn, const std::string &msgTag){ // the trimChars belong to NativeCharSize chars subset, the logic should work, even // if string has a mixture of NativeCharSize and MultiNativeCharSize encoded chars. // Chances are utf-16 and utf-32 also have similar characteristics wrt thier -// NativeCharSize encoded chars (ie fully encoded within single 16bit and 32bit value -// respectively), and so equivalent semantic applies to them also. +// NativeCharSize encoded chars (ie those fully encoded within single 16bit and 32bit +// value respectively), and so equivalent semantic applies to them also. // // ALERT: Given that this simple minded logic, works at individual NativeCharSize level // only, If trimChars involve variable length MultiNativeCharSize encoded chars, then @@ -146,7 +147,7 @@ void dumphex_string(const TString &sIn, const std::string &msgTag){ // * given that different variable length MultiNativeCharSize encoded chars may have // some common NativeCharSize subparts (bytes in case of utf-8) between them, if one // of these chars is at either end of the string and another char is in trimChars, -// then string may get partially trimmed. +// then string may get partially trimmed wrt such a char at either end. // template TString str_trim_dumb(TString sin, const TString &trimChars=" \t\n") { @@ -188,7 +189,7 @@ std::string str_trim_oversmart(std::string sIn, const std::string &trimChars=" \ // NOTE:UTF8: This will work provided the string being trimmed as well the chars // being trimmed are made up of 1byte encoded chars in case of utf8 encoding space. // If the string being trimmed includes multibyte (ie MultiNativeCharSize) encoded -// characters at the end, then trimming can mess things up, if you have multibyte +// characters at either end, then trimming can mess things up, if you have multibyte // encoded utf-8 chars in the trimChars set. // // Currently given that SimpCfg only uses this with NativeCharSize chars in the @@ -268,7 +269,7 @@ std::string str(std::vector values) { } -// **** **** **** SimpCfg related helpers **** **** **** // +// **** **** **** the SimpCfg **** **** **** // typedef std::variant SimpCfgData; @@ -500,6 +501,10 @@ class SimpCfg { #ifdef SC_TEST_PRG + +// **** **** **** some simple test code **** **** **** // + + void check_string() { std::vector vStandard = { "123", "1अ3" }; std::cout << "**** string **** " << vStandard.size() << std::endl; From 93115a9733b3d7003ff572684687b8ece372a979 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sun, 5 May 2024 15:05:14 +0530 Subject: [PATCH 136/217] ChatON: initial go at OrionStar Ai chat model template Got from its tokenizer config json. Also same found in existing hardcoded template in default chat apply template logic of llamacpp --- examples/chaton_meta.json | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/examples/chaton_meta.json b/examples/chaton_meta.json index 1214319a1f738..ab0d73cb48a37 100644 --- a/examples/chaton_meta.json +++ b/examples/chaton_meta.json @@ -320,6 +320,36 @@ "systemuser-system-has-end": true, "systemuser-1st-user-has-begin": true, "systemuser-1st-user-has-prefix": true + }, + "orion": { + "global": { + "begin": "", + "end": "" + }, + "system": { + "begin": "", + "prefix": "Human: ", + "suffix": "\n\n", + "end": "" + }, + "user": { + "begin": "", + "prefix": "Human: ", + "suffix": "\n\nAssistant: ", + "end": "" + }, + "assistant": { + "begin": "", + "prefix": "", + "suffix": "", + "end": "" + }, + "reverse-prompt": "", + "systemuser-system-has-suffix": true, + "systemuser-system-has-end": true, + "systemuser-1st-user-has-begin": false, + "systemuser-1st-user-has-prefix": false } + } From 0f8f2a18c28a574394d622db32e53eef3ad1bea5 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sun, 5 May 2024 15:18:24 +0530 Subject: [PATCH 137/217] ChatON:chat template for OpenChat in meta.json initial go The first model seen, based on templates added till now into meta json file, that needs a Global Begin. From tokenizer_config json file, it appears like even system role should have a appropriate prefix, unlike what is seen in hardcoded default chat apply template of llama.cpp and chat jinja template. --- examples/chaton_meta.json | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/examples/chaton_meta.json b/examples/chaton_meta.json index ab0d73cb48a37..c1e3229f0a9f4 100644 --- a/examples/chaton_meta.json +++ b/examples/chaton_meta.json @@ -349,6 +349,35 @@ "systemuser-system-has-end": true, "systemuser-1st-user-has-begin": false, "systemuser-1st-user-has-prefix": false + }, + "openchat": { + "global": { + "begin": "", + "end": "" + }, + "system": { + "begin": "", + "prefix": "GPT4 Correct System: ", + "suffix": "<|end_of_turn|>", + "end": "" + }, + "user": { + "begin": "", + "prefix": "GPT4 Correct User: ", + "suffix": "<|end_of_turn|>", + "end": "" + }, + "assistant": { + "begin": "", + "prefix": "GPT4 Correct Assistant: ", + "suffix": "<|end_of_turn|>", + "end": "" + }, + "reverse-prompt": "<|end_of_turn|>", + "systemuser-system-has-suffix": true, + "systemuser-system-has-end": true, + "systemuser-1st-user-has-begin": true, + "systemuser-1st-user-has-prefix": true } } From b875b029791b4b5a4b189822030f72705cde026c Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sun, 5 May 2024 15:44:39 +0530 Subject: [PATCH 138/217] ChatON:Initial go at vicuna chat template in meta.json Have looked at tokenizer_config.json, jinja file and default hardcoded template in llama.cpp. This is also one of the models where a Global BoS is needed. NOTE: Have taken the liberty to also add a SYSTEM: prefix wrt system message, even thou default vicuna doesnt seem to need, but vicuna-orca seems to need, so that both models can be driven from same chat template config. I am assuming the system prefix should not create any problem even in default vicuna, however if it does create a problem one can duplicate the existing vicuna block in chaton_meta.json and make the system prefix empty in it. --- examples/chaton_meta.json | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/examples/chaton_meta.json b/examples/chaton_meta.json index c1e3229f0a9f4..70c05359d17ae 100644 --- a/examples/chaton_meta.json +++ b/examples/chaton_meta.json @@ -378,6 +378,35 @@ "systemuser-system-has-end": true, "systemuser-1st-user-has-begin": true, "systemuser-1st-user-has-prefix": true + }, + "vicuna": { + "global": { + "begin": "", + "end": "" + }, + "system": { + "begin": "", + "prefix": "SYSTEM: ", + "suffix": "\n\n", + "end": "" + }, + "user": { + "begin": "", + "prefix": "USER: ", + "suffix": "\n", + "end": "" + }, + "assistant": { + "begin": "", + "prefix": "ASSISTANT: ", + "suffix": "\n", + "end": "" + }, + "reverse-prompt": "", + "systemuser-system-has-suffix": true, + "systemuser-system-has-end": true, + "systemuser-1st-user-has-begin": true, + "systemuser-1st-user-has-prefix": true } } From f6a86cd2090be921375e9d1038f3e2298ca48ee8 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Tue, 7 May 2024 10:29:16 +0530 Subject: [PATCH 139/217] ChatON: Update the Note a bit --- common/chaton.hpp | 36 ++++++++++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index a14f3351faa67..6bd8250deca26 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -56,12 +56,16 @@ * * however in many of these models, the subsequent user messages will have the * BeginOfSentenceTag and or RolePrefixTag. * + * * Some models may require a BoS for a group of messages, independent of BoS (if any) + * wrt individual roles. + * * * ## The Strategy * * The template meta data json file allows the user to specify the above mentioned tags wrt - * each of the Role. Depending on whether a given model uses a given tag or not you either - * specify the required tag or else you specify a empty string. + * each of the Role as well as any global tag for a group of messages. Depending on whether + * a given model uses/needs a given tag or not you either specify the required tag or else + * you specify a empty string. * * A tag could be a single word or multiple words, and may include newline char specified * using \n and so on. The tag is always demarcated using double quotes and thus also allows @@ -80,6 +84,8 @@ * the assistant's suffix or end tag or to the user's begin or prefix tag, depending on what * is generated by the model at the end of its response. * + * Currently flags for trimming wrt user text (be it wrt system or user role) is not added. + * * * ## The JSON File * @@ -149,20 +155,38 @@ * These always adds any role specific begin+prefix and suffix+end around * the passed message. * + * ## other uses be it wrt llama.cpp-as-library or examples/server or ... + * + * This module exposes a c-api which is equivalent to the current hardcoded + * templating logic's llama_chat_apply_template. So any program using llama.cpp's + * chat templating logic can be easily migrated to make use of this generic code + * with text based config file based flow. + * + * If a program doesnt want to bring in json dependency into their project, + * there is also common/simpcfg.hpp, which provides a simple text based config + * file format, along with the corresponding parser for the same. This can be + * modified to work with simpcfg easily, if needed. * * ## Adding support for new model / chat-handshake-template-standard * * 1. Add suitable entries in json for that model/standard - * 2. Try to reuse the generic flow in chaton-tmpl-apply, as much as possible, - * before trying to add a custom logic. + * This in itself should work for most of the models. + * + * 2. If some new model introduces a totally different kind of chat-templating + * tag inter/intra mixing, Try to reuse and update the generic flow in + * chaton-tmpl-apply, as much as possible, before trying to add any custom logic. + * * If you update the generic flow, cross check if existing json files will * need to be updated or not. * * * ## Notes * - * Look at the sample chaton_meta.json in examples folder for how the above may apply - * * llama2, llama3, gemma, chatml, zephyr, deepseek(normal and coder), monarch, mistral + * Look at the sample chaton_meta.json in examples folder for how the above may apply to + * the different llm's out there like + * + * * llama2, llama3, gemma, zephyr, deepseek(normal and coder), monarch, mistral, phi3 + * * chatml, command-r, orion, openchat, vicuna * */ From 04b4a15177fa0cc1f9528b0fc952f18171f8894a Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Tue, 7 May 2024 11:08:47 +0530 Subject: [PATCH 140/217] ChatON: Initial go at chat-template-apply c-api with parts info --- common/chaton.hpp | 53 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/common/chaton.hpp b/common/chaton.hpp index 6bd8250deca26..c16be0ee07c63 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -541,6 +541,7 @@ inline int32_t chaton_tmpl_apply( // If the passed char array is smaller than that required for the tagged messages string, // * part of the tagged messages string which fits within dest buffer is copied // * the returned value, indicates the size of the actual tagged message +// // NOTE: // * ideally the passed char array should be able to fit the tagged messages string + 0|null char. // * if the return value from this function is larger than or equal to destLength, @@ -572,6 +573,58 @@ inline int32_t chaton_tmpl_apply_capi( return taggedLength; } +// +// In addition to the semantic provided by chaton_tmpl_apply_capi +// this additionally also returns info about the parts that make up +// the returned tagged message. +// +// partTypes and partLengths should be arrays that can accomodate the +// same number of elements belonging to its respective type. +// Inturn the pNumParts should point to a int which specifies the +// number of elements. +// If the generated tagged message has more parts than the specified +// *pNumParts, then the logic copies partTypes and partLengths to the +// specified length/NumOfParts only. Parallely it updates *pNumParts +// to the actual needed length (not including any terminating null char or so). +// +inline int32_t chaton_tmpl_apply_ex_capi( + const char *tmpl, + const struct llama_chat_message *msgs, + const size_t numMsgs, + bool alertAssistantAtEnd, + char *dest, + int32_t destLength, + char *partTypes, + int32_t *partLengths, + int32_t *pNumParts + ) { + if ((tmpl == nullptr) || (dest == nullptr)) { + return -1; + } + std::vector vMsgs; + for(size_t i=0; i lens; + int32_t taggedLength = chaton_tmpl_apply_ex(tmpl, vMsgs, taggedMsgs, types, lens, alertAssistantAtEnd); + if (taggedLength <= 0) { + return taggedLength; + } + if (destLength > 0) { + strlcpy(dest, taggedMsgs.c_str(), destLength); + } + if (*pNumParts > 0) { + strlcpy(partTypes, types.c_str(), *pNumParts); + for(int i=0; i < *pNumParts; i++) { + partLengths[i] = lens[i]; + } + } + *pNumParts = types.length(); + return taggedLength; +} + /** * if tmpl is From 7c288d3dfcaed0113d720b879088ec841b8b0e2f Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Tue, 7 May 2024 11:32:20 +0530 Subject: [PATCH 141/217] ChatON: Rename to partstypes for consistency --- common/chaton.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index c16be0ee07c63..aa9ade4aa2fee 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -264,7 +264,7 @@ class ChatParts { return allin; } - std::string get_types() { + std::string get_partstypes() { return types; } @@ -376,7 +376,7 @@ inline bool chaton_tmpl_apply_single_ex( cp.dump(); tagged = cp.str(); LOGLN("DBUG:%s:%s:%s:%s", __func__, tmpl.c_str(), role.c_str(), tagged.c_str()); - types = cp.get_types(); + types = cp.get_partstypes(); lens = cp.get_partslens(); return true; } @@ -512,7 +512,7 @@ inline bool chaton_tmpl_apply_ex( tagged = cp.str(); LOGLN("DBUG:%s:%s:%s", __func__, tmpl.c_str(), tagged.c_str()); LOGLN("DBUG:%s:%s:CntSys[%d]:CntUsr[%d]:CntOthers[%d]", __func__, tmpl.c_str(), cntSystem, cntUser, cntOthers); - types = cp.get_types(); + types = cp.get_partstypes(); lens = cp.get_partslens(); return true; } From 43a3a91b0372ad4f51e57b3a9d861f1d2bc8a649 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Tue, 7 May 2024 11:44:25 +0530 Subject: [PATCH 142/217] ChatON: Cleanup/Refine initial go at tmpl_apply_ex_capi --- common/chaton.hpp | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index aa9ade4aa2fee..74a9a38fb767f 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -608,7 +608,10 @@ inline int32_t chaton_tmpl_apply_ex_capi( std::string taggedMsgs; std::string types; std::vector lens; - int32_t taggedLength = chaton_tmpl_apply_ex(tmpl, vMsgs, taggedMsgs, types, lens, alertAssistantAtEnd); + if (!chaton_tmpl_apply_ex(tmpl, vMsgs, taggedMsgs, types, lens, alertAssistantAtEnd)) { + return -1; + } + int32_t taggedLength = taggedMsgs.size(); if (taggedLength <= 0) { return taggedLength; } @@ -616,9 +619,13 @@ inline int32_t chaton_tmpl_apply_ex_capi( strlcpy(dest, taggedMsgs.c_str(), destLength); } if (*pNumParts > 0) { - strlcpy(partTypes, types.c_str(), *pNumParts); - for(int i=0; i < *pNumParts; i++) { - partLengths[i] = lens[i]; + if (partTypes != nullptr) { + strlcpy(partTypes, types.c_str(), *pNumParts); + } + if (partLengths != nullptr) { + for(int i=0; i < *pNumParts; i++) { + partLengths[i] = lens[i]; + } } } *pNumParts = types.length(); From 0852f3b7ec627d3cc1f9ef9281610777874fd663 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Tue, 7 May 2024 11:46:40 +0530 Subject: [PATCH 143/217] ChatON:ExCApi: Rename for consistency --- common/chaton.hpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index 74a9a38fb767f..b34129c387a4b 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -594,8 +594,8 @@ inline int32_t chaton_tmpl_apply_ex_capi( bool alertAssistantAtEnd, char *dest, int32_t destLength, - char *partTypes, - int32_t *partLengths, + char *partsTypes, + int32_t *partsLengths, int32_t *pNumParts ) { if ((tmpl == nullptr) || (dest == nullptr)) { @@ -619,12 +619,12 @@ inline int32_t chaton_tmpl_apply_ex_capi( strlcpy(dest, taggedMsgs.c_str(), destLength); } if (*pNumParts > 0) { - if (partTypes != nullptr) { - strlcpy(partTypes, types.c_str(), *pNumParts); + if (partsTypes != nullptr) { + strlcpy(partsTypes, types.c_str(), *pNumParts); } - if (partLengths != nullptr) { + if (partsLengths != nullptr) { for(int i=0; i < *pNumParts; i++) { - partLengths[i] = lens[i]; + partsLengths[i] = lens[i]; } } } From b3a56545d62e6b6fd2f9c2df46c786fe7c398399 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Tue, 7 May 2024 11:49:43 +0530 Subject: [PATCH 144/217] ChatON:Reposition alertAssistantAtEnd flag for consistency --- common/chaton.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index b34129c387a4b..395057f2c798f 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -446,10 +446,10 @@ inline size_t chat_tmpl_apply_single_capi( inline bool chaton_tmpl_apply_ex( const std::string &tmpl, const std::vector &msgs, + bool alertAssistantAtEnd, std::string &tagged, std::string &types, - std::vector &lens, - bool alertAssistantAtEnd + std::vector &lens ) { if (!chaton_tmpl_exists(tmpl)) { return false; @@ -528,7 +528,7 @@ inline int32_t chaton_tmpl_apply( ) { std::string types; std::vector lens; - if (!chaton_tmpl_apply_ex(tmpl, msgs, tagged, types, lens, alertAssistantAtEnd)) { + if (!chaton_tmpl_apply_ex(tmpl, msgs, alertAssistantAtEnd, tagged, types, lens)) { return -1; } return tagged.size(); @@ -608,7 +608,7 @@ inline int32_t chaton_tmpl_apply_ex_capi( std::string taggedMsgs; std::string types; std::vector lens; - if (!chaton_tmpl_apply_ex(tmpl, vMsgs, taggedMsgs, types, lens, alertAssistantAtEnd)) { + if (!chaton_tmpl_apply_ex(tmpl, vMsgs, alertAssistantAtEnd, taggedMsgs, types, lens)) { return -1; } int32_t taggedLength = taggedMsgs.size(); From 76791bad63cc2e2c273a876810c23af4a3c49ce5 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Tue, 7 May 2024 12:35:46 +0530 Subject: [PATCH 145/217] ChatON:Fix partsLengths to int32_t type, instead of int so that the size of the elements is explicit and fixed, so that it is inturn in sync with the fixed int size specified wrt the c-api, even with any c compilers with different idea about int. avoid some ununsed vars, need to update compile flags later to enable corresponding warnings. --- common/chaton.hpp | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index 395057f2c798f..1e9eb4328e5c0 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -268,8 +268,8 @@ class ChatParts { return types; } - std::vector get_partslens() { - std::vector lens = {}; + std::vector get_partslens() { + std::vector lens = {}; for(auto part: parts) { lens.push_back(part.length()); } @@ -362,7 +362,7 @@ inline bool chaton_tmpl_apply_single_ex( const std::string &content, std::string &tagged, std::string &types, - std::vector &lens + std::vector &lens ) { if (!chaton_tmpl_exists(tmpl)) { return false; @@ -392,7 +392,7 @@ inline size_t chaton_tmpl_apply_single( std::string &tagged ) { std::string types; - std::vector lens; + std::vector lens; if (!chaton_tmpl_apply_single_ex(tmpl, role, content, tagged, types, lens)) { return -1; } @@ -418,8 +418,6 @@ inline size_t chat_tmpl_apply_single_capi( const size_t destLength ) { std::string tagged; - std::string types; - std::vector lens; auto taggedLength = chaton_tmpl_apply_single(tmpl, role, content, tagged); if (taggedLength <= 0) { return taggedLength; @@ -449,7 +447,7 @@ inline bool chaton_tmpl_apply_ex( bool alertAssistantAtEnd, std::string &tagged, std::string &types, - std::vector &lens + std::vector &lens ) { if (!chaton_tmpl_exists(tmpl)) { return false; @@ -527,7 +525,7 @@ inline int32_t chaton_tmpl_apply( std::string &tagged ) { std::string types; - std::vector lens; + std::vector lens; if (!chaton_tmpl_apply_ex(tmpl, msgs, alertAssistantAtEnd, tagged, types, lens)) { return -1; } @@ -578,12 +576,12 @@ inline int32_t chaton_tmpl_apply_capi( // this additionally also returns info about the parts that make up // the returned tagged message. // -// partTypes and partLengths should be arrays that can accomodate the +// partsTypes and partsLengths should be arrays that can accomodate the // same number of elements belonging to its respective type. // Inturn the pNumParts should point to a int which specifies the // number of elements. // If the generated tagged message has more parts than the specified -// *pNumParts, then the logic copies partTypes and partLengths to the +// *pNumParts, then the logic copies partsTypes and partsLengths to the // specified length/NumOfParts only. Parallely it updates *pNumParts // to the actual needed length (not including any terminating null char or so). // @@ -607,7 +605,7 @@ inline int32_t chaton_tmpl_apply_ex_capi( } std::string taggedMsgs; std::string types; - std::vector lens; + std::vector lens; if (!chaton_tmpl_apply_ex(tmpl, vMsgs, alertAssistantAtEnd, taggedMsgs, types, lens)) { return -1; } @@ -623,9 +621,7 @@ inline int32_t chaton_tmpl_apply_ex_capi( strlcpy(partsTypes, types.c_str(), *pNumParts); } if (partsLengths != nullptr) { - for(int i=0; i < *pNumParts; i++) { - partsLengths[i] = lens[i]; - } + memcpy(partsLengths, lens.data(), (*pNumParts)*4); } } *pNumParts = types.length(); From 8dfa31bb910d1c2b6ee4a899806fbc2376eddbd8 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Wed, 8 May 2024 16:50:13 +0530 Subject: [PATCH 146/217] ChatON: Make c-api wrappers a bit robust incl some cross checks If the tagged message will be of 0 length, ensure that the passed dest char* array, has null inserted appropriately. Check that user has passed a non-null pNumParts. Dont hard code int32_t size, pick using sizeof --- common/chaton.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index 1e9eb4328e5c0..9fa6614615a5b 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -562,7 +562,7 @@ inline int32_t chaton_tmpl_apply_capi( } std::string taggedMsgs; int32_t taggedLength = chaton_tmpl_apply(tmpl, vMsgs, alertAssistantAtEnd, taggedMsgs); - if (taggedLength <= 0) { + if (taggedLength < 0) { return taggedLength; } if (destLength > 0) { @@ -596,7 +596,7 @@ inline int32_t chaton_tmpl_apply_ex_capi( int32_t *partsLengths, int32_t *pNumParts ) { - if ((tmpl == nullptr) || (dest == nullptr)) { + if ((tmpl == nullptr) || (dest == nullptr) || (pNumParts == nullptr)) { return -1; } std::vector vMsgs; @@ -610,7 +610,7 @@ inline int32_t chaton_tmpl_apply_ex_capi( return -1; } int32_t taggedLength = taggedMsgs.size(); - if (taggedLength <= 0) { + if (taggedLength < 0) { return taggedLength; } if (destLength > 0) { @@ -621,7 +621,7 @@ inline int32_t chaton_tmpl_apply_ex_capi( strlcpy(partsTypes, types.c_str(), *pNumParts); } if (partsLengths != nullptr) { - memcpy(partsLengths, lens.data(), (*pNumParts)*4); + memcpy(partsLengths, lens.data(), (*pNumParts)*sizeof(int32_t)); } } *pNumParts = types.length(); From b6da7d9c9dd5b82e3e017ca2f7d9e0e031f3e1b1 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Wed, 8 May 2024 18:29:48 +0530 Subject: [PATCH 147/217] ChatON: tokenize keeping in mind the taggedMessage subparts Initial go --- common/chaton.hpp | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/common/chaton.hpp b/common/chaton.hpp index 9fa6614615a5b..c7cf5de2bf7a7 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -628,6 +628,51 @@ inline int32_t chaton_tmpl_apply_ex_capi( return taggedLength; } +// Copied from common.cpp +std::vector chaton_llama_tokenize( + const struct llama_model * model, + const std::string & text, + bool add_special, + bool parse_special) { + LOGLN("DBUG:%s:%s:special[add:%d, parse:%d]", __func__, text.c_str(), add_special, parse_special); + // upper limit for the number of tokens + int n_tokens = text.length() + 2 * add_special; + std::vector result(n_tokens); + n_tokens = llama_tokenize(model, text.data(), text.length(), result.data(), result.size(), add_special, parse_special); + if (n_tokens < 0) { + result.resize(-n_tokens); + int check = llama_tokenize(model, text.data(), text.length(), result.data(), result.size(), add_special, parse_special); + GGML_ASSERT(check == -n_tokens); + } else { + result.resize(n_tokens); + } + return result; +} + +// Tokenize the passed taggedText, keeping in mind the subparts within and +// inturn whether to parse special tokens in them or not (partsTypes). +std::vector chaton_llama_tokenize_ex( + const llama_context *ctx, + const std::string &taggedText, + const std::string &partsTypes, + const std::vector &partsLengths, + bool addSpecial + ) { + std::vector tokens; + int iPart = 0; + int iStart = 0; + for(auto partLen: partsLengths) { + auto partType = partsTypes[iPart]; + iPart += 1; + auto msgPart = taggedText.substr(iStart, partLen); + iStart += partLen; + auto parseSpecial = partType == ChatParts::S ? true : false; + auto curTokens = chaton_llama_tokenize(llama_get_model(ctx), msgPart, addSpecial, parseSpecial); + tokens.insert(tokens.end(), curTokens.begin(), curTokens.end()); + } + return tokens; +} + /** * if tmpl is From 868ab608f0e55abb6ce23ffd09a20b4c0fadfee1 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Wed, 8 May 2024 18:42:22 +0530 Subject: [PATCH 148/217] ChatON: Add forceParseSpecial flag to subparts aware tokenizing --- common/chaton.hpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index c7cf5de2bf7a7..c4ffd6e58c8a4 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -651,12 +651,15 @@ std::vector chaton_llama_tokenize( // Tokenize the passed taggedText, keeping in mind the subparts within and // inturn whether to parse special tokens in them or not (partsTypes). +// If you want to parse special tokens in the taggedText, independent of what +// partsTypes specifies, then set forceParseSpecial to true. std::vector chaton_llama_tokenize_ex( const llama_context *ctx, const std::string &taggedText, const std::string &partsTypes, const std::vector &partsLengths, - bool addSpecial + bool addSpecial, + bool forceParseSpecial ) { std::vector tokens; int iPart = 0; @@ -667,6 +670,7 @@ std::vector chaton_llama_tokenize_ex( auto msgPart = taggedText.substr(iStart, partLen); iStart += partLen; auto parseSpecial = partType == ChatParts::S ? true : false; + parseSpecial |= forceParseSpecial; auto curTokens = chaton_llama_tokenize(llama_get_model(ctx), msgPart, addSpecial, parseSpecial); tokens.insert(tokens.end(), curTokens.begin(), curTokens.end()); } From 0d81ffe6eb788f1911956544b6cdd19badd09120 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Wed, 8 May 2024 19:06:51 +0530 Subject: [PATCH 149/217] Tests:ChatON: Add partial skeleton wrt subparts tokenizing --- tests/test-chat-template-chaton.cpp | 34 +++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tests/test-chat-template-chaton.cpp b/tests/test-chat-template-chaton.cpp index 8cf60edf54da5..8c7a26489b69a 100644 --- a/tests/test-chat-template-chaton.cpp +++ b/tests/test-chat-template-chaton.cpp @@ -133,6 +133,40 @@ static void check_chaton(std::string &metaJson) { } +static void check_chaton_ex(std::string &metaJson) { + std::vector formatted_chat(1024); + int32_t res; + + chaton_meta_load(metaJson); + for(auto tmplId: templateIds) { + formatted_chat.resize(1024); + std::cout << "\n----------" << tmplId << "---------------\n"; + if (!chaton_meta_ok(tmplId)) { + exit(1); + } + std::vector partsTypes; + std::vector partsLens; + int numParts = message_count * 3; + int numPartsCrossCheck = numParts; + partsLens.resize(numParts); + partsTypes.resize(numParts); + res = chaton_tmpl_apply_ex_capi(tmplId.c_str(), conversation, message_count, true, formatted_chat.data(), formatted_chat.size(), partsTypes.data(), partsLens.data(), &numParts); + assert(res > 0); + assert(numParts <= numPartsCrossCheck); + formatted_chat.resize(res); + std::string output(formatted_chat.data(), formatted_chat.size()); + std::cout << "*** TAGGEDMSG ***\n" << output; + std::cout << "\n-----------------------------------------\n"; + partsLens.resize(numParts); + partsTypes.resize(numParts); + std::string sPartsTypes(partsTypes.data(), partsTypes.size()); + chaton_llama_tokenize_ex(nullptr, output, sPartsTypes, partsLens, false, false); + std::cout << "\n----------" << tmplId << "---------------\n"; + } + +} + + int main(int argc, char **argv) { if (argc != 2) { std::cout << argv[0] << " meta.json" << std::endl; From a49697b4885d3caf46068fc333d442a1827f70a4 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Wed, 8 May 2024 19:18:35 +0530 Subject: [PATCH 150/217] ChatON: Keep compiler happy simbly --- common/chaton.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index c4ffd6e58c8a4..686466cf9dcdf 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -629,7 +629,7 @@ inline int32_t chaton_tmpl_apply_ex_capi( } // Copied from common.cpp -std::vector chaton_llama_tokenize( +inline std::vector chaton_llama_tokenize( const struct llama_model * model, const std::string & text, bool add_special, @@ -653,7 +653,7 @@ std::vector chaton_llama_tokenize( // inturn whether to parse special tokens in them or not (partsTypes). // If you want to parse special tokens in the taggedText, independent of what // partsTypes specifies, then set forceParseSpecial to true. -std::vector chaton_llama_tokenize_ex( +inline std::vector chaton_llama_tokenize_ex( const llama_context *ctx, const std::string &taggedText, const std::string &partsTypes, From 8fe8231313b996bce1d6c5ae608b8a291cabbdff Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Wed, 8 May 2024 19:36:39 +0530 Subject: [PATCH 151/217] ChatON:SubPartsAwareTokenizePath: Allow extract subparts testing --- common/chaton.hpp | 8 ++++++-- tests/test-chat-template-chaton.cpp | 3 ++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index 686466cf9dcdf..c278246ee490e 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -635,6 +635,10 @@ inline std::vector chaton_llama_tokenize( bool add_special, bool parse_special) { LOGLN("DBUG:%s:%s:special[add:%d, parse:%d]", __func__, text.c_str(), add_special, parse_special); + if (model == nullptr) { + LOG_TEELN("ERRR:%s:Model NOT Provided:%s:special[add:%d, parse:%d]", __func__, text.c_str(), add_special, parse_special); + return std::vector{}; + } // upper limit for the number of tokens int n_tokens = text.length() + 2 * add_special; std::vector result(n_tokens); @@ -654,7 +658,7 @@ inline std::vector chaton_llama_tokenize( // If you want to parse special tokens in the taggedText, independent of what // partsTypes specifies, then set forceParseSpecial to true. inline std::vector chaton_llama_tokenize_ex( - const llama_context *ctx, + const struct llama_model *model, const std::string &taggedText, const std::string &partsTypes, const std::vector &partsLengths, @@ -671,7 +675,7 @@ inline std::vector chaton_llama_tokenize_ex( iStart += partLen; auto parseSpecial = partType == ChatParts::S ? true : false; parseSpecial |= forceParseSpecial; - auto curTokens = chaton_llama_tokenize(llama_get_model(ctx), msgPart, addSpecial, parseSpecial); + auto curTokens = chaton_llama_tokenize(model, msgPart, addSpecial, parseSpecial); tokens.insert(tokens.end(), curTokens.begin(), curTokens.end()); } return tokens; diff --git a/tests/test-chat-template-chaton.cpp b/tests/test-chat-template-chaton.cpp index 8c7a26489b69a..ce65cc31ea525 100644 --- a/tests/test-chat-template-chaton.cpp +++ b/tests/test-chat-template-chaton.cpp @@ -173,6 +173,7 @@ int main(int argc, char **argv) { exit(1); } std::string metaJson(argv[1]); - check_chaton(metaJson); + //check_chaton(metaJson); + check_chaton_ex(metaJson); return 0; } From 1f9a0eb8ce94b963cf5c99e8571615d040b3d9c7 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Fri, 10 May 2024 21:10:44 +0530 Subject: [PATCH 152/217] ChatON: Remove unneeded iostream --- common/chaton.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index c278246ee490e..8e2983cdf81b3 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -192,7 +192,6 @@ #include #include -#include #include #include "log.h" From fe27902964f39b19953bd50d2540fc42f2b5e5a8 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Fri, 10 May 2024 21:32:19 +0530 Subject: [PATCH 153/217] SimpCfg: Avoid iostream/cout and format for direct library use It appears like std::format is not supported in older g++/lib still in wide use like current debian stable, so avoiding same wrt direct library use. Allow for empty VAARGS NOTE: However test program mode of the same uses cout and format --- common/simpcfg.hpp | 38 ++++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index 58725f930ab63..93d0b7539c298 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -51,22 +51,25 @@ #include #include #include -#include #include -#include #define SC_DEBUG #undef SC_DEBUG_VERBOSE + #define SC_TEST_PRG #ifdef SC_TEST_PRG -#define LINFO_LN(FMT, ...) fprintf(stdout, FMT"\n", __VA_ARGS__) -#define LDBUG_LN(FMT, ...) fprintf(stderr, FMT"\n", __VA_ARGS__) -#define LERRR_LN(FMT, ...) fprintf(stderr, FMT"\n", __VA_ARGS__) -#define LWARN_LN(FMT, ...) fprintf(stderr, FMT"\n", __VA_ARGS__) +#include +#include +#define LINFO_LN(FMT, ...) fprintf(stdout, FMT"\n", ##__VA_ARGS__) +#define LDBUG(FMT, ...) fprintf(stderr, FMT, ##__VA_ARGS__) +#define LDBUG_LN(FMT, ...) fprintf(stderr, FMT"\n", ##__VA_ARGS__) +#define LERRR_LN(FMT, ...) fprintf(stderr, FMT"\n", ##__VA_ARGS__) +#define LWARN_LN(FMT, ...) fprintf(stderr, FMT"\n", ##__VA_ARGS__) #else #include "log.h" #define LINFO_LN LOG_TEELN +#define LDBUG LOG #define LDBUG_LN LOGLN #define LERRR_LN LOG_TEELN #define LWARN_LN LOG_TEELN @@ -101,20 +104,22 @@ size_t mbs_to_wcs(std::wstring &wDest, const std::string &sSrc) { template void dumphex_string(const TString &sIn, const std::string &msgTag){ - std::cout << msgTag << "[ "; + LDBUG("%s[ ", msgTag.c_str()); for(auto c: sIn) { auto cSize = sizeof(c); if (cSize == 1) { - std::cout << std::format("{:02x}, ", (uint8_t)c); + LDBUG("%02x, ", (uint8_t)c); } else if (cSize == 2) { - std::cout << std::format("{:04x}, ", (uint16_t)c); + LDBUG("%04x, ", (uint16_t)c); } else if (cSize == 4) { - std::cout << std::format("{:08x}, ", (uint32_t)c); + LDBUG("%08x, ", (uint32_t)c); } else { - throw std::runtime_error( std::format("ERRR:{}:Unsupported char type with size [{}]", __func__, cSize) ); + std::stringstream ss; + ss << "ERRR:" << __func__ << ":Unsupported char type with size [" << cSize << "]"; + throw std::runtime_error( ss.str().c_str() ); } } - std::cout << " ]" << std::endl; + LDBUG_LN(" ]"); } // Remove chars from begin and end of the passed string, provided the char @@ -152,8 +157,8 @@ void dumphex_string(const TString &sIn, const std::string &msgTag){ template TString str_trim_dumb(TString sin, const TString &trimChars=" \t\n") { #ifdef SC_DEBUG_VERBOSE - dumphex_string(sin, "DBUG:TrimDumb:Str:"); - dumphex_string(trimChars, "DBUG:TrimDumb:Tim:"); + dumphex_string(sin, "DBUG:StrTrimDumb:Str:"); + dumphex_string(trimChars, "DBUG:StrTrimDumb:TrimChars:"); #endif sin.erase(sin.find_last_not_of(trimChars)+1); sin.erase(0, sin.find_first_not_of(trimChars)); @@ -231,8 +236,8 @@ TString str_tolower(const TString &sin) { sout.resize(sin.size()); std::transform(sin.begin(), sin.end(), sout.begin(), [](auto c)->auto {return std::tolower(c);}); #ifdef SC_DEBUG_VERBOSE - dumphex_string(sin, std::format("DBUG:{}:in:", __func__)); - dumphex_string(sout, std::format("DBUG:{}:out:", __func__)); + dumphex_string(sin, "DBUG:StrToLower:in:"); + dumphex_string(sout, "DBUG:StrToLower:out:"); #endif return sout; } @@ -568,6 +573,7 @@ void check_wstring_cout() { } void check_nonenglish() { + std::cout << "**** non english **** " << std::endl; std::vector vTest1 = { "\n\tAഅअಅ\n\t", "\n\tAഅअಅ " }; for (auto sTest: vTest1) { std::string sGotDumb = str_trim_dumb(sTest, {" \n\t"}); From c0506f94bf3bcb5a64cd02026976dddc15f4b568 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sat, 11 May 2024 00:17:08 +0530 Subject: [PATCH 154/217] SimpCfg: Allow for direct initialization lists based init This should pave way for having a default chat templates dataset in the code, without needing to load it from a config file, if one doesnt want to. TODO: allow for loading config from json into simpcfg, so that a program which uses llama.cpp can decide, whether it is ok with what is already there in the internal dataset, or allow for loading template info at runtime using the simpcfg's simple text file or additionally include the json code to load template info at runtime from json file. --- common/simpcfg.hpp | 48 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 39 insertions(+), 9 deletions(-) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index 93d0b7539c298..694f822cf40bc 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -279,16 +279,19 @@ std::string str(std::vector values) { typedef std::variant SimpCfgData; typedef std::vector MultiPart; +typedef std::map> SimpCfgMapMapVariant; class SimpCfg { private: - std::map> mapV = {}; + SimpCfgMapMapVariant mapV = {}; std::regex rInt {R"(^[-+]?\d+$)"}; std::regex rFloat {R"(^[-+]?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?$)"}; public: + SimpCfg(SimpCfgMapMapVariant defaultMap) : mapV(defaultMap) {} + static std::string joiner(const MultiPart& parts) { std::stringstream joined; int iCnt = 0; @@ -354,6 +357,8 @@ class SimpCfg { set_double(group, keyParts, dvalue); } + // Dump info about the specified group. + // If group is empty, then dump info about all groups maintained in this instance. void dump(const std::string &group) { for (auto gm: mapV) { if (!group.empty() && (gm.first != group)) { @@ -604,16 +609,28 @@ void check_strings() { SimpCfg::locale_restore(sSavedLocale); } -int main(int argc, char **argv) { - if (argc != 2) { - LERRR_LN("USAGE:%s simp.cfg", argv[0]); - exit(1); - } +void sc_inited() { + SimpCfg sc = {{ + {"Group1",{ + {"testkey11", 11}, + {"testkey12", true} + }}, + {"Group2", { + {"key21", "val21"}, + {"key22", 22}, + {"key23", 2.3} + }} + }}; + + std::cout << "**** sc inited **** " << std::endl; + sc.dump(""); - check_strings(); +} - std::string fname {argv[1]}; - SimpCfg sc; +void sc_set(const std::string &fname) { + + std::cout << "**** sc set **** " << std::endl; + SimpCfg sc = {{}}; sc.load(fname); sc.dump(""); @@ -646,6 +663,19 @@ int main(int argc, char **argv) { sc.set_string("testme", {"keyA301", "2"}, "AkashaGanga"); sc.get_vector("testme", {"keyA300"}, {1, 2, 3}); sc.get_vector("testme", {"keyA301"}, { "yes 1", "No 2", "very well 3" }); +} + +int main(int argc, char **argv) { + if (argc != 2) { + LERRR_LN("USAGE:%s simp.cfg", argv[0]); + exit(1); + } + + check_strings(); + sc_inited(); + std::string fname {argv[1]}; + sc_set(fname); + return 0; } #endif From 86b842b1727c4765146d277069f8d0d600a92c34 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sat, 11 May 2024 10:57:32 +0530 Subject: [PATCH 155/217] GroupKV: Duplicate SimpCfg to chop down into GroupKV IE a minimal MapOfMapOfVariant, with some basic helpers. This can be the basis of a ChatTemplates object, as well as SimpCfg built on top of it. --- common/groupkv.hpp | 681 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 681 insertions(+) create mode 100644 common/groupkv.hpp diff --git a/common/groupkv.hpp b/common/groupkv.hpp new file mode 100644 index 0000000000000..694f822cf40bc --- /dev/null +++ b/common/groupkv.hpp @@ -0,0 +1,681 @@ +#pragma once + +/** + * Provides a simple direct 1-level only config file logic + * by Humans for All + * + * ## File format + * + * It can consist of multiple config groups. + * * the group name needs to start at the begining of the line. + * Each group can inturn contain multiple config fields (key:value pairs) wrt that group. + * * the group fields need to have 1 or more space at the begining of line. + * + * ## Supported data types + * + * The fields can have values belonging to ane one of the below types + * * strings - enclosed in double quotes + * this is also the fallback catch all type, but dont rely on this behaviour. + * * int - using decimal number system + * * float - needs to have a decimal point and or e/E + * if decimal point is used, there should be atleast one decimal number on its either side + * * bool - either true or false + * + * It tries to provide a crude expanded form of array wrt any of the above supported types. + * For this one needs to define keys using the pattern TheKeyName-0, TheKeyName-1, .... + * + * ## Additional notes + * + * NativeCharSize encoded char refers to chars which fit within the size of char type in a given + * type of c++ string or base bitsize of a encoding standard, like 1 byte in case of std::string, + * utf-8, ... + * * example english alphabets in utf-8 encoding space are 1byte chars, in its variable length + * encoding space. + * + * MultiNativeCharSize encoded char refers to chars which occupy multiple base-char-bit-size of + * a c++ string type or char encoding standard. + * * example indian scripts alphabets in utf-8 encoding space occupy multiple bytes in its variable + * length encoding space. + * + * Sane variable length encoding - refers to encoding where the values of NativeCharSized chars of + * a char encoding space cant overlap with values in NativeCharSize subparts of MultiNativeCharSized + * chars of the same char encoding standard. + * * utf-8 shows this behaviour + * * chances are utf-16 and utf-32 also show this behaviour (need to cross check once) + */ + +#include +#include +#include +#include +#include +#include +#include +#include + + +#define SC_DEBUG +#undef SC_DEBUG_VERBOSE + +#define SC_TEST_PRG +#ifdef SC_TEST_PRG +#include +#include +#define LINFO_LN(FMT, ...) fprintf(stdout, FMT"\n", ##__VA_ARGS__) +#define LDBUG(FMT, ...) fprintf(stderr, FMT, ##__VA_ARGS__) +#define LDBUG_LN(FMT, ...) fprintf(stderr, FMT"\n", ##__VA_ARGS__) +#define LERRR_LN(FMT, ...) fprintf(stderr, FMT"\n", ##__VA_ARGS__) +#define LWARN_LN(FMT, ...) fprintf(stderr, FMT"\n", ##__VA_ARGS__) +#else +#include "log.h" +#define LINFO_LN LOG_TEELN +#define LDBUG LOG +#define LDBUG_LN LOGLN +#define LERRR_LN LOG_TEELN +#define LWARN_LN LOG_TEELN +#endif + +#undef SC_STR_OVERSMART +#ifdef SC_STR_OVERSMART +#define str_trim str_trim_oversmart +#else +#define str_trim str_trim_dumb +#endif + + +// **** **** **** String related helpers **** **** **** // + + +size_t wcs_to_mbs(std::string &sDest, const std::wstring &wSrc) { + std::mbstate_t mbState = std::mbstate_t(); + const wchar_t *wSrcP = wSrc.c_str(); + auto reqLen = std::wcsrtombs(nullptr, &wSrcP, 0, &mbState); + sDest.resize(reqLen); + return std::wcsrtombs(sDest.data(), &wSrcP, sDest.length(), &mbState); +} + +size_t mbs_to_wcs(std::wstring &wDest, const std::string &sSrc) { + std::mbstate_t mbState = std::mbstate_t(); + const char *sSrcP = sSrc.c_str(); + auto reqLen = std::mbsrtowcs(nullptr, &sSrcP, 0, &mbState); + wDest.resize(reqLen); + return std::mbsrtowcs(wDest.data(), &sSrcP, wDest.length(), &mbState); +} + +template +void dumphex_string(const TString &sIn, const std::string &msgTag){ + LDBUG("%s[ ", msgTag.c_str()); + for(auto c: sIn) { + auto cSize = sizeof(c); + if (cSize == 1) { + LDBUG("%02x, ", (uint8_t)c); + } else if (cSize == 2) { + LDBUG("%04x, ", (uint16_t)c); + } else if (cSize == 4) { + LDBUG("%08x, ", (uint32_t)c); + } else { + std::stringstream ss; + ss << "ERRR:" << __func__ << ":Unsupported char type with size [" << cSize << "]"; + throw std::runtime_error( ss.str().c_str() ); + } + } + LDBUG_LN(" ]"); +} + +// Remove chars from begin and end of the passed string, provided the char +// belongs to one of the chars in trimChars. +// +// NOTE: This will work perfectly provided the string being trimmed as well as +// chars being trimmed are made up of NativeCharSize chars from same encoded space. +// For utf-8, this means the ascii equivalent 1byteSized chars of utf8 and not +// variable length MultiNativeCharSize (ie multibye in case of utf-8) ones. +// NOTE: It will also work, if atleast either end of string as well as trimChars +// have NativeCharSize chars from their encoding space, rather than variable +// length MultiNativeCharSize based chars if any. There needs to be NativeCharSized +// chars beyond any chars that get trimmed, on either side. +// +// NOTE: Given the way UTF-8 char encoding is designed, where NativeCharSize 1byte +// encoded chars are fully unique and dont overlap with any bytes from any of the +// variable length MultiNativeCharSize encoded chars in the utf-8 space, so as long as +// the trimChars belong to NativeCharSize chars subset, the logic should work, even +// if string has a mixture of NativeCharSize and MultiNativeCharSize encoded chars. +// Chances are utf-16 and utf-32 also have similar characteristics wrt thier +// NativeCharSize encoded chars (ie those fully encoded within single 16bit and 32bit +// value respectively), and so equivalent semantic applies to them also. +// +// ALERT: Given that this simple minded logic, works at individual NativeCharSize level +// only, If trimChars involve variable length MultiNativeCharSize encoded chars, then +// * because different NativeCharSize subparts (bytes in case of utf-8) from different +// MultiNativeCharSize trim chars when clubbed together can map to some other new char +// in a variable length encoded char space, if there is that new char at either end +// of the string, it may get trimmed, because of the possibility of mix up mentioned. +// * given that different variable length MultiNativeCharSize encoded chars may have +// some common NativeCharSize subparts (bytes in case of utf-8) between them, if one +// of these chars is at either end of the string and another char is in trimChars, +// then string may get partially trimmed wrt such a char at either end. +// +template +TString str_trim_dumb(TString sin, const TString &trimChars=" \t\n") { +#ifdef SC_DEBUG_VERBOSE + dumphex_string(sin, "DBUG:StrTrimDumb:Str:"); + dumphex_string(trimChars, "DBUG:StrTrimDumb:TrimChars:"); +#endif + sin.erase(sin.find_last_not_of(trimChars)+1); + sin.erase(0, sin.find_first_not_of(trimChars)); + return sin; +} + +// Remove chars from begin and end of the passed string, provided the char belongs +// to one of the chars in trimChars. +// NOTE: Internally converts to wchar/wstring to try and support proper trimming, +// wrt possibly more languages, to some extent. IE even if the passed string +// contains multibyte encoded characters in it in utf-8 space (ie MultiNativeCharSize), +// it may get converted to NativeCharSize chars in the expanded wchar_t encoding space, +// thus leading to fixed NativeCharSize driven logic itself handling things sufficiently. +// Look at str_trim_dumb comments for additional aspects. +std::string str_trim_oversmart(std::string sIn, const std::string &trimChars=" \t\n") { + std::wstring wIn; + mbs_to_wcs(wIn, sIn); + std::wstring wTrimChars; + mbs_to_wcs(wTrimChars, trimChars); + auto wOut = str_trim_dumb(wIn, wTrimChars); + std::string sOut; + wcs_to_mbs(sOut, wOut); + return sOut; +} + +// Remove atmost 1 char at the begin and 1 char at the end of the passed string, +// provided the char belongs to one of the chars in trimChars. +// +// NOTE: Chars being trimmed (ie in trimChars) needs to be part of NativeCharSize +// subset of the string's encoded char space, to avoid mix up when working with +// strings which can be utf-8/utf-16/utf-32/sane-variable-length encoded strings. +// +// NOTE:UTF8: This will work provided the string being trimmed as well the chars +// being trimmed are made up of 1byte encoded chars in case of utf8 encoding space. +// If the string being trimmed includes multibyte (ie MultiNativeCharSize) encoded +// characters at either end, then trimming can mess things up, if you have multibyte +// encoded utf-8 chars in the trimChars set. +// +// Currently given that SimpCfg only uses this with NativeCharSize chars in the +// trimChars and most of the platforms are likely to be using utf-8 based char +// space (which is a realtively sane variable length char encoding from this +// logics perspective), so not providing oversmart variant. +// +template +TString str_trim_single(TString sin, const TString& trimChars=" \t\n") { + if (sin.empty()) return sin; + for(auto c: trimChars) { + if (c == sin.front()) { + sin = sin.substr(1, TString::npos); + break; + } + } + if (sin.empty()) return sin; + for(auto c: trimChars) { + if (c == sin.back()) { + sin = sin.substr(0, sin.length()-1); + break; + } + } + return sin; +} + +// Convert to lower case, if language has upper and lower case semantic +// +// This works for fixed size encoded char spaces. +// +// For variable length encoded char spaces, it can work +// * if one is doing the conversion for languages which fit into NativeCharSized chars in it +// * AND if one is working with a sane variable length encoding standard +// * ex: this will work if trying to do the conversion for english language within utf-8 +// +template +TString str_tolower(const TString &sin) { + TString sout; + sout.resize(sin.size()); + std::transform(sin.begin(), sin.end(), sout.begin(), [](auto c)->auto {return std::tolower(c);}); +#ifdef SC_DEBUG_VERBOSE + dumphex_string(sin, "DBUG:StrToLower:in:"); + dumphex_string(sout, "DBUG:StrToLower:out:"); +#endif + return sout; +} + +void str_compare_dump(const std::string &s1, const std::string &s2) { + LDBUG_LN("DBUG:%s:%s:Len:%zu", __func__, s1.c_str(), s1.length()); + LDBUG_LN("DBUG:%s:%s:Len:%zu", __func__, s2.c_str(), s2.length()); + int minLen = s1.length() < s2.length() ? s1.length() : s2.length(); + for(int i=0; i +std::string str(TypeWithStrSupp value) { + std::stringstream ss; + ss << value; + return ss.str(); +} + +template +std::string str(std::vector values) { + std::stringstream ss; + ss << "[ "; + int cnt = 0; + for(auto value: values) { + cnt += 1; + if (cnt != 1) ss << ", "; + ss << value; + } + ss << " ]"; + return ss.str(); +} + + +// **** **** **** the SimpCfg **** **** **** // + + +typedef std::variant SimpCfgData; +typedef std::vector MultiPart; +typedef std::map> SimpCfgMapMapVariant; + +class SimpCfg { + +private: + SimpCfgMapMapVariant mapV = {}; + std::regex rInt {R"(^[-+]?\d+$)"}; + std::regex rFloat {R"(^[-+]?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?$)"}; + +public: + + SimpCfg(SimpCfgMapMapVariant defaultMap) : mapV(defaultMap) {} + + static std::string joiner(const MultiPart& parts) { + std::stringstream joined; + int iCnt = 0; + for(auto part: parts) { + if (iCnt != 0) { + joined << "-"; + } + iCnt += 1; + joined << part; + } + return joined.str(); + } + + std::string to_str(const SimpCfgData &value) { + auto visitor = [](auto value) -> auto { + std::stringstream ss; + ss << value; + return ss.str(); + }; + return std::visit(visitor, value); + } + + template + void set_value(const std::string &group, const MultiPart &keyParts, const SupportedDataType &value, const std::string &callerName="") { + auto key = joiner(keyParts); + auto &gm = mapV[group]; + gm[key] = value; +#ifdef SC_DEBUG + LDBUG_LN("DBUG:SC:%s_%s:%s:%s:%s", __func__, callerName.c_str(), group.c_str(), key.c_str(), to_str(value).c_str()); +#endif + } + + void set_string(const std::string &group, const MultiPart &keyParts, const std::string &value) { + set_value(group, keyParts, value, __func__); + } + + void set_bool(const std::string &group, const MultiPart &keyParts, bool value) { + set_value(group, keyParts, value, __func__); + } + + void set_bool(const std::string &group, const MultiPart &keyParts, const std::string &value) { + std::string sValue = str_tolower(value); + bool bValue = sValue == "true" ? true : false; + //LDBUG_LN("DBUG:%s:%s:%s:%d", __func__, value.c_str(), sValue.c_str(), bValue); + set_bool(group, keyParts, bValue); + } + + void set_int64(const std::string &group, const MultiPart &keyParts, int64_t value) { + set_value(group, keyParts, value, __func__); + } + + void set_int64(const std::string &group, const MultiPart &keyParts, std::string &value) { + auto ivalue = strtoll(value.c_str(), nullptr, 0); + set_int64(group, keyParts, ivalue); + } + + void set_double(const std::string &group, const MultiPart &keyParts, double value) { + set_value(group, keyParts, value, __func__); + } + + void set_double(const std::string &group, const MultiPart &keyParts, std::string &value) { + auto dvalue = strtod(value.c_str(), nullptr); + set_double(group, keyParts, dvalue); + } + + // Dump info about the specified group. + // If group is empty, then dump info about all groups maintained in this instance. + void dump(const std::string &group) { + for (auto gm: mapV) { + if (!group.empty() && (gm.first != group)) { + LINFO_LN("INFO:SC:%s:%s:Skipping...", __func__, gm.first.c_str()); + continue; + } + for(auto k: gm.second) { + LINFO_LN("DBUG:SC:%s:%s:Iterate:%s:%s", __func__, gm.first.c_str(), k.first.c_str(), to_str(k.second).c_str()); + } + } + } + + template + SupportedDataType get_value(const std::string &group, const MultiPart &keyParts, const SupportedDataType &defaultValue, const std::string &callerName="") { + auto key = joiner(keyParts); + auto gm = mapV[group]; + if (gm.find(key) == gm.end()) { +#ifdef SC_DEBUG + LWARN_LN("WARN:SC:%s_%s:%s:%s:%s[default]", __func__, callerName.c_str(), group.c_str(), key.c_str(), to_str(defaultValue).c_str()); +#endif + return defaultValue; + } + auto value = gm[key]; +#ifdef SC_DEBUG + LDBUG_LN("DBUG:SC:%s_%s:%s:%s:%s", __func__, callerName.c_str(), group.c_str(), key.c_str(), to_str(value).c_str()); +#endif + return std::get(value); + } + + std::string get_string(const std::string &group, const MultiPart &keyParts, const std::string &defaultValue) { + return get_value(group, keyParts, defaultValue, __func__); + } + + bool get_bool(const std::string &group, const MultiPart &keyParts, bool defaultValue) { + return get_value(group, keyParts, defaultValue, __func__); + } + + int64_t get_int64(const std::string &group, const MultiPart &keyParts, int64_t defaultValue) { + return get_value(group, keyParts, defaultValue, __func__); + } + + double get_double(const std::string &group, const MultiPart &keyParts, double defaultValue) { + return get_value(group, keyParts, defaultValue, __func__); + } + + + template + std::vector get_vector(const std::string &group, const MultiPart &keyParts, const std::vector &defaultValue, const std::string &callerName="") { + auto key = joiner(keyParts); + auto gm = mapV[group]; + std::vector array; + int i = 0; + while(true) { + std::stringstream ssArrayKey; + ssArrayKey << key << "-" << i; + auto arrayKey = ssArrayKey.str(); + if (gm.find(arrayKey) == gm.end()) { + break; + } + array.push_back(std::get(gm[arrayKey])); + i += 1; + } + if (array.empty()) { + LWARN_LN("DBUG:SC:%s_%s:%s:%s:%s[default]", __func__, callerName.c_str(), group.c_str(), key.c_str(), str(defaultValue).c_str()); + return defaultValue; + } + LDBUG_LN("DBUG:SC:%s_%s:%s:%s:%s", __func__, callerName.c_str(), group.c_str(), key.c_str(), str(array).c_str()); + return array; + } + + static void locale_prepare(std::string &sSavedLocale) { + sSavedLocale = std::setlocale(LC_ALL, nullptr); + auto sUpdatedLocale = std::setlocale(LC_ALL, "en_US.UTF-8"); + LDBUG_LN("DBUG:%s:Locale:Prev:%s:Cur:%s", __func__, sSavedLocale.c_str(), sUpdatedLocale); + } + + static void locale_restore(const std::string &sSavedLocale) { + auto sCurLocale = std::setlocale(LC_ALL, sSavedLocale.c_str()); + LDBUG_LN("DBUG:%s:Locale:Requested:%s:Got:%s", __func__, sSavedLocale.c_str(), sCurLocale); + } + + void load(const std::string &fname) { + std::ifstream f {fname}; + if (!f) { + LERRR_LN("ERRR:SC:%s:%s:failed to load...", __func__, fname.c_str()); + throw std::runtime_error { "ERRR:SimpCfg:File not found" }; + } else { + LDBUG_LN("DBUG:SC:%s:%s", __func__, fname.c_str()); + } + std::string group; + int iLine = 0; + while(!f.eof()) { + iLine += 1; + std::string curL; + getline(f, curL); + if (curL.empty()) { + continue; + } + if (curL[0] == '#') { + continue; + } + bool bGroup = !isspace(curL[0]); + curL = str_trim(curL); + if (bGroup) { + curL = str_trim_single(curL, {"\""}); + group = curL; + LDBUG_LN("DBUG:SC:%s:group:%s", __func__, group.c_str()); + continue; + } + auto dPos = curL.find(':'); + if (dPos == std::string::npos) { + LERRR_LN("ERRR:SC:%s:%d:invalid key value line:%s", __func__, iLine, curL.c_str()); + throw std::runtime_error { "ERRR:SimpCfg:Invalid key value line" }; + } + auto dEnd = curL.length() - dPos; + if ((dPos == 0) || (dEnd < 2)) { + LERRR_LN("ERRR:SC:%s:%d:invalid key value line:%s", __func__, iLine, curL.c_str()); + throw std::runtime_error { "ERRR:SimpCfg:Invalid key value line" }; + } + std::string key = curL.substr(0, dPos); + key = str_trim(key); + key = str_trim_single(key, {"\""}); + std::string value = curL.substr(dPos+1); + value = str_trim(value); + value = str_trim(value, {","}); + std::string vtype = "bool"; + auto valueLower = str_tolower(value); + if ((valueLower.compare("true") == 0) || (valueLower == "false")) { + set_bool(group, {key}, value); + } else if (std::regex_match(value, rInt)) { + vtype = "int"; + set_int64(group, {key}, value); + } else if (std::regex_match(value, rFloat)) { + vtype = "float"; + set_double(group, {key}, value); + } else { + vtype = "string"; + if (!value.empty() && (value.front() != '"')) { + LWARN_LN("WARN:SC:%s:%d:%s:k:%s:v:%s:is this string?", __func__, iLine, group.c_str(), key.c_str(), value.c_str()); + } + value = str_trim_single(value, {"\""}); + set_string(group, {key}, value); + } + //LDBUG_LN("DBUG:SC:%s:%d:kv:%s:%s:%s:%s", __func__, iLine, group.c_str(), key.c_str(), vtype.c_str(), value.c_str()); + } + } + +}; + + +#ifdef SC_TEST_PRG + + +// **** **** **** some simple test code **** **** **** // + + +void check_string() { + std::vector vStandard = { "123", "1अ3" }; + std::cout << "**** string **** " << vStandard.size() << std::endl; + for(auto sCur: vStandard) { + std::cout << std::format("string: [{}] len[{}] size[{}]", sCur, sCur.length(), sCur.size()) << std::endl; + int i = 0; + for(auto c: sCur) { + std::cout << std::format("string:{}:pos:{}:char:{}[0x{:x}]\n", sCur, i, c, (uint8_t)c); + i += 1; + } + } +} + +void check_u8string() { + std::vector vU8s = { u8"123", u8"1अ3" }; + std::cout << "**** u8string **** " << vU8s.size() << std::endl; + for(auto sCur: vU8s) { + std::string sCurx (sCur.begin(), sCur.end()); + std::cout << std::format("u8string: [{}] len[{}] size[{}]", sCurx, sCur.length(), sCur.size()) << std::endl; + int i = 0; + for(auto c: sCur) { + //std::cout << c << std::endl; + std::cout << std::format("u8string:{}:pos:{}:char:{}[0x{:x}]\n", sCurx, i, (unsigned char)c, (unsigned char)c); + i += 1; + } + } +} + +void check_wstring_wcout() { + std::wcout.imbue(std::locale("en_US.UTF-8")); + std::vector vWide = { L"123", L"1अ3" }; + std::cout << "**** wstring wcout **** " << vWide.size() << std::endl; + for(auto sCur: vWide) { + std::wcout << sCur << std::endl; + std::wcout << std::format(L"wstring: [{}] len[{}] size[{}]", sCur, sCur.length(), sCur.size()) << std::endl; + int i = 0; + for(auto c: sCur) { + std::wcout << std::format(L"wstring:{}:pos:{}:char:{}[0x{:x}]\n", sCur, i, c, c); + i += 1; + } + } +} + +void check_wstring_cout() { + std::vector vWide = { L"123", L"1अ3" }; + std::cout << "**** wstring cout **** " << vWide.size() << std::endl; + for(auto sCur: vWide) { + std::string sCury; + wcs_to_mbs(sCury, sCur); + std::cout << std::format("wstring: [{}] len[{}] size[{}]", sCury, sCur.length(), sCur.size()) << std::endl; + int i = 0; + for(auto c: sCur) { + std::wstringstream wsc; + wsc << c; + std::string ssc; + wcs_to_mbs(ssc, wsc.str()); + std::cout << std::format("wstring:{}:pos:{}:char:{}[0x{:x}]\n", sCury, i, ssc, (uint32_t)c); + i += 1; + } + } +} + +void check_nonenglish() { + std::cout << "**** non english **** " << std::endl; + std::vector vTest1 = { "\n\tAഅअಅ\n\t", "\n\tAഅअಅ " }; + for (auto sTest: vTest1) { + std::string sGotDumb = str_trim_dumb(sTest, {" \n\t"}); + std::string sGotOSmart = str_trim_oversmart(sTest, {" \n\t"}); + std::string sLower = str_tolower(sTest); + std::cout << std::format("{}: Test1 [{}]\n\tTrimDumb[{}]\n\tTrimOverSmart[{}]\n\tLowerDumb[{}]", __func__, sTest, sGotDumb, sGotOSmart, sLower) << std::endl; + } + // The string "\n\tthis र remove 0s and अs at end 000रअ0\xa4अ ", + // * will mess up str_trim_dumb, + // * but will rightly trigger a exception with oversmart. + std::vector vTest2 = { "\n\t this र remove 0s at end 000 ", "\n\tthis र remove 0s, अs, ഇs at end 000रअ0अ ", "\n\tthis र remove 0s, अs, ഇs at end 000रअ0इअ "}; + std::string trimChars = {" \n\tഇ0अ"}; + for (auto sTest: vTest2) { + std::string sGotDumb = str_trim_dumb(sTest, trimChars); + std::string sGotOSmart = str_trim_oversmart(sTest, trimChars); + std::cout << std::format("{}: Test2 [{}]\n\tDumb[{}]\n\tOverSmart[{}]", __func__, sTest, sGotDumb, sGotOSmart) << std::endl; + } +} + +void check_strings() { + std::string sSavedLocale; + SimpCfg::locale_prepare(sSavedLocale); + check_string(); + check_u8string(); + //check_wstring_wcout(); + check_wstring_cout(); + check_nonenglish(); + SimpCfg::locale_restore(sSavedLocale); +} + +void sc_inited() { + SimpCfg sc = {{ + {"Group1",{ + {"testkey11", 11}, + {"testkey12", true} + }}, + {"Group2", { + {"key21", "val21"}, + {"key22", 22}, + {"key23", 2.3} + }} + }}; + + std::cout << "**** sc inited **** " << std::endl; + sc.dump(""); + +} + +void sc_set(const std::string &fname) { + + std::cout << "**** sc set **** " << std::endl; + SimpCfg sc = {{}}; + sc.load(fname); + sc.dump(""); + + sc.get_bool("testme", {"key101b"}, false); + sc.get_string("testme", {"key101s"}, "Not found"); + sc.get_int64("testme", {"key101i"}, 123456); + sc.get_double("testme", {"key101d"}, 123456.789); + + sc.set_bool("testme", {"key201b"}, true); + sc.set_string("testme", {"key201s"}, "hello world"); + sc.set_int64("testme", {"key201i"}, 987654); + sc.set_double("testme", {"key201d"}, 9988.7766); + + sc.dump("testme"); + sc.get_bool("testme", {"key201b"}, false); + sc.get_string("testme", {"key201s"}, "Not found"); + sc.get_int64("testme", {"key201i"}, 123456); + sc.get_double("testme", {"key201d"}, 123456.789); + + sc.get_string("mistral", {"system-prefix"}, "Not found"); + sc.get_string("\"mistral\"", {"\"system-prefix\""}, "Not found"); + + sc.get_vector("testme", {"keyA100"}, {1, 2, 3}); + sc.get_vector("testme", {"keyA100"}, { "A", "അ", "अ", "ಅ" }); + sc.set_int64("testme", {"keyA300-0"}, 330); + sc.set_int64("testme", {"keyA300-1"}, 331); + sc.set_int64("testme", {"keyA300-2"}, 332); + sc.set_string("testme", {"keyA301-0"}, "India"); + sc.set_value("testme", {"keyA301", "1"}, "World"); + sc.set_string("testme", {"keyA301", "2"}, "AkashaGanga"); + sc.get_vector("testme", {"keyA300"}, {1, 2, 3}); + sc.get_vector("testme", {"keyA301"}, { "yes 1", "No 2", "very well 3" }); +} + +int main(int argc, char **argv) { + if (argc != 2) { + LERRR_LN("USAGE:%s simp.cfg", argv[0]); + exit(1); + } + + check_strings(); + sc_inited(); + std::string fname {argv[1]}; + sc_set(fname); + + return 0; +} +#endif From d764a9d3950e5eb63682c61b56a05d94e80c37f5 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sat, 11 May 2024 11:32:06 +0530 Subject: [PATCH 156/217] GroupKV: Simplify code to the minimal needed for GroupKV - P1 --- common/groupkv.hpp | 499 ++------------------------------------------- 1 file changed, 19 insertions(+), 480 deletions(-) diff --git a/common/groupkv.hpp b/common/groupkv.hpp index 694f822cf40bc..147bb85aefb58 100644 --- a/common/groupkv.hpp +++ b/common/groupkv.hpp @@ -1,64 +1,27 @@ #pragma once /** - * Provides a simple direct 1-level only config file logic + * Allows grouping key-value pairs into groups. * by Humans for All * - * ## File format + * Allows one to maintain a bunch of groups, where each group contains key-value pairs in them. + * The values could belong to any one of these types ie strings, int, float and bool. * - * It can consist of multiple config groups. - * * the group name needs to start at the begining of the line. - * Each group can inturn contain multiple config fields (key:value pairs) wrt that group. - * * the group fields need to have 1 or more space at the begining of line. - * - * ## Supported data types - * - * The fields can have values belonging to ane one of the below types - * * strings - enclosed in double quotes - * this is also the fallback catch all type, but dont rely on this behaviour. - * * int - using decimal number system - * * float - needs to have a decimal point and or e/E - * if decimal point is used, there should be atleast one decimal number on its either side - * * bool - either true or false - * - * It tries to provide a crude expanded form of array wrt any of the above supported types. + * It also tries to provide a crude expanded form of array wrt any of the above supported types. * For this one needs to define keys using the pattern TheKeyName-0, TheKeyName-1, .... - * - * ## Additional notes - * - * NativeCharSize encoded char refers to chars which fit within the size of char type in a given - * type of c++ string or base bitsize of a encoding standard, like 1 byte in case of std::string, - * utf-8, ... - * * example english alphabets in utf-8 encoding space are 1byte chars, in its variable length - * encoding space. - * - * MultiNativeCharSize encoded char refers to chars which occupy multiple base-char-bit-size of - * a c++ string type or char encoding standard. - * * example indian scripts alphabets in utf-8 encoding space occupy multiple bytes in its variable - * length encoding space. - * - * Sane variable length encoding - refers to encoding where the values of NativeCharSized chars of - * a char encoding space cant overlap with values in NativeCharSize subparts of MultiNativeCharSized - * chars of the same char encoding standard. - * * utf-8 shows this behaviour - * * chances are utf-16 and utf-32 also show this behaviour (need to cross check once) */ #include #include #include -#include -#include #include #include -#include -#define SC_DEBUG -#undef SC_DEBUG_VERBOSE +#define GKV_DEBUG -#define SC_TEST_PRG -#ifdef SC_TEST_PRG +#define GKV_TEST_PRG +#ifdef GKV_TEST_PRG #include #include #define LINFO_LN(FMT, ...) fprintf(stdout, FMT"\n", ##__VA_ARGS__) @@ -75,222 +38,20 @@ #define LWARN_LN LOG_TEELN #endif -#undef SC_STR_OVERSMART -#ifdef SC_STR_OVERSMART -#define str_trim str_trim_oversmart -#else -#define str_trim str_trim_dumb -#endif - - -// **** **** **** String related helpers **** **** **** // - - -size_t wcs_to_mbs(std::string &sDest, const std::wstring &wSrc) { - std::mbstate_t mbState = std::mbstate_t(); - const wchar_t *wSrcP = wSrc.c_str(); - auto reqLen = std::wcsrtombs(nullptr, &wSrcP, 0, &mbState); - sDest.resize(reqLen); - return std::wcsrtombs(sDest.data(), &wSrcP, sDest.length(), &mbState); -} - -size_t mbs_to_wcs(std::wstring &wDest, const std::string &sSrc) { - std::mbstate_t mbState = std::mbstate_t(); - const char *sSrcP = sSrc.c_str(); - auto reqLen = std::mbsrtowcs(nullptr, &sSrcP, 0, &mbState); - wDest.resize(reqLen); - return std::mbsrtowcs(wDest.data(), &sSrcP, wDest.length(), &mbState); -} - -template -void dumphex_string(const TString &sIn, const std::string &msgTag){ - LDBUG("%s[ ", msgTag.c_str()); - for(auto c: sIn) { - auto cSize = sizeof(c); - if (cSize == 1) { - LDBUG("%02x, ", (uint8_t)c); - } else if (cSize == 2) { - LDBUG("%04x, ", (uint16_t)c); - } else if (cSize == 4) { - LDBUG("%08x, ", (uint32_t)c); - } else { - std::stringstream ss; - ss << "ERRR:" << __func__ << ":Unsupported char type with size [" << cSize << "]"; - throw std::runtime_error( ss.str().c_str() ); - } - } - LDBUG_LN(" ]"); -} - -// Remove chars from begin and end of the passed string, provided the char -// belongs to one of the chars in trimChars. -// -// NOTE: This will work perfectly provided the string being trimmed as well as -// chars being trimmed are made up of NativeCharSize chars from same encoded space. -// For utf-8, this means the ascii equivalent 1byteSized chars of utf8 and not -// variable length MultiNativeCharSize (ie multibye in case of utf-8) ones. -// NOTE: It will also work, if atleast either end of string as well as trimChars -// have NativeCharSize chars from their encoding space, rather than variable -// length MultiNativeCharSize based chars if any. There needs to be NativeCharSized -// chars beyond any chars that get trimmed, on either side. -// -// NOTE: Given the way UTF-8 char encoding is designed, where NativeCharSize 1byte -// encoded chars are fully unique and dont overlap with any bytes from any of the -// variable length MultiNativeCharSize encoded chars in the utf-8 space, so as long as -// the trimChars belong to NativeCharSize chars subset, the logic should work, even -// if string has a mixture of NativeCharSize and MultiNativeCharSize encoded chars. -// Chances are utf-16 and utf-32 also have similar characteristics wrt thier -// NativeCharSize encoded chars (ie those fully encoded within single 16bit and 32bit -// value respectively), and so equivalent semantic applies to them also. -// -// ALERT: Given that this simple minded logic, works at individual NativeCharSize level -// only, If trimChars involve variable length MultiNativeCharSize encoded chars, then -// * because different NativeCharSize subparts (bytes in case of utf-8) from different -// MultiNativeCharSize trim chars when clubbed together can map to some other new char -// in a variable length encoded char space, if there is that new char at either end -// of the string, it may get trimmed, because of the possibility of mix up mentioned. -// * given that different variable length MultiNativeCharSize encoded chars may have -// some common NativeCharSize subparts (bytes in case of utf-8) between them, if one -// of these chars is at either end of the string and another char is in trimChars, -// then string may get partially trimmed wrt such a char at either end. -// -template -TString str_trim_dumb(TString sin, const TString &trimChars=" \t\n") { -#ifdef SC_DEBUG_VERBOSE - dumphex_string(sin, "DBUG:StrTrimDumb:Str:"); - dumphex_string(trimChars, "DBUG:StrTrimDumb:TrimChars:"); -#endif - sin.erase(sin.find_last_not_of(trimChars)+1); - sin.erase(0, sin.find_first_not_of(trimChars)); - return sin; -} - -// Remove chars from begin and end of the passed string, provided the char belongs -// to one of the chars in trimChars. -// NOTE: Internally converts to wchar/wstring to try and support proper trimming, -// wrt possibly more languages, to some extent. IE even if the passed string -// contains multibyte encoded characters in it in utf-8 space (ie MultiNativeCharSize), -// it may get converted to NativeCharSize chars in the expanded wchar_t encoding space, -// thus leading to fixed NativeCharSize driven logic itself handling things sufficiently. -// Look at str_trim_dumb comments for additional aspects. -std::string str_trim_oversmart(std::string sIn, const std::string &trimChars=" \t\n") { - std::wstring wIn; - mbs_to_wcs(wIn, sIn); - std::wstring wTrimChars; - mbs_to_wcs(wTrimChars, trimChars); - auto wOut = str_trim_dumb(wIn, wTrimChars); - std::string sOut; - wcs_to_mbs(sOut, wOut); - return sOut; -} -// Remove atmost 1 char at the begin and 1 char at the end of the passed string, -// provided the char belongs to one of the chars in trimChars. -// -// NOTE: Chars being trimmed (ie in trimChars) needs to be part of NativeCharSize -// subset of the string's encoded char space, to avoid mix up when working with -// strings which can be utf-8/utf-16/utf-32/sane-variable-length encoded strings. -// -// NOTE:UTF8: This will work provided the string being trimmed as well the chars -// being trimmed are made up of 1byte encoded chars in case of utf8 encoding space. -// If the string being trimmed includes multibyte (ie MultiNativeCharSize) encoded -// characters at either end, then trimming can mess things up, if you have multibyte -// encoded utf-8 chars in the trimChars set. -// -// Currently given that SimpCfg only uses this with NativeCharSize chars in the -// trimChars and most of the platforms are likely to be using utf-8 based char -// space (which is a realtively sane variable length char encoding from this -// logics perspective), so not providing oversmart variant. -// -template -TString str_trim_single(TString sin, const TString& trimChars=" \t\n") { - if (sin.empty()) return sin; - for(auto c: trimChars) { - if (c == sin.front()) { - sin = sin.substr(1, TString::npos); - break; - } - } - if (sin.empty()) return sin; - for(auto c: trimChars) { - if (c == sin.back()) { - sin = sin.substr(0, sin.length()-1); - break; - } - } - return sin; -} - -// Convert to lower case, if language has upper and lower case semantic -// -// This works for fixed size encoded char spaces. -// -// For variable length encoded char spaces, it can work -// * if one is doing the conversion for languages which fit into NativeCharSized chars in it -// * AND if one is working with a sane variable length encoding standard -// * ex: this will work if trying to do the conversion for english language within utf-8 -// -template -TString str_tolower(const TString &sin) { - TString sout; - sout.resize(sin.size()); - std::transform(sin.begin(), sin.end(), sout.begin(), [](auto c)->auto {return std::tolower(c);}); -#ifdef SC_DEBUG_VERBOSE - dumphex_string(sin, "DBUG:StrToLower:in:"); - dumphex_string(sout, "DBUG:StrToLower:out:"); -#endif - return sout; -} - -void str_compare_dump(const std::string &s1, const std::string &s2) { - LDBUG_LN("DBUG:%s:%s:Len:%zu", __func__, s1.c_str(), s1.length()); - LDBUG_LN("DBUG:%s:%s:Len:%zu", __func__, s2.c_str(), s2.length()); - int minLen = s1.length() < s2.length() ? s1.length() : s2.length(); - for(int i=0; i -std::string str(TypeWithStrSupp value) { - std::stringstream ss; - ss << value; - return ss.str(); -} - -template -std::string str(std::vector values) { - std::stringstream ss; - ss << "[ "; - int cnt = 0; - for(auto value: values) { - cnt += 1; - if (cnt != 1) ss << ", "; - ss << value; - } - ss << " ]"; - return ss.str(); -} - - -// **** **** **** the SimpCfg **** **** **** // - - -typedef std::variant SimpCfgData; +typedef std::variant GroupKVData; typedef std::vector MultiPart; -typedef std::map> SimpCfgMapMapVariant; +typedef std::map> GroupKVMapMapVariant; -class SimpCfg { +class GroupKV { private: - SimpCfgMapMapVariant mapV = {}; - std::regex rInt {R"(^[-+]?\d+$)"}; - std::regex rFloat {R"(^[-+]?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?$)"}; + + GroupKVMapMapVariant mapV = {}; public: - SimpCfg(SimpCfgMapMapVariant defaultMap) : mapV(defaultMap) {} + GroupKV(GroupKVMapMapVariant defaultMap) : mapV(defaultMap) {} static std::string joiner(const MultiPart& parts) { std::stringstream joined; @@ -305,7 +66,7 @@ class SimpCfg { return joined.str(); } - std::string to_str(const SimpCfgData &value) { + std::string to_str(const GroupKVData &value) { auto visitor = [](auto value) -> auto { std::stringstream ss; ss << value; @@ -319,44 +80,11 @@ class SimpCfg { auto key = joiner(keyParts); auto &gm = mapV[group]; gm[key] = value; -#ifdef SC_DEBUG +#ifdef GKV_DEBUG LDBUG_LN("DBUG:SC:%s_%s:%s:%s:%s", __func__, callerName.c_str(), group.c_str(), key.c_str(), to_str(value).c_str()); #endif } - void set_string(const std::string &group, const MultiPart &keyParts, const std::string &value) { - set_value(group, keyParts, value, __func__); - } - - void set_bool(const std::string &group, const MultiPart &keyParts, bool value) { - set_value(group, keyParts, value, __func__); - } - - void set_bool(const std::string &group, const MultiPart &keyParts, const std::string &value) { - std::string sValue = str_tolower(value); - bool bValue = sValue == "true" ? true : false; - //LDBUG_LN("DBUG:%s:%s:%s:%d", __func__, value.c_str(), sValue.c_str(), bValue); - set_bool(group, keyParts, bValue); - } - - void set_int64(const std::string &group, const MultiPart &keyParts, int64_t value) { - set_value(group, keyParts, value, __func__); - } - - void set_int64(const std::string &group, const MultiPart &keyParts, std::string &value) { - auto ivalue = strtoll(value.c_str(), nullptr, 0); - set_int64(group, keyParts, ivalue); - } - - void set_double(const std::string &group, const MultiPart &keyParts, double value) { - set_value(group, keyParts, value, __func__); - } - - void set_double(const std::string &group, const MultiPart &keyParts, std::string &value) { - auto dvalue = strtod(value.c_str(), nullptr); - set_double(group, keyParts, dvalue); - } - // Dump info about the specified group. // If group is empty, then dump info about all groups maintained in this instance. void dump(const std::string &group) { @@ -376,35 +104,18 @@ class SimpCfg { auto key = joiner(keyParts); auto gm = mapV[group]; if (gm.find(key) == gm.end()) { -#ifdef SC_DEBUG +#ifdef GKV_DEBUG LWARN_LN("WARN:SC:%s_%s:%s:%s:%s[default]", __func__, callerName.c_str(), group.c_str(), key.c_str(), to_str(defaultValue).c_str()); #endif return defaultValue; } auto value = gm[key]; -#ifdef SC_DEBUG +#ifdef GKV_DEBUG LDBUG_LN("DBUG:SC:%s_%s:%s:%s:%s", __func__, callerName.c_str(), group.c_str(), key.c_str(), to_str(value).c_str()); #endif return std::get(value); } - std::string get_string(const std::string &group, const MultiPart &keyParts, const std::string &defaultValue) { - return get_value(group, keyParts, defaultValue, __func__); - } - - bool get_bool(const std::string &group, const MultiPart &keyParts, bool defaultValue) { - return get_value(group, keyParts, defaultValue, __func__); - } - - int64_t get_int64(const std::string &group, const MultiPart &keyParts, int64_t defaultValue) { - return get_value(group, keyParts, defaultValue, __func__); - } - - double get_double(const std::string &group, const MultiPart &keyParts, double defaultValue) { - return get_value(group, keyParts, defaultValue, __func__); - } - - template std::vector get_vector(const std::string &group, const MultiPart &keyParts, const std::vector &defaultValue, const std::string &callerName="") { auto key = joiner(keyParts); @@ -429,188 +140,17 @@ class SimpCfg { return array; } - static void locale_prepare(std::string &sSavedLocale) { - sSavedLocale = std::setlocale(LC_ALL, nullptr); - auto sUpdatedLocale = std::setlocale(LC_ALL, "en_US.UTF-8"); - LDBUG_LN("DBUG:%s:Locale:Prev:%s:Cur:%s", __func__, sSavedLocale.c_str(), sUpdatedLocale); - } - - static void locale_restore(const std::string &sSavedLocale) { - auto sCurLocale = std::setlocale(LC_ALL, sSavedLocale.c_str()); - LDBUG_LN("DBUG:%s:Locale:Requested:%s:Got:%s", __func__, sSavedLocale.c_str(), sCurLocale); - } - - void load(const std::string &fname) { - std::ifstream f {fname}; - if (!f) { - LERRR_LN("ERRR:SC:%s:%s:failed to load...", __func__, fname.c_str()); - throw std::runtime_error { "ERRR:SimpCfg:File not found" }; - } else { - LDBUG_LN("DBUG:SC:%s:%s", __func__, fname.c_str()); - } - std::string group; - int iLine = 0; - while(!f.eof()) { - iLine += 1; - std::string curL; - getline(f, curL); - if (curL.empty()) { - continue; - } - if (curL[0] == '#') { - continue; - } - bool bGroup = !isspace(curL[0]); - curL = str_trim(curL); - if (bGroup) { - curL = str_trim_single(curL, {"\""}); - group = curL; - LDBUG_LN("DBUG:SC:%s:group:%s", __func__, group.c_str()); - continue; - } - auto dPos = curL.find(':'); - if (dPos == std::string::npos) { - LERRR_LN("ERRR:SC:%s:%d:invalid key value line:%s", __func__, iLine, curL.c_str()); - throw std::runtime_error { "ERRR:SimpCfg:Invalid key value line" }; - } - auto dEnd = curL.length() - dPos; - if ((dPos == 0) || (dEnd < 2)) { - LERRR_LN("ERRR:SC:%s:%d:invalid key value line:%s", __func__, iLine, curL.c_str()); - throw std::runtime_error { "ERRR:SimpCfg:Invalid key value line" }; - } - std::string key = curL.substr(0, dPos); - key = str_trim(key); - key = str_trim_single(key, {"\""}); - std::string value = curL.substr(dPos+1); - value = str_trim(value); - value = str_trim(value, {","}); - std::string vtype = "bool"; - auto valueLower = str_tolower(value); - if ((valueLower.compare("true") == 0) || (valueLower == "false")) { - set_bool(group, {key}, value); - } else if (std::regex_match(value, rInt)) { - vtype = "int"; - set_int64(group, {key}, value); - } else if (std::regex_match(value, rFloat)) { - vtype = "float"; - set_double(group, {key}, value); - } else { - vtype = "string"; - if (!value.empty() && (value.front() != '"')) { - LWARN_LN("WARN:SC:%s:%d:%s:k:%s:v:%s:is this string?", __func__, iLine, group.c_str(), key.c_str(), value.c_str()); - } - value = str_trim_single(value, {"\""}); - set_string(group, {key}, value); - } - //LDBUG_LN("DBUG:SC:%s:%d:kv:%s:%s:%s:%s", __func__, iLine, group.c_str(), key.c_str(), vtype.c_str(), value.c_str()); - } - } - }; -#ifdef SC_TEST_PRG +#ifdef GKV_TEST_PRG // **** **** **** some simple test code **** **** **** // -void check_string() { - std::vector vStandard = { "123", "1अ3" }; - std::cout << "**** string **** " << vStandard.size() << std::endl; - for(auto sCur: vStandard) { - std::cout << std::format("string: [{}] len[{}] size[{}]", sCur, sCur.length(), sCur.size()) << std::endl; - int i = 0; - for(auto c: sCur) { - std::cout << std::format("string:{}:pos:{}:char:{}[0x{:x}]\n", sCur, i, c, (uint8_t)c); - i += 1; - } - } -} - -void check_u8string() { - std::vector vU8s = { u8"123", u8"1अ3" }; - std::cout << "**** u8string **** " << vU8s.size() << std::endl; - for(auto sCur: vU8s) { - std::string sCurx (sCur.begin(), sCur.end()); - std::cout << std::format("u8string: [{}] len[{}] size[{}]", sCurx, sCur.length(), sCur.size()) << std::endl; - int i = 0; - for(auto c: sCur) { - //std::cout << c << std::endl; - std::cout << std::format("u8string:{}:pos:{}:char:{}[0x{:x}]\n", sCurx, i, (unsigned char)c, (unsigned char)c); - i += 1; - } - } -} - -void check_wstring_wcout() { - std::wcout.imbue(std::locale("en_US.UTF-8")); - std::vector vWide = { L"123", L"1अ3" }; - std::cout << "**** wstring wcout **** " << vWide.size() << std::endl; - for(auto sCur: vWide) { - std::wcout << sCur << std::endl; - std::wcout << std::format(L"wstring: [{}] len[{}] size[{}]", sCur, sCur.length(), sCur.size()) << std::endl; - int i = 0; - for(auto c: sCur) { - std::wcout << std::format(L"wstring:{}:pos:{}:char:{}[0x{:x}]\n", sCur, i, c, c); - i += 1; - } - } -} - -void check_wstring_cout() { - std::vector vWide = { L"123", L"1अ3" }; - std::cout << "**** wstring cout **** " << vWide.size() << std::endl; - for(auto sCur: vWide) { - std::string sCury; - wcs_to_mbs(sCury, sCur); - std::cout << std::format("wstring: [{}] len[{}] size[{}]", sCury, sCur.length(), sCur.size()) << std::endl; - int i = 0; - for(auto c: sCur) { - std::wstringstream wsc; - wsc << c; - std::string ssc; - wcs_to_mbs(ssc, wsc.str()); - std::cout << std::format("wstring:{}:pos:{}:char:{}[0x{:x}]\n", sCury, i, ssc, (uint32_t)c); - i += 1; - } - } -} - -void check_nonenglish() { - std::cout << "**** non english **** " << std::endl; - std::vector vTest1 = { "\n\tAഅअಅ\n\t", "\n\tAഅअಅ " }; - for (auto sTest: vTest1) { - std::string sGotDumb = str_trim_dumb(sTest, {" \n\t"}); - std::string sGotOSmart = str_trim_oversmart(sTest, {" \n\t"}); - std::string sLower = str_tolower(sTest); - std::cout << std::format("{}: Test1 [{}]\n\tTrimDumb[{}]\n\tTrimOverSmart[{}]\n\tLowerDumb[{}]", __func__, sTest, sGotDumb, sGotOSmart, sLower) << std::endl; - } - // The string "\n\tthis र remove 0s and अs at end 000रअ0\xa4अ ", - // * will mess up str_trim_dumb, - // * but will rightly trigger a exception with oversmart. - std::vector vTest2 = { "\n\t this र remove 0s at end 000 ", "\n\tthis र remove 0s, अs, ഇs at end 000रअ0अ ", "\n\tthis र remove 0s, अs, ഇs at end 000रअ0इअ "}; - std::string trimChars = {" \n\tഇ0अ"}; - for (auto sTest: vTest2) { - std::string sGotDumb = str_trim_dumb(sTest, trimChars); - std::string sGotOSmart = str_trim_oversmart(sTest, trimChars); - std::cout << std::format("{}: Test2 [{}]\n\tDumb[{}]\n\tOverSmart[{}]", __func__, sTest, sGotDumb, sGotOSmart) << std::endl; - } -} - -void check_strings() { - std::string sSavedLocale; - SimpCfg::locale_prepare(sSavedLocale); - check_string(); - check_u8string(); - //check_wstring_wcout(); - check_wstring_cout(); - check_nonenglish(); - SimpCfg::locale_restore(sSavedLocale); -} - void sc_inited() { - SimpCfg sc = {{ + GroupKV sc = {{ {"Group1",{ {"testkey11", 11}, {"testkey12", true} @@ -671,7 +211,6 @@ int main(int argc, char **argv) { exit(1); } - check_strings(); sc_inited(); std::string fname {argv[1]}; sc_set(fname); From 7d7c59ec50a139c5c17cf53bd0ebede28160c052 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sat, 11 May 2024 11:57:27 +0530 Subject: [PATCH 157/217] GroupKV:Simplify:P2: Rename tags, Make debug logs conditional Rename all the log messages to have GKV and not SC. The log messages in get_vector made conditional to GKV_DEBUG, this was missed out earlier in simpcfg itself. --- common/groupkv.hpp | 106 ++++++++++++++++++++------------------------- 1 file changed, 48 insertions(+), 58 deletions(-) diff --git a/common/groupkv.hpp b/common/groupkv.hpp index 147bb85aefb58..4902cf997e7b7 100644 --- a/common/groupkv.hpp +++ b/common/groupkv.hpp @@ -81,7 +81,7 @@ class GroupKV { auto &gm = mapV[group]; gm[key] = value; #ifdef GKV_DEBUG - LDBUG_LN("DBUG:SC:%s_%s:%s:%s:%s", __func__, callerName.c_str(), group.c_str(), key.c_str(), to_str(value).c_str()); + LDBUG_LN("DBUG:GKV:%s_%s:%s:%s:%s", __func__, callerName.c_str(), group.c_str(), key.c_str(), to_str(value).c_str()); #endif } @@ -90,11 +90,11 @@ class GroupKV { void dump(const std::string &group) { for (auto gm: mapV) { if (!group.empty() && (gm.first != group)) { - LINFO_LN("INFO:SC:%s:%s:Skipping...", __func__, gm.first.c_str()); + LINFO_LN("INFO:GKV:%s:%s:Skipping...", __func__, gm.first.c_str()); continue; } for(auto k: gm.second) { - LINFO_LN("DBUG:SC:%s:%s:Iterate:%s:%s", __func__, gm.first.c_str(), k.first.c_str(), to_str(k.second).c_str()); + LINFO_LN("DBUG:GKV:%s:%s:Iterate:%s:%s", __func__, gm.first.c_str(), k.first.c_str(), to_str(k.second).c_str()); } } } @@ -105,13 +105,13 @@ class GroupKV { auto gm = mapV[group]; if (gm.find(key) == gm.end()) { #ifdef GKV_DEBUG - LWARN_LN("WARN:SC:%s_%s:%s:%s:%s[default]", __func__, callerName.c_str(), group.c_str(), key.c_str(), to_str(defaultValue).c_str()); + LWARN_LN("WARN:GKV:%s_%s:%s:%s:%s[default]", __func__, callerName.c_str(), group.c_str(), key.c_str(), to_str(defaultValue).c_str()); #endif return defaultValue; } auto value = gm[key]; #ifdef GKV_DEBUG - LDBUG_LN("DBUG:SC:%s_%s:%s:%s:%s", __func__, callerName.c_str(), group.c_str(), key.c_str(), to_str(value).c_str()); + LDBUG_LN("DBUG:GKV:%s_%s:%s:%s:%s", __func__, callerName.c_str(), group.c_str(), key.c_str(), to_str(value).c_str()); #endif return std::get(value); } @@ -133,10 +133,14 @@ class GroupKV { i += 1; } if (array.empty()) { - LWARN_LN("DBUG:SC:%s_%s:%s:%s:%s[default]", __func__, callerName.c_str(), group.c_str(), key.c_str(), str(defaultValue).c_str()); +#ifdef GKV_DEBUG + LWARN_LN("DBUG:GKV:%s_%s:%s:%s:%s[default]", __func__, callerName.c_str(), group.c_str(), key.c_str(), str(defaultValue).c_str()); +#endif return defaultValue; } +#ifdef GKV_DEBUG LDBUG_LN("DBUG:SC:%s_%s:%s:%s:%s", __func__, callerName.c_str(), group.c_str(), key.c_str(), str(array).c_str()); +#endif return array; } @@ -146,11 +150,8 @@ class GroupKV { #ifdef GKV_TEST_PRG -// **** **** **** some simple test code **** **** **** // - - -void sc_inited() { - GroupKV sc = {{ +void gkv_inited() { + GroupKV gkv = {{ {"Group1",{ {"testkey11", 11}, {"testkey12", true} @@ -162,59 +163,48 @@ void sc_inited() { }} }}; - std::cout << "**** sc inited **** " << std::endl; - sc.dump(""); + std::cout << "**** gkv inited **** " << std::endl; + gkv.dump(""); } -void sc_set(const std::string &fname) { - - std::cout << "**** sc set **** " << std::endl; - SimpCfg sc = {{}}; - sc.load(fname); - sc.dump(""); - - sc.get_bool("testme", {"key101b"}, false); - sc.get_string("testme", {"key101s"}, "Not found"); - sc.get_int64("testme", {"key101i"}, 123456); - sc.get_double("testme", {"key101d"}, 123456.789); - - sc.set_bool("testme", {"key201b"}, true); - sc.set_string("testme", {"key201s"}, "hello world"); - sc.set_int64("testme", {"key201i"}, 987654); - sc.set_double("testme", {"key201d"}, 9988.7766); - - sc.dump("testme"); - sc.get_bool("testme", {"key201b"}, false); - sc.get_string("testme", {"key201s"}, "Not found"); - sc.get_int64("testme", {"key201i"}, 123456); - sc.get_double("testme", {"key201d"}, 123456.789); - - sc.get_string("mistral", {"system-prefix"}, "Not found"); - sc.get_string("\"mistral\"", {"\"system-prefix\""}, "Not found"); - - sc.get_vector("testme", {"keyA100"}, {1, 2, 3}); - sc.get_vector("testme", {"keyA100"}, { "A", "അ", "अ", "ಅ" }); - sc.set_int64("testme", {"keyA300-0"}, 330); - sc.set_int64("testme", {"keyA300-1"}, 331); - sc.set_int64("testme", {"keyA300-2"}, 332); - sc.set_string("testme", {"keyA301-0"}, "India"); - sc.set_value("testme", {"keyA301", "1"}, "World"); - sc.set_string("testme", {"keyA301", "2"}, "AkashaGanga"); - sc.get_vector("testme", {"keyA300"}, {1, 2, 3}); - sc.get_vector("testme", {"keyA301"}, { "yes 1", "No 2", "very well 3" }); +void gkv_set() { + + std::cout << "**** gkv set **** " << std::endl; + GroupKV gkv = {{}}; + gkv.dump(""); + + gkv.get_value("testme", {"key101b"}, false); + gkv.get_value("testme", {"key101s"}, "Not found"); + gkv.get_value("testme", {"key101i"}, 123456); + gkv.get_value("testme", {"key101d"}, 123456.789); + + gkv.set_value("testme", {"key201b"}, true); + gkv.set_value("testme", {"key201s"}, "hello world"); + gkv.set_value("testme", {"key201i"}, 987654); + gkv.set_value("testme", {"key201d"}, 9988.7766); + + gkv.dump("testme"); + gkv.get_value("testme", {"key201b"}, false); + gkv.get_value("testme", {"key201s"}, "Not found"); + gkv.get_value("testme", {"key201i"}, 123456); + gkv.get_value("testme", {"key201d"}, 123456.789); + + gkv.get_vector("testme", {"keyA100"}, {1, 2, 3}); + gkv.get_vector("testme", {"keyA100"}, { "A", "അ", "अ", "ಅ" }); + gkv.set_value("testme", {"keyA300-0"}, 330); + gkv.set_value("testme", {"keyA300-1"}, 331); + gkv.set_value("testme", {"keyA300-2"}, 332); + gkv.set_value("testme", {"keyA301-0"}, "India"); + gkv.set_value("testme", {"keyA301", "1"}, "World"); + gkv.set_value("testme", {"keyA301", "2"}, "AkashaGanga"); + gkv.get_vector("testme", {"keyA300"}, {1, 2, 3}); + gkv.get_vector("testme", {"keyA301"}, { "yes 1", "No 2", "very well 3" }); } int main(int argc, char **argv) { - if (argc != 2) { - LERRR_LN("USAGE:%s simp.cfg", argv[0]); - exit(1); - } - - sc_inited(); - std::string fname {argv[1]}; - sc_set(fname); - + gkv_inited(); + gkv_set(); return 0; } #endif From 0342124946fbd30598f141e0c15738d7e519f9c1 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sat, 11 May 2024 12:27:42 +0530 Subject: [PATCH 158/217] GroupKV: Add to_str wrt vectors, help avoid compiler confusion --- common/groupkv.hpp | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/common/groupkv.hpp b/common/groupkv.hpp index 4902cf997e7b7..748016d793226 100644 --- a/common/groupkv.hpp +++ b/common/groupkv.hpp @@ -75,6 +75,21 @@ class GroupKV { return std::visit(visitor, value); } + template + std::string to_str(std::vector values) { + std::stringstream ss; + ss << "[ "; + int cnt = 0; + for(auto value: values) { + cnt += 1; + if (cnt != 1) ss << ", "; + ss << value; + } + ss << " ]"; + return ss.str(); + } + + template void set_value(const std::string &group, const MultiPart &keyParts, const SupportedDataType &value, const std::string &callerName="") { auto key = joiner(keyParts); @@ -134,12 +149,12 @@ class GroupKV { } if (array.empty()) { #ifdef GKV_DEBUG - LWARN_LN("DBUG:GKV:%s_%s:%s:%s:%s[default]", __func__, callerName.c_str(), group.c_str(), key.c_str(), str(defaultValue).c_str()); + LWARN_LN("DBUG:GKV:%s_%s:%s:%s:%s[default]", __func__, callerName.c_str(), group.c_str(), key.c_str(), to_str(defaultValue).c_str()); #endif return defaultValue; } #ifdef GKV_DEBUG - LDBUG_LN("DBUG:SC:%s_%s:%s:%s:%s", __func__, callerName.c_str(), group.c_str(), key.c_str(), str(array).c_str()); + LDBUG_LN("DBUG:SC:%s_%s:%s:%s:%s", __func__, callerName.c_str(), group.c_str(), key.c_str(), to_str(array).c_str()); #endif return array; } @@ -176,7 +191,7 @@ void gkv_set() { gkv.get_value("testme", {"key101b"}, false); gkv.get_value("testme", {"key101s"}, "Not found"); - gkv.get_value("testme", {"key101i"}, 123456); + gkv.get_value("testme", {"key101i"}, 123456); gkv.get_value("testme", {"key101d"}, 123456.789); gkv.set_value("testme", {"key201b"}, true); @@ -187,7 +202,7 @@ void gkv_set() { gkv.dump("testme"); gkv.get_value("testme", {"key201b"}, false); gkv.get_value("testme", {"key201s"}, "Not found"); - gkv.get_value("testme", {"key201i"}, 123456); + gkv.get_value("testme", {"key201i"}, 123456LL); gkv.get_value("testme", {"key201d"}, 123456.789); gkv.get_vector("testme", {"keyA100"}, {1, 2, 3}); From 7f03dd0d4bdbf0270d2e33f2947543c1559e2607 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sat, 11 May 2024 12:45:58 +0530 Subject: [PATCH 159/217] GroupKV: Add int32_t to variant list, to simplify int use So that no need to explicitly specify or LL wrt int literals, which dont need 64bit space by default. Which also means one shouldnt/cant mix up type of value stored and default type specified when getting. --- common/groupkv.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/common/groupkv.hpp b/common/groupkv.hpp index 748016d793226..1fd5f3e52753d 100644 --- a/common/groupkv.hpp +++ b/common/groupkv.hpp @@ -39,7 +39,7 @@ #endif -typedef std::variant GroupKVData; +typedef std::variant GroupKVData; typedef std::vector MultiPart; typedef std::map> GroupKVMapMapVariant; @@ -191,7 +191,7 @@ void gkv_set() { gkv.get_value("testme", {"key101b"}, false); gkv.get_value("testme", {"key101s"}, "Not found"); - gkv.get_value("testme", {"key101i"}, 123456); + gkv.get_value("testme", {"key101i"}, 123456); gkv.get_value("testme", {"key101d"}, 123456.789); gkv.set_value("testme", {"key201b"}, true); @@ -202,7 +202,7 @@ void gkv_set() { gkv.dump("testme"); gkv.get_value("testme", {"key201b"}, false); gkv.get_value("testme", {"key201s"}, "Not found"); - gkv.get_value("testme", {"key201i"}, 123456LL); + gkv.get_value("testme", {"key201i"}, 123456); gkv.get_value("testme", {"key201d"}, 123456.789); gkv.get_vector("testme", {"keyA100"}, {1, 2, 3}); @@ -213,7 +213,7 @@ void gkv_set() { gkv.set_value("testme", {"keyA301-0"}, "India"); gkv.set_value("testme", {"keyA301", "1"}, "World"); gkv.set_value("testme", {"keyA301", "2"}, "AkashaGanga"); - gkv.get_vector("testme", {"keyA300"}, {1, 2, 3}); + gkv.get_vector("testme", {"keyA300"}, {1, 2, 3}); gkv.get_vector("testme", {"keyA301"}, { "yes 1", "No 2", "very well 3" }); } From fdefb39518bb44137e9a161c90348603e76e2fc9 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sat, 11 May 2024 13:21:41 +0530 Subject: [PATCH 160/217] GroupKV:Make LDBUG macros conditional, avoid condition at usage site Also change LWARN to LDBUG wrt previously GKV_DEBUG conditional code --- common/groupkv.hpp | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/common/groupkv.hpp b/common/groupkv.hpp index 1fd5f3e52753d..407b03496758d 100644 --- a/common/groupkv.hpp +++ b/common/groupkv.hpp @@ -25,15 +25,25 @@ #include #include #define LINFO_LN(FMT, ...) fprintf(stdout, FMT"\n", ##__VA_ARGS__) +#ifdef GKV_DEBUG #define LDBUG(FMT, ...) fprintf(stderr, FMT, ##__VA_ARGS__) #define LDBUG_LN(FMT, ...) fprintf(stderr, FMT"\n", ##__VA_ARGS__) +#else +#define LDBUG_LN(...) +#define LDBUG(...) +#endif #define LERRR_LN(FMT, ...) fprintf(stderr, FMT"\n", ##__VA_ARGS__) #define LWARN_LN(FMT, ...) fprintf(stderr, FMT"\n", ##__VA_ARGS__) #else #include "log.h" #define LINFO_LN LOG_TEELN +#ifdef GKV_DEBUG #define LDBUG LOG #define LDBUG_LN LOGLN +#else +#define LDBUG_LN(...) +#define LDBUG(...) +#endif #define LERRR_LN LOG_TEELN #define LWARN_LN LOG_TEELN #endif @@ -95,9 +105,7 @@ class GroupKV { auto key = joiner(keyParts); auto &gm = mapV[group]; gm[key] = value; -#ifdef GKV_DEBUG LDBUG_LN("DBUG:GKV:%s_%s:%s:%s:%s", __func__, callerName.c_str(), group.c_str(), key.c_str(), to_str(value).c_str()); -#endif } // Dump info about the specified group. @@ -119,15 +127,11 @@ class GroupKV { auto key = joiner(keyParts); auto gm = mapV[group]; if (gm.find(key) == gm.end()) { -#ifdef GKV_DEBUG - LWARN_LN("WARN:GKV:%s_%s:%s:%s:%s[default]", __func__, callerName.c_str(), group.c_str(), key.c_str(), to_str(defaultValue).c_str()); -#endif + LDBUG_LN("WARN:GKV:%s_%s:%s:%s:%s[default]", __func__, callerName.c_str(), group.c_str(), key.c_str(), to_str(defaultValue).c_str()); return defaultValue; } auto value = gm[key]; -#ifdef GKV_DEBUG LDBUG_LN("DBUG:GKV:%s_%s:%s:%s:%s", __func__, callerName.c_str(), group.c_str(), key.c_str(), to_str(value).c_str()); -#endif return std::get(value); } @@ -148,14 +152,10 @@ class GroupKV { i += 1; } if (array.empty()) { -#ifdef GKV_DEBUG - LWARN_LN("DBUG:GKV:%s_%s:%s:%s:%s[default]", __func__, callerName.c_str(), group.c_str(), key.c_str(), to_str(defaultValue).c_str()); -#endif + LDBUG_LN("WARN:GKV:%s_%s:%s:%s:%s[default]", __func__, callerName.c_str(), group.c_str(), key.c_str(), to_str(defaultValue).c_str()); return defaultValue; } -#ifdef GKV_DEBUG - LDBUG_LN("DBUG:SC:%s_%s:%s:%s:%s", __func__, callerName.c_str(), group.c_str(), key.c_str(), to_str(array).c_str()); -#endif + LDBUG_LN("DBUG:GKV:%s_%s:%s:%s:%s", __func__, callerName.c_str(), group.c_str(), key.c_str(), to_str(array).c_str()); return array; } From dde72df9d34b2880ec809f7f96097c87641766a9 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sat, 11 May 2024 18:23:06 +0530 Subject: [PATCH 161/217] GroupKV: Rename the internal map --- common/groupkv.hpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/common/groupkv.hpp b/common/groupkv.hpp index 407b03496758d..7459c79b0a117 100644 --- a/common/groupkv.hpp +++ b/common/groupkv.hpp @@ -57,11 +57,11 @@ class GroupKV { private: - GroupKVMapMapVariant mapV = {}; + GroupKVMapMapVariant gkv = {}; public: - GroupKV(GroupKVMapMapVariant defaultMap) : mapV(defaultMap) {} + GroupKV(GroupKVMapMapVariant defaultMap) : gkv(defaultMap) {} static std::string joiner(const MultiPart& parts) { std::stringstream joined; @@ -103,7 +103,7 @@ class GroupKV { template void set_value(const std::string &group, const MultiPart &keyParts, const SupportedDataType &value, const std::string &callerName="") { auto key = joiner(keyParts); - auto &gm = mapV[group]; + auto &gm = gkv[group]; gm[key] = value; LDBUG_LN("DBUG:GKV:%s_%s:%s:%s:%s", __func__, callerName.c_str(), group.c_str(), key.c_str(), to_str(value).c_str()); } @@ -111,7 +111,7 @@ class GroupKV { // Dump info about the specified group. // If group is empty, then dump info about all groups maintained in this instance. void dump(const std::string &group) { - for (auto gm: mapV) { + for (auto gm: gkv) { if (!group.empty() && (gm.first != group)) { LINFO_LN("INFO:GKV:%s:%s:Skipping...", __func__, gm.first.c_str()); continue; @@ -125,7 +125,7 @@ class GroupKV { template SupportedDataType get_value(const std::string &group, const MultiPart &keyParts, const SupportedDataType &defaultValue, const std::string &callerName="") { auto key = joiner(keyParts); - auto gm = mapV[group]; + auto gm = gkv[group]; if (gm.find(key) == gm.end()) { LDBUG_LN("WARN:GKV:%s_%s:%s:%s:%s[default]", __func__, callerName.c_str(), group.c_str(), key.c_str(), to_str(defaultValue).c_str()); return defaultValue; @@ -138,7 +138,7 @@ class GroupKV { template std::vector get_vector(const std::string &group, const MultiPart &keyParts, const std::vector &defaultValue, const std::string &callerName="") { auto key = joiner(keyParts); - auto gm = mapV[group]; + auto gm = gkv[group]; std::vector array; int i = 0; while(true) { From f294fddf430452e33c9c3284ce3d9c5df316b981 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sat, 11 May 2024 19:18:19 +0530 Subject: [PATCH 162/217] GroupKV: Add group_exists checker --- common/groupkv.hpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/common/groupkv.hpp b/common/groupkv.hpp index 7459c79b0a117..fbaa9dcba03a3 100644 --- a/common/groupkv.hpp +++ b/common/groupkv.hpp @@ -99,6 +99,12 @@ class GroupKV { return ss.str(); } + bool group_exists(const std::string &group) { + if (gkv.find(group) == gkv.end()) { + return false; + } + return true; + } template void set_value(const std::string &group, const MultiPart &keyParts, const SupportedDataType &value, const std::string &callerName="") { From e999934e91c5fab57af46cc5dfebdd5b18c906d5 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sat, 11 May 2024 19:18:50 +0530 Subject: [PATCH 163/217] ChatON:WIP: initial go at GroupKV based flow, instead of json --- common/chaton.hpp | 71 +++++++++++++++++++++++------------------------ 1 file changed, 34 insertions(+), 37 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index 8e2983cdf81b3..f5dfde1102687 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -192,7 +192,7 @@ #include #include -#include +#include #include "log.h" #include "llama.h" @@ -213,10 +213,12 @@ const auto K_SYSTEMUSER_1ST_USER_HAS_BEGIN = "systemuser-1st-user-has-begin"; const auto K_SYSTEMUSER_1ST_USER_HAS_PREFIX = "systemuser-1st-user-has-prefix"; const auto K_REVERSE_PROMPT = "reverse-prompt"; - +#define CHATON_JSON +#ifdef CHATON_JSON +#include using json = nlohmann::ordered_json; +#endif -json conMeta; /** @@ -296,37 +298,40 @@ class ChatParts { }; -inline bool chaton_meta_load(std::string &fname) { - if (conMeta != nullptr) { - LOGXLN("WARN:%s:ChatOn Meta: overwriting???", __func__); - } + +class ChatTemplates : public GroupKV { + +}; + +ChatTemplates gCT; + +#ifdef CHATON_JSON + +inline bool chaton_meta_load(const std::string &fname) { std::ifstream f(fname); - conMeta = json::parse(f); + json conMeta = json::parse(f); + for(auto curTmpl: conMeta) { + curTmpl.dump(); + } return true; } +#endif + + inline bool chaton_tmpl_exists(const std::string &tmpl) { - if (conMeta == nullptr) { - LOG_TEELN("ERRR:%s:ChatOnMeta: Not loaded yet...", __func__); - return false; - } - try { - auto tmplData = conMeta[tmpl]; - return true; - } catch (json::exception &err) { - LOG_TEELN("WARN:%s:ChatOnMeta: tmpl[%s] not found...", __func__, tmpl.c_str()); + if (!gCT.group_exists(tmpl)) { + LOG_TEELN("WARN:%s: tmpl[%s] not found...", __func__, tmpl.c_str()); return false; } + return true; } inline std::string chaton_tmpl_role_kv(const std::string &tmpl, const std::string &role, const std::vector &keys) { std::string got = ""; std::string sKeys = ""; for(auto key: keys) { - try { - got += conMeta[tmpl][role][key]; - } catch (json::exception &err) { - } + got += gCT.get_value(tmpl, {role, key}, ""); sKeys += "+"; sKeys += key; } @@ -335,13 +340,13 @@ inline std::string chaton_tmpl_role_kv(const std::string &tmpl, const std::strin } inline std::string chaton_tmpl_kv(const std::string &tmpl, const std::string &key) { - std::string got = conMeta[tmpl][key]; + std::string got = gCT.get_value(tmpl, {key}, ""); LOGLN("DBUG:%s:%s:%s:%s", __func__, tmpl.c_str(), key.c_str(), got.c_str()); return got; } inline bool chaton_tmpl_kv_bool(const std::string &tmpl, const std::string &key) { - bool got = conMeta[tmpl][key]; + bool got = gCT.get_value(tmpl, {key}, false); LOGLN("DBUG:%s:%s:%s:%d", __func__, tmpl.c_str(), key.c_str(), got); return got; } @@ -471,10 +476,10 @@ inline bool chaton_tmpl_apply_ex( } else if (role == K_USER) { cntUser += 1; if ((cntSystem == 1) && (cntUser == 1)) { - if (conMeta[tmpl][K_SYSTEMUSER_1ST_USER_HAS_BEGIN]) { + if (chaton_tmpl_kv_bool(tmpl, K_SYSTEMUSER_1ST_USER_HAS_BEGIN)) { cp.add_part(ChatParts::S, begin); } - if (conMeta[tmpl][K_SYSTEMUSER_1ST_USER_HAS_PREFIX]) { + if (chaton_tmpl_kv_bool(tmpl, K_SYSTEMUSER_1ST_USER_HAS_PREFIX)) { cp.add_part(ChatParts::S, prefix); } } else { @@ -687,16 +692,13 @@ inline std::vector chaton_llama_tokenize_ex( * * chaton-template-id, then dump contents related to that specific chat-handshake-template-standard */ inline bool _chaton_meta_dump(std::string &tmpl) { - json theJson; - if (tmpl.empty()) { - theJson = conMeta; - } else { - theJson = conMeta[tmpl]; - if (theJson.empty()) { + if (!tmpl.empty()) { + if (!gCT.group_exists(tmpl)) { LOGXLN("ERRR:%s:Specified template-id [%s] not found", __func__, tmpl.c_str()); return false; } } + gCT.dump(tmpl); LOGXLN("\n\nINFO:%s:ChatOn Meta:%s:\n%s", __func__, tmpl.c_str(), theJson.dump(4).c_str()); if (!tmpl.empty()) { std::string globalBegin = conMeta[tmpl][K_GLOBAL][K_BEGIN]; @@ -745,13 +747,8 @@ inline bool _chaton_meta_dump(std::string &tmpl) { } /** - * Check that a meta-json file has been loaded. - * Verify that specified chaton-template-id contains required fields in meta-json, using meta-dump + * Verify that specified chaton-template-id contains required fields using meta-dump */ inline bool chaton_meta_ok(std::string &tmpl) { - if (conMeta == nullptr) { - LOG_TEELN("ERRR:%s:%s:ChatOn Meta: Not loaded yet...", __func__, tmpl.c_str()); - return false; - } return _chaton_meta_dump(tmpl); } From 9d4450d51a161d4d34720c543996d1bc60fb94b5 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sat, 11 May 2024 19:43:34 +0530 Subject: [PATCH 164/217] GroupKV: Let dump return a string, rather than printing/logging --- common/groupkv.hpp | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/common/groupkv.hpp b/common/groupkv.hpp index fbaa9dcba03a3..e6a94f3c4bad7 100644 --- a/common/groupkv.hpp +++ b/common/groupkv.hpp @@ -116,16 +116,19 @@ class GroupKV { // Dump info about the specified group. // If group is empty, then dump info about all groups maintained in this instance. - void dump(const std::string &group) { + std::string dump(const std::string &group, const std::string &msgTag = "") { + std::stringstream ss; for (auto gm: gkv) { if (!group.empty() && (gm.first != group)) { - LINFO_LN("INFO:GKV:%s:%s:Skipping...", __func__, gm.first.c_str()); + LDBUG_LN("DBUG:GKV:%s:%s:%s:Skipping...", __func__, msgTag.c_str(), gm.first.c_str()); continue; } + ss << "\n" << msgTag << gm.first << ":\n"; for(auto k: gm.second) { - LINFO_LN("DBUG:GKV:%s:%s:Iterate:%s:%s", __func__, gm.first.c_str(), k.first.c_str(), to_str(k.second).c_str()); + ss << msgTag << "\t" << k.first << ":" << to_str(k.second) << "\n"; } } + return ss.str(); } template @@ -185,7 +188,7 @@ void gkv_inited() { }}; std::cout << "**** gkv inited **** " << std::endl; - gkv.dump(""); + std::cout << gkv.dump("", "INFO:GKV:Inited:") << std::endl; } @@ -193,7 +196,7 @@ void gkv_set() { std::cout << "**** gkv set **** " << std::endl; GroupKV gkv = {{}}; - gkv.dump(""); + std::cout << gkv.dump("", "INFO:GKV:Set:Initial:") << std::endl; gkv.get_value("testme", {"key101b"}, false); gkv.get_value("testme", {"key101s"}, "Not found"); @@ -205,7 +208,7 @@ void gkv_set() { gkv.set_value("testme", {"key201i"}, 987654); gkv.set_value("testme", {"key201d"}, 9988.7766); - gkv.dump("testme"); + std::cout << gkv.dump("testme", "INFO:GKV:Set:After testme set:") << std::endl; gkv.get_value("testme", {"key201b"}, false); gkv.get_value("testme", {"key201s"}, "Not found"); gkv.get_value("testme", {"key201i"}, 123456); From 484c710eab5830967ab2bd4ec70cbbaeabd5ab02 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sat, 11 May 2024 20:46:10 +0530 Subject: [PATCH 165/217] GroupKV:Add GetValue which throws exception --- common/groupkv.hpp | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/common/groupkv.hpp b/common/groupkv.hpp index e6a94f3c4bad7..62fe6d3ee3305 100644 --- a/common/groupkv.hpp +++ b/common/groupkv.hpp @@ -132,18 +132,30 @@ class GroupKV { } template - SupportedDataType get_value(const std::string &group, const MultiPart &keyParts, const SupportedDataType &defaultValue, const std::string &callerName="") { + SupportedDataType get_value(const std::string &group, const MultiPart &keyParts) { auto key = joiner(keyParts); auto gm = gkv[group]; if (gm.find(key) == gm.end()) { - LDBUG_LN("WARN:GKV:%s_%s:%s:%s:%s[default]", __func__, callerName.c_str(), group.c_str(), key.c_str(), to_str(defaultValue).c_str()); - return defaultValue; + std::stringstream ss; + ss << "WARN:GKV:" << __func__ << ":" << group << ":Key [" << key << "] not found"; + throw std::range_error(ss.str()); } auto value = gm[key]; - LDBUG_LN("DBUG:GKV:%s_%s:%s:%s:%s", __func__, callerName.c_str(), group.c_str(), key.c_str(), to_str(value).c_str()); return std::get(value); } + template + SupportedDataType get_value(const std::string &group, const MultiPart &keyParts, const SupportedDataType &defaultValue, const std::string &callerName="") { + try { + auto value = get_value(group, keyParts); + LDBUG_LN("DBUG:GKV:%s_%s:%s:%s:%s", __func__, callerName.c_str(), group.c_str(), to_str(keyParts).c_str(), to_str(value).c_str()); + return value; + } catch (std::exception &e) { + } + LDBUG_LN("WARN:GKV:%s_%s:%s:%s:%s[default]", __func__, callerName.c_str(), group.c_str(), to_str(keyParts).c_str(), to_str(defaultValue).c_str()); + return defaultValue; + } + template std::vector get_vector(const std::string &group, const MultiPart &keyParts, const std::vector &defaultValue, const std::string &callerName="") { auto key = joiner(keyParts); From 4a9a6ce256a1a0e2ee55204dbc69aaf96ae279d0 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sat, 11 May 2024 22:47:27 +0530 Subject: [PATCH 166/217] ChatON: ChatONMetaDump switch to GKV/ChatTemplates based flow --- common/chaton.hpp | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index f5dfde1102687..37fcd162de111 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -690,6 +690,8 @@ inline std::vector chaton_llama_tokenize_ex( * if tmpl is * * empty string, then dump the full loaded chaton-meta * * chaton-template-id, then dump contents related to that specific chat-handshake-template-standard + * NOTE: It uses the exception raising get_value to check if the tags related keys are present + * wrt the specified template-standard/model-id or not. */ inline bool _chaton_meta_dump(std::string &tmpl) { if (!tmpl.empty()) { @@ -698,23 +700,22 @@ inline bool _chaton_meta_dump(std::string &tmpl) { return false; } } - gCT.dump(tmpl); - LOGXLN("\n\nINFO:%s:ChatOn Meta:%s:\n%s", __func__, tmpl.c_str(), theJson.dump(4).c_str()); + LOGXLN("\n\nINFO:%s:%s:\n%s", __func__, tmpl.c_str(), gCT.dump(tmpl, "INFO:ChatOnMetaDump:").c_str()); if (!tmpl.empty()) { - std::string globalBegin = conMeta[tmpl][K_GLOBAL][K_BEGIN]; - std::string globalEnd = conMeta[tmpl][K_GLOBAL][K_END]; - std::string systemBegin = conMeta[tmpl][K_SYSTEM][K_BEGIN]; - std::string systemPrefix = conMeta[tmpl][K_SYSTEM][K_PREFIX]; - std::string systemSuffix = conMeta[tmpl][K_SYSTEM][K_SUFFIX]; - std::string systemEnd = conMeta[tmpl][K_SYSTEM][K_END]; - std::string userBegin = conMeta[tmpl][K_USER][K_BEGIN]; - std::string userPrefix = conMeta[tmpl][K_USER][K_PREFIX]; - std::string userSuffix = conMeta[tmpl][K_USER][K_SUFFIX]; - std::string userEnd = conMeta[tmpl][K_USER][K_END]; - std::string assistantBegin = conMeta[tmpl][K_ASSISTANT][K_BEGIN]; - std::string assistantPrefix = conMeta[tmpl][K_ASSISTANT][K_PREFIX]; - std::string assistantSuffix = conMeta[tmpl][K_ASSISTANT][K_SUFFIX]; - std::string assistantEnd = conMeta[tmpl][K_ASSISTANT][K_END]; + std::string globalBegin = gCT.get_value(tmpl, { K_GLOBAL, K_BEGIN }); + std::string globalEnd = gCT.get_value(tmpl, { K_GLOBAL, K_END }); + std::string systemBegin = gCT.get_value(tmpl, { K_SYSTEM, K_BEGIN }); + std::string systemPrefix = gCT.get_value(tmpl, { K_SYSTEM, K_PREFIX }); + std::string systemSuffix = gCT.get_value(tmpl, { K_SYSTEM, K_SUFFIX }); + std::string systemEnd = gCT.get_value(tmpl, { K_SYSTEM, K_END }); + std::string userBegin = gCT.get_value(tmpl, { K_USER, K_BEGIN }); + std::string userPrefix = gCT.get_value(tmpl, { K_USER, K_PREFIX }); + std::string userSuffix = gCT.get_value(tmpl, { K_USER, K_SUFFIX }); + std::string userEnd = gCT.get_value(tmpl, { K_USER, K_END }); + std::string assistantBegin = gCT.get_value(tmpl, { K_ASSISTANT, K_BEGIN }); + std::string assistantPrefix = gCT.get_value(tmpl, { K_ASSISTANT, K_PREFIX }); + std::string assistantSuffix = gCT.get_value(tmpl, { K_ASSISTANT, K_SUFFIX }); + std::string assistantEnd = gCT.get_value(tmpl, { K_ASSISTANT, K_END }); LOGXLN("INFO:%s:%s:%s", __func__, "global->begin", globalBegin.c_str()); LOGXLN("INFO:%s:%s:%s", __func__, "global->end", globalEnd.c_str()); From d9959b74e7e08dc5a4926d9612ae16a7f254d9a2 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sat, 11 May 2024 23:30:12 +0530 Subject: [PATCH 167/217] GroupKV: Get ready for use in llama.cpp ++ Avoid defining GKV_TEST_PRG, used for self testing, by default Add it to common library --- common/CMakeLists.txt | 1 + common/groupkv.hpp | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index fe865abab708d..a07566c0e13ad 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -66,6 +66,7 @@ add_library(${TARGET} STATIC ngram-cache.h ngram-cache.cpp chaton.hpp + groupkv.hpp ) if (BUILD_SHARED_LIBS) diff --git a/common/groupkv.hpp b/common/groupkv.hpp index 62fe6d3ee3305..1e5f4628ce188 100644 --- a/common/groupkv.hpp +++ b/common/groupkv.hpp @@ -20,7 +20,7 @@ #define GKV_DEBUG -#define GKV_TEST_PRG +//#define GKV_TEST_PRG #ifdef GKV_TEST_PRG #include #include From b944d04d08a33e9bff14ea078362da63143fa91c Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sat, 11 May 2024 23:31:53 +0530 Subject: [PATCH 168/217] ChatON: Add constructor for ChatTemplates which chains into GKV --- common/chaton.hpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index 37fcd162de111..06646c372e04f 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -301,9 +301,13 @@ class ChatParts { class ChatTemplates : public GroupKV { +public: + + ChatTemplates(GroupKVMapMapVariant defaultMap) : GroupKV(defaultMap) {} + }; -ChatTemplates gCT; +ChatTemplates gCT = {{}}; #ifdef CHATON_JSON From b9d9700de379e12d8bb07008be42e37eaa86605c Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sat, 11 May 2024 23:36:02 +0530 Subject: [PATCH 169/217] CMakeLists.txt: Compile C++ code for -std=c++20 --- CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index aa65b0d6c3d04..5dca1fe6ccbd1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -837,6 +837,8 @@ function(get_flags CCID CCVER) set(GF_CXX_FLAGS ${CXX_FLAGS} PARENT_SCOPE) endfunction() +list(APPEND CXX_FLAGS -std=c++20) + if (LLAMA_FATAL_WARNINGS) if (CMAKE_CXX_COMPILER_ID MATCHES "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") list(APPEND C_FLAGS -Werror) From 2efc09f2d042a60ca2befad38a7ed2b95d5101cd Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sun, 12 May 2024 00:41:15 +0530 Subject: [PATCH 170/217] ChatON: Unnecessarily indirect nlohmann json code used for exploring/testing commited just for future reference --- common/chaton.hpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index 06646c372e04f..dd8488906ab36 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -314,8 +314,14 @@ ChatTemplates gCT = {{}}; inline bool chaton_meta_load(const std::string &fname) { std::ifstream f(fname); json conMeta = json::parse(f); + for(auto it=conMeta.begin(); it != conMeta.end(); ++it) { + std::cout << it.key() << std::endl; + std::cout << it.value() << std::endl; + auto curMeta = conMeta[it.key()]; + std::cout << curMeta.dump(4) << std::endl; + } for(auto curTmpl: conMeta) { - curTmpl.dump(); + std::cout << curTmpl.dump(4) << std::endl; } return true; } From 444d2ccf9c1fd3b5eab7efb79b6c2b14434899c4 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sun, 12 May 2024 01:15:04 +0530 Subject: [PATCH 171/217] ChatON:LoadJSON: ChatTemplates - global/system/user/assistant Manually iterate the json object items using begin-end explicitly, because the implicit iteration for loop related helpers for the used json lib gives only the values and not a key-value pair. --- common/chaton.hpp | 54 +++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 47 insertions(+), 7 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index dd8488906ab36..4a27d93e80f2d 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -315,14 +315,54 @@ inline bool chaton_meta_load(const std::string &fname) { std::ifstream f(fname); json conMeta = json::parse(f); for(auto it=conMeta.begin(); it != conMeta.end(); ++it) { - std::cout << it.key() << std::endl; - std::cout << it.value() << std::endl; - auto curMeta = conMeta[it.key()]; - std::cout << curMeta.dump(4) << std::endl; - } - for(auto curTmpl: conMeta) { - std::cout << curTmpl.dump(4) << std::endl; + + auto group = it.key(); + auto curTmpl = conMeta[group]; + + std::string globalBegin = curTmpl[K_GLOBAL][K_BEGIN]; + gCT.set_value(group, { K_GLOBAL, K_BEGIN }, globalBegin); + + std::string globalEnd = curTmpl[K_GLOBAL][K_END]; + gCT.set_value(group, { K_GLOBAL, K_END }, globalEnd); + + std::string systemBegin = curTmpl[K_SYSTEM][K_BEGIN]; + gCT.set_value(group, { K_SYSTEM, K_BEGIN }, systemBegin); + + std::string systemPrefix = curTmpl[K_SYSTEM][K_PREFIX]; + gCT.set_value(group, { K_SYSTEM, K_PREFIX }, systemPrefix); + + std::string systemSuffix = curTmpl[K_SYSTEM][K_SUFFIX]; + gCT.set_value(group, { K_SYSTEM, K_SUFFIX }, systemSuffix); + + std::string systemEnd = curTmpl[K_SYSTEM][K_END]; + gCT.set_value(group, { K_SYSTEM, K_END }, systemEnd); + + std::string userBegin = curTmpl[K_USER][K_BEGIN]; + gCT.set_value(group, { K_USER, K_BEGIN }, userBegin); + + std::string userPrefix = curTmpl[K_USER][K_PREFIX]; + gCT.set_value(group, { K_USER, K_PREFIX }, userPrefix); + + std::string userSuffix = curTmpl[K_USER][K_SUFFIX]; + gCT.set_value(group, { K_USER, K_SUFFIX }, userSuffix); + + std::string userEnd = curTmpl[K_USER][K_END]; + gCT.set_value(group, { K_USER, K_END }, userEnd); + + std::string assistantBegin = curTmpl[K_ASSISTANT][K_BEGIN]; + gCT.set_value(group, { K_ASSISTANT, K_BEGIN }, assistantBegin); + + std::string assistantPrefix = curTmpl[K_ASSISTANT][K_PREFIX]; + gCT.set_value(group, { K_ASSISTANT, K_PREFIX }, assistantPrefix); + + std::string assistantSuffix = curTmpl[K_ASSISTANT][K_SUFFIX]; + gCT.set_value(group, { K_ASSISTANT, K_SUFFIX }, assistantSuffix); + + std::string assistantEnd = curTmpl[K_ASSISTANT][K_END]; + gCT.set_value(group, { K_ASSISTANT, K_END }, assistantEnd); + } + LOGXLN("%s", gCT.dump("", "DBUG:ChatONMetaLoad:ChatTemplates:").c_str()); return true; } From 1574201f71c625871e82b632e854548554a2ddcd Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sun, 12 May 2024 01:36:03 +0530 Subject: [PATCH 172/217] ChatON:LoadJSon:ChatTemplates: revPrompt, system-user flags WIP:NOTE: Initial go converting from json driven flow to ChatTemplatesGroupKV related flow done. Needs to be tested. A optional helper added to load ChatTemplates from a specified json file. Need to add a compile time initialized MapOfMapOfVariants wrt the chat template details of models/standards already known to the program. So that one can use the llama.cpp and this new chat template logic, even without json dependency, if one doesnt want to. --- common/chaton.hpp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/common/chaton.hpp b/common/chaton.hpp index 4a27d93e80f2d..38df0790ee3be 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -361,6 +361,20 @@ inline bool chaton_meta_load(const std::string &fname) { std::string assistantEnd = curTmpl[K_ASSISTANT][K_END]; gCT.set_value(group, { K_ASSISTANT, K_END }, assistantEnd); + + std::string reversePrompt = curTmpl[K_REVERSE_PROMPT]; + gCT.set_value(group, { K_REVERSE_PROMPT }, reversePrompt); + + bool systemHasSuffix = curTmpl[K_SYSTEMUSER_SYSTEM_HAS_SUFFIX]; + gCT.set_value(group, { K_SYSTEMUSER_SYSTEM_HAS_SUFFIX }, systemHasSuffix); + bool systemHasEnd = curTmpl[K_SYSTEMUSER_SYSTEM_HAS_END]; + gCT.set_value(group, { K_SYSTEMUSER_SYSTEM_HAS_END }, systemHasEnd); + + bool userHasBegin = curTmpl[K_SYSTEMUSER_1ST_USER_HAS_BEGIN]; + gCT.set_value(group, { K_SYSTEMUSER_1ST_USER_HAS_BEGIN }, userHasBegin); + bool userHasPrefix = curTmpl[K_SYSTEMUSER_1ST_USER_HAS_PREFIX]; + gCT.set_value(group, { K_SYSTEMUSER_1ST_USER_HAS_PREFIX }, userHasPrefix); + } LOGXLN("%s", gCT.dump("", "DBUG:ChatONMetaLoad:ChatTemplates:").c_str()); return true; From 0c21a0084f061cce79b647e7be3b284ef346a2c4 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sun, 12 May 2024 12:29:16 +0530 Subject: [PATCH 173/217] ChatON:p1: meta json to hpp conversion - Initial skeleton load the json file and put the template ids --- scripts/chaton-meta-json-to-hpp.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100755 scripts/chaton-meta-json-to-hpp.py diff --git a/scripts/chaton-meta-json-to-hpp.py b/scripts/chaton-meta-json-to-hpp.py new file mode 100755 index 0000000000000..a1272deb235f3 --- /dev/null +++ b/scripts/chaton-meta-json-to-hpp.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 +# Convert chaton meta json file to c++ hpp file +# by Humans for All + +import sys +import json + +fp=open(sys.argv[1]) +j=json.load(fp) +print("{") +for tmpl in j: + print("\t{{ \"{}\", ".format(tmpl)) + From 078e04d32bb8f45c80cf42f2de50c3c1d8694647 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sun, 12 May 2024 12:40:33 +0530 Subject: [PATCH 174/217] ChatON:P2:meta json to hpp conversion - add k-v pairs skeleton --- scripts/chaton-meta-json-to-hpp.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/scripts/chaton-meta-json-to-hpp.py b/scripts/chaton-meta-json-to-hpp.py index a1272deb235f3..a5e68d823d94d 100755 --- a/scripts/chaton-meta-json-to-hpp.py +++ b/scripts/chaton-meta-json-to-hpp.py @@ -5,9 +5,17 @@ import sys import json + +def kv(j, tmpl, k1, k2, comma): + print("\t\t{{ \"{}\", \"{}\" }}{}".format("{}-{}".format(k1,k2), j[tmpl][k1][k2], comma)) + + fp=open(sys.argv[1]) j=json.load(fp) print("{") for tmpl in j: - print("\t{{ \"{}\", ".format(tmpl)) + print("\t{{ \"{}\", {{".format(tmpl)) + kv(j, tmpl, "global", "begin", ",") + kv(j, tmpl, "global", "end", ",") + print("\t}},") From 7b5fb0a2fa2f1df3422b7a8cea83bd8b418f5dea Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sun, 12 May 2024 13:06:22 +0530 Subject: [PATCH 175/217] ChatON:P3:meta json to hpp: Retain esc seqs and more kv pairs Use repr to retain the escape sequences in the read string. And parallely skip the single quote around strings wrt repr. Bring in more k-v pairs wrt chaton_meta.json --- scripts/chaton-meta-json-to-hpp.py | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/scripts/chaton-meta-json-to-hpp.py b/scripts/chaton-meta-json-to-hpp.py index a5e68d823d94d..2ee0a065a13f8 100755 --- a/scripts/chaton-meta-json-to-hpp.py +++ b/scripts/chaton-meta-json-to-hpp.py @@ -7,15 +7,31 @@ def kv(j, tmpl, k1, k2, comma): - print("\t\t{{ \"{}\", \"{}\" }}{}".format("{}-{}".format(k1,k2), j[tmpl][k1][k2], comma)) + print("\t\t{{ \"{}\", \"{}\" }}{}".format("{}-{}".format(k1,k2), repr(j[tmpl][k1][k2])[1:-1], comma)) fp=open(sys.argv[1]) j=json.load(fp) print("{") for tmpl in j: - print("\t{{ \"{}\", {{".format(tmpl)) - kv(j, tmpl, "global", "begin", ",") - kv(j, tmpl, "global", "end", ",") - print("\t}},") - + print("\t{{ \"{}\", {{".format(tmpl)) + + kv(j, tmpl, "global", "begin", ",") + kv(j, tmpl, "global", "end", ",") + + kv(j, tmpl, "system", "begin", ",") + kv(j, tmpl, "system", "prefix", ",") + kv(j, tmpl, "system", "suffix", ",") + kv(j, tmpl, "system", "end", ",") + + kv(j, tmpl, "user", "begin", ",") + kv(j, tmpl, "user", "prefix", ",") + kv(j, tmpl, "user", "suffix", ",") + kv(j, tmpl, "user", "end", ",") + + kv(j, tmpl, "assistant", "begin", ",") + kv(j, tmpl, "assistant", "prefix", ",") + kv(j, tmpl, "assistant", "suffix", ",") + kv(j, tmpl, "assistant", "end", ",") + + print("\t}},") From b5b274a44ba9d8ecf862f6481a6189a4207e0d49 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sun, 12 May 2024 13:21:17 +0530 Subject: [PATCH 176/217] ChatON:P4:meta json to hpp: Insert kv bool Rename kv helpers to match their semantic. * whether working with string or bool value * whether two keys or a single key Add support for kv with bool value inturn add the kv boolean pairs used in the chaton_meta.json file Add the closing bracket --- scripts/chaton-meta-json-to-hpp.py | 42 +++++++++++++++++++----------- 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/scripts/chaton-meta-json-to-hpp.py b/scripts/chaton-meta-json-to-hpp.py index 2ee0a065a13f8..95368eef5e9cf 100755 --- a/scripts/chaton-meta-json-to-hpp.py +++ b/scripts/chaton-meta-json-to-hpp.py @@ -6,32 +6,44 @@ import json -def kv(j, tmpl, k1, k2, comma): +def kkv_str(j, tmpl, k1, k2, comma): print("\t\t{{ \"{}\", \"{}\" }}{}".format("{}-{}".format(k1,k2), repr(j[tmpl][k1][k2])[1:-1], comma)) +def kv_bool(j, tmpl, k1, comma): + print("\t\t{{ \"{}\", {} }}{}".format(k1, repr(j[tmpl][k1]).lower(), comma)) + fp=open(sys.argv[1]) j=json.load(fp) print("{") + for tmpl in j: print("\t{{ \"{}\", {{".format(tmpl)) - kv(j, tmpl, "global", "begin", ",") - kv(j, tmpl, "global", "end", ",") + kkv_str(j, tmpl, "global", "begin", ",") + kkv_str(j, tmpl, "global", "end", ",") + + kkv_str(j, tmpl, "system", "begin", ",") + kkv_str(j, tmpl, "system", "prefix", ",") + kkv_str(j, tmpl, "system", "suffix", ",") + kkv_str(j, tmpl, "system", "end", ",") - kv(j, tmpl, "system", "begin", ",") - kv(j, tmpl, "system", "prefix", ",") - kv(j, tmpl, "system", "suffix", ",") - kv(j, tmpl, "system", "end", ",") + kkv_str(j, tmpl, "user", "begin", ",") + kkv_str(j, tmpl, "user", "prefix", ",") + kkv_str(j, tmpl, "user", "suffix", ",") + kkv_str(j, tmpl, "user", "end", ",") - kv(j, tmpl, "user", "begin", ",") - kv(j, tmpl, "user", "prefix", ",") - kv(j, tmpl, "user", "suffix", ",") - kv(j, tmpl, "user", "end", ",") + kkv_str(j, tmpl, "assistant", "begin", ",") + kkv_str(j, tmpl, "assistant", "prefix", ",") + kkv_str(j, tmpl, "assistant", "suffix", ",") + kkv_str(j, tmpl, "assistant", "end", ",") - kv(j, tmpl, "assistant", "begin", ",") - kv(j, tmpl, "assistant", "prefix", ",") - kv(j, tmpl, "assistant", "suffix", ",") - kv(j, tmpl, "assistant", "end", ",") + kv_bool(j, tmpl, "systemuser-system-has-suffix", ",") + kv_bool(j, tmpl, "systemuser-system-has-end", ",") + kv_bool(j, tmpl, "systemuser-1st-user-has-begin", ",") + kv_bool(j, tmpl, "systemuser-1st-user-has-prefix", ",") print("\t}},") + +print("}") + From b8590e3e5722cf047af075aae660227946f81733 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sun, 12 May 2024 13:50:22 +0530 Subject: [PATCH 177/217] ChatON:P5:meta json to hpp: Add required c++ inc and global var Also comment to indicate that the hpp file is auto converted from the chaton_meta.json file --- scripts/chaton-meta-json-to-hpp.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/chaton-meta-json-to-hpp.py b/scripts/chaton-meta-json-to-hpp.py index 95368eef5e9cf..6c08ca4db9d4d 100755 --- a/scripts/chaton-meta-json-to-hpp.py +++ b/scripts/chaton-meta-json-to-hpp.py @@ -15,7 +15,8 @@ def kv_bool(j, tmpl, k1, comma): fp=open(sys.argv[1]) j=json.load(fp) -print("{") +print("//This is auto created/converted from chaton-meta-json file") +print("#pragma once\n\n#include \"chaton.hpp\"\n\nChatTemplates gCT = {{") for tmpl in j: print("\t{{ \"{}\", {{".format(tmpl)) @@ -45,5 +46,5 @@ def kv_bool(j, tmpl, k1, comma): print("\t}},") -print("}") +print("}};") From a3285e8e25c0864ab7d1e0f9e91b8096ac28571c Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sun, 12 May 2024 13:56:02 +0530 Subject: [PATCH 178/217] ChatON:Include auto converted ChatONMeta.hpp chat template data This should allow for using this generic chat templating code flow along with the included chat template data, without needing to load any json file at runtime. However If user wants to change the already included chat template data, or add new chat template standard/model related data, one can explicitly load json file. TODO: Need to cross check this flow once, but logically should work --- common/chaton.hpp | 3 +- common/chaton_meta.hpp | 287 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 289 insertions(+), 1 deletion(-) create mode 100644 common/chaton_meta.hpp diff --git a/common/chaton.hpp b/common/chaton.hpp index 38df0790ee3be..4f6436dbb5f2a 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -307,7 +307,8 @@ class ChatTemplates : public GroupKV { }; -ChatTemplates gCT = {{}}; +#include "chaton_meta.hpp" +//ChatTemplates gCT = {{}}; #ifdef CHATON_JSON diff --git a/common/chaton_meta.hpp b/common/chaton_meta.hpp new file mode 100644 index 0000000000000..3db8c08bec877 --- /dev/null +++ b/common/chaton_meta.hpp @@ -0,0 +1,287 @@ +//This is auto created/converted from chaton-meta-json file +#pragma once + +#include "chaton.hpp" + +ChatTemplates gCT = {{ + { "llama2", { + { "global-begin", "" }, + { "global-end", "" }, + { "system-begin", "" }, + { "system-prefix", "[INST] <>\n" }, + { "system-suffix", "\n<> " }, + { "system-end", "[/INST]\n\n" }, + { "user-begin", "" }, + { "user-prefix", "[INST] " }, + { "user-suffix", " [/INST]\n\n" }, + { "user-end", "" }, + { "assistant-begin", "" }, + { "assistant-prefix", "" }, + { "assistant-suffix", "" }, + { "assistant-end", "" }, + { "systemuser-system-has-suffix", true }, + { "systemuser-system-has-end", false }, + { "systemuser-1st-user-has-begin", false }, + { "systemuser-1st-user-has-prefix", false }, + }}, + { "llama3", { + { "global-begin", "" }, + { "global-end", "" }, + { "system-begin", "" }, + { "system-prefix", "<|start_header_id|>system<|end_header_id|>\n" }, + { "system-suffix", "<|eot_id|>\n\n" }, + { "system-end", "" }, + { "user-begin", "" }, + { "user-prefix", "<|start_header_id|>user<|end_header_id|>\n" }, + { "user-suffix", "<|eot_id|>\n\n" }, + { "user-end", "" }, + { "assistant-begin", "" }, + { "assistant-prefix", "<|start_header_id|>assistant<|end_header_id|>\n" }, + { "assistant-suffix", "<|eot_id|>\n\n" }, + { "assistant-end", "" }, + { "systemuser-system-has-suffix", true }, + { "systemuser-system-has-end", true }, + { "systemuser-1st-user-has-begin", true }, + { "systemuser-1st-user-has-prefix", true }, + }}, + { "chatml", { + { "global-begin", "" }, + { "global-end", "" }, + { "system-begin", "" }, + { "system-prefix", "<|im_start|>system\n" }, + { "system-suffix", "<|im_end|>\n" }, + { "system-end", "" }, + { "user-begin", "" }, + { "user-prefix", "<|im_start|>user\n" }, + { "user-suffix", "<|im_end|>\n" }, + { "user-end", "" }, + { "assistant-begin", "" }, + { "assistant-prefix", "<|im_start|>assistant\n" }, + { "assistant-suffix", "<|im_end|>\n" }, + { "assistant-end", "" }, + { "systemuser-system-has-suffix", true }, + { "systemuser-system-has-end", true }, + { "systemuser-1st-user-has-begin", true }, + { "systemuser-1st-user-has-prefix", true }, + }}, + { "zephyr", { + { "global-begin", "" }, + { "global-end", "" }, + { "system-begin", "" }, + { "system-prefix", "<|system|>\n" }, + { "system-suffix", "" }, + { "system-end", "\n" }, + { "user-begin", "" }, + { "user-prefix", "<|user|>\n" }, + { "user-suffix", "" }, + { "user-end", "\n" }, + { "assistant-begin", "" }, + { "assistant-prefix", "<|assistant|>\n" }, + { "assistant-suffix", "" }, + { "assistant-end", "\n" }, + { "systemuser-system-has-suffix", true }, + { "systemuser-system-has-end", true }, + { "systemuser-1st-user-has-begin", true }, + { "systemuser-1st-user-has-prefix", true }, + }}, + { "gemma", { + { "global-begin", "" }, + { "global-end", "" }, + { "system-begin", "" }, + { "system-prefix", "user\n" }, + { "system-suffix", "\n" }, + { "system-end", "" }, + { "user-begin", "" }, + { "user-prefix", "user\n" }, + { "user-suffix", "\n" }, + { "user-end", "" }, + { "assistant-begin", "" }, + { "assistant-prefix", "model\n" }, + { "assistant-suffix", "\n" }, + { "assistant-end", "" }, + { "systemuser-system-has-suffix", true }, + { "systemuser-system-has-end", false }, + { "systemuser-1st-user-has-begin", true }, + { "systemuser-1st-user-has-prefix", true }, + }}, + { "deepseek-coder", { + { "global-begin", "" }, + { "global-end", "" }, + { "system-begin", "\n<｜begin▁of▁sentence｜>" }, + { "system-prefix", "" }, + { "system-suffix", "\n" }, + { "system-end", "" }, + { "user-begin", "" }, + { "user-prefix", "### Instruction:\n" }, + { "user-suffix", "\n" }, + { "user-end", "" }, + { "assistant-begin", "" }, + { "assistant-prefix", "### Response:\n" }, + { "assistant-suffix", "\n<|EOT|>\n" }, + { "assistant-end", "" }, + { "systemuser-system-has-suffix", true }, + { "systemuser-system-has-end", false }, + { "systemuser-1st-user-has-begin", false }, + { "systemuser-1st-user-has-prefix", true }, + }}, + { "deepseek", { + { "global-begin", "" }, + { "global-end", "" }, + { "system-begin", "<｜begin▁of▁sentence｜>" }, + { "system-prefix", "" }, + { "system-suffix", "\n\n" }, + { "system-end", "" }, + { "user-begin", "<｜begin▁of▁sentence｜>" }, + { "user-prefix", "User: " }, + { "user-suffix", "\n\n" }, + { "user-end", "" }, + { "assistant-begin", "" }, + { "assistant-prefix", "Assistant: " }, + { "assistant-suffix", " <｜end▁of▁sentence｜>\n" }, + { "assistant-end", "" }, + { "systemuser-system-has-suffix", true }, + { "systemuser-system-has-end", false }, + { "systemuser-1st-user-has-begin", false }, + { "systemuser-1st-user-has-prefix", true }, + }}, + { "monarch", { + { "global-begin", "" }, + { "global-end", "" }, + { "system-begin", "" }, + { "system-prefix", "system\n" }, + { "system-suffix", "" }, + { "system-end", "\n" }, + { "user-begin", "" }, + { "user-prefix", "user\n" }, + { "user-suffix", "" }, + { "user-end", "\n" }, + { "assistant-begin", "" }, + { "assistant-prefix", "assistant\n" }, + { "assistant-suffix", "" }, + { "assistant-end", "\n" }, + { "systemuser-system-has-suffix", true }, + { "systemuser-system-has-end", true }, + { "systemuser-1st-user-has-begin", true }, + { "systemuser-1st-user-has-prefix", true }, + }}, + { "mistral", { + { "global-begin", "" }, + { "global-end", "" }, + { "system-begin", "" }, + { "system-prefix", "[INST] " }, + { "system-suffix", " [/INST]" }, + { "system-end", "\n" }, + { "user-begin", "" }, + { "user-prefix", "[INST] " }, + { "user-suffix", " [/INST]" }, + { "user-end", "\n" }, + { "assistant-begin", "" }, + { "assistant-prefix", "" }, + { "assistant-suffix", "" }, + { "assistant-end", " \n" }, + { "systemuser-system-has-suffix", false }, + { "systemuser-system-has-end", true }, + { "systemuser-1st-user-has-begin", false }, + { "systemuser-1st-user-has-prefix", false }, + }}, + { "phi3", { + { "global-begin", "" }, + { "global-end", "" }, + { "system-begin", "" }, + { "system-prefix", "<|system|>\n" }, + { "system-suffix", "<|end|>\n" }, + { "system-end", "" }, + { "user-begin", "" }, + { "user-prefix", "<|user|>\n" }, + { "user-suffix", "<|end|>\n" }, + { "user-end", "" }, + { "assistant-begin", "" }, + { "assistant-prefix", "<|assistant|>\n" }, + { "assistant-suffix", "<|end|>\n" }, + { "assistant-end", "" }, + { "systemuser-system-has-suffix", true }, + { "systemuser-system-has-end", true }, + { "systemuser-1st-user-has-begin", true }, + { "systemuser-1st-user-has-prefix", true }, + }}, + { "command-r", { + { "global-begin", "" }, + { "global-end", "" }, + { "system-begin", "" }, + { "system-prefix", "<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>" }, + { "system-suffix", "<|END_OF_TURN_TOKEN|>" }, + { "system-end", "" }, + { "user-begin", "" }, + { "user-prefix", "<|START_OF_TURN_TOKEN|><|USER_TOKEN|>" }, + { "user-suffix", "<|END_OF_TURN_TOKEN|>" }, + { "user-end", "" }, + { "assistant-begin", "" }, + { "assistant-prefix", "<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>" }, + { "assistant-suffix", "<|END_OF_TURN_TOKEN|>" }, + { "assistant-end", "" }, + { "systemuser-system-has-suffix", true }, + { "systemuser-system-has-end", true }, + { "systemuser-1st-user-has-begin", true }, + { "systemuser-1st-user-has-prefix", true }, + }}, + { "orion", { + { "global-begin", "" }, + { "global-end", "" }, + { "system-begin", "" }, + { "system-prefix", "Human: " }, + { "system-suffix", "\n\n" }, + { "system-end", "" }, + { "user-begin", "" }, + { "user-prefix", "Human: " }, + { "user-suffix", "\n\nAssistant: " }, + { "user-end", "" }, + { "assistant-begin", "" }, + { "assistant-prefix", "" }, + { "assistant-suffix", "" }, + { "assistant-end", "" }, + { "systemuser-system-has-suffix", true }, + { "systemuser-system-has-end", true }, + { "systemuser-1st-user-has-begin", false }, + { "systemuser-1st-user-has-prefix", false }, + }}, + { "openchat", { + { "global-begin", "" }, + { "global-end", "" }, + { "system-begin", "" }, + { "system-prefix", "GPT4 Correct System: " }, + { "system-suffix", "<|end_of_turn|>" }, + { "system-end", "" }, + { "user-begin", "" }, + { "user-prefix", "GPT4 Correct User: " }, + { "user-suffix", "<|end_of_turn|>" }, + { "user-end", "" }, + { "assistant-begin", "" }, + { "assistant-prefix", "GPT4 Correct Assistant: " }, + { "assistant-suffix", "<|end_of_turn|>" }, + { "assistant-end", "" }, + { "systemuser-system-has-suffix", true }, + { "systemuser-system-has-end", true }, + { "systemuser-1st-user-has-begin", true }, + { "systemuser-1st-user-has-prefix", true }, + }}, + { "vicuna", { + { "global-begin", "" }, + { "global-end", "" }, + { "system-begin", "" }, + { "system-prefix", "SYSTEM: " }, + { "system-suffix", "\n\n" }, + { "system-end", "" }, + { "user-begin", "" }, + { "user-prefix", "USER: " }, + { "user-suffix", "\n" }, + { "user-end", "" }, + { "assistant-begin", "" }, + { "assistant-prefix", "ASSISTANT: " }, + { "assistant-suffix", "\n" }, + { "assistant-end", "" }, + { "systemuser-system-has-suffix", true }, + { "systemuser-system-has-end", true }, + { "systemuser-1st-user-has-begin", true }, + { "systemuser-1st-user-has-prefix", true }, + }}, +}}; From 4232ec1fb9a24d4696b6faa4386e4728a9bd99ab Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sun, 12 May 2024 14:53:37 +0530 Subject: [PATCH 179/217] Main: Load json meta file only if specified This should be ok, given that there is a version of the chat tmpl meta data already included with the library. So only if user wants to change the chat template info wrt a existing model/template-standard or add a new one, then there is need to pass a json file with info for that model/standard. --- examples/main/main.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/main/main.cpp b/examples/main/main.cpp index 591d83b353869..a600d16b55b44 100644 --- a/examples/main/main.cpp +++ b/examples/main/main.cpp @@ -143,7 +143,9 @@ int main(int argc, char ** argv) { atexit([]() { console::cleanup(); }); if (params.chaton) { - chaton_meta_load(params.chaton_meta_json); + if (!params.chaton_meta_json.empty()) { + chaton_meta_load(params.chaton_meta_json); + } if (!chaton_meta_ok(params.chaton_template_id)) { exit(1); } From f94fed92d31439f0e3ef66ace79fbf251a0052e5 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sun, 12 May 2024 15:59:37 +0530 Subject: [PATCH 180/217] ChatON+MetaHpp: Had forgotten to conv reverse-prompt Also has dump was using get_value calls with fallback to default, so it wasnt identifying the missed field. Have fixed both of those. Also reconverted meta json file. Misc: interesting avesham and aattam --- common/chaton.hpp | 15 ++++++++++----- common/chaton_meta.hpp | 14 ++++++++++++++ scripts/chaton-meta-json-to-hpp.py | 5 +++++ 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index 4f6436dbb5f2a..83ba104d2c083 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -781,6 +781,11 @@ inline bool _chaton_meta_dump(std::string &tmpl) { std::string assistantPrefix = gCT.get_value(tmpl, { K_ASSISTANT, K_PREFIX }); std::string assistantSuffix = gCT.get_value(tmpl, { K_ASSISTANT, K_SUFFIX }); std::string assistantEnd = gCT.get_value(tmpl, { K_ASSISTANT, K_END }); + std::string reversePrompt = gCT.get_value(tmpl, { K_REVERSE_PROMPT }); + bool systemHasSuffix = gCT.get_value(tmpl, { K_SYSTEMUSER_SYSTEM_HAS_SUFFIX }); + bool systemHasEnd = gCT.get_value(tmpl, { K_SYSTEMUSER_SYSTEM_HAS_END }); + bool userHasBegin = gCT.get_value(tmpl, { K_SYSTEMUSER_1ST_USER_HAS_BEGIN }); + bool userHasPrefix = gCT.get_value(tmpl, { K_SYSTEMUSER_1ST_USER_HAS_PREFIX }); LOGXLN("INFO:%s:%s:%s", __func__, "global->begin", globalBegin.c_str()); LOGXLN("INFO:%s:%s:%s", __func__, "global->end", globalEnd.c_str()); @@ -796,11 +801,11 @@ inline bool _chaton_meta_dump(std::string &tmpl) { LOGXLN("INFO:%s:%s:%s", __func__, "assistant->prefix", assistantPrefix.c_str()); LOGXLN("INFO:%s:%s:%s", __func__, "assistant->suffix", assistantSuffix.c_str()); LOGXLN("INFO:%s:%s:%s", __func__, "assistant->end", assistantEnd.c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, K_REVERSE_PROMPT, chaton_tmpl_kv(tmpl, K_REVERSE_PROMPT).c_str()); - LOGXLN("INFO:%s:%s:%d", __func__, K_SYSTEMUSER_SYSTEM_HAS_SUFFIX, chaton_tmpl_kv_bool(tmpl, K_SYSTEMUSER_SYSTEM_HAS_SUFFIX)); - LOGXLN("INFO:%s:%s:%d", __func__, K_SYSTEMUSER_SYSTEM_HAS_END, chaton_tmpl_kv_bool(tmpl, K_SYSTEMUSER_SYSTEM_HAS_END)); - LOGXLN("INFO:%s:%s:%d", __func__, K_SYSTEMUSER_1ST_USER_HAS_BEGIN, chaton_tmpl_kv_bool(tmpl, K_SYSTEMUSER_1ST_USER_HAS_BEGIN)); - LOGXLN("INFO:%s:%s:%d", __func__, K_SYSTEMUSER_1ST_USER_HAS_PREFIX, chaton_tmpl_kv_bool(tmpl, K_SYSTEMUSER_1ST_USER_HAS_PREFIX)); + LOGXLN("INFO:%s:%s:%s", __func__, K_REVERSE_PROMPT, reversePrompt.c_str()); + LOGXLN("INFO:%s:%s:%d", __func__, K_SYSTEMUSER_SYSTEM_HAS_SUFFIX, systemHasSuffix); + LOGXLN("INFO:%s:%s:%d", __func__, K_SYSTEMUSER_SYSTEM_HAS_END, systemHasEnd); + LOGXLN("INFO:%s:%s:%d", __func__, K_SYSTEMUSER_1ST_USER_HAS_BEGIN, userHasBegin); + LOGXLN("INFO:%s:%s:%d", __func__, K_SYSTEMUSER_1ST_USER_HAS_PREFIX, userHasPrefix); if (!userEnd.empty()) { LOG_TEELN("WARN:%s:User->End seems to be set to [%s], do cross check if this is proper and needed", __func__, userEnd.c_str()); diff --git a/common/chaton_meta.hpp b/common/chaton_meta.hpp index 3db8c08bec877..3516d5bb91e80 100644 --- a/common/chaton_meta.hpp +++ b/common/chaton_meta.hpp @@ -19,6 +19,7 @@ ChatTemplates gCT = {{ { "assistant-prefix", "" }, { "assistant-suffix", "" }, { "assistant-end", "" }, + { "reverse-prompt", "" }, { "systemuser-system-has-suffix", true }, { "systemuser-system-has-end", false }, { "systemuser-1st-user-has-begin", false }, @@ -39,6 +40,7 @@ ChatTemplates gCT = {{ { "assistant-prefix", "<|start_header_id|>assistant<|end_header_id|>\n" }, { "assistant-suffix", "<|eot_id|>\n\n" }, { "assistant-end", "" }, + { "reverse-prompt", "<|eot_id|>" }, { "systemuser-system-has-suffix", true }, { "systemuser-system-has-end", true }, { "systemuser-1st-user-has-begin", true }, @@ -59,6 +61,7 @@ ChatTemplates gCT = {{ { "assistant-prefix", "<|im_start|>assistant\n" }, { "assistant-suffix", "<|im_end|>\n" }, { "assistant-end", "" }, + { "reverse-prompt", "<|im_start|>user\n" }, { "systemuser-system-has-suffix", true }, { "systemuser-system-has-end", true }, { "systemuser-1st-user-has-begin", true }, @@ -79,6 +82,7 @@ ChatTemplates gCT = {{ { "assistant-prefix", "<|assistant|>\n" }, { "assistant-suffix", "" }, { "assistant-end", "\n" }, + { "reverse-prompt", "" }, { "systemuser-system-has-suffix", true }, { "systemuser-system-has-end", true }, { "systemuser-1st-user-has-begin", true }, @@ -99,6 +103,7 @@ ChatTemplates gCT = {{ { "assistant-prefix", "model\n" }, { "assistant-suffix", "\n" }, { "assistant-end", "" }, + { "reverse-prompt", "" }, { "systemuser-system-has-suffix", true }, { "systemuser-system-has-end", false }, { "systemuser-1st-user-has-begin", true }, @@ -119,6 +124,7 @@ ChatTemplates gCT = {{ { "assistant-prefix", "### Response:\n" }, { "assistant-suffix", "\n<|EOT|>\n" }, { "assistant-end", "" }, + { "reverse-prompt", "<|EOT|>" }, { "systemuser-system-has-suffix", true }, { "systemuser-system-has-end", false }, { "systemuser-1st-user-has-begin", false }, @@ -139,6 +145,7 @@ ChatTemplates gCT = {{ { "assistant-prefix", "Assistant: " }, { "assistant-suffix", " <｜end▁of▁sentence｜>\n" }, { "assistant-end", "" }, + { "reverse-prompt", "<｜end▁of▁sentence｜>" }, { "systemuser-system-has-suffix", true }, { "systemuser-system-has-end", false }, { "systemuser-1st-user-has-begin", false }, @@ -159,6 +166,7 @@ ChatTemplates gCT = {{ { "assistant-prefix", "assistant\n" }, { "assistant-suffix", "" }, { "assistant-end", "\n" }, + { "reverse-prompt", "" }, { "systemuser-system-has-suffix", true }, { "systemuser-system-has-end", true }, { "systemuser-1st-user-has-begin", true }, @@ -179,6 +187,7 @@ ChatTemplates gCT = {{ { "assistant-prefix", "" }, { "assistant-suffix", "" }, { "assistant-end", " \n" }, + { "reverse-prompt", "" }, { "systemuser-system-has-suffix", false }, { "systemuser-system-has-end", true }, { "systemuser-1st-user-has-begin", false }, @@ -199,6 +208,7 @@ ChatTemplates gCT = {{ { "assistant-prefix", "<|assistant|>\n" }, { "assistant-suffix", "<|end|>\n" }, { "assistant-end", "" }, + { "reverse-prompt", "<|end|>" }, { "systemuser-system-has-suffix", true }, { "systemuser-system-has-end", true }, { "systemuser-1st-user-has-begin", true }, @@ -219,6 +229,7 @@ ChatTemplates gCT = {{ { "assistant-prefix", "<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>" }, { "assistant-suffix", "<|END_OF_TURN_TOKEN|>" }, { "assistant-end", "" }, + { "reverse-prompt", "<|END_OF_TURN_TOKEN|>" }, { "systemuser-system-has-suffix", true }, { "systemuser-system-has-end", true }, { "systemuser-1st-user-has-begin", true }, @@ -239,6 +250,7 @@ ChatTemplates gCT = {{ { "assistant-prefix", "" }, { "assistant-suffix", "" }, { "assistant-end", "" }, + { "reverse-prompt", "" }, { "systemuser-system-has-suffix", true }, { "systemuser-system-has-end", true }, { "systemuser-1st-user-has-begin", false }, @@ -259,6 +271,7 @@ ChatTemplates gCT = {{ { "assistant-prefix", "GPT4 Correct Assistant: " }, { "assistant-suffix", "<|end_of_turn|>" }, { "assistant-end", "" }, + { "reverse-prompt", "<|end_of_turn|>" }, { "systemuser-system-has-suffix", true }, { "systemuser-system-has-end", true }, { "systemuser-1st-user-has-begin", true }, @@ -279,6 +292,7 @@ ChatTemplates gCT = {{ { "assistant-prefix", "ASSISTANT: " }, { "assistant-suffix", "\n" }, { "assistant-end", "" }, + { "reverse-prompt", "" }, { "systemuser-system-has-suffix", true }, { "systemuser-system-has-end", true }, { "systemuser-1st-user-has-begin", true }, diff --git a/scripts/chaton-meta-json-to-hpp.py b/scripts/chaton-meta-json-to-hpp.py index 6c08ca4db9d4d..d72c4cdf5858b 100755 --- a/scripts/chaton-meta-json-to-hpp.py +++ b/scripts/chaton-meta-json-to-hpp.py @@ -9,6 +9,9 @@ def kkv_str(j, tmpl, k1, k2, comma): print("\t\t{{ \"{}\", \"{}\" }}{}".format("{}-{}".format(k1,k2), repr(j[tmpl][k1][k2])[1:-1], comma)) +def kv_str(j, tmpl, k1, comma): + print("\t\t{{ \"{}\", \"{}\" }}{}".format(k1, repr(j[tmpl][k1])[1:-1], comma)) + def kv_bool(j, tmpl, k1, comma): print("\t\t{{ \"{}\", {} }}{}".format(k1, repr(j[tmpl][k1]).lower(), comma)) @@ -39,6 +42,8 @@ def kv_bool(j, tmpl, k1, comma): kkv_str(j, tmpl, "assistant", "suffix", ",") kkv_str(j, tmpl, "assistant", "end", ",") + kv_str(j, tmpl, "reverse-prompt", ",") + kv_bool(j, tmpl, "systemuser-system-has-suffix", ",") kv_bool(j, tmpl, "systemuser-system-has-end", ",") kv_bool(j, tmpl, "systemuser-1st-user-has-begin", ",") From 4eae05a6b7c7ef52cce6cc120205a9048d5750da Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sun, 12 May 2024 17:08:03 +0530 Subject: [PATCH 181/217] ChatON: json access helper which raises exception if key missing --- common/chaton.hpp | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/common/chaton.hpp b/common/chaton.hpp index 83ba104d2c083..e40b0420ef7fd 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -312,6 +312,25 @@ class ChatTemplates : public GroupKV { #ifdef CHATON_JSON +inline std::string json_get_str(json &j, std::vector &keys, const std::string &msgTag) { + json curJ = j; + std::stringstream skey; + int i = 0; + for(auto key: keys) { + if (i != 0) skey << "-"; + i += 1; + skey << key; + if (curJ.contains(key)) { + curJ = curJ[key]; + } else { + std::stringstream ss; + ss << "ERRR:ChatON:" << __func__ << msgTag << ":Key [" << skey.str() << "] is missing"; + throw std::runtime_error(ss.str()); + } + } + return curJ; +} + inline bool chaton_meta_load(const std::string &fname) { std::ifstream f(fname); json conMeta = json::parse(f); From 0249c07e6b786e49bb9fd92f453a48644a226935 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sun, 12 May 2024 17:44:13 +0530 Subject: [PATCH 182/217] ChatON:Switch to json_get_str to help identify missing keys better The json library generates less informative exception message, which doesnt help one identify which key is missing, so switch to the new json_get_str helper added in the last commit. It generates more informative exception message. --- common/chaton.hpp | 45 +++++++++++++++++---------------------------- 1 file changed, 17 insertions(+), 28 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index e40b0420ef7fd..882766b51919d 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -312,7 +312,7 @@ class ChatTemplates : public GroupKV { #ifdef CHATON_JSON -inline std::string json_get_str(json &j, std::vector &keys, const std::string &msgTag) { +inline std::string json_get_str(json &j, std::vector keys, const std::string &msgTag) { json curJ = j; std::stringstream skey; int i = 0; @@ -324,7 +324,7 @@ inline std::string json_get_str(json &j, std::vector &keys, const s curJ = curJ[key]; } else { std::stringstream ss; - ss << "ERRR:ChatON:" << __func__ << msgTag << ":Key [" << skey.str() << "] is missing"; + ss << "ERRR:ChatON:" << __func__ << ":" << msgTag << ":Key [" << skey.str() << "] is missing"; throw std::runtime_error(ss.str()); } } @@ -339,50 +339,39 @@ inline bool chaton_meta_load(const std::string &fname) { auto group = it.key(); auto curTmpl = conMeta[group]; - std::string globalBegin = curTmpl[K_GLOBAL][K_BEGIN]; + std::string globalBegin = json_get_str(curTmpl, { K_GLOBAL, K_BEGIN }, group); gCT.set_value(group, { K_GLOBAL, K_BEGIN }, globalBegin); - - std::string globalEnd = curTmpl[K_GLOBAL][K_END]; + std::string globalEnd = json_get_str(curTmpl, { K_GLOBAL, K_END }, group); gCT.set_value(group, { K_GLOBAL, K_END }, globalEnd); - std::string systemBegin = curTmpl[K_SYSTEM][K_BEGIN]; + std::string systemBegin = json_get_str(curTmpl, { K_SYSTEM, K_BEGIN }, group); gCT.set_value(group, { K_SYSTEM, K_BEGIN }, systemBegin); - - std::string systemPrefix = curTmpl[K_SYSTEM][K_PREFIX]; + std::string systemPrefix = json_get_str(curTmpl, { K_SYSTEM, K_PREFIX }, group); gCT.set_value(group, { K_SYSTEM, K_PREFIX }, systemPrefix); - - std::string systemSuffix = curTmpl[K_SYSTEM][K_SUFFIX]; + std::string systemSuffix = json_get_str(curTmpl, { K_SYSTEM, K_SUFFIX }, group); gCT.set_value(group, { K_SYSTEM, K_SUFFIX }, systemSuffix); - - std::string systemEnd = curTmpl[K_SYSTEM][K_END]; + std::string systemEnd = json_get_str(curTmpl, { K_SYSTEM, K_END }, group); gCT.set_value(group, { K_SYSTEM, K_END }, systemEnd); - std::string userBegin = curTmpl[K_USER][K_BEGIN]; + std::string userBegin = json_get_str(curTmpl, { K_USER, K_BEGIN }, group); gCT.set_value(group, { K_USER, K_BEGIN }, userBegin); - - std::string userPrefix = curTmpl[K_USER][K_PREFIX]; + std::string userPrefix = json_get_str(curTmpl, { K_USER, K_PREFIX }, group); gCT.set_value(group, { K_USER, K_PREFIX }, userPrefix); - - std::string userSuffix = curTmpl[K_USER][K_SUFFIX]; + std::string userSuffix = json_get_str(curTmpl, { K_USER, K_SUFFIX }, group); gCT.set_value(group, { K_USER, K_SUFFIX }, userSuffix); - - std::string userEnd = curTmpl[K_USER][K_END]; + std::string userEnd = json_get_str(curTmpl, { K_USER, K_END }, group); gCT.set_value(group, { K_USER, K_END }, userEnd); - std::string assistantBegin = curTmpl[K_ASSISTANT][K_BEGIN]; + std::string assistantBegin = json_get_str(curTmpl, { K_ASSISTANT, K_BEGIN }, group); gCT.set_value(group, { K_ASSISTANT, K_BEGIN }, assistantBegin); - - std::string assistantPrefix = curTmpl[K_ASSISTANT][K_PREFIX]; + std::string assistantPrefix = json_get_str(curTmpl, { K_ASSISTANT, K_PREFIX }, group); gCT.set_value(group, { K_ASSISTANT, K_PREFIX }, assistantPrefix); - - std::string assistantSuffix = curTmpl[K_ASSISTANT][K_SUFFIX]; + std::string assistantSuffix = json_get_str(curTmpl, { K_ASSISTANT, K_SUFFIX }, group); gCT.set_value(group, { K_ASSISTANT, K_SUFFIX }, assistantSuffix); - - std::string assistantEnd = curTmpl[K_ASSISTANT][K_END]; + std::string assistantEnd = json_get_str(curTmpl, { K_ASSISTANT, K_END }, group); gCT.set_value(group, { K_ASSISTANT, K_END }, assistantEnd); - - std::string reversePrompt = curTmpl[K_REVERSE_PROMPT]; + std::string reversePrompt = json_get_str(curTmpl, { K_REVERSE_PROMPT }, group); gCT.set_value(group, { K_REVERSE_PROMPT }, reversePrompt); bool systemHasSuffix = curTmpl[K_SYSTEMUSER_SYSTEM_HAS_SUFFIX]; From 470b8885f32e518aa0d87c23814b5be646feea55 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sun, 12 May 2024 18:19:18 +0530 Subject: [PATCH 183/217] ChatON: Switch to templated json_get for str/bool/etal --- common/chaton.hpp | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index 882766b51919d..8b0a3c5b41b34 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -312,7 +312,8 @@ class ChatTemplates : public GroupKV { #ifdef CHATON_JSON -inline std::string json_get_str(json &j, std::vector keys, const std::string &msgTag) { +template +inline SupportedType json_get(json &j, std::vector keys, const std::string &msgTag) { json curJ = j; std::stringstream skey; int i = 0; @@ -339,39 +340,39 @@ inline bool chaton_meta_load(const std::string &fname) { auto group = it.key(); auto curTmpl = conMeta[group]; - std::string globalBegin = json_get_str(curTmpl, { K_GLOBAL, K_BEGIN }, group); + std::string globalBegin = json_get(curTmpl, { K_GLOBAL, K_BEGIN }, group); gCT.set_value(group, { K_GLOBAL, K_BEGIN }, globalBegin); - std::string globalEnd = json_get_str(curTmpl, { K_GLOBAL, K_END }, group); + std::string globalEnd = json_get(curTmpl, { K_GLOBAL, K_END }, group); gCT.set_value(group, { K_GLOBAL, K_END }, globalEnd); - std::string systemBegin = json_get_str(curTmpl, { K_SYSTEM, K_BEGIN }, group); + std::string systemBegin = json_get(curTmpl, { K_SYSTEM, K_BEGIN }, group); gCT.set_value(group, { K_SYSTEM, K_BEGIN }, systemBegin); - std::string systemPrefix = json_get_str(curTmpl, { K_SYSTEM, K_PREFIX }, group); + std::string systemPrefix = json_get(curTmpl, { K_SYSTEM, K_PREFIX }, group); gCT.set_value(group, { K_SYSTEM, K_PREFIX }, systemPrefix); - std::string systemSuffix = json_get_str(curTmpl, { K_SYSTEM, K_SUFFIX }, group); + std::string systemSuffix = json_get(curTmpl, { K_SYSTEM, K_SUFFIX }, group); gCT.set_value(group, { K_SYSTEM, K_SUFFIX }, systemSuffix); - std::string systemEnd = json_get_str(curTmpl, { K_SYSTEM, K_END }, group); + std::string systemEnd = json_get(curTmpl, { K_SYSTEM, K_END }, group); gCT.set_value(group, { K_SYSTEM, K_END }, systemEnd); - std::string userBegin = json_get_str(curTmpl, { K_USER, K_BEGIN }, group); + std::string userBegin = json_get(curTmpl, { K_USER, K_BEGIN }, group); gCT.set_value(group, { K_USER, K_BEGIN }, userBegin); - std::string userPrefix = json_get_str(curTmpl, { K_USER, K_PREFIX }, group); + std::string userPrefix = json_get(curTmpl, { K_USER, K_PREFIX }, group); gCT.set_value(group, { K_USER, K_PREFIX }, userPrefix); - std::string userSuffix = json_get_str(curTmpl, { K_USER, K_SUFFIX }, group); + std::string userSuffix = json_get(curTmpl, { K_USER, K_SUFFIX }, group); gCT.set_value(group, { K_USER, K_SUFFIX }, userSuffix); - std::string userEnd = json_get_str(curTmpl, { K_USER, K_END }, group); + std::string userEnd = json_get(curTmpl, { K_USER, K_END }, group); gCT.set_value(group, { K_USER, K_END }, userEnd); - std::string assistantBegin = json_get_str(curTmpl, { K_ASSISTANT, K_BEGIN }, group); + std::string assistantBegin = json_get(curTmpl, { K_ASSISTANT, K_BEGIN }, group); gCT.set_value(group, { K_ASSISTANT, K_BEGIN }, assistantBegin); - std::string assistantPrefix = json_get_str(curTmpl, { K_ASSISTANT, K_PREFIX }, group); + std::string assistantPrefix = json_get(curTmpl, { K_ASSISTANT, K_PREFIX }, group); gCT.set_value(group, { K_ASSISTANT, K_PREFIX }, assistantPrefix); - std::string assistantSuffix = json_get_str(curTmpl, { K_ASSISTANT, K_SUFFIX }, group); + std::string assistantSuffix = json_get(curTmpl, { K_ASSISTANT, K_SUFFIX }, group); gCT.set_value(group, { K_ASSISTANT, K_SUFFIX }, assistantSuffix); - std::string assistantEnd = json_get_str(curTmpl, { K_ASSISTANT, K_END }, group); + std::string assistantEnd = json_get(curTmpl, { K_ASSISTANT, K_END }, group); gCT.set_value(group, { K_ASSISTANT, K_END }, assistantEnd); - std::string reversePrompt = json_get_str(curTmpl, { K_REVERSE_PROMPT }, group); + std::string reversePrompt = json_get(curTmpl, { K_REVERSE_PROMPT }, group); gCT.set_value(group, { K_REVERSE_PROMPT }, reversePrompt); bool systemHasSuffix = curTmpl[K_SYSTEMUSER_SYSTEM_HAS_SUFFIX]; From db2ffabb1878c5558cfb2c9be2de71834fcee329 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sun, 12 May 2024 18:24:02 +0530 Subject: [PATCH 184/217] ChatON: use templated json_get when loading bool key-value fields With this now even loading chaton_meta.json file will generate more informative exception, so that user can know which field is missing, if any. --- common/chaton.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index 8b0a3c5b41b34..60209266b301a 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -375,14 +375,14 @@ inline bool chaton_meta_load(const std::string &fname) { std::string reversePrompt = json_get(curTmpl, { K_REVERSE_PROMPT }, group); gCT.set_value(group, { K_REVERSE_PROMPT }, reversePrompt); - bool systemHasSuffix = curTmpl[K_SYSTEMUSER_SYSTEM_HAS_SUFFIX]; + bool systemHasSuffix = json_get(curTmpl, { K_SYSTEMUSER_SYSTEM_HAS_SUFFIX }, group); gCT.set_value(group, { K_SYSTEMUSER_SYSTEM_HAS_SUFFIX }, systemHasSuffix); - bool systemHasEnd = curTmpl[K_SYSTEMUSER_SYSTEM_HAS_END]; + bool systemHasEnd = json_get(curTmpl, { K_SYSTEMUSER_SYSTEM_HAS_END }, group); gCT.set_value(group, { K_SYSTEMUSER_SYSTEM_HAS_END }, systemHasEnd); - bool userHasBegin = curTmpl[K_SYSTEMUSER_1ST_USER_HAS_BEGIN]; + bool userHasBegin = json_get(curTmpl, { K_SYSTEMUSER_1ST_USER_HAS_BEGIN }, group); gCT.set_value(group, { K_SYSTEMUSER_1ST_USER_HAS_BEGIN }, userHasBegin); - bool userHasPrefix = curTmpl[K_SYSTEMUSER_1ST_USER_HAS_PREFIX]; + bool userHasPrefix = json_get(curTmpl, { K_SYSTEMUSER_1ST_USER_HAS_PREFIX }, group); gCT.set_value(group, { K_SYSTEMUSER_1ST_USER_HAS_PREFIX }, userHasPrefix); } From 604821838354e8b2947625523709718d7219c121 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sun, 12 May 2024 21:58:59 +0530 Subject: [PATCH 185/217] SimpCFG: COnvert to GroupKV extended version Reuse the code already moved into GroupKV Add explicit get and set wrt int32_t, which was added after move to GroupKV wrt basic MapOfMapOfVariant logic. --- common/simpcfg.hpp | 142 +++++++-------------------------------------- 1 file changed, 20 insertions(+), 122 deletions(-) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index 694f822cf40bc..3290a801ded94 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -53,27 +53,13 @@ #include #include +#include "groupkv.hpp" + #define SC_DEBUG #undef SC_DEBUG_VERBOSE #define SC_TEST_PRG -#ifdef SC_TEST_PRG -#include -#include -#define LINFO_LN(FMT, ...) fprintf(stdout, FMT"\n", ##__VA_ARGS__) -#define LDBUG(FMT, ...) fprintf(stderr, FMT, ##__VA_ARGS__) -#define LDBUG_LN(FMT, ...) fprintf(stderr, FMT"\n", ##__VA_ARGS__) -#define LERRR_LN(FMT, ...) fprintf(stderr, FMT"\n", ##__VA_ARGS__) -#define LWARN_LN(FMT, ...) fprintf(stderr, FMT"\n", ##__VA_ARGS__) -#else -#include "log.h" -#define LINFO_LN LOG_TEELN -#define LDBUG LOG -#define LDBUG_LN LOGLN -#define LERRR_LN LOG_TEELN -#define LWARN_LN LOG_TEELN -#endif #undef SC_STR_OVERSMART #ifdef SC_STR_OVERSMART @@ -259,70 +245,20 @@ std::string str(TypeWithStrSupp value) { return ss.str(); } -template -std::string str(std::vector values) { - std::stringstream ss; - ss << "[ "; - int cnt = 0; - for(auto value: values) { - cnt += 1; - if (cnt != 1) ss << ", "; - ss << value; - } - ss << " ]"; - return ss.str(); -} - // **** **** **** the SimpCfg **** **** **** // -typedef std::variant SimpCfgData; -typedef std::vector MultiPart; -typedef std::map> SimpCfgMapMapVariant; - -class SimpCfg { +class SimpCfg : public GroupKV { private: - SimpCfgMapMapVariant mapV = {}; std::regex rInt {R"(^[-+]?\d+$)"}; std::regex rFloat {R"(^[-+]?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?$)"}; public: - SimpCfg(SimpCfgMapMapVariant defaultMap) : mapV(defaultMap) {} + SimpCfg(GroupKVMapMapVariant defaultMap) : GroupKV(defaultMap) {} - static std::string joiner(const MultiPart& parts) { - std::stringstream joined; - int iCnt = 0; - for(auto part: parts) { - if (iCnt != 0) { - joined << "-"; - } - iCnt += 1; - joined << part; - } - return joined.str(); - } - - std::string to_str(const SimpCfgData &value) { - auto visitor = [](auto value) -> auto { - std::stringstream ss; - ss << value; - return ss.str(); - }; - return std::visit(visitor, value); - } - - template - void set_value(const std::string &group, const MultiPart &keyParts, const SupportedDataType &value, const std::string &callerName="") { - auto key = joiner(keyParts); - auto &gm = mapV[group]; - gm[key] = value; -#ifdef SC_DEBUG - LDBUG_LN("DBUG:SC:%s_%s:%s:%s:%s", __func__, callerName.c_str(), group.c_str(), key.c_str(), to_str(value).c_str()); -#endif - } void set_string(const std::string &group, const MultiPart &keyParts, const std::string &value) { set_value(group, keyParts, value, __func__); @@ -339,6 +275,15 @@ class SimpCfg { set_bool(group, keyParts, bValue); } + void set_int32(const std::string &group, const MultiPart &keyParts, int32_t value) { + set_value(group, keyParts, value, __func__); + } + + void set_int32(const std::string &group, const MultiPart &keyParts, std::string &value) { + auto ivalue = strtol(value.c_str(), nullptr, 0); + set_int32(group, keyParts, ivalue); + } + void set_int64(const std::string &group, const MultiPart &keyParts, int64_t value) { set_value(group, keyParts, value, __func__); } @@ -357,36 +302,6 @@ class SimpCfg { set_double(group, keyParts, dvalue); } - // Dump info about the specified group. - // If group is empty, then dump info about all groups maintained in this instance. - void dump(const std::string &group) { - for (auto gm: mapV) { - if (!group.empty() && (gm.first != group)) { - LINFO_LN("INFO:SC:%s:%s:Skipping...", __func__, gm.first.c_str()); - continue; - } - for(auto k: gm.second) { - LINFO_LN("DBUG:SC:%s:%s:Iterate:%s:%s", __func__, gm.first.c_str(), k.first.c_str(), to_str(k.second).c_str()); - } - } - } - - template - SupportedDataType get_value(const std::string &group, const MultiPart &keyParts, const SupportedDataType &defaultValue, const std::string &callerName="") { - auto key = joiner(keyParts); - auto gm = mapV[group]; - if (gm.find(key) == gm.end()) { -#ifdef SC_DEBUG - LWARN_LN("WARN:SC:%s_%s:%s:%s:%s[default]", __func__, callerName.c_str(), group.c_str(), key.c_str(), to_str(defaultValue).c_str()); -#endif - return defaultValue; - } - auto value = gm[key]; -#ifdef SC_DEBUG - LDBUG_LN("DBUG:SC:%s_%s:%s:%s:%s", __func__, callerName.c_str(), group.c_str(), key.c_str(), to_str(value).c_str()); -#endif - return std::get(value); - } std::string get_string(const std::string &group, const MultiPart &keyParts, const std::string &defaultValue) { return get_value(group, keyParts, defaultValue, __func__); @@ -396,6 +311,10 @@ class SimpCfg { return get_value(group, keyParts, defaultValue, __func__); } + int32_t get_int32(const std::string &group, const MultiPart &keyParts, int32_t defaultValue) { + return get_value(group, keyParts, defaultValue, __func__); + } + int64_t get_int64(const std::string &group, const MultiPart &keyParts, int64_t defaultValue) { return get_value(group, keyParts, defaultValue, __func__); } @@ -405,30 +324,6 @@ class SimpCfg { } - template - std::vector get_vector(const std::string &group, const MultiPart &keyParts, const std::vector &defaultValue, const std::string &callerName="") { - auto key = joiner(keyParts); - auto gm = mapV[group]; - std::vector array; - int i = 0; - while(true) { - std::stringstream ssArrayKey; - ssArrayKey << key << "-" << i; - auto arrayKey = ssArrayKey.str(); - if (gm.find(arrayKey) == gm.end()) { - break; - } - array.push_back(std::get(gm[arrayKey])); - i += 1; - } - if (array.empty()) { - LWARN_LN("DBUG:SC:%s_%s:%s:%s:%s[default]", __func__, callerName.c_str(), group.c_str(), key.c_str(), str(defaultValue).c_str()); - return defaultValue; - } - LDBUG_LN("DBUG:SC:%s_%s:%s:%s:%s", __func__, callerName.c_str(), group.c_str(), key.c_str(), str(array).c_str()); - return array; - } - static void locale_prepare(std::string &sSavedLocale) { sSavedLocale = std::setlocale(LC_ALL, nullptr); auto sUpdatedLocale = std::setlocale(LC_ALL, "en_US.UTF-8"); @@ -511,6 +406,9 @@ class SimpCfg { #ifdef SC_TEST_PRG +#include +#include + // **** **** **** some simple test code **** **** **** // From f2dd1263fd24c7549a09c0d7504d0812a9c6be13 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sun, 12 May 2024 22:10:40 +0530 Subject: [PATCH 186/217] GroupKV: Move test code into its own file in tests --- common/groupkv.hpp | 79 ----------------------------------- tests/CMakeLists.txt | 1 + tests/test-chaton-groupkv.cpp | 66 +++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 79 deletions(-) create mode 100644 tests/test-chaton-groupkv.cpp diff --git a/common/groupkv.hpp b/common/groupkv.hpp index 1e5f4628ce188..b76edddfcea3f 100644 --- a/common/groupkv.hpp +++ b/common/groupkv.hpp @@ -20,21 +20,6 @@ #define GKV_DEBUG -//#define GKV_TEST_PRG -#ifdef GKV_TEST_PRG -#include -#include -#define LINFO_LN(FMT, ...) fprintf(stdout, FMT"\n", ##__VA_ARGS__) -#ifdef GKV_DEBUG -#define LDBUG(FMT, ...) fprintf(stderr, FMT, ##__VA_ARGS__) -#define LDBUG_LN(FMT, ...) fprintf(stderr, FMT"\n", ##__VA_ARGS__) -#else -#define LDBUG_LN(...) -#define LDBUG(...) -#endif -#define LERRR_LN(FMT, ...) fprintf(stderr, FMT"\n", ##__VA_ARGS__) -#define LWARN_LN(FMT, ...) fprintf(stderr, FMT"\n", ##__VA_ARGS__) -#else #include "log.h" #define LINFO_LN LOG_TEELN #ifdef GKV_DEBUG @@ -46,7 +31,6 @@ #endif #define LERRR_LN LOG_TEELN #define LWARN_LN LOG_TEELN -#endif typedef std::variant GroupKVData; @@ -181,66 +165,3 @@ class GroupKV { } }; - - -#ifdef GKV_TEST_PRG - - -void gkv_inited() { - GroupKV gkv = {{ - {"Group1",{ - {"testkey11", 11}, - {"testkey12", true} - }}, - {"Group2", { - {"key21", "val21"}, - {"key22", 22}, - {"key23", 2.3} - }} - }}; - - std::cout << "**** gkv inited **** " << std::endl; - std::cout << gkv.dump("", "INFO:GKV:Inited:") << std::endl; - -} - -void gkv_set() { - - std::cout << "**** gkv set **** " << std::endl; - GroupKV gkv = {{}}; - std::cout << gkv.dump("", "INFO:GKV:Set:Initial:") << std::endl; - - gkv.get_value("testme", {"key101b"}, false); - gkv.get_value("testme", {"key101s"}, "Not found"); - gkv.get_value("testme", {"key101i"}, 123456); - gkv.get_value("testme", {"key101d"}, 123456.789); - - gkv.set_value("testme", {"key201b"}, true); - gkv.set_value("testme", {"key201s"}, "hello world"); - gkv.set_value("testme", {"key201i"}, 987654); - gkv.set_value("testme", {"key201d"}, 9988.7766); - - std::cout << gkv.dump("testme", "INFO:GKV:Set:After testme set:") << std::endl; - gkv.get_value("testme", {"key201b"}, false); - gkv.get_value("testme", {"key201s"}, "Not found"); - gkv.get_value("testme", {"key201i"}, 123456); - gkv.get_value("testme", {"key201d"}, 123456.789); - - gkv.get_vector("testme", {"keyA100"}, {1, 2, 3}); - gkv.get_vector("testme", {"keyA100"}, { "A", "അ", "अ", "ಅ" }); - gkv.set_value("testme", {"keyA300-0"}, 330); - gkv.set_value("testme", {"keyA300-1"}, 331); - gkv.set_value("testme", {"keyA300-2"}, 332); - gkv.set_value("testme", {"keyA301-0"}, "India"); - gkv.set_value("testme", {"keyA301", "1"}, "World"); - gkv.set_value("testme", {"keyA301", "2"}, "AkashaGanga"); - gkv.get_vector("testme", {"keyA300"}, {1, 2, 3}); - gkv.get_vector("testme", {"keyA301"}, { "yes 1", "No 2", "very well 3" }); -} - -int main(int argc, char **argv) { - gkv_inited(); - gkv_set(); - return 0; -} -#endif diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 7c766612f7b84..5d94b17bfbc9e 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -117,6 +117,7 @@ llama_target_and_test(test-quantize-perf.cpp) llama_target_and_test(test-sampling.cpp) llama_target_and_test(test-chat-template.cpp) llama_target_and_test(test-chat-template-chaton.cpp) +llama_target_and_test(test-chaton-groupkv.cpp) llama_target_and_test(test-grammar-parser.cpp) llama_target_and_test(test-llama-grammar.cpp) diff --git a/tests/test-chaton-groupkv.cpp b/tests/test-chaton-groupkv.cpp new file mode 100644 index 0000000000000..84eb63a21a331 --- /dev/null +++ b/tests/test-chaton-groupkv.cpp @@ -0,0 +1,66 @@ +// +// Test GroupKV +// + +#include "groupkv.hpp" + + +static void gkv_inited() { + GroupKV gkv = {{ + {"Group1",{ + {"testkey11", 11}, + {"testkey12", true} + }}, + {"Group2", { + {"key21", "val21"}, + {"key22", 22}, + {"key23", 2.3} + }} + }}; + + std::cout << "**** gkv inited **** " << std::endl; + std::cout << gkv.dump("", "INFO:GKV:Inited:") << std::endl; + +} + +static void gkv_set() { + + std::cout << "**** gkv set **** " << std::endl; + GroupKV gkv = {{}}; + std::cout << gkv.dump("", "INFO:GKV:Set:Initial:") << std::endl; + + gkv.get_value("testme", {"key101b"}, false); + gkv.get_value("testme", {"key101s"}, "Not found"); + gkv.get_value("testme", {"key101i"}, 123456); + gkv.get_value("testme", {"key101d"}, 123456.789); + + gkv.set_value("testme", {"key201b"}, true); + gkv.set_value("testme", {"key201s"}, "hello world"); + gkv.set_value("testme", {"key201i"}, 987654); + gkv.set_value("testme", {"key201d"}, 9988.7766); + + std::cout << gkv.dump("testme", "INFO:GKV:Set:After testme set:") << std::endl; + gkv.get_value("testme", {"key201b"}, false); + gkv.get_value("testme", {"key201s"}, "Not found"); + gkv.get_value("testme", {"key201i"}, 123456); + gkv.get_value("testme", {"key201d"}, 123456.789); + + gkv.get_vector("testme", {"keyA100"}, {1, 2, 3}); + gkv.get_vector("testme", {"keyA100"}, { "A", "അ", "अ", "ಅ" }); + gkv.set_value("testme", {"keyA300-0"}, 330); + gkv.set_value("testme", {"keyA300-1"}, 331); + gkv.set_value("testme", {"keyA300-2"}, 332); + gkv.set_value("testme", {"keyA301-0"}, "India"); + gkv.set_value("testme", {"keyA301", "1"}, "World"); + gkv.set_value("testme", {"keyA301", "2"}, "AkashaGanga"); + gkv.get_vector("testme", {"keyA300"}, {1, 2, 3}); + gkv.get_vector("testme", {"keyA301"}, { "yes 1", "No 2", "very well 3" }); +} + +int main(int argc, char **argv) { + log_set_target(log_filename_generator("main", "log")); + log_dump_cmdline(argc, argv); + gkv_inited(); + gkv_set(); + return 0; +} From 3d33d62924fb6bcb1323b43ecf1afcfb264a38a5 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sun, 12 May 2024 22:44:58 +0530 Subject: [PATCH 187/217] SimpCfg: Move testing code into its own file in tests Also set functions to inline or static as appropriate --- common/simpcfg.hpp | 193 ++-------------------------------- tests/CMakeLists.txt | 1 + tests/test-chaton-simpcfg.cpp | 176 +++++++++++++++++++++++++++++++ 3 files changed, 185 insertions(+), 185 deletions(-) create mode 100644 tests/test-chaton-simpcfg.cpp diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index 3290a801ded94..ab61d7483450d 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -59,8 +59,6 @@ #define SC_DEBUG #undef SC_DEBUG_VERBOSE -#define SC_TEST_PRG - #undef SC_STR_OVERSMART #ifdef SC_STR_OVERSMART #define str_trim str_trim_oversmart @@ -72,7 +70,7 @@ // **** **** **** String related helpers **** **** **** // -size_t wcs_to_mbs(std::string &sDest, const std::wstring &wSrc) { +inline size_t wcs_to_mbs(std::string &sDest, const std::wstring &wSrc) { std::mbstate_t mbState = std::mbstate_t(); const wchar_t *wSrcP = wSrc.c_str(); auto reqLen = std::wcsrtombs(nullptr, &wSrcP, 0, &mbState); @@ -80,7 +78,7 @@ size_t wcs_to_mbs(std::string &sDest, const std::wstring &wSrc) { return std::wcsrtombs(sDest.data(), &wSrcP, sDest.length(), &mbState); } -size_t mbs_to_wcs(std::wstring &wDest, const std::string &sSrc) { +inline size_t mbs_to_wcs(std::wstring &wDest, const std::string &sSrc) { std::mbstate_t mbState = std::mbstate_t(); const char *sSrcP = sSrc.c_str(); auto reqLen = std::mbsrtowcs(nullptr, &sSrcP, 0, &mbState); @@ -89,7 +87,7 @@ size_t mbs_to_wcs(std::wstring &wDest, const std::string &sSrc) { } template -void dumphex_string(const TString &sIn, const std::string &msgTag){ +inline void dumphex_string(const TString &sIn, const std::string &msgTag){ LDBUG("%s[ ", msgTag.c_str()); for(auto c: sIn) { auto cSize = sizeof(c); @@ -141,7 +139,7 @@ void dumphex_string(const TString &sIn, const std::string &msgTag){ // then string may get partially trimmed wrt such a char at either end. // template -TString str_trim_dumb(TString sin, const TString &trimChars=" \t\n") { +inline TString str_trim_dumb(TString sin, const TString &trimChars=" \t\n") { #ifdef SC_DEBUG_VERBOSE dumphex_string(sin, "DBUG:StrTrimDumb:Str:"); dumphex_string(trimChars, "DBUG:StrTrimDumb:TrimChars:"); @@ -159,7 +157,7 @@ TString str_trim_dumb(TString sin, const TString &trimChars=" \t\n") { // it may get converted to NativeCharSize chars in the expanded wchar_t encoding space, // thus leading to fixed NativeCharSize driven logic itself handling things sufficiently. // Look at str_trim_dumb comments for additional aspects. -std::string str_trim_oversmart(std::string sIn, const std::string &trimChars=" \t\n") { +inline std::string str_trim_oversmart(std::string sIn, const std::string &trimChars=" \t\n") { std::wstring wIn; mbs_to_wcs(wIn, sIn); std::wstring wTrimChars; @@ -189,7 +187,7 @@ std::string str_trim_oversmart(std::string sIn, const std::string &trimChars=" \ // logics perspective), so not providing oversmart variant. // template -TString str_trim_single(TString sin, const TString& trimChars=" \t\n") { +inline TString str_trim_single(TString sin, const TString& trimChars=" \t\n") { if (sin.empty()) return sin; for(auto c: trimChars) { if (c == sin.front()) { @@ -217,7 +215,7 @@ TString str_trim_single(TString sin, const TString& trimChars=" \t\n") { // * ex: this will work if trying to do the conversion for english language within utf-8 // template -TString str_tolower(const TString &sin) { +inline TString str_tolower(const TString &sin) { TString sout; sout.resize(sin.size()); std::transform(sin.begin(), sin.end(), sout.begin(), [](auto c)->auto {return std::tolower(c);}); @@ -228,7 +226,7 @@ TString str_tolower(const TString &sin) { return sout; } -void str_compare_dump(const std::string &s1, const std::string &s2) { +inline void str_compare_dump(const std::string &s1, const std::string &s2) { LDBUG_LN("DBUG:%s:%s:Len:%zu", __func__, s1.c_str(), s1.length()); LDBUG_LN("DBUG:%s:%s:Len:%zu", __func__, s2.c_str(), s2.length()); int minLen = s1.length() < s2.length() ? s1.length() : s2.length(); @@ -402,178 +400,3 @@ class SimpCfg : public GroupKV { } }; - - -#ifdef SC_TEST_PRG - -#include -#include - - -// **** **** **** some simple test code **** **** **** // - - -void check_string() { - std::vector vStandard = { "123", "1अ3" }; - std::cout << "**** string **** " << vStandard.size() << std::endl; - for(auto sCur: vStandard) { - std::cout << std::format("string: [{}] len[{}] size[{}]", sCur, sCur.length(), sCur.size()) << std::endl; - int i = 0; - for(auto c: sCur) { - std::cout << std::format("string:{}:pos:{}:char:{}[0x{:x}]\n", sCur, i, c, (uint8_t)c); - i += 1; - } - } -} - -void check_u8string() { - std::vector vU8s = { u8"123", u8"1अ3" }; - std::cout << "**** u8string **** " << vU8s.size() << std::endl; - for(auto sCur: vU8s) { - std::string sCurx (sCur.begin(), sCur.end()); - std::cout << std::format("u8string: [{}] len[{}] size[{}]", sCurx, sCur.length(), sCur.size()) << std::endl; - int i = 0; - for(auto c: sCur) { - //std::cout << c << std::endl; - std::cout << std::format("u8string:{}:pos:{}:char:{}[0x{:x}]\n", sCurx, i, (unsigned char)c, (unsigned char)c); - i += 1; - } - } -} - -void check_wstring_wcout() { - std::wcout.imbue(std::locale("en_US.UTF-8")); - std::vector vWide = { L"123", L"1अ3" }; - std::cout << "**** wstring wcout **** " << vWide.size() << std::endl; - for(auto sCur: vWide) { - std::wcout << sCur << std::endl; - std::wcout << std::format(L"wstring: [{}] len[{}] size[{}]", sCur, sCur.length(), sCur.size()) << std::endl; - int i = 0; - for(auto c: sCur) { - std::wcout << std::format(L"wstring:{}:pos:{}:char:{}[0x{:x}]\n", sCur, i, c, c); - i += 1; - } - } -} - -void check_wstring_cout() { - std::vector vWide = { L"123", L"1अ3" }; - std::cout << "**** wstring cout **** " << vWide.size() << std::endl; - for(auto sCur: vWide) { - std::string sCury; - wcs_to_mbs(sCury, sCur); - std::cout << std::format("wstring: [{}] len[{}] size[{}]", sCury, sCur.length(), sCur.size()) << std::endl; - int i = 0; - for(auto c: sCur) { - std::wstringstream wsc; - wsc << c; - std::string ssc; - wcs_to_mbs(ssc, wsc.str()); - std::cout << std::format("wstring:{}:pos:{}:char:{}[0x{:x}]\n", sCury, i, ssc, (uint32_t)c); - i += 1; - } - } -} - -void check_nonenglish() { - std::cout << "**** non english **** " << std::endl; - std::vector vTest1 = { "\n\tAഅअಅ\n\t", "\n\tAഅअಅ " }; - for (auto sTest: vTest1) { - std::string sGotDumb = str_trim_dumb(sTest, {" \n\t"}); - std::string sGotOSmart = str_trim_oversmart(sTest, {" \n\t"}); - std::string sLower = str_tolower(sTest); - std::cout << std::format("{}: Test1 [{}]\n\tTrimDumb[{}]\n\tTrimOverSmart[{}]\n\tLowerDumb[{}]", __func__, sTest, sGotDumb, sGotOSmart, sLower) << std::endl; - } - // The string "\n\tthis र remove 0s and अs at end 000रअ0\xa4अ ", - // * will mess up str_trim_dumb, - // * but will rightly trigger a exception with oversmart. - std::vector vTest2 = { "\n\t this र remove 0s at end 000 ", "\n\tthis र remove 0s, अs, ഇs at end 000रअ0अ ", "\n\tthis र remove 0s, अs, ഇs at end 000रअ0इअ "}; - std::string trimChars = {" \n\tഇ0अ"}; - for (auto sTest: vTest2) { - std::string sGotDumb = str_trim_dumb(sTest, trimChars); - std::string sGotOSmart = str_trim_oversmart(sTest, trimChars); - std::cout << std::format("{}: Test2 [{}]\n\tDumb[{}]\n\tOverSmart[{}]", __func__, sTest, sGotDumb, sGotOSmart) << std::endl; - } -} - -void check_strings() { - std::string sSavedLocale; - SimpCfg::locale_prepare(sSavedLocale); - check_string(); - check_u8string(); - //check_wstring_wcout(); - check_wstring_cout(); - check_nonenglish(); - SimpCfg::locale_restore(sSavedLocale); -} - -void sc_inited() { - SimpCfg sc = {{ - {"Group1",{ - {"testkey11", 11}, - {"testkey12", true} - }}, - {"Group2", { - {"key21", "val21"}, - {"key22", 22}, - {"key23", 2.3} - }} - }}; - - std::cout << "**** sc inited **** " << std::endl; - sc.dump(""); - -} - -void sc_set(const std::string &fname) { - - std::cout << "**** sc set **** " << std::endl; - SimpCfg sc = {{}}; - sc.load(fname); - sc.dump(""); - - sc.get_bool("testme", {"key101b"}, false); - sc.get_string("testme", {"key101s"}, "Not found"); - sc.get_int64("testme", {"key101i"}, 123456); - sc.get_double("testme", {"key101d"}, 123456.789); - - sc.set_bool("testme", {"key201b"}, true); - sc.set_string("testme", {"key201s"}, "hello world"); - sc.set_int64("testme", {"key201i"}, 987654); - sc.set_double("testme", {"key201d"}, 9988.7766); - - sc.dump("testme"); - sc.get_bool("testme", {"key201b"}, false); - sc.get_string("testme", {"key201s"}, "Not found"); - sc.get_int64("testme", {"key201i"}, 123456); - sc.get_double("testme", {"key201d"}, 123456.789); - - sc.get_string("mistral", {"system-prefix"}, "Not found"); - sc.get_string("\"mistral\"", {"\"system-prefix\""}, "Not found"); - - sc.get_vector("testme", {"keyA100"}, {1, 2, 3}); - sc.get_vector("testme", {"keyA100"}, { "A", "അ", "अ", "ಅ" }); - sc.set_int64("testme", {"keyA300-0"}, 330); - sc.set_int64("testme", {"keyA300-1"}, 331); - sc.set_int64("testme", {"keyA300-2"}, 332); - sc.set_string("testme", {"keyA301-0"}, "India"); - sc.set_value("testme", {"keyA301", "1"}, "World"); - sc.set_string("testme", {"keyA301", "2"}, "AkashaGanga"); - sc.get_vector("testme", {"keyA300"}, {1, 2, 3}); - sc.get_vector("testme", {"keyA301"}, { "yes 1", "No 2", "very well 3" }); -} - -int main(int argc, char **argv) { - if (argc != 2) { - LERRR_LN("USAGE:%s simp.cfg", argv[0]); - exit(1); - } - - check_strings(); - sc_inited(); - std::string fname {argv[1]}; - sc_set(fname); - - return 0; -} -#endif diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 5d94b17bfbc9e..6a237d36f4285 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -118,6 +118,7 @@ llama_target_and_test(test-sampling.cpp) llama_target_and_test(test-chat-template.cpp) llama_target_and_test(test-chat-template-chaton.cpp) llama_target_and_test(test-chaton-groupkv.cpp) +llama_target_and_test(test-chaton-simpcfg.cpp) llama_target_and_test(test-grammar-parser.cpp) llama_target_and_test(test-llama-grammar.cpp) diff --git a/tests/test-chaton-simpcfg.cpp b/tests/test-chaton-simpcfg.cpp new file mode 100644 index 0000000000000..4c0b5699a3cf6 --- /dev/null +++ b/tests/test-chaton-simpcfg.cpp @@ -0,0 +1,176 @@ +// +// Test SimpCfg +// + +#include "simpcfg.hpp" + +#include +#include + + +static void check_string() { + std::vector vStandard = { "123", "1अ3" }; + std::cout << "**** string **** " << vStandard.size() << std::endl; + for(auto sCur: vStandard) { + std::cout << std::format("string: [{}] len[{}] size[{}]", sCur, sCur.length(), sCur.size()) << std::endl; + int i = 0; + for(auto c: sCur) { + std::cout << std::format("string:{}:pos:{}:char:{}[0x{:x}]\n", sCur, i, c, (uint8_t)c); + i += 1; + } + } +} + +static void check_u8string() { + std::vector vU8s = { u8"123", u8"1अ3" }; + std::cout << "**** u8string **** " << vU8s.size() << std::endl; + for(auto sCur: vU8s) { + std::string sCurx (sCur.begin(), sCur.end()); + std::cout << std::format("u8string: [{}] len[{}] size[{}]", sCurx, sCur.length(), sCur.size()) << std::endl; + int i = 0; + for(auto c: sCur) { + //std::cout << c << std::endl; + std::cout << std::format("u8string:{}:pos:{}:char:{}[0x{:x}]\n", sCurx, i, (unsigned char)c, (unsigned char)c); + i += 1; + } + } +} + +static void check_wstring_wcout() { + std::wcout.imbue(std::locale("en_US.UTF-8")); + std::vector vWide = { L"123", L"1अ3" }; + std::cout << "**** wstring wcout **** " << vWide.size() << std::endl; + for(auto sCur: vWide) { + std::wcout << sCur << std::endl; + std::wcout << std::format(L"wstring: [{}] len[{}] size[{}]", sCur, sCur.length(), sCur.size()) << std::endl; + int i = 0; + for(auto c: sCur) { + std::wcout << std::format(L"wstring:{}:pos:{}:char:{}[0x{:x}]\n", sCur, i, c, c); + i += 1; + } + } +} + +static void check_wstring_cout() { + std::vector vWide = { L"123", L"1अ3" }; + std::cout << "**** wstring cout **** " << vWide.size() << std::endl; + for(auto sCur: vWide) { + std::string sCury; + wcs_to_mbs(sCury, sCur); + std::cout << std::format("wstring: [{}] len[{}] size[{}]", sCury, sCur.length(), sCur.size()) << std::endl; + int i = 0; + for(auto c: sCur) { + std::wstringstream wsc; + wsc << c; + std::string ssc; + wcs_to_mbs(ssc, wsc.str()); + std::cout << std::format("wstring:{}:pos:{}:char:{}[0x{:x}]\n", sCury, i, ssc, (uint32_t)c); + i += 1; + } + } +} + +static void check_nonenglish() { + std::cout << "**** non english **** " << std::endl; + std::vector vTest1 = { "\n\tAഅअಅ\n\t", "\n\tAഅअಅ " }; + for (auto sTest: vTest1) { + std::string sGotDumb = str_trim_dumb(sTest, {" \n\t"}); + std::string sGotOSmart = str_trim_oversmart(sTest, {" \n\t"}); + std::string sLower = str_tolower(sTest); + std::cout << std::format("{}: Test1 [{}]\n\tTrimDumb[{}]\n\tTrimOverSmart[{}]\n\tLowerDumb[{}]", __func__, sTest, sGotDumb, sGotOSmart, sLower) << std::endl; + } + // The string "\n\tthis र remove 0s and अs at end 000रअ0\xa4अ ", + // * will mess up str_trim_dumb, + // * but will rightly trigger a exception with oversmart. + std::vector vTest2 = { "\n\t this र remove 0s at end 000 ", "\n\tthis र remove 0s, अs, ഇs at end 000रअ0अ ", "\n\tthis र remove 0s, अs, ഇs at end 000रअ0इअ "}; + std::string trimChars = {" \n\tഇ0अ"}; + for (auto sTest: vTest2) { + std::string sGotDumb = str_trim_dumb(sTest, trimChars); + std::string sGotOSmart = str_trim_oversmart(sTest, trimChars); + std::cout << std::format("{}: Test2 [{}]\n\tDumb[{}]\n\tOverSmart[{}]", __func__, sTest, sGotDumb, sGotOSmart) << std::endl; + } +} + +static void check_strings() { + std::string sSavedLocale; + SimpCfg::locale_prepare(sSavedLocale); + check_string(); + check_u8string(); + //check_wstring_wcout(); + check_wstring_cout(); + check_nonenglish(); + SimpCfg::locale_restore(sSavedLocale); +} + +static void sc_inited() { + SimpCfg sc = {{ + {"Group1",{ + {"testkey11", 11}, + {"testkey12", true} + }}, + {"Group2", { + {"key21", "val21"}, + {"key22", 22}, + {"key23", 2.3} + }} + }}; + + std::cout << "**** sc inited **** " << std::endl; + sc.dump(""); + +} + +static void sc_set(const std::string &fname) { + + std::cout << "**** sc set **** " << std::endl; + SimpCfg sc = {{}}; + sc.load(fname); + sc.dump(""); + + sc.get_bool("testme", {"key101b"}, false); + sc.get_string("testme", {"key101s"}, "Not found"); + sc.get_int64("testme", {"key101i"}, 123456); + sc.get_double("testme", {"key101d"}, 123456.789); + + sc.set_bool("testme", {"key201b"}, true); + sc.set_string("testme", {"key201s"}, "hello world"); + sc.set_int64("testme", {"key201i"}, 987654); + sc.set_double("testme", {"key201d"}, 9988.7766); + + sc.dump("testme"); + sc.get_bool("testme", {"key201b"}, false); + sc.get_string("testme", {"key201s"}, "Not found"); + sc.get_int64("testme", {"key201i"}, 123456); + sc.get_double("testme", {"key201d"}, 123456.789); + + sc.get_string("mistral", {"system-prefix"}, "Not found"); + sc.get_string("\"mistral\"", {"\"system-prefix\""}, "Not found"); + + sc.get_vector("testme", {"keyA100"}, {1, 2, 3}); + sc.get_vector("testme", {"keyA100"}, { "A", "അ", "अ", "ಅ" }); + sc.set_int64("testme", {"keyA300-0"}, 330); + sc.set_int64("testme", {"keyA300-1"}, 331); + sc.set_int64("testme", {"keyA300-2"}, 332); + sc.set_string("testme", {"keyA301-0"}, "India"); + sc.set_value("testme", {"keyA301", "1"}, "World"); + sc.set_string("testme", {"keyA301", "2"}, "AkashaGanga"); + sc.get_vector("testme", {"keyA300"}, {1, 2, 3}); + sc.get_vector("testme", {"keyA301"}, { "yes 1", "No 2", "very well 3" }); +} + +int main(int argc, char **argv) { + if (argc != 2) { + LERRR_LN("USAGE:%s simp.cfg", argv[0]); + exit(1); + } + + log_set_target(log_filename_generator("main", "log")); + log_dump_cmdline(argc, argv); + + check_strings(); + sc_inited(); + std::string fname {argv[1]}; + sc_set(fname); + + return 0; +} From 9249649fb30fb3d79c49b7273743e9383eac0d19 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Sun, 12 May 2024 23:59:27 +0530 Subject: [PATCH 188/217] ChatON+TestPrgs: Use specific log files --- tests/test-chaton-groupkv.cpp | 2 +- tests/test-chaton-simpcfg.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test-chaton-groupkv.cpp b/tests/test-chaton-groupkv.cpp index 84eb63a21a331..fd984f760e288 100644 --- a/tests/test-chaton-groupkv.cpp +++ b/tests/test-chaton-groupkv.cpp @@ -58,7 +58,7 @@ static void gkv_set() { } int main(int argc, char **argv) { - log_set_target(log_filename_generator("main", "log")); + log_set_target(log_filename_generator("chaton-groupkv", "log")); log_dump_cmdline(argc, argv); gkv_inited(); gkv_set(); diff --git a/tests/test-chaton-simpcfg.cpp b/tests/test-chaton-simpcfg.cpp index 4c0b5699a3cf6..d0cfef6060c5e 100644 --- a/tests/test-chaton-simpcfg.cpp +++ b/tests/test-chaton-simpcfg.cpp @@ -164,7 +164,7 @@ int main(int argc, char **argv) { exit(1); } - log_set_target(log_filename_generator("main", "log")); + log_set_target(log_filename_generator("chaton-simpcfg", "log")); log_dump_cmdline(argc, argv); check_strings(); From 857570f8f82c8e7f2efdc9099aa31c3cb7a8b5ab Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Mon, 13 May 2024 00:20:58 +0530 Subject: [PATCH 189/217] SimpCfgTest: Update dump usage to GKV return string semantic --- tests/test-chaton-simpcfg.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test-chaton-simpcfg.cpp b/tests/test-chaton-simpcfg.cpp index d0cfef6060c5e..9d7ffc3d56e1e 100644 --- a/tests/test-chaton-simpcfg.cpp +++ b/tests/test-chaton-simpcfg.cpp @@ -116,7 +116,7 @@ static void sc_inited() { }}; std::cout << "**** sc inited **** " << std::endl; - sc.dump(""); + std::cout << sc.dump("") << std::endl; } @@ -125,7 +125,7 @@ static void sc_set(const std::string &fname) { std::cout << "**** sc set **** " << std::endl; SimpCfg sc = {{}}; sc.load(fname); - sc.dump(""); + std::cout << sc.dump("", "INFO:SC:Set:AfterLoad:") << std::endl; sc.get_bool("testme", {"key101b"}, false); sc.get_string("testme", {"key101s"}, "Not found"); @@ -137,7 +137,7 @@ static void sc_set(const std::string &fname) { sc.set_int64("testme", {"key201i"}, 987654); sc.set_double("testme", {"key201d"}, 9988.7766); - sc.dump("testme"); + std::cout << sc.dump("testme", "INFO:SC:Set:AfterSet:") << std::endl; sc.get_bool("testme", {"key201b"}, false); sc.get_string("testme", {"key201s"}, "Not found"); sc.get_int64("testme", {"key201i"}, 123456); From d5b0bfbaec3702b474be42077bfe4ab1ba1633da Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Mon, 13 May 2024 00:33:02 +0530 Subject: [PATCH 190/217] SimpCfg: Remove now unused SC_DEBUG, rather GroupKV uses equiv The code which was using SC_DEBUG moved to GroupKV and inturn GKV_DEBUG --- common/simpcfg.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index ab61d7483450d..0fca387d79a84 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -56,7 +56,6 @@ #include "groupkv.hpp" -#define SC_DEBUG #undef SC_DEBUG_VERBOSE #undef SC_STR_OVERSMART From eb7554ca3bd2f33cb299edb320a45fd5dd0579e2 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Mon, 13 May 2024 10:37:14 +0530 Subject: [PATCH 191/217] ChatON: Avoid -> to match simpcfg as well as corresponding keys --- common/chaton.hpp | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index 60209266b301a..3da99fd9f051d 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -94,7 +94,7 @@ * * The fields that make up a given chat-handshake-template-standard include * - * * global-> begin & end + * * global -> begin & end * * * system -> begin, prefix, suffix & end * @@ -796,20 +796,20 @@ inline bool _chaton_meta_dump(std::string &tmpl) { bool userHasBegin = gCT.get_value(tmpl, { K_SYSTEMUSER_1ST_USER_HAS_BEGIN }); bool userHasPrefix = gCT.get_value(tmpl, { K_SYSTEMUSER_1ST_USER_HAS_PREFIX }); - LOGXLN("INFO:%s:%s:%s", __func__, "global->begin", globalBegin.c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "global->end", globalEnd.c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "system->begin", systemBegin.c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "system->prefix", systemPrefix.c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "system->suffix", systemSuffix.c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "system->end", systemEnd.c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "user->begin", userBegin.c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "user->prefix", userPrefix.c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "user->suffix", userSuffix.c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "user->end", userEnd.c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "assistant->begin", assistantBegin.c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "assistant->prefix", assistantPrefix.c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "assistant->suffix", assistantSuffix.c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "assistant->end", assistantEnd.c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "global-begin", globalBegin.c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "global-end", globalEnd.c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "system-begin", systemBegin.c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "system-prefix", systemPrefix.c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "system-suffix", systemSuffix.c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "system-end", systemEnd.c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "user-begin", userBegin.c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "user-prefix", userPrefix.c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "user-suffix", userSuffix.c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "user-end", userEnd.c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "assistant-begin", assistantBegin.c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "assistant-prefix", assistantPrefix.c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "assistant-suffix", assistantSuffix.c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "assistant-end", assistantEnd.c_str()); LOGXLN("INFO:%s:%s:%s", __func__, K_REVERSE_PROMPT, reversePrompt.c_str()); LOGXLN("INFO:%s:%s:%d", __func__, K_SYSTEMUSER_SYSTEM_HAS_SUFFIX, systemHasSuffix); LOGXLN("INFO:%s:%s:%d", __func__, K_SYSTEMUSER_SYSTEM_HAS_END, systemHasEnd); @@ -817,10 +817,10 @@ inline bool _chaton_meta_dump(std::string &tmpl) { LOGXLN("INFO:%s:%s:%d", __func__, K_SYSTEMUSER_1ST_USER_HAS_PREFIX, userHasPrefix); if (!userEnd.empty()) { - LOG_TEELN("WARN:%s:User->End seems to be set to [%s], do cross check if this is proper and needed", __func__, userEnd.c_str()); + LOG_TEELN("WARN:%s:User-End seems to be set to [%s], do cross check if this is proper and needed", __func__, userEnd.c_str()); } if (!assistantBegin.empty()) { - LOG_TEELN("WARN:%s:Assistant->Begin seems to be set to [%s], do cross check if this is proper and needed", __func__, assistantBegin.c_str()); + LOG_TEELN("WARN:%s:Assistant-Begin seems to be set to [%s], do cross check if this is proper and needed", __func__, assistantBegin.c_str()); } } return true; From 184ac322e33a425d1d53f5a56768ed279ed27285 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Mon, 13 May 2024 16:04:42 +0530 Subject: [PATCH 192/217] ChatON: Make json_get efficient and flexible wrt its calling Also explicitly indicate that we are looking at a chain of keys --- common/chaton.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index 3da99fd9f051d..7a633c13a81f9 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -313,7 +313,7 @@ class ChatTemplates : public GroupKV { #ifdef CHATON_JSON template -inline SupportedType json_get(json &j, std::vector keys, const std::string &msgTag) { +inline SupportedType json_get(json &j, const std::vector &keys, const std::string &msgTag) { json curJ = j; std::stringstream skey; int i = 0; @@ -325,7 +325,7 @@ inline SupportedType json_get(json &j, std::vector keys, const std: curJ = curJ[key]; } else { std::stringstream ss; - ss << "ERRR:ChatON:" << __func__ << ":" << msgTag << ":Key [" << skey.str() << "] is missing"; + ss << "ERRR:ChatON:" << __func__ << ":" << msgTag << ":KeyChain [" << skey.str() << "] is missing"; throw std::runtime_error(ss.str()); } } From 0cfe99076dc930377021b2a40bc69271ac0d9740 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Mon, 13 May 2024 16:51:07 +0530 Subject: [PATCH 193/217] ChatON:ChatTemplates: TmplExists, TmplGetKey, TmplRoleGetKeys ChatTemplate directly supports these now, as well as the existing global instance based corresponding helpers depend on same. --- common/chaton.hpp | 93 ++++++++++++++++++++++++++---------------- examples/main/main.cpp | 8 ++-- 2 files changed, 61 insertions(+), 40 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index 7a633c13a81f9..13ec3a8530205 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -305,6 +305,43 @@ class ChatTemplates : public GroupKV { ChatTemplates(GroupKVMapMapVariant defaultMap) : GroupKV(defaultMap) {} + /** + * Check if the specified chat-template exists or not. + * NOTE: This doesnt cross check, if the template inturn contains all the required fields or not. + */ + bool tmpl_exists(const std::string &tmpl) { + if (!group_exists(tmpl)) { + LOG_TEELN("WARN:CT:%s: tmpl[%s] not found...", __func__, tmpl.c_str()); + return false; + } + return true; + } + + /** + * For the specified chat-template, get the value associated with the specified key/field. + */ + template + SupportedDataType tmpl_getkey(const std::string &tmpl, const std::string &key, const SupportedDataType &defaultValue) { + return get_value(tmpl, {key}, defaultValue, "CTTmplGetKey"); + } + + /** + * For the specified chat-template and the role within, cumulate the values of the specified keys/fields + * and return the same. + */ + std::string tmpl_role_getkeys(const std::string &tmpl, const std::string &role, const std::vector &keys) { + std::string got = ""; + std::string sKeys = ""; + for(auto key: keys) { + got += get_value(tmpl, {role, key}, "", "CTTmplRoleGetKeys"); + sKeys += "+"; + sKeys += key; + } + LDBUG_LN("DBUG:CT:%s:%s:%s:%s:%s", __func__, tmpl.c_str(), role.c_str(), sKeys.c_str(), got.c_str()); + return got; + } + + }; #include "chaton_meta.hpp" @@ -394,35 +431,19 @@ inline bool chaton_meta_load(const std::string &fname) { inline bool chaton_tmpl_exists(const std::string &tmpl) { - if (!gCT.group_exists(tmpl)) { - LOG_TEELN("WARN:%s: tmpl[%s] not found...", __func__, tmpl.c_str()); - return false; - } - return true; + return gCT.tmpl_exists(tmpl); } -inline std::string chaton_tmpl_role_kv(const std::string &tmpl, const std::string &role, const std::vector &keys) { - std::string got = ""; - std::string sKeys = ""; - for(auto key: keys) { - got += gCT.get_value(tmpl, {role, key}, ""); - sKeys += "+"; - sKeys += key; - } - LOGLN("DBUG:%s:%s:%s:%s:%s", __func__, tmpl.c_str(), role.c_str(), sKeys.c_str(), got.c_str()); - return got; +inline std::string chaton_tmpl_role_getkeys(const std::string &tmpl, const std::string &role, const std::vector &keys) { + return gCT.tmpl_role_getkeys(tmpl, role, keys); } -inline std::string chaton_tmpl_kv(const std::string &tmpl, const std::string &key) { - std::string got = gCT.get_value(tmpl, {key}, ""); - LOGLN("DBUG:%s:%s:%s:%s", __func__, tmpl.c_str(), key.c_str(), got.c_str()); - return got; +inline std::string chaton_tmpl_getkey_str(const std::string &tmpl, const std::string &key) { + return gCT.tmpl_getkey(tmpl, {key}, ""); } -inline bool chaton_tmpl_kv_bool(const std::string &tmpl, const std::string &key) { - bool got = gCT.get_value(tmpl, {key}, false); - LOGLN("DBUG:%s:%s:%s:%d", __func__, tmpl.c_str(), key.c_str(), got); - return got; +inline bool chaton_tmpl_getkey_bool(const std::string &tmpl, const std::string &key) { + return gCT.tmpl_getkey(tmpl, {key}, false); } @@ -446,8 +467,8 @@ inline bool chaton_tmpl_apply_single_ex( return false; } ChatParts cp = {}; - std::string beginPrefix = chaton_tmpl_role_kv(tmpl, role, {K_BEGIN, K_PREFIX}); - std::string suffixEnd = chaton_tmpl_role_kv(tmpl, role, {K_SUFFIX, K_END}); + std::string beginPrefix = chaton_tmpl_role_getkeys(tmpl, role, {K_BEGIN, K_PREFIX}); + std::string suffixEnd = chaton_tmpl_role_getkeys(tmpl, role, {K_SUFFIX, K_END}); cp.add_part(ChatParts::S, beginPrefix); cp.add_part(ChatParts::N, content); cp.add_part(ChatParts::S, suffixEnd); @@ -531,7 +552,7 @@ inline bool chaton_tmpl_apply_ex( return false; } ChatParts cp = {}; - std::string globalBegin = chaton_tmpl_role_kv(tmpl, K_GLOBAL, {K_BEGIN}); + std::string globalBegin = chaton_tmpl_role_getkeys(tmpl, K_GLOBAL, {K_BEGIN}); cp.add_part(ChatParts::S, globalBegin); int cntSystem = 0; int cntUser = 0; @@ -539,10 +560,10 @@ inline bool chaton_tmpl_apply_ex( for(const auto msg: msgs) { auto role = msg->role; auto content = msg->content; - std::string begin = chaton_tmpl_role_kv(tmpl, role, {K_BEGIN}); - auto prefix = chaton_tmpl_role_kv(tmpl, role, {K_PREFIX}); - auto suffix = chaton_tmpl_role_kv(tmpl, role, {K_SUFFIX}); - auto end = chaton_tmpl_role_kv(tmpl, role, {K_END}); + std::string begin = chaton_tmpl_role_getkeys(tmpl, role, {K_BEGIN}); + auto prefix = chaton_tmpl_role_getkeys(tmpl, role, {K_PREFIX}); + auto suffix = chaton_tmpl_role_getkeys(tmpl, role, {K_SUFFIX}); + auto end = chaton_tmpl_role_getkeys(tmpl, role, {K_END}); if (role == K_SYSTEM) { cntSystem += 1; cp.add_part(ChatParts::S, begin); @@ -550,10 +571,10 @@ inline bool chaton_tmpl_apply_ex( } else if (role == K_USER) { cntUser += 1; if ((cntSystem == 1) && (cntUser == 1)) { - if (chaton_tmpl_kv_bool(tmpl, K_SYSTEMUSER_1ST_USER_HAS_BEGIN)) { + if (chaton_tmpl_getkey_bool(tmpl, K_SYSTEMUSER_1ST_USER_HAS_BEGIN)) { cp.add_part(ChatParts::S, begin); } - if (chaton_tmpl_kv_bool(tmpl, K_SYSTEMUSER_1ST_USER_HAS_PREFIX)) { + if (chaton_tmpl_getkey_bool(tmpl, K_SYSTEMUSER_1ST_USER_HAS_PREFIX)) { cp.add_part(ChatParts::S, prefix); } } else { @@ -567,10 +588,10 @@ inline bool chaton_tmpl_apply_ex( } cp.add_part(ChatParts::N, content); if (role == K_SYSTEM) { - if (chaton_tmpl_kv_bool(tmpl, K_SYSTEMUSER_SYSTEM_HAS_SUFFIX)) { + if (chaton_tmpl_getkey_bool(tmpl, K_SYSTEMUSER_SYSTEM_HAS_SUFFIX)) { cp.add_part(ChatParts::S, suffix); } - if (chaton_tmpl_kv_bool(tmpl, K_SYSTEMUSER_SYSTEM_HAS_END)) { + if (chaton_tmpl_getkey_bool(tmpl, K_SYSTEMUSER_SYSTEM_HAS_END)) { cp.add_part(ChatParts::S, end); } } else { @@ -579,10 +600,10 @@ inline bool chaton_tmpl_apply_ex( } } if (alertAssistantAtEnd) { - auto assistantBeginPrefix = chaton_tmpl_role_kv(tmpl, K_ASSISTANT, {K_BEGIN, K_PREFIX}); + auto assistantBeginPrefix = chaton_tmpl_role_getkeys(tmpl, K_ASSISTANT, {K_BEGIN, K_PREFIX}); cp.add_part(ChatParts::S, assistantBeginPrefix); } - auto globalEnd = chaton_tmpl_role_kv(tmpl, K_GLOBAL, {K_END}); + auto globalEnd = chaton_tmpl_role_getkeys(tmpl, K_GLOBAL, {K_END}); cp.add_part(ChatParts::S, globalEnd); cp.dump(); tagged = cp.str(); diff --git a/examples/main/main.cpp b/examples/main/main.cpp index a600d16b55b44..1a982d4ce3966 100644 --- a/examples/main/main.cpp +++ b/examples/main/main.cpp @@ -380,13 +380,13 @@ int main(int argc, char ** argv) { } // chaton mode - const auto chaton_assitant_prefix = ::llama_tokenize(ctx, chaton_tmpl_role_kv(params.chaton_template_id, K_ASSISTANT, {K_BEGIN, K_PREFIX}), false, true); + const auto chaton_assitant_prefix = ::llama_tokenize(ctx, chaton_tmpl_role_getkeys(params.chaton_template_id, K_ASSISTANT, {K_BEGIN, K_PREFIX}), false, true); if (params.chaton) { params.interactive = true; // may remove later, by requiring user to explicitly request interactive mode params.interactive_first = true; - params.input_prefix = chaton_tmpl_role_kv(params.chaton_template_id, K_USER, {K_BEGIN, K_PREFIX}); - params.input_suffix = chaton_tmpl_role_kv(params.chaton_template_id, K_USER, {K_SUFFIX, K_END}); - params.antiprompt.emplace_back(chaton_tmpl_kv(params.chaton_template_id, K_REVERSE_PROMPT)); + params.input_prefix = chaton_tmpl_role_getkeys(params.chaton_template_id, K_USER, {K_BEGIN, K_PREFIX}); + params.input_suffix = chaton_tmpl_role_getkeys(params.chaton_template_id, K_USER, {K_SUFFIX, K_END}); + params.antiprompt.emplace_back(chaton_tmpl_getkey_str(params.chaton_template_id, K_REVERSE_PROMPT)); } // enable interactive mode if interactive start is specified From efbb87dba6afb5c9a1212573bd1546cf51551daa Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Mon, 13 May 2024 17:50:15 +0530 Subject: [PATCH 194/217] ChatON:ChatTemplates:TmplBasicCheck --- common/chaton.hpp | 103 +++++++++++++++++++++++++--------------------- 1 file changed, 56 insertions(+), 47 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index 13ec3a8530205..32bce26a17469 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -317,6 +317,60 @@ class ChatTemplates : public GroupKV { return true; } + /** + * Check if all expected keys/fields are present wrt the specified chat-template. + * If any key/field is missing, expect a exception. + */ + bool tmpl_basiccheck(const std::string &tmpl, std::stringstream &ss) { + std::string globalBegin = get_value(tmpl, { K_GLOBAL, K_BEGIN }); + std::string globalEnd = get_value(tmpl, { K_GLOBAL, K_END }); + std::string systemBegin = get_value(tmpl, { K_SYSTEM, K_BEGIN }); + std::string systemPrefix = get_value(tmpl, { K_SYSTEM, K_PREFIX }); + std::string systemSuffix = get_value(tmpl, { K_SYSTEM, K_SUFFIX }); + std::string systemEnd = get_value(tmpl, { K_SYSTEM, K_END }); + std::string userBegin = get_value(tmpl, { K_USER, K_BEGIN }); + std::string userPrefix = get_value(tmpl, { K_USER, K_PREFIX }); + std::string userSuffix = get_value(tmpl, { K_USER, K_SUFFIX }); + std::string userEnd = get_value(tmpl, { K_USER, K_END }); + std::string assistantBegin = get_value(tmpl, { K_ASSISTANT, K_BEGIN }); + std::string assistantPrefix = get_value(tmpl, { K_ASSISTANT, K_PREFIX }); + std::string assistantSuffix = get_value(tmpl, { K_ASSISTANT, K_SUFFIX }); + std::string assistantEnd = get_value(tmpl, { K_ASSISTANT, K_END }); + std::string reversePrompt = get_value(tmpl, { K_REVERSE_PROMPT }); + bool systemHasSuffix = get_value(tmpl, { K_SYSTEMUSER_SYSTEM_HAS_SUFFIX }); + bool systemHasEnd = get_value(tmpl, { K_SYSTEMUSER_SYSTEM_HAS_END }); + bool userHasBegin = get_value(tmpl, { K_SYSTEMUSER_1ST_USER_HAS_BEGIN }); + bool userHasPrefix = get_value(tmpl, { K_SYSTEMUSER_1ST_USER_HAS_PREFIX }); + + LOGXLN("INFO:%s:%s:%s", __func__, "global-begin", globalBegin.c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "global-end", globalEnd.c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "system-begin", systemBegin.c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "system-prefix", systemPrefix.c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "system-suffix", systemSuffix.c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "system-end", systemEnd.c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "user-begin", userBegin.c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "user-prefix", userPrefix.c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "user-suffix", userSuffix.c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "user-end", userEnd.c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "assistant-begin", assistantBegin.c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "assistant-prefix", assistantPrefix.c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "assistant-suffix", assistantSuffix.c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, "assistant-end", assistantEnd.c_str()); + LOGXLN("INFO:%s:%s:%s", __func__, K_REVERSE_PROMPT, reversePrompt.c_str()); + LOGXLN("INFO:%s:%s:%d", __func__, K_SYSTEMUSER_SYSTEM_HAS_SUFFIX, systemHasSuffix); + LOGXLN("INFO:%s:%s:%d", __func__, K_SYSTEMUSER_SYSTEM_HAS_END, systemHasEnd); + LOGXLN("INFO:%s:%s:%d", __func__, K_SYSTEMUSER_1ST_USER_HAS_BEGIN, userHasBegin); + LOGXLN("INFO:%s:%s:%d", __func__, K_SYSTEMUSER_1ST_USER_HAS_PREFIX, userHasPrefix); + + if (!userEnd.empty()) { + LOG_TEELN("WARN:%s:User-End seems to be set to [%s], do cross check if this is proper and needed", __func__, userEnd.c_str()); + } + if (!assistantBegin.empty()) { + LOG_TEELN("WARN:%s:Assistant-Begin seems to be set to [%s], do cross check if this is proper and needed", __func__, assistantBegin.c_str()); + } + + } + /** * For the specified chat-template, get the value associated with the specified key/field. */ @@ -790,59 +844,14 @@ inline std::vector chaton_llama_tokenize_ex( */ inline bool _chaton_meta_dump(std::string &tmpl) { if (!tmpl.empty()) { - if (!gCT.group_exists(tmpl)) { + if (!gCT.tmpl_exists(tmpl)) { LOGXLN("ERRR:%s:Specified template-id [%s] not found", __func__, tmpl.c_str()); return false; } } LOGXLN("\n\nINFO:%s:%s:\n%s", __func__, tmpl.c_str(), gCT.dump(tmpl, "INFO:ChatOnMetaDump:").c_str()); if (!tmpl.empty()) { - std::string globalBegin = gCT.get_value(tmpl, { K_GLOBAL, K_BEGIN }); - std::string globalEnd = gCT.get_value(tmpl, { K_GLOBAL, K_END }); - std::string systemBegin = gCT.get_value(tmpl, { K_SYSTEM, K_BEGIN }); - std::string systemPrefix = gCT.get_value(tmpl, { K_SYSTEM, K_PREFIX }); - std::string systemSuffix = gCT.get_value(tmpl, { K_SYSTEM, K_SUFFIX }); - std::string systemEnd = gCT.get_value(tmpl, { K_SYSTEM, K_END }); - std::string userBegin = gCT.get_value(tmpl, { K_USER, K_BEGIN }); - std::string userPrefix = gCT.get_value(tmpl, { K_USER, K_PREFIX }); - std::string userSuffix = gCT.get_value(tmpl, { K_USER, K_SUFFIX }); - std::string userEnd = gCT.get_value(tmpl, { K_USER, K_END }); - std::string assistantBegin = gCT.get_value(tmpl, { K_ASSISTANT, K_BEGIN }); - std::string assistantPrefix = gCT.get_value(tmpl, { K_ASSISTANT, K_PREFIX }); - std::string assistantSuffix = gCT.get_value(tmpl, { K_ASSISTANT, K_SUFFIX }); - std::string assistantEnd = gCT.get_value(tmpl, { K_ASSISTANT, K_END }); - std::string reversePrompt = gCT.get_value(tmpl, { K_REVERSE_PROMPT }); - bool systemHasSuffix = gCT.get_value(tmpl, { K_SYSTEMUSER_SYSTEM_HAS_SUFFIX }); - bool systemHasEnd = gCT.get_value(tmpl, { K_SYSTEMUSER_SYSTEM_HAS_END }); - bool userHasBegin = gCT.get_value(tmpl, { K_SYSTEMUSER_1ST_USER_HAS_BEGIN }); - bool userHasPrefix = gCT.get_value(tmpl, { K_SYSTEMUSER_1ST_USER_HAS_PREFIX }); - - LOGXLN("INFO:%s:%s:%s", __func__, "global-begin", globalBegin.c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "global-end", globalEnd.c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "system-begin", systemBegin.c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "system-prefix", systemPrefix.c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "system-suffix", systemSuffix.c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "system-end", systemEnd.c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "user-begin", userBegin.c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "user-prefix", userPrefix.c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "user-suffix", userSuffix.c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "user-end", userEnd.c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "assistant-begin", assistantBegin.c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "assistant-prefix", assistantPrefix.c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "assistant-suffix", assistantSuffix.c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "assistant-end", assistantEnd.c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, K_REVERSE_PROMPT, reversePrompt.c_str()); - LOGXLN("INFO:%s:%s:%d", __func__, K_SYSTEMUSER_SYSTEM_HAS_SUFFIX, systemHasSuffix); - LOGXLN("INFO:%s:%s:%d", __func__, K_SYSTEMUSER_SYSTEM_HAS_END, systemHasEnd); - LOGXLN("INFO:%s:%s:%d", __func__, K_SYSTEMUSER_1ST_USER_HAS_BEGIN, userHasBegin); - LOGXLN("INFO:%s:%s:%d", __func__, K_SYSTEMUSER_1ST_USER_HAS_PREFIX, userHasPrefix); - - if (!userEnd.empty()) { - LOG_TEELN("WARN:%s:User-End seems to be set to [%s], do cross check if this is proper and needed", __func__, userEnd.c_str()); - } - if (!assistantBegin.empty()) { - LOG_TEELN("WARN:%s:Assistant-Begin seems to be set to [%s], do cross check if this is proper and needed", __func__, assistantBegin.c_str()); - } + gCT.tmpl_basiccheck(tmpl); } return true; } From fe0c9ce646c738ea7598d247ecef0299878a855b Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Tue, 14 May 2024 00:08:33 +0530 Subject: [PATCH 195/217] ChatON:BasicCheck+:return a string with info, dont directly log --- common/chaton.hpp | 90 +++++++++++++++++++++++++++-------------------- 1 file changed, 52 insertions(+), 38 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index 32bce26a17469..aabd398fcf795 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -320,55 +320,69 @@ class ChatTemplates : public GroupKV { /** * Check if all expected keys/fields are present wrt the specified chat-template. * If any key/field is missing, expect a exception. + * + * Additionally also return a string containing info about all the fields. */ - bool tmpl_basiccheck(const std::string &tmpl, std::stringstream &ss) { + bool tmpl_basiccheck(const std::string &tmpl, std::stringstream &ss, const std::string &msgTag) { + + if (!tmpl_exists(tmpl)) { + LOGXLN("ERRR:CT:%s:Specified template-id [%s] not found", msgTag.c_str(), tmpl.c_str()); + return false; + } + std::string globalBegin = get_value(tmpl, { K_GLOBAL, K_BEGIN }); std::string globalEnd = get_value(tmpl, { K_GLOBAL, K_END }); + std::string systemBegin = get_value(tmpl, { K_SYSTEM, K_BEGIN }); std::string systemPrefix = get_value(tmpl, { K_SYSTEM, K_PREFIX }); std::string systemSuffix = get_value(tmpl, { K_SYSTEM, K_SUFFIX }); std::string systemEnd = get_value(tmpl, { K_SYSTEM, K_END }); + std::string userBegin = get_value(tmpl, { K_USER, K_BEGIN }); std::string userPrefix = get_value(tmpl, { K_USER, K_PREFIX }); std::string userSuffix = get_value(tmpl, { K_USER, K_SUFFIX }); std::string userEnd = get_value(tmpl, { K_USER, K_END }); + std::string assistantBegin = get_value(tmpl, { K_ASSISTANT, K_BEGIN }); std::string assistantPrefix = get_value(tmpl, { K_ASSISTANT, K_PREFIX }); std::string assistantSuffix = get_value(tmpl, { K_ASSISTANT, K_SUFFIX }); std::string assistantEnd = get_value(tmpl, { K_ASSISTANT, K_END }); + std::string reversePrompt = get_value(tmpl, { K_REVERSE_PROMPT }); + bool systemHasSuffix = get_value(tmpl, { K_SYSTEMUSER_SYSTEM_HAS_SUFFIX }); bool systemHasEnd = get_value(tmpl, { K_SYSTEMUSER_SYSTEM_HAS_END }); bool userHasBegin = get_value(tmpl, { K_SYSTEMUSER_1ST_USER_HAS_BEGIN }); bool userHasPrefix = get_value(tmpl, { K_SYSTEMUSER_1ST_USER_HAS_PREFIX }); - LOGXLN("INFO:%s:%s:%s", __func__, "global-begin", globalBegin.c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "global-end", globalEnd.c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "system-begin", systemBegin.c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "system-prefix", systemPrefix.c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "system-suffix", systemSuffix.c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "system-end", systemEnd.c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "user-begin", userBegin.c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "user-prefix", userPrefix.c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "user-suffix", userSuffix.c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "user-end", userEnd.c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "assistant-begin", assistantBegin.c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "assistant-prefix", assistantPrefix.c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "assistant-suffix", assistantSuffix.c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, "assistant-end", assistantEnd.c_str()); - LOGXLN("INFO:%s:%s:%s", __func__, K_REVERSE_PROMPT, reversePrompt.c_str()); - LOGXLN("INFO:%s:%s:%d", __func__, K_SYSTEMUSER_SYSTEM_HAS_SUFFIX, systemHasSuffix); - LOGXLN("INFO:%s:%s:%d", __func__, K_SYSTEMUSER_SYSTEM_HAS_END, systemHasEnd); - LOGXLN("INFO:%s:%s:%d", __func__, K_SYSTEMUSER_1ST_USER_HAS_BEGIN, userHasBegin); - LOGXLN("INFO:%s:%s:%d", __func__, K_SYSTEMUSER_1ST_USER_HAS_PREFIX, userHasPrefix); + ss << msgTag << ":" + tmpl + ":" << "global-begin" << ":" << globalBegin << std::endl; + ss << msgTag << ":" + tmpl + ":" << "global-end" << ":" << globalEnd << std::endl; + ss << msgTag << ":" + tmpl + ":" << "system-begin" << ":" << systemBegin << std::endl; + ss << msgTag << ":" + tmpl + ":" << "system-prefix" << ":" << systemPrefix << std::endl; + ss << msgTag << ":" + tmpl + ":" << "system-suffix" << ":" << systemSuffix << std::endl; + ss << msgTag << ":" + tmpl + ":" << "system-end" << ":" << systemEnd << std::endl; + ss << msgTag << ":" + tmpl + ":" << "user-begin" << ":" << userBegin << std::endl; + ss << msgTag << ":" + tmpl + ":" << "user-prefix" << ":" << userPrefix << std::endl; + ss << msgTag << ":" + tmpl + ":" << "user-suffix" << ":" << userSuffix << std::endl; + ss << msgTag << ":" + tmpl + ":" << "user-end" << ":" << userEnd << std::endl; + ss << msgTag << ":" + tmpl + ":" << "assistant-begin" << ":" << assistantBegin << std::endl; + ss << msgTag << ":" + tmpl + ":" << "assistant-prefix" << ":" << assistantPrefix << std::endl; + ss << msgTag << ":" + tmpl + ":" << "assistant-suffix" << ":" << assistantSuffix << std::endl; + ss << msgTag << ":" + tmpl + ":" << "assistant-end" << ":" << assistantEnd << std::endl; + ss << msgTag << ":" + tmpl + ":" << K_REVERSE_PROMPT << ":" << reversePrompt << std::endl; + ss << msgTag << ":" + tmpl + ":" << K_SYSTEMUSER_SYSTEM_HAS_SUFFIX << ":" << systemHasSuffix << std::endl; + ss << msgTag << ":" + tmpl + ":" << K_SYSTEMUSER_SYSTEM_HAS_END << ":" << systemHasEnd << std::endl; + ss << msgTag << ":" + tmpl + ":" << K_SYSTEMUSER_1ST_USER_HAS_BEGIN << ":" << userHasBegin << std::endl; + ss << msgTag << ":" + tmpl + ":" << K_SYSTEMUSER_1ST_USER_HAS_PREFIX << ":" << userHasPrefix << std::endl; if (!userEnd.empty()) { - LOG_TEELN("WARN:%s:User-End seems to be set to [%s], do cross check if this is proper and needed", __func__, userEnd.c_str()); + LOG_TEELN("WARN:CT:%s:User-End seems to be set to [%s], do cross check if this is proper and needed", msgTag.c_str(), userEnd.c_str()); } if (!assistantBegin.empty()) { - LOG_TEELN("WARN:%s:Assistant-Begin seems to be set to [%s], do cross check if this is proper and needed", __func__, assistantBegin.c_str()); + LOG_TEELN("WARN:CT:%s:Assistant-Begin seems to be set to [%s], do cross check if this is proper and needed", msgTag.c_str(), assistantBegin.c_str()); } + return true; } /** @@ -836,29 +850,29 @@ inline std::vector chaton_llama_tokenize_ex( /** - * if tmpl is - * * empty string, then dump the full loaded chaton-meta - * * chaton-template-id, then dump contents related to that specific chat-handshake-template-standard - * NOTE: It uses the exception raising get_value to check if the tags related keys are present - * wrt the specified template-standard/model-id or not. + * Dump the full loaded chaton templates data + * Additionally if a chaton-template-id is specified + * dump contents related to that specific chat-handshake-template-standard + * NOTE: It uses the exception raising get_value to check if all the required + * keys/fields are present wrt the specified template-standard/model-id or not. */ -inline bool _chaton_meta_dump(std::string &tmpl) { - if (!tmpl.empty()) { - if (!gCT.tmpl_exists(tmpl)) { - LOGXLN("ERRR:%s:Specified template-id [%s] not found", __func__, tmpl.c_str()); - return false; - } +inline bool _chaton_meta_validate_dump(std::string &tmpl) { + LOGXLN("\n\nINFO:%s:%s:\n%s", __func__, tmpl.c_str(), gCT.dump(tmpl, "INFO:ChatOnMetaValidateDump:").c_str()); + if (tmpl.empty()) { + return true; } - LOGXLN("\n\nINFO:%s:%s:\n%s", __func__, tmpl.c_str(), gCT.dump(tmpl, "INFO:ChatOnMetaDump:").c_str()); - if (!tmpl.empty()) { - gCT.tmpl_basiccheck(tmpl); + std::stringstream ss; + if (gCT.tmpl_basiccheck(tmpl, ss, "INFO:ChatOnMetaValidateDump")) { + LOGXLN("%s", ss.str().c_str()); + } else { + return false; } return true; } /** - * Verify that specified chaton-template-id contains required fields using meta-dump + * Verify that specified chaton-template-id contains required fields using meta-validate-dump */ inline bool chaton_meta_ok(std::string &tmpl) { - return _chaton_meta_dump(tmpl); + return _chaton_meta_validate_dump(tmpl); } From 8165bd4035f305090aa9d8d161daf4d19704fb96 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Tue, 14 May 2024 00:44:47 +0530 Subject: [PATCH 196/217] ChatON:WIP:chaton_tmpl_apply_single build on multi msg tagging To avoid having to duplicate any hardcoding in future, wrt any new model/chat-template-standard, at multiple locations, remove the single message templating code with a wrapper which does the same but using the multi-msg templating helper. --- common/chaton.hpp | 103 ++++++++++------------------------------------ 1 file changed, 22 insertions(+), 81 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index aabd398fcf795..94c8b4c8c971a 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -515,87 +515,6 @@ inline bool chaton_tmpl_getkey_bool(const std::string &tmpl, const std::string & } -// Given the template standard, role and a message, this returns -// a tagged message, types string and lens vector wrt the parts that make up the returned string -// -// * a string containing the tagged message -// * role-(begin+prefix) + msg + role-(suffix+end) -// * a string where the chars contain info about -// type of sub-strings/parts that make up the tagged message. -// * a vector of ints, which give the length of each part in the tagged message. -inline bool chaton_tmpl_apply_single_ex( - const std::string &tmpl, - const std::string &role, - const std::string &content, - std::string &tagged, - std::string &types, - std::vector &lens - ) { - if (!chaton_tmpl_exists(tmpl)) { - return false; - } - ChatParts cp = {}; - std::string beginPrefix = chaton_tmpl_role_getkeys(tmpl, role, {K_BEGIN, K_PREFIX}); - std::string suffixEnd = chaton_tmpl_role_getkeys(tmpl, role, {K_SUFFIX, K_END}); - cp.add_part(ChatParts::S, beginPrefix); - cp.add_part(ChatParts::N, content); - cp.add_part(ChatParts::S, suffixEnd); - cp.dump(); - tagged = cp.str(); - LOGLN("DBUG:%s:%s:%s:%s", __func__, tmpl.c_str(), role.c_str(), tagged.c_str()); - types = cp.get_partstypes(); - lens = cp.get_partslens(); - return true; -} - -// Given the template standard, role and a message, this returns the tagged message. -// -// * a string containing the tagged message -// * role-(begin+prefix) + msg + role-(suffix+end) -inline size_t chaton_tmpl_apply_single( - const std::string &tmpl, - const std::string &role, - const std::string &content, - std::string &tagged - ) { - std::string types; - std::vector lens; - if (!chaton_tmpl_apply_single_ex(tmpl, role, content, tagged, types, lens)) { - return -1; - } - return tagged.size(); -} - -/** - * Apply chat-handshake-template for the specified template standard and role. - * If the passed char array is smaller than that required for the tagged message, - * * part of the tagged message which fits within dest buffer is copied - * * the returned value, indicates the size of the actual tagged message - * NOTE: - * * ideally the passed char array should be able to fit the tagged message+0|null char. - * * if the return value from this function is larger than or equal to destLength, - * then you will have to increase the size of the dest buffer, and call this - * function a second time, to ensure that one gets the full tagged message. - */ -inline size_t chat_tmpl_apply_single_capi( - const char *tmpl, - const char *role, - const char *content, - char *dest, - const size_t destLength - ) { - std::string tagged; - auto taggedLength = chaton_tmpl_apply_single(tmpl, role, content, tagged); - if (taggedLength <= 0) { - return taggedLength; - } - if (dest && (destLength > 0)) { - strlcpy(dest, tagged.c_str(), destLength); - } - return taggedLength; -} - - // Given the template standard and a bunch of messages including their roles, this returns // tagged messages, types string and lens vector. Returned types string and lens vector help // identify the parts of the tagged msgs string, which relate to passed msgs and added tags. @@ -699,6 +618,28 @@ inline int32_t chaton_tmpl_apply( return tagged.size(); } +// +// Given the template standard, role and a message, this creates the tagged message. +// +// string containing the tagged message +// * role-(begin+prefix) + msg + role-(suffix+end) +// +inline size_t chaton_tmpl_apply_single( + const std::string &tmpl, + const std::string &role, + const std::string &content, + bool alertAssistantAtEnd, + std::string &tagged + ) { + std::string types; + std::vector lens; + llama_chat_message cm {role.c_str(), content.c_str()}; + if (!chaton_tmpl_apply_ex(tmpl, {&cm}, alertAssistantAtEnd, tagged, types, lens)) { + return -1; + } + return tagged.size(); +} + // Given the template standard and a bunch of messages including their roles, this returns // the tagged messages as a string. // global-begin + 1 or more [[role-begin] + [role-prefix] + msg + [role-suffix] +[role-end]] + global-end From 3fcaf199671a70b0293362e962c03bd665fe49dc Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Tue, 14 May 2024 00:58:16 +0530 Subject: [PATCH 197/217] ChatON+:Multi4Single: applyGlobalIfAny flag wrt templating api Given that now the multi chat templating logic itself is used to apply chat templating/tagging to a single chat message, so give flexibility of deciding whether global tags if any should be applied or not wrt the core tagging logic. examples/main inturn updated to not apply global tags if any wrt the system message. Also the user messages already dont apply global tags if any, as its currently implemented to build on the existing in-prefix/suffix and anitprompt flow. --- common/chaton.hpp | 23 +++++++++++++++-------- examples/main/main.cpp | 2 +- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index 94c8b4c8c971a..3e69c5fdaaf9c 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -531,6 +531,7 @@ inline bool chaton_tmpl_apply_ex( const std::string &tmpl, const std::vector &msgs, bool alertAssistantAtEnd, + bool applyGlobalIfAny, std::string &tagged, std::string &types, std::vector &lens @@ -539,8 +540,10 @@ inline bool chaton_tmpl_apply_ex( return false; } ChatParts cp = {}; - std::string globalBegin = chaton_tmpl_role_getkeys(tmpl, K_GLOBAL, {K_BEGIN}); - cp.add_part(ChatParts::S, globalBegin); + if (applyGlobalIfAny) { + std::string globalBegin = chaton_tmpl_role_getkeys(tmpl, K_GLOBAL, {K_BEGIN}); + cp.add_part(ChatParts::S, globalBegin); + } int cntSystem = 0; int cntUser = 0; int cntOthers = 0; @@ -590,8 +593,10 @@ inline bool chaton_tmpl_apply_ex( auto assistantBeginPrefix = chaton_tmpl_role_getkeys(tmpl, K_ASSISTANT, {K_BEGIN, K_PREFIX}); cp.add_part(ChatParts::S, assistantBeginPrefix); } - auto globalEnd = chaton_tmpl_role_getkeys(tmpl, K_GLOBAL, {K_END}); - cp.add_part(ChatParts::S, globalEnd); + if (applyGlobalIfAny) { + auto globalEnd = chaton_tmpl_role_getkeys(tmpl, K_GLOBAL, {K_END}); + cp.add_part(ChatParts::S, globalEnd); + } cp.dump(); tagged = cp.str(); LOGLN("DBUG:%s:%s:%s", __func__, tmpl.c_str(), tagged.c_str()); @@ -608,11 +613,12 @@ inline int32_t chaton_tmpl_apply( const std::string &tmpl, const std::vector &msgs, bool alertAssistantAtEnd, + bool applyGlobalIfAny, std::string &tagged ) { std::string types; std::vector lens; - if (!chaton_tmpl_apply_ex(tmpl, msgs, alertAssistantAtEnd, tagged, types, lens)) { + if (!chaton_tmpl_apply_ex(tmpl, msgs, alertAssistantAtEnd, applyGlobalIfAny, tagged, types, lens)) { return -1; } return tagged.size(); @@ -629,12 +635,13 @@ inline size_t chaton_tmpl_apply_single( const std::string &role, const std::string &content, bool alertAssistantAtEnd, + bool applyGlobalIfAny, std::string &tagged ) { std::string types; std::vector lens; llama_chat_message cm {role.c_str(), content.c_str()}; - if (!chaton_tmpl_apply_ex(tmpl, {&cm}, alertAssistantAtEnd, tagged, types, lens)) { + if (!chaton_tmpl_apply_ex(tmpl, {&cm}, alertAssistantAtEnd, applyGlobalIfAny, tagged, types, lens)) { return -1; } return tagged.size(); @@ -669,7 +676,7 @@ inline int32_t chaton_tmpl_apply_capi( vMsgs.push_back(&msgs[i]); } std::string taggedMsgs; - int32_t taggedLength = chaton_tmpl_apply(tmpl, vMsgs, alertAssistantAtEnd, taggedMsgs); + int32_t taggedLength = chaton_tmpl_apply(tmpl, vMsgs, alertAssistantAtEnd, true, taggedMsgs); if (taggedLength < 0) { return taggedLength; } @@ -714,7 +721,7 @@ inline int32_t chaton_tmpl_apply_ex_capi( std::string taggedMsgs; std::string types; std::vector lens; - if (!chaton_tmpl_apply_ex(tmpl, vMsgs, alertAssistantAtEnd, taggedMsgs, types, lens)) { + if (!chaton_tmpl_apply_ex(tmpl, vMsgs, alertAssistantAtEnd, true, taggedMsgs, types, lens)) { return -1; } int32_t taggedLength = taggedMsgs.size(); diff --git a/examples/main/main.cpp b/examples/main/main.cpp index 1a982d4ce3966..f86592fc7da41 100644 --- a/examples/main/main.cpp +++ b/examples/main/main.cpp @@ -266,7 +266,7 @@ int main(int argc, char ** argv) { params.prompt = "<|im_start|>system\n" + params.prompt + "<|im_end|>"; } if (params.chaton) { - chaton_tmpl_apply_single(params.chaton_template_id, K_SYSTEM, params.prompt, params.prompt); + chaton_tmpl_apply_single(params.chaton_template_id, K_SYSTEM, params.prompt, false, false, params.prompt); } embd_inp = ::llama_tokenize(ctx, params.prompt, true, true); } else { From 6e13c0c87e490670b7af254f6e123118d5a7bbd2 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Tue, 14 May 2024 01:19:04 +0530 Subject: [PATCH 198/217] ChatON:Control SystemMsgSuffix+End tags only wrt 1st system msg Make it similar to user-begin+prefix control. ie only wrt 1st msg of respective type. --- common/chaton.hpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index 3e69c5fdaaf9c..3547881e43ce9 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -578,10 +578,15 @@ inline bool chaton_tmpl_apply_ex( } cp.add_part(ChatParts::N, content); if (role == K_SYSTEM) { - if (chaton_tmpl_getkey_bool(tmpl, K_SYSTEMUSER_SYSTEM_HAS_SUFFIX)) { + if (cntSystem == 1) { + if (chaton_tmpl_getkey_bool(tmpl, K_SYSTEMUSER_SYSTEM_HAS_SUFFIX)) { + cp.add_part(ChatParts::S, suffix); + } + if (chaton_tmpl_getkey_bool(tmpl, K_SYSTEMUSER_SYSTEM_HAS_END)) { + cp.add_part(ChatParts::S, end); + } + } else { cp.add_part(ChatParts::S, suffix); - } - if (chaton_tmpl_getkey_bool(tmpl, K_SYSTEMUSER_SYSTEM_HAS_END)) { cp.add_part(ChatParts::S, end); } } else { From 600653dae289b8fbc34a87e17e0da623f6500e53 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Tue, 14 May 2024 01:27:24 +0530 Subject: [PATCH 199/217] ChatON:Optional control of MsgCntBasedTagging Use same to bypass any msg count based tagging behaviour for the single message tagging through its helper wrapper. --- common/chaton.hpp | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index 3547881e43ce9..96d25c0453e6d 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -534,7 +534,9 @@ inline bool chaton_tmpl_apply_ex( bool applyGlobalIfAny, std::string &tagged, std::string &types, - std::vector &lens + std::vector &lens, + int curSystemMsgCnt = 0, + int curUserMsgCnt = 0 ) { if (!chaton_tmpl_exists(tmpl)) { return false; @@ -544,8 +546,8 @@ inline bool chaton_tmpl_apply_ex( std::string globalBegin = chaton_tmpl_role_getkeys(tmpl, K_GLOBAL, {K_BEGIN}); cp.add_part(ChatParts::S, globalBegin); } - int cntSystem = 0; - int cntUser = 0; + int cntSystem = curSystemMsgCnt; + int cntUser = curUserMsgCnt; int cntOthers = 0; for(const auto msg: msgs) { auto role = msg->role; @@ -629,6 +631,7 @@ inline int32_t chaton_tmpl_apply( return tagged.size(); } +const int BYPASS_MSGCNT = 101; // // Given the template standard, role and a message, this creates the tagged message. // @@ -646,7 +649,7 @@ inline size_t chaton_tmpl_apply_single( std::string types; std::vector lens; llama_chat_message cm {role.c_str(), content.c_str()}; - if (!chaton_tmpl_apply_ex(tmpl, {&cm}, alertAssistantAtEnd, applyGlobalIfAny, tagged, types, lens)) { + if (!chaton_tmpl_apply_ex(tmpl, {&cm}, alertAssistantAtEnd, applyGlobalIfAny, tagged, types, lens, BYPASS_MSGCNT, BYPASS_MSGCNT)) { return -1; } return tagged.size(); From 4dfd10a40d4db55cc3a25572d36269a32f408d84 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Tue, 14 May 2024 01:49:38 +0530 Subject: [PATCH 200/217] ChatON: Move core templating/tagging code into ChatTemplates class However still retain the wrappers, which work with a predefined global instance of ChatTemplates. --- common/chaton.hpp | 191 ++++++++++++++++++++++++++-------------------- 1 file changed, 107 insertions(+), 84 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index 96d25c0453e6d..ba377f2b93eaa 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -409,6 +409,108 @@ class ChatTemplates : public GroupKV { return got; } + /** + * Given the template standard and a bunch of messages including their roles, this returns + * tagged messages, subPartsTypes string and subPartsLens vector. The returned subParts + * types string and lens vector help identify the parts of the tagged msgs string, + * which relate to passed msgs and added tags. + * + * * a string containing the tagged messages + * * global-begin + 1 or more [[role-begin] + [role-prefix] + msg + [role-suffix] +[role-end]] + global-end + * * a string where the chars contain info about + * type of sub-strings/parts that make up the tagged messages string. + * * a vector of ints, which give the length of each part in the tagged messages string. + * + * If a combination of system-user messages is passed, then tags between the 1st system and + * the 1st user message, is based on the flags set wrt the corresponding template standard. + * If you dont want this behaviour, pass non 0 values wrt the optional cntSystemMsgCnt and + * cntUserMsgCnt arguments. + */ + bool chaton_tmpl_apply_ex( + const std::string &tmpl, + const std::vector &msgs, + bool alertAssistantAtEnd, + bool applyGlobalIfAny, + std::string &tagged, + std::string &types, + std::vector &lens, + int curSystemMsgCnt = 0, + int curUserMsgCnt = 0 + ) { + if (!tmpl_exists(tmpl)) { + return false; + } + ChatParts cp = {}; + if (applyGlobalIfAny) { + std::string globalBegin = tmpl_role_getkeys(tmpl, K_GLOBAL, {K_BEGIN}); + cp.add_part(ChatParts::S, globalBegin); + } + int cntSystem = curSystemMsgCnt; + int cntUser = curUserMsgCnt; + int cntOthers = 0; + for(const auto msg: msgs) { + auto role = msg->role; + auto content = msg->content; + std::string begin = tmpl_role_getkeys(tmpl, role, {K_BEGIN}); + auto prefix = tmpl_role_getkeys(tmpl, role, {K_PREFIX}); + auto suffix = tmpl_role_getkeys(tmpl, role, {K_SUFFIX}); + auto end = tmpl_role_getkeys(tmpl, role, {K_END}); + if (role == K_SYSTEM) { + cntSystem += 1; + cp.add_part(ChatParts::S, begin); + cp.add_part(ChatParts::S, prefix); + } else if (role == K_USER) { + cntUser += 1; + if ((cntSystem == 1) && (cntUser == 1)) { + if (tmpl_getkey(tmpl, K_SYSTEMUSER_1ST_USER_HAS_BEGIN, true)) { + cp.add_part(ChatParts::S, begin); + } + if (tmpl_getkey(tmpl, K_SYSTEMUSER_1ST_USER_HAS_PREFIX, true)) { + cp.add_part(ChatParts::S, prefix); + } + } else { + cp.add_part(ChatParts::S, begin); + cp.add_part(ChatParts::S, prefix); + } + } else { + cntOthers += 1; + cp.add_part(ChatParts::S, begin); + cp.add_part(ChatParts::S, prefix); + } + cp.add_part(ChatParts::N, content); + if (role == K_SYSTEM) { + if (cntSystem == 1) { + if (tmpl_getkey(tmpl, K_SYSTEMUSER_SYSTEM_HAS_SUFFIX, true)) { + cp.add_part(ChatParts::S, suffix); + } + if (tmpl_getkey(tmpl, K_SYSTEMUSER_SYSTEM_HAS_END, true)) { + cp.add_part(ChatParts::S, end); + } + } else { + cp.add_part(ChatParts::S, suffix); + cp.add_part(ChatParts::S, end); + } + } else { + cp.add_part(ChatParts::S, suffix); + cp.add_part(ChatParts::S, end); + } + } + if (alertAssistantAtEnd) { + auto assistantBeginPrefix = tmpl_role_getkeys(tmpl, K_ASSISTANT, {K_BEGIN, K_PREFIX}); + cp.add_part(ChatParts::S, assistantBeginPrefix); + } + if (applyGlobalIfAny) { + auto globalEnd = tmpl_role_getkeys(tmpl, K_GLOBAL, {K_END}); + cp.add_part(ChatParts::S, globalEnd); + } + cp.dump(); + tagged = cp.str(); + LOGLN("DBUG:CT:%s:%s:%s", __func__, tmpl.c_str(), tagged.c_str()); + LOGLN("DBUG:CT:%s:%s:CntSys[%d]:CntUsr[%d]:CntOthers[%d]", __func__, tmpl.c_str(), cntSystem, cntUser, cntOthers); + types = cp.get_partstypes(); + lens = cp.get_partslens(); + return true; + } }; @@ -516,17 +618,10 @@ inline bool chaton_tmpl_getkey_bool(const std::string &tmpl, const std::string & // Given the template standard and a bunch of messages including their roles, this returns -// tagged messages, types string and lens vector. Returned types string and lens vector help -// identify the parts of the tagged msgs string, which relate to passed msgs and added tags. -// -// * a string containing the tagged messages -// * global-begin + 1 or more [[role-begin] + [role-prefix] + msg + [role-suffix] +[role-end]] + global-end -// * a string where the chars contain info about -// type of sub-strings/parts that make up the tagged messages string. -// * a vector of ints, which give the length of each part in the tagged messages string. +// the tagged messages as a string. +// global-begin + 1 or more [[role-begin] + [role-prefix] + msg + [role-suffix] +[role-end]] + global-end // -// if a combination of system-user messages is passed, then tags between the system -// and the 1st user message, is based on the flags set wrt the corresponding template standard. +// Additionally also return info about the parts that make up the tagged message. inline bool chaton_tmpl_apply_ex( const std::string &tmpl, const std::vector &msgs, @@ -538,79 +633,7 @@ inline bool chaton_tmpl_apply_ex( int curSystemMsgCnt = 0, int curUserMsgCnt = 0 ) { - if (!chaton_tmpl_exists(tmpl)) { - return false; - } - ChatParts cp = {}; - if (applyGlobalIfAny) { - std::string globalBegin = chaton_tmpl_role_getkeys(tmpl, K_GLOBAL, {K_BEGIN}); - cp.add_part(ChatParts::S, globalBegin); - } - int cntSystem = curSystemMsgCnt; - int cntUser = curUserMsgCnt; - int cntOthers = 0; - for(const auto msg: msgs) { - auto role = msg->role; - auto content = msg->content; - std::string begin = chaton_tmpl_role_getkeys(tmpl, role, {K_BEGIN}); - auto prefix = chaton_tmpl_role_getkeys(tmpl, role, {K_PREFIX}); - auto suffix = chaton_tmpl_role_getkeys(tmpl, role, {K_SUFFIX}); - auto end = chaton_tmpl_role_getkeys(tmpl, role, {K_END}); - if (role == K_SYSTEM) { - cntSystem += 1; - cp.add_part(ChatParts::S, begin); - cp.add_part(ChatParts::S, prefix); - } else if (role == K_USER) { - cntUser += 1; - if ((cntSystem == 1) && (cntUser == 1)) { - if (chaton_tmpl_getkey_bool(tmpl, K_SYSTEMUSER_1ST_USER_HAS_BEGIN)) { - cp.add_part(ChatParts::S, begin); - } - if (chaton_tmpl_getkey_bool(tmpl, K_SYSTEMUSER_1ST_USER_HAS_PREFIX)) { - cp.add_part(ChatParts::S, prefix); - } - } else { - cp.add_part(ChatParts::S, begin); - cp.add_part(ChatParts::S, prefix); - } - } else { - cntOthers += 1; - cp.add_part(ChatParts::S, begin); - cp.add_part(ChatParts::S, prefix); - } - cp.add_part(ChatParts::N, content); - if (role == K_SYSTEM) { - if (cntSystem == 1) { - if (chaton_tmpl_getkey_bool(tmpl, K_SYSTEMUSER_SYSTEM_HAS_SUFFIX)) { - cp.add_part(ChatParts::S, suffix); - } - if (chaton_tmpl_getkey_bool(tmpl, K_SYSTEMUSER_SYSTEM_HAS_END)) { - cp.add_part(ChatParts::S, end); - } - } else { - cp.add_part(ChatParts::S, suffix); - cp.add_part(ChatParts::S, end); - } - } else { - cp.add_part(ChatParts::S, suffix); - cp.add_part(ChatParts::S, end); - } - } - if (alertAssistantAtEnd) { - auto assistantBeginPrefix = chaton_tmpl_role_getkeys(tmpl, K_ASSISTANT, {K_BEGIN, K_PREFIX}); - cp.add_part(ChatParts::S, assistantBeginPrefix); - } - if (applyGlobalIfAny) { - auto globalEnd = chaton_tmpl_role_getkeys(tmpl, K_GLOBAL, {K_END}); - cp.add_part(ChatParts::S, globalEnd); - } - cp.dump(); - tagged = cp.str(); - LOGLN("DBUG:%s:%s:%s", __func__, tmpl.c_str(), tagged.c_str()); - LOGLN("DBUG:%s:%s:CntSys[%d]:CntUsr[%d]:CntOthers[%d]", __func__, tmpl.c_str(), cntSystem, cntUser, cntOthers); - types = cp.get_partstypes(); - lens = cp.get_partslens(); - return true; + return gCT.chaton_tmpl_apply_ex(tmpl, msgs, alertAssistantAtEnd, applyGlobalIfAny, tagged, types, lens, curSystemMsgCnt, curUserMsgCnt); } // Given the template standard and a bunch of messages including their roles, this returns @@ -751,7 +774,7 @@ inline int32_t chaton_tmpl_apply_ex_capi( return taggedLength; } -// Copied from common.cpp +// Copied from common.cpp, updated wrt model and logging flow. inline std::vector chaton_llama_tokenize( const struct llama_model * model, const std::string & text, From 28ddd2c474436dcf0f0d7698c813cdf32bf5e447 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Tue, 14 May 2024 02:11:10 +0530 Subject: [PATCH 201/217] ChatON: ChatParts dump returns info str rather than direct logging --- common/chaton.hpp | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index ba377f2b93eaa..94c91e9d81355 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -281,19 +281,21 @@ class ChatParts { return typeid(*this).name(); } - void dump() { + std::string dump(const std::string &msgTag) { + std::stringstream ss; std::string me = name() + ":" + __func__; - LOGXLN("INFO:%s:NumTypes:%zu", me.c_str(), types.length()); - LOGXLN("INFO:%s:NumParts:%zu", me.c_str(), parts.size()); - LOGXLN("INFO:%s:StrLength:%zu", me.c_str(), str().length()); + ss << msgTag << ":NumTypes:" << types.length() << std::endl; + ss << msgTag << ":NumParts:" << parts.size() << std::endl; + ss << msgTag << ":StrLength:" << str().length() << std::endl; if (parts.size() != types.length()) { - LOG_TEELN("DBUG:%s:Mismatch between parts and types", me.c_str()); + LOG_TEELN("DBUG:%s:Mismatch between parts[%zu] and types[%zu]", me.c_str(), parts.size(), types.length()); } int i = 0; for(auto part: parts) { - LOGXLN("INFO:%s:%c:%s", me.c_str(), types[i], part.c_str()); + ss << msgTag << ":Part:" << i << ":" << types[i] << ":" << part << std::endl; i += 1; } + return ss.str(); } }; @@ -503,7 +505,7 @@ class ChatTemplates : public GroupKV { auto globalEnd = tmpl_role_getkeys(tmpl, K_GLOBAL, {K_END}); cp.add_part(ChatParts::S, globalEnd); } - cp.dump(); + LDBUG_LN("DBUG:CT:%s", cp.dump("INFO:ChatOnTmplApplyEx").c_str()); tagged = cp.str(); LOGLN("DBUG:CT:%s:%s:%s", __func__, tmpl.c_str(), tagged.c_str()); LOGLN("DBUG:CT:%s:%s:CntSys[%d]:CntUsr[%d]:CntOthers[%d]", __func__, tmpl.c_str(), cntSystem, cntUser, cntOthers); From bb9ce52b111f965c1de8fd4d4c1772ea8972ad2d Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Tue, 14 May 2024 18:25:58 +0530 Subject: [PATCH 202/217] ChatON+: ValidateDump dumps All, wrapped in optional LDBUG_LN GroupKV dump adds needed ":" seperator on its own, so calling functions can just pass the tag string they want in the log without worrying about any demarkation. --- common/chaton.hpp | 4 ++-- common/groupkv.hpp | 4 ++-- tests/test-chaton-groupkv.cpp | 2 +- tests/test-chaton-simpcfg.cpp | 6 +++--- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index 94c91e9d81355..ec0e48fd81ccc 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -595,7 +595,7 @@ inline bool chaton_meta_load(const std::string &fname) { gCT.set_value(group, { K_SYSTEMUSER_1ST_USER_HAS_PREFIX }, userHasPrefix); } - LOGXLN("%s", gCT.dump("", "DBUG:ChatONMetaLoad:ChatTemplates:").c_str()); + LOGXLN("%s", gCT.dump("", "DBUG:ChatONMetaLoad:ChatTemplates").c_str()); return true; } @@ -838,7 +838,7 @@ inline std::vector chaton_llama_tokenize_ex( * keys/fields are present wrt the specified template-standard/model-id or not. */ inline bool _chaton_meta_validate_dump(std::string &tmpl) { - LOGXLN("\n\nINFO:%s:%s:\n%s", __func__, tmpl.c_str(), gCT.dump(tmpl, "INFO:ChatOnMetaValidateDump:").c_str()); + LDBUG_LN("\n\nINFO:%s:%s:\n%s", __func__, tmpl.c_str(), gCT.dump("", "INFO:ChatOnMetaValidateDump").c_str()); if (tmpl.empty()) { return true; } diff --git a/common/groupkv.hpp b/common/groupkv.hpp index b76edddfcea3f..a4dd3d3aa558c 100644 --- a/common/groupkv.hpp +++ b/common/groupkv.hpp @@ -107,9 +107,9 @@ class GroupKV { LDBUG_LN("DBUG:GKV:%s:%s:%s:Skipping...", __func__, msgTag.c_str(), gm.first.c_str()); continue; } - ss << "\n" << msgTag << gm.first << ":\n"; + ss << "\n" << msgTag + ":" << gm.first << ":\n"; for(auto k: gm.second) { - ss << msgTag << "\t" << k.first << ":" << to_str(k.second) << "\n"; + ss << msgTag + ":" << "\t" << k.first << ":" << to_str(k.second) << "\n"; } } return ss.str(); diff --git a/tests/test-chaton-groupkv.cpp b/tests/test-chaton-groupkv.cpp index fd984f760e288..93e277f5c13f7 100644 --- a/tests/test-chaton-groupkv.cpp +++ b/tests/test-chaton-groupkv.cpp @@ -19,7 +19,7 @@ static void gkv_inited() { }}; std::cout << "**** gkv inited **** " << std::endl; - std::cout << gkv.dump("", "INFO:GKV:Inited:") << std::endl; + std::cout << gkv.dump("", "INFO:GKV:Inited") << std::endl; } diff --git a/tests/test-chaton-simpcfg.cpp b/tests/test-chaton-simpcfg.cpp index 9d7ffc3d56e1e..634e5d5c20387 100644 --- a/tests/test-chaton-simpcfg.cpp +++ b/tests/test-chaton-simpcfg.cpp @@ -116,7 +116,7 @@ static void sc_inited() { }}; std::cout << "**** sc inited **** " << std::endl; - std::cout << sc.dump("") << std::endl; + std::cout << sc.dump("", "INFO:SC:Inited") << std::endl; } @@ -125,7 +125,7 @@ static void sc_set(const std::string &fname) { std::cout << "**** sc set **** " << std::endl; SimpCfg sc = {{}}; sc.load(fname); - std::cout << sc.dump("", "INFO:SC:Set:AfterLoad:") << std::endl; + std::cout << sc.dump("", "INFO:SC:Set:AfterLoad") << std::endl; sc.get_bool("testme", {"key101b"}, false); sc.get_string("testme", {"key101s"}, "Not found"); @@ -137,7 +137,7 @@ static void sc_set(const std::string &fname) { sc.set_int64("testme", {"key201i"}, 987654); sc.set_double("testme", {"key201d"}, 9988.7766); - std::cout << sc.dump("testme", "INFO:SC:Set:AfterSet:") << std::endl; + std::cout << sc.dump("testme", "INFO:SC:Set:AfterSet") << std::endl; sc.get_bool("testme", {"key201b"}, false); sc.get_string("testme", {"key201s"}, "Not found"); sc.get_int64("testme", {"key201i"}, 123456); From bd5c39e0f0a87cfd6939b502ba6915501b7a358a Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Tue, 14 May 2024 21:15:02 +0530 Subject: [PATCH 203/217] ChatOn+GroupKV: Cleanup a bit, including using debug logging --- common/chaton.hpp | 26 +++++++++++++------------- common/groupkv.hpp | 6 ++++-- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index ec0e48fd81ccc..b7132e61dd082 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -311,9 +311,9 @@ class ChatTemplates : public GroupKV { * Check if the specified chat-template exists or not. * NOTE: This doesnt cross check, if the template inturn contains all the required fields or not. */ - bool tmpl_exists(const std::string &tmpl) { + bool tmpl_exists(const std::string &tmpl, const std::string &msgTag="") { if (!group_exists(tmpl)) { - LOG_TEELN("WARN:CT:%s: tmpl[%s] not found...", __func__, tmpl.c_str()); + LOG_TEELN("WARN:CT:%s:%s:Specified template-id [%s] not found...", __func__, msgTag.c_str(), tmpl.c_str()); return false; } return true; @@ -327,8 +327,7 @@ class ChatTemplates : public GroupKV { */ bool tmpl_basiccheck(const std::string &tmpl, std::stringstream &ss, const std::string &msgTag) { - if (!tmpl_exists(tmpl)) { - LOGXLN("ERRR:CT:%s:Specified template-id [%s] not found", msgTag.c_str(), tmpl.c_str()); + if (!tmpl_exists(tmpl, msgTag)) { return false; } @@ -507,8 +506,8 @@ class ChatTemplates : public GroupKV { } LDBUG_LN("DBUG:CT:%s", cp.dump("INFO:ChatOnTmplApplyEx").c_str()); tagged = cp.str(); - LOGLN("DBUG:CT:%s:%s:%s", __func__, tmpl.c_str(), tagged.c_str()); - LOGLN("DBUG:CT:%s:%s:CntSys[%d]:CntUsr[%d]:CntOthers[%d]", __func__, tmpl.c_str(), cntSystem, cntUser, cntOthers); + LDBUG_LN("DBUG:CT:%s:%s:%s", __func__, tmpl.c_str(), tagged.c_str()); + LDBUG_LN("DBUG:CT:%s:CntSys[%d]:CntUsr[%d]:CntOthers[%d]", __func__, cntSystem, cntUser, cntOthers); types = cp.get_partstypes(); lens = cp.get_partslens(); return true; @@ -595,7 +594,7 @@ inline bool chaton_meta_load(const std::string &fname) { gCT.set_value(group, { K_SYSTEMUSER_1ST_USER_HAS_PREFIX }, userHasPrefix); } - LOGXLN("%s", gCT.dump("", "DBUG:ChatONMetaLoad:ChatTemplates").c_str()); + LDBUG_LN("%s", gCT.dump("", "DBUG:ChatONMetaLoad:ChatTemplates").c_str()); return true; } @@ -782,7 +781,7 @@ inline std::vector chaton_llama_tokenize( const std::string & text, bool add_special, bool parse_special) { - LOGLN("DBUG:%s:%s:special[add:%d, parse:%d]", __func__, text.c_str(), add_special, parse_special); + LDBUG_LN("DBUG:%s:%s:special[add:%d, parse:%d]", __func__, text.c_str(), add_special, parse_special); if (model == nullptr) { LOG_TEELN("ERRR:%s:Model NOT Provided:%s:special[add:%d, parse:%d]", __func__, text.c_str(), add_special, parse_special); return std::vector{}; @@ -831,11 +830,12 @@ inline std::vector chaton_llama_tokenize_ex( /** - * Dump the full loaded chaton templates data - * Additionally if a chaton-template-id is specified - * dump contents related to that specific chat-handshake-template-standard - * NOTE: It uses the exception raising get_value to check if all the required - * keys/fields are present wrt the specified template-standard/model-id or not. + * Validate specified chaton-template-id and inturn dump the contents + * related to that specific chat-handshake-template-standard. + * ALERT: If no template-id is specified, it is ignored with a warning. + * NOTE: It optionally dumps the full loaded chaton templates data + * NOTE: It uses tmpl_basiccheck, which raises exception, if all the required + * keys/fields are not present wrt the specified template-standard/model-id. */ inline bool _chaton_meta_validate_dump(std::string &tmpl) { LDBUG_LN("\n\nINFO:%s:%s:\n%s", __func__, tmpl.c_str(), gCT.dump("", "INFO:ChatOnMetaValidateDump").c_str()); diff --git a/common/groupkv.hpp b/common/groupkv.hpp index a4dd3d3aa558c..b2a574bf1ca6b 100644 --- a/common/groupkv.hpp +++ b/common/groupkv.hpp @@ -18,11 +18,11 @@ #include -#define GKV_DEBUG +#define GKV_DEBUGLOG_ON #include "log.h" #define LINFO_LN LOG_TEELN -#ifdef GKV_DEBUG +#ifdef GKV_DEBUGLOG_ON #define LDBUG LOG #define LDBUG_LN LOGLN #else @@ -115,6 +115,7 @@ class GroupKV { return ss.str(); } + // If the specified key is missing, an exception will be thrown. template SupportedDataType get_value(const std::string &group, const MultiPart &keyParts) { auto key = joiner(keyParts); @@ -128,6 +129,7 @@ class GroupKV { return std::get(value); } + // If the specified key is missing, then the provided default value will be returned. template SupportedDataType get_value(const std::string &group, const MultiPart &keyParts, const SupportedDataType &defaultValue, const std::string &callerName="") { try { From f8c0b474ec0a2003e2636a1f7ff5b5a05bdc9f5d Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Tue, 14 May 2024 21:34:53 +0530 Subject: [PATCH 204/217] ChatON+:RenameTo chaton_meta_load_json to match semantic Also add simple note wrt itself and its helper. --- common/chaton.hpp | 6 +++++- examples/main/main.cpp | 2 +- tests/test-chat-template-chaton.cpp | 4 ++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index b7132e61dd082..cc8380d5a1a76 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -515,11 +515,14 @@ class ChatTemplates : public GroupKV { }; +// The compiled-in configurable template data (the meta) #include "chaton_meta.hpp" //ChatTemplates gCT = {{}}; #ifdef CHATON_JSON +// Get value corresponding to the specified hierarchy/chain of keys. +// Also throw a more informative exception, if it is not found. template inline SupportedType json_get(json &j, const std::vector &keys, const std::string &msgTag) { json curJ = j; @@ -540,7 +543,8 @@ inline SupportedType json_get(json &j, const std::vector &keys return curJ; } -inline bool chaton_meta_load(const std::string &fname) { +// Update/Extend the compiled-in configurable template data (the meta) from the specified json file. +inline bool chaton_meta_load_json(const std::string &fname) { std::ifstream f(fname); json conMeta = json::parse(f); for(auto it=conMeta.begin(); it != conMeta.end(); ++it) { diff --git a/examples/main/main.cpp b/examples/main/main.cpp index f86592fc7da41..39fdcdc548a56 100644 --- a/examples/main/main.cpp +++ b/examples/main/main.cpp @@ -144,7 +144,7 @@ int main(int argc, char ** argv) { if (params.chaton) { if (!params.chaton_meta_json.empty()) { - chaton_meta_load(params.chaton_meta_json); + chaton_meta_load_json(params.chaton_meta_json); } if (!chaton_meta_ok(params.chaton_template_id)) { exit(1); diff --git a/tests/test-chat-template-chaton.cpp b/tests/test-chat-template-chaton.cpp index ce65cc31ea525..e14266ad560b9 100644 --- a/tests/test-chat-template-chaton.cpp +++ b/tests/test-chat-template-chaton.cpp @@ -116,7 +116,7 @@ static void check_chaton(std::string &metaJson) { std::vector formatted_chat(1024); int32_t res; - chaton_meta_load(metaJson); + chaton_meta_load_json(metaJson); for(auto tmplId: templateIds) { formatted_chat.resize(1024); std::cout << "\n----------" << tmplId << "---------------\n"; @@ -137,7 +137,7 @@ static void check_chaton_ex(std::string &metaJson) { std::vector formatted_chat(1024); int32_t res; - chaton_meta_load(metaJson); + chaton_meta_load_json(metaJson); for(auto tmplId: templateIds) { formatted_chat.resize(1024); std::cout << "\n----------" << tmplId << "---------------\n"; From 8975de996bf08b25e2846a45cd18432b3d42b9a3 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Tue, 14 May 2024 19:39:17 +0530 Subject: [PATCH 205/217] ChatON: Update Notes to match the updated semantics and flows The initial version was rooted around a json object, while the new version is rooted around a MapOfMapOfVariant (GroupKV), which could be preloaded with chat templates info at compile time itself and used as is. Or optionally one could allow the configurable template data to be extended/updated at runtime from a text(/SimpCfg)/json file. --- common/chaton.hpp | 102 +++++++++++++++++++++++++++++----------------- 1 file changed, 65 insertions(+), 37 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index cc8380d5a1a76..b26bb237e639a 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -2,21 +2,22 @@ /** * - * Generic tagging logic + text config file based chat templates handling + * Generic tagging logic + configurable template data based chat templates handling * by Humans for All * * ## Overview * * Helps chat with models, by tagging chat messages based on the specified * chat-handshake-template-standard. This uses a generic tagging code driven - * by a json meta data file, which specifies the handshake template details. + * by configurable template data which is either builtin or loaded from text/ + * json file, which specifies the handshake template details. * * This can be used by * - * * main, to build on existing interactive flow and its in-prefix, in-suffix + * * examples/main, to build on existing interactive flow and its in-prefix, in-suffix * and antiprompt/reverse-prompt * - * * server, by replacing its existing llama_chat_apply_template with the + * * examples/server|..., by replacing its existing llama_chat_apply_template with the * equivalent helper here. * * @@ -62,7 +63,7 @@ * * ## The Strategy * - * The template meta data json file allows the user to specify the above mentioned tags wrt + * The configurable template data allows the user to specify the above mentioned tags wrt * each of the Role as well as any global tag for a group of messages. Depending on whether * a given model uses/needs a given tag or not you either specify the required tag or else * you specify a empty string. @@ -71,13 +72,13 @@ * using \n and so on. The tag is always demarcated using double quotes and thus also allows * spaces at the begining or end of the tag, if needed. * - * In order to account for the conditionality of tags between the system message and the 1st - * user message, flags are provided to explicitly control whether each of these possible tags - * is used by a specific model or not, as part of its template info. + * In order to account for the conditionality of tags between the system message and the + * following 1st user message, flags are provided to explicitly control whether each of + * these possible tags is used by a specific model or not, as part of its template info. * - * The Roles are identified in the json file using "system", "user" and "assistant". However - * the model may use different words to identify these roles, in which case setup RolePrefix - * and or RoleSuffix appropriately. + * The Roles are identified in the configurable template data using "system", "user" and + * "assistant". However the model may use different words to identify these roles, in which + * case setup RolePrefix and or RoleSuffix appropriately. * * To identify that model is finished with generating response to user query, depending on * the model's handshake template standard, one will need to set the reverse-prompt to either @@ -87,7 +88,7 @@ * Currently flags for trimming wrt user text (be it wrt system or user role) is not added. * * - * ## The JSON File + * ## Configurable template data and related optional text/JSON file * * Can contain the template info wrt multiple models/handshake-standards. And inturn each * unique template is identified by a unique template id string. @@ -107,25 +108,38 @@ * * systemuser-system-has-suffix, systemuser-system-has-end, * systemuser-1st-user-has-begin and systemuser-1st-user-has-prefix * - * + * By default one can preload at compile time. Additionally one could update/load + * more at runtime. A compile time optionally enabled load from json helper is + * provided. For any reason, if one doesnt want to use the json based mechanism, + * and instead wants a simple mechanism for runtime updating/loading, one could + * update ChatTemplates to extend from SimpCfg and inturn use its load from a + * simple text file based flow. + * + * * ## Usage * - * One needs to load the json file containing the template meta data and inturn call the - * other helper functions as needed. + * One could use the logic along with compile time builtin configurable template data as is + * or one could optionally load configurable template data from a text/json file containing + * the template meta data and inturn call the other helper functions as needed. + * + * NOTE: One could either make do with a pre-compiled chat templates info, or allow users + * to update/modify/override the pre-compiled info and or extend with info for new models + * or chat-handshake-template-standards at runtime. * * Inturn one can use the helper functions to either extract a given tag or to apply all * tags specified wrt a given role to the passed message or to apply tags as needed for * a bunch of messages in one go. * - * The individual message tagging helper, will apply all tags specified wrt that role. + * The single message tagging helper setup to apply all tags specified wrt that role. * - * The multiple messages tagging helper chaton-tmpl-apply, will look at the boolean flags - * when tagging the passed messages. In this the system suffix, system end, user begin and - * user prefix get included only if corresponding flag is set. + * The multiple messages tagging helper chaton-tmpl-apply[-ex][-capi], will look at the + * boolean flags when tagging the passed messages. In this the system suffix, system end, + * user begin and user prefix get included only if corresponding flag is set, the 1st time + * system + user message is encountered. * - * Both the single and multi messages tagging helpers provide two versions. + * The multi messages tagging is provided in two versions. * * one which returns a single string which contains the tagged message(s) - * * one which returns + * * one which returns [ex version] * * [tagged msg] the string containing the tagged message(s) * * [parts lengths] an array of integers, which specifies the part lengths, * which divides the returned string into parts. @@ -133,6 +147,12 @@ * part is a normal part which needs to be tokenized without parse_special * or is a special part which needs to be tokenized with parse-special. * + * A single message wrapper is provided for the simple (no extended) version. + * + * chaton_llama_tokenize_ex is provided to show how the extended helpers additional + * subparts info wrt tagged message could be used to tokenize with and without + * parse_special to the appropriate subparts that make up the tagged message. + * * * ## example/main * @@ -146,12 +166,14 @@ * * the reverse-prompt to map to antiprompt * * wrt tokenization * * the user specified system prompt is tokenized with parse_special flag. - * * however the user messages are tokenized without parse_special flag. + * * however the user messages are tokenized with/without parse_special flag, + * based on interactive-specials. * * Currently Main doesnt use chaton-tmpl-apply, but only * * chaton-tmpl-apply-single (for system prompt) and - * * chaton-tmpl-role-kv which maps the user prefix, suffix and reverse-prompt - * to in-prefix, in-suffix and antiprompt of main. + * * chaton_tmpl_role_getkeys, used to map the user prefix and suffix + * to in-prefix, in-suffix of main. + * * chaton_tmpl_getkey_str, used to map reverse-prompt to main's antiprompt. * These always adds any role specific begin+prefix and suffix+end around * the passed message. * @@ -163,20 +185,25 @@ * with text based config file based flow. * * If a program doesnt want to bring in json dependency into their project, - * there is also common/simpcfg.hpp, which provides a simple text based config - * file format, along with the corresponding parser for the same. This can be - * modified to work with simpcfg easily, if needed. + * one can make do with the pre initialized configurable template data which + * is compiled in. + * + * Additionally, if runtime configurability required without json dependency, + * the ChatTemplates can be updated to extend SimpCfg from common/simpcfg.hpp, + * which provides a simple text based config file format, along with the + * corresponding parser for the same. This should be relatively easy, if needed. * * ## Adding support for new model / chat-handshake-template-standard * - * 1. Add suitable entries in json for that model/standard - * This in itself should work for most of the models. + * 1. Add suitable entries wrt configurable template data, either as part of the + * compile time builtin initialisation or the text/json file loaded at runtime, + * for that model/standard. This in itself should work for most of the models. * * 2. If some new model introduces a totally different kind of chat-templating * tag inter/intra mixing, Try to reuse and update the generic flow in - * chaton-tmpl-apply, as much as possible, before trying to add any custom logic. + * chaton-tmpl-apply-ex, as much as possible, before trying to add any custom logic. * - * If you update the generic flow, cross check if existing json files will + * If you update the generic flow, cross check if existing text/json files will * need to be updated or not. * * @@ -411,16 +438,17 @@ class ChatTemplates : public GroupKV { } /** - * Given the template standard and a bunch of messages including their roles, this returns - * tagged messages, subPartsTypes string and subPartsLens vector. The returned subParts - * types string and lens vector help identify the parts of the tagged msgs string, - * which relate to passed msgs and added tags. + * Given the template model/standard id and a bunch of messages including their roles, + * this returns tagged messages, subPartsTypes string and subPartsLens vector. + * The returned subParts types string and lens vector help identify the parts of the + * tagged msgs string, which relate to passed msgs and added tags. * * * a string containing the tagged messages - * * global-begin + 1 or more [[role-begin] + [role-prefix] + msg + [role-suffix] +[role-end]] + global-end + * [global-begin] + 1 or more [[role-begin] + [role-prefix] + msg + [role-suffix] +[role-end]] + [global-end] * * a string where the chars contain info about * type of sub-strings/parts that make up the tagged messages string. - * * a vector of ints, which give the length of each part in the tagged messages string. + * * a vector of ints, + * which give the length of each part in the tagged messages string. * * If a combination of system-user messages is passed, then tags between the 1st system and * the 1st user message, is based on the flags set wrt the corresponding template standard. From 14c28e717e3e6db1de4d31b2db28578c6a86c530 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Wed, 15 May 2024 02:11:26 +0530 Subject: [PATCH 206/217] GroupKV+: dump cleanup - forgot to commit earlier --- tests/test-chaton-groupkv.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test-chaton-groupkv.cpp b/tests/test-chaton-groupkv.cpp index 93e277f5c13f7..150adf4f67888 100644 --- a/tests/test-chaton-groupkv.cpp +++ b/tests/test-chaton-groupkv.cpp @@ -27,7 +27,7 @@ static void gkv_set() { std::cout << "**** gkv set **** " << std::endl; GroupKV gkv = {{}}; - std::cout << gkv.dump("", "INFO:GKV:Set:Initial:") << std::endl; + std::cout << gkv.dump("", "INFO:GKV:Set:Initial") << std::endl; gkv.get_value("testme", {"key101b"}, false); gkv.get_value("testme", {"key101s"}, "Not found"); @@ -39,7 +39,7 @@ static void gkv_set() { gkv.set_value("testme", {"key201i"}, 987654); gkv.set_value("testme", {"key201d"}, 9988.7766); - std::cout << gkv.dump("testme", "INFO:GKV:Set:After testme set:") << std::endl; + std::cout << gkv.dump("testme", "INFO:GKV:Set:After testme set") << std::endl; gkv.get_value("testme", {"key201b"}, false); gkv.get_value("testme", {"key201s"}, "Not found"); gkv.get_value("testme", {"key201i"}, 123456); From a3d641b55549e0092163023c97ccbd977817706e Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Wed, 15 May 2024 02:26:51 +0530 Subject: [PATCH 207/217] ChatON: Move loading from json file into its own file Any program which wants to use json file to update/extend the chaton's configurable template data, can include this new file chaton_json.hpp, to get the reqd functionality. Update chaton_meta_ok, _chaton_meta_validate_dump and chaton_meta_load_json to either work with a passed ChatTemplates instance, or fallback to the compiled-in global instance of same. --- common/chaton.hpp | 114 ++++------------------------ common/chaton_json.hpp | 100 ++++++++++++++++++++++++ examples/main/main.cpp | 2 +- tests/test-chat-template-chaton.cpp | 2 +- 4 files changed, 117 insertions(+), 101 deletions(-) create mode 100644 common/chaton_json.hpp diff --git a/common/chaton.hpp b/common/chaton.hpp index b26bb237e639a..af19727d8a8b2 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -240,12 +240,6 @@ const auto K_SYSTEMUSER_1ST_USER_HAS_BEGIN = "systemuser-1st-user-has-begin"; const auto K_SYSTEMUSER_1ST_USER_HAS_PREFIX = "systemuser-1st-user-has-prefix"; const auto K_REVERSE_PROMPT = "reverse-prompt"; -#define CHATON_JSON -#ifdef CHATON_JSON -#include -using json = nlohmann::ordered_json; -#endif - /** @@ -547,91 +541,6 @@ class ChatTemplates : public GroupKV { #include "chaton_meta.hpp" //ChatTemplates gCT = {{}}; -#ifdef CHATON_JSON - -// Get value corresponding to the specified hierarchy/chain of keys. -// Also throw a more informative exception, if it is not found. -template -inline SupportedType json_get(json &j, const std::vector &keys, const std::string &msgTag) { - json curJ = j; - std::stringstream skey; - int i = 0; - for(auto key: keys) { - if (i != 0) skey << "-"; - i += 1; - skey << key; - if (curJ.contains(key)) { - curJ = curJ[key]; - } else { - std::stringstream ss; - ss << "ERRR:ChatON:" << __func__ << ":" << msgTag << ":KeyChain [" << skey.str() << "] is missing"; - throw std::runtime_error(ss.str()); - } - } - return curJ; -} - -// Update/Extend the compiled-in configurable template data (the meta) from the specified json file. -inline bool chaton_meta_load_json(const std::string &fname) { - std::ifstream f(fname); - json conMeta = json::parse(f); - for(auto it=conMeta.begin(); it != conMeta.end(); ++it) { - - auto group = it.key(); - auto curTmpl = conMeta[group]; - - std::string globalBegin = json_get(curTmpl, { K_GLOBAL, K_BEGIN }, group); - gCT.set_value(group, { K_GLOBAL, K_BEGIN }, globalBegin); - std::string globalEnd = json_get(curTmpl, { K_GLOBAL, K_END }, group); - gCT.set_value(group, { K_GLOBAL, K_END }, globalEnd); - - std::string systemBegin = json_get(curTmpl, { K_SYSTEM, K_BEGIN }, group); - gCT.set_value(group, { K_SYSTEM, K_BEGIN }, systemBegin); - std::string systemPrefix = json_get(curTmpl, { K_SYSTEM, K_PREFIX }, group); - gCT.set_value(group, { K_SYSTEM, K_PREFIX }, systemPrefix); - std::string systemSuffix = json_get(curTmpl, { K_SYSTEM, K_SUFFIX }, group); - gCT.set_value(group, { K_SYSTEM, K_SUFFIX }, systemSuffix); - std::string systemEnd = json_get(curTmpl, { K_SYSTEM, K_END }, group); - gCT.set_value(group, { K_SYSTEM, K_END }, systemEnd); - - std::string userBegin = json_get(curTmpl, { K_USER, K_BEGIN }, group); - gCT.set_value(group, { K_USER, K_BEGIN }, userBegin); - std::string userPrefix = json_get(curTmpl, { K_USER, K_PREFIX }, group); - gCT.set_value(group, { K_USER, K_PREFIX }, userPrefix); - std::string userSuffix = json_get(curTmpl, { K_USER, K_SUFFIX }, group); - gCT.set_value(group, { K_USER, K_SUFFIX }, userSuffix); - std::string userEnd = json_get(curTmpl, { K_USER, K_END }, group); - gCT.set_value(group, { K_USER, K_END }, userEnd); - - std::string assistantBegin = json_get(curTmpl, { K_ASSISTANT, K_BEGIN }, group); - gCT.set_value(group, { K_ASSISTANT, K_BEGIN }, assistantBegin); - std::string assistantPrefix = json_get(curTmpl, { K_ASSISTANT, K_PREFIX }, group); - gCT.set_value(group, { K_ASSISTANT, K_PREFIX }, assistantPrefix); - std::string assistantSuffix = json_get(curTmpl, { K_ASSISTANT, K_SUFFIX }, group); - gCT.set_value(group, { K_ASSISTANT, K_SUFFIX }, assistantSuffix); - std::string assistantEnd = json_get(curTmpl, { K_ASSISTANT, K_END }, group); - gCT.set_value(group, { K_ASSISTANT, K_END }, assistantEnd); - - std::string reversePrompt = json_get(curTmpl, { K_REVERSE_PROMPT }, group); - gCT.set_value(group, { K_REVERSE_PROMPT }, reversePrompt); - - bool systemHasSuffix = json_get(curTmpl, { K_SYSTEMUSER_SYSTEM_HAS_SUFFIX }, group); - gCT.set_value(group, { K_SYSTEMUSER_SYSTEM_HAS_SUFFIX }, systemHasSuffix); - bool systemHasEnd = json_get(curTmpl, { K_SYSTEMUSER_SYSTEM_HAS_END }, group); - gCT.set_value(group, { K_SYSTEMUSER_SYSTEM_HAS_END }, systemHasEnd); - - bool userHasBegin = json_get(curTmpl, { K_SYSTEMUSER_1ST_USER_HAS_BEGIN }, group); - gCT.set_value(group, { K_SYSTEMUSER_1ST_USER_HAS_BEGIN }, userHasBegin); - bool userHasPrefix = json_get(curTmpl, { K_SYSTEMUSER_1ST_USER_HAS_PREFIX }, group); - gCT.set_value(group, { K_SYSTEMUSER_1ST_USER_HAS_PREFIX }, userHasPrefix); - - } - LDBUG_LN("%s", gCT.dump("", "DBUG:ChatONMetaLoad:ChatTemplates").c_str()); - return true; -} - -#endif - inline bool chaton_tmpl_exists(const std::string &tmpl) { return gCT.tmpl_exists(tmpl); @@ -862,20 +771,25 @@ inline std::vector chaton_llama_tokenize_ex( /** - * Validate specified chaton-template-id and inturn dump the contents - * related to that specific chat-handshake-template-standard. + * Validate specified chaton-template-id and inturn dump the contents related to that + * specific chat-handshake-template-standard, wrt the specified ChatTemplates. + * If ct is nullptr, then map to the compiled-in ChatTemplates global instance. + * * ALERT: If no template-id is specified, it is ignored with a warning. * NOTE: It optionally dumps the full loaded chaton templates data * NOTE: It uses tmpl_basiccheck, which raises exception, if all the required * keys/fields are not present wrt the specified template-standard/model-id. */ -inline bool _chaton_meta_validate_dump(std::string &tmpl) { - LDBUG_LN("\n\nINFO:%s:%s:\n%s", __func__, tmpl.c_str(), gCT.dump("", "INFO:ChatOnMetaValidateDump").c_str()); +inline bool _chaton_meta_validate_dump(std::string &tmpl, ChatTemplates *ct=nullptr) { + if (ct == nullptr) { + ct = &gCT; + } + LDBUG_LN("\n\nINFO:%s:%s:\n%s", __func__, tmpl.c_str(), ct->dump("", "INFO:ChatOnMetaValidateDump").c_str()); if (tmpl.empty()) { return true; } std::stringstream ss; - if (gCT.tmpl_basiccheck(tmpl, ss, "INFO:ChatOnMetaValidateDump")) { + if (ct->tmpl_basiccheck(tmpl, ss, "INFO:ChatOnMetaValidateDump")) { LOGXLN("%s", ss.str().c_str()); } else { return false; @@ -884,8 +798,10 @@ inline bool _chaton_meta_validate_dump(std::string &tmpl) { } /** - * Verify that specified chaton-template-id contains required fields using meta-validate-dump + * In the passed ChatTemplates instance, verify that specified chaton-template-id + * contains required fields using meta-validate-dump. + * If ct is nullptr, then map to the compiled-in ChatTemplates global instance. */ -inline bool chaton_meta_ok(std::string &tmpl) { - return _chaton_meta_validate_dump(tmpl); +inline bool chaton_meta_ok(std::string &tmpl, ChatTemplates *ct=nullptr) { + return _chaton_meta_validate_dump(tmpl, ct); } diff --git a/common/chaton_json.hpp b/common/chaton_json.hpp new file mode 100644 index 0000000000000..7915adcc43533 --- /dev/null +++ b/common/chaton_json.hpp @@ -0,0 +1,100 @@ +#pragma once + +/** + * Helper to load chaton's configurable template data from json file + * By Humans for All + * + * Any program which wants to load configurable template data from json file, + * can include this file to get the needed helpers for same. +*/ + +#include "chaton.hpp" + +#include +using json = nlohmann::ordered_json; + + +// Get value corresponding to the specified hierarchy/chain of keys. +// Also throw a more informative exception, if it is not found. +template +inline SupportedType json_get(json &j, const std::vector &keys, const std::string &msgTag) { + json curJ = j; + std::stringstream skey; + int i = 0; + for(auto key: keys) { + if (i != 0) skey << "-"; + i += 1; + skey << key; + if (curJ.contains(key)) { + curJ = curJ[key]; + } else { + std::stringstream ss; + ss << "ERRR:ChatON:" << __func__ << ":" << msgTag << ":KeyChain [" << skey.str() << "] is missing"; + throw std::runtime_error(ss.str()); + } + } + return curJ; +} + +// Update/Extend the configurable template data in specified ChatTemplates instance from the specified json file. +// If nullptr is passed wrt ct, then update/extend the global compiled-in configurable template data. +inline bool chaton_meta_load_json(const std::string &fname, ChatTemplates *ct=nullptr) { + if (ct == nullptr) { + ct = &gCT; + } + std::ifstream f(fname); + json conMeta = json::parse(f); + for(auto it=conMeta.begin(); it != conMeta.end(); ++it) { + + auto group = it.key(); + auto curTmpl = conMeta[group]; + + std::string globalBegin = json_get(curTmpl, { K_GLOBAL, K_BEGIN }, group); + ct->set_value(group, { K_GLOBAL, K_BEGIN }, globalBegin); + std::string globalEnd = json_get(curTmpl, { K_GLOBAL, K_END }, group); + ct->set_value(group, { K_GLOBAL, K_END }, globalEnd); + + std::string systemBegin = json_get(curTmpl, { K_SYSTEM, K_BEGIN }, group); + ct->set_value(group, { K_SYSTEM, K_BEGIN }, systemBegin); + std::string systemPrefix = json_get(curTmpl, { K_SYSTEM, K_PREFIX }, group); + ct->set_value(group, { K_SYSTEM, K_PREFIX }, systemPrefix); + std::string systemSuffix = json_get(curTmpl, { K_SYSTEM, K_SUFFIX }, group); + ct->set_value(group, { K_SYSTEM, K_SUFFIX }, systemSuffix); + std::string systemEnd = json_get(curTmpl, { K_SYSTEM, K_END }, group); + ct->set_value(group, { K_SYSTEM, K_END }, systemEnd); + + std::string userBegin = json_get(curTmpl, { K_USER, K_BEGIN }, group); + ct->set_value(group, { K_USER, K_BEGIN }, userBegin); + std::string userPrefix = json_get(curTmpl, { K_USER, K_PREFIX }, group); + ct->set_value(group, { K_USER, K_PREFIX }, userPrefix); + std::string userSuffix = json_get(curTmpl, { K_USER, K_SUFFIX }, group); + ct->set_value(group, { K_USER, K_SUFFIX }, userSuffix); + std::string userEnd = json_get(curTmpl, { K_USER, K_END }, group); + ct->set_value(group, { K_USER, K_END }, userEnd); + + std::string assistantBegin = json_get(curTmpl, { K_ASSISTANT, K_BEGIN }, group); + ct->set_value(group, { K_ASSISTANT, K_BEGIN }, assistantBegin); + std::string assistantPrefix = json_get(curTmpl, { K_ASSISTANT, K_PREFIX }, group); + ct->set_value(group, { K_ASSISTANT, K_PREFIX }, assistantPrefix); + std::string assistantSuffix = json_get(curTmpl, { K_ASSISTANT, K_SUFFIX }, group); + ct->set_value(group, { K_ASSISTANT, K_SUFFIX }, assistantSuffix); + std::string assistantEnd = json_get(curTmpl, { K_ASSISTANT, K_END }, group); + ct->set_value(group, { K_ASSISTANT, K_END }, assistantEnd); + + std::string reversePrompt = json_get(curTmpl, { K_REVERSE_PROMPT }, group); + ct->set_value(group, { K_REVERSE_PROMPT }, reversePrompt); + + bool systemHasSuffix = json_get(curTmpl, { K_SYSTEMUSER_SYSTEM_HAS_SUFFIX }, group); + ct->set_value(group, { K_SYSTEMUSER_SYSTEM_HAS_SUFFIX }, systemHasSuffix); + bool systemHasEnd = json_get(curTmpl, { K_SYSTEMUSER_SYSTEM_HAS_END }, group); + ct->set_value(group, { K_SYSTEMUSER_SYSTEM_HAS_END }, systemHasEnd); + + bool userHasBegin = json_get(curTmpl, { K_SYSTEMUSER_1ST_USER_HAS_BEGIN }, group); + ct->set_value(group, { K_SYSTEMUSER_1ST_USER_HAS_BEGIN }, userHasBegin); + bool userHasPrefix = json_get(curTmpl, { K_SYSTEMUSER_1ST_USER_HAS_PREFIX }, group); + ct->set_value(group, { K_SYSTEMUSER_1ST_USER_HAS_PREFIX }, userHasPrefix); + + } + LDBUG_LN("%s", ct->dump("", "DBUG:ChatONMetaLoad:ChatTemplates").c_str()); + return true; +} diff --git a/examples/main/main.cpp b/examples/main/main.cpp index 39fdcdc548a56..7035029682f8a 100644 --- a/examples/main/main.cpp +++ b/examples/main/main.cpp @@ -1,5 +1,5 @@ #include "common.h" -#include "chaton.hpp" +#include "chaton_json.hpp" #include "console.h" #include "llama.h" diff --git a/tests/test-chat-template-chaton.cpp b/tests/test-chat-template-chaton.cpp index e14266ad560b9..7e352738c94f5 100644 --- a/tests/test-chat-template-chaton.cpp +++ b/tests/test-chat-template-chaton.cpp @@ -7,7 +7,7 @@ #include #include "llama.h" -#include "chaton.hpp" +#include "chaton_json.hpp" std::vector templateIds = { "llama2", "llama3", "chatml", From 4a15989000949df49c94f2cbb319f142a78366cb Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Wed, 15 May 2024 03:38:41 +0530 Subject: [PATCH 208/217] ChatON: Forgot this note earlier --- common/chaton.hpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/common/chaton.hpp b/common/chaton.hpp index af19727d8a8b2..17bd59be73661 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -603,6 +603,9 @@ const int BYPASS_MSGCNT = 101; // string containing the tagged message // * role-(begin+prefix) + msg + role-(suffix+end) // +// ALERT: This currently assumes/behaves as if the system or user message it is working on +// is a non-1st message belonging to that role. +// inline size_t chaton_tmpl_apply_single( const std::string &tmpl, const std::string &role, From dc03a7134a0689a5a2ea090c2aafecccfe716ed1 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Wed, 15 May 2024 12:56:05 +0530 Subject: [PATCH 209/217] CMakeLists: base std::variantC++17, specificTest std::formatC++20 --- CMakeLists.txt | 2 +- tests/CMakeLists.txt | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5dca1fe6ccbd1..a1eec0af29576 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -837,7 +837,7 @@ function(get_flags CCID CCVER) set(GF_CXX_FLAGS ${CXX_FLAGS} PARENT_SCOPE) endfunction() -list(APPEND CXX_FLAGS -std=c++20) +list(APPEND CXX_FLAGS -std=c++17) if (LLAMA_FATAL_WARNINGS) if (CMAKE_CXX_COMPILER_ID MATCHES "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 6a237d36f4285..d1e701b0ae3f8 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -119,6 +119,9 @@ llama_target_and_test(test-chat-template.cpp) llama_target_and_test(test-chat-template-chaton.cpp) llama_target_and_test(test-chaton-groupkv.cpp) llama_target_and_test(test-chaton-simpcfg.cpp) +#target_compile_features(test-chaton-simpcfg PUBLIC cxx_std_20) +target_compile_options(test-chaton-simpcfg PUBLIC -std=c++20) + llama_target_and_test(test-grammar-parser.cpp) llama_target_and_test(test-llama-grammar.cpp) From 4f5add68c666da0ed28eb4a5402800a563c5176f Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Wed, 15 May 2024 13:36:27 +0530 Subject: [PATCH 210/217] GroupKV:Dump/Log type of the variant instance also --- common/groupkv.hpp | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/common/groupkv.hpp b/common/groupkv.hpp index b2a574bf1ca6b..220ee9b33b49f 100644 --- a/common/groupkv.hpp +++ b/common/groupkv.hpp @@ -60,6 +60,21 @@ class GroupKV { return joined.str(); } + std::string the_type(const GroupKVData &value) { + if (std::holds_alternative(value)) { + return "string"; + } else if (std::holds_alternative(value)) { + return "bool"; + } else if (std::holds_alternative(value)) { + return "int32_t"; + } else if (std::holds_alternative(value)) { + return "int64_t"; + } else if (std::holds_alternative(value)) { + return "double"; + } + return "unknown"; + } + std::string to_str(const GroupKVData &value) { auto visitor = [](auto value) -> auto { std::stringstream ss; @@ -95,7 +110,7 @@ class GroupKV { auto key = joiner(keyParts); auto &gm = gkv[group]; gm[key] = value; - LDBUG_LN("DBUG:GKV:%s_%s:%s:%s:%s", __func__, callerName.c_str(), group.c_str(), key.c_str(), to_str(value).c_str()); + LDBUG_LN("DBUG:GKV:%s_%s:%s:%s:%s:%s", __func__, callerName.c_str(), group.c_str(), key.c_str(), the_type(value).c_str(), to_str(value).c_str()); } // Dump info about the specified group. @@ -109,7 +124,11 @@ class GroupKV { } ss << "\n" << msgTag + ":" << gm.first << ":\n"; for(auto k: gm.second) { +#ifdef GKV_DEBUGLOG_ON + ss << msgTag + ":" << "\t" << k.first << ":" + the_type(k.second) + ":" << to_str(k.second) << "\n"; +#else ss << msgTag + ":" << "\t" << k.first << ":" << to_str(k.second) << "\n"; +#endif } } return ss.str(); @@ -134,11 +153,11 @@ class GroupKV { SupportedDataType get_value(const std::string &group, const MultiPart &keyParts, const SupportedDataType &defaultValue, const std::string &callerName="") { try { auto value = get_value(group, keyParts); - LDBUG_LN("DBUG:GKV:%s_%s:%s:%s:%s", __func__, callerName.c_str(), group.c_str(), to_str(keyParts).c_str(), to_str(value).c_str()); + LDBUG_LN("DBUG:GKV:%s_%s:%s:%s:%s:%s", __func__, callerName.c_str(), group.c_str(), to_str(keyParts).c_str(), the_type(value).c_str(), to_str(value).c_str()); return value; } catch (std::exception &e) { } - LDBUG_LN("WARN:GKV:%s_%s:%s:%s:%s[default]", __func__, callerName.c_str(), group.c_str(), to_str(keyParts).c_str(), to_str(defaultValue).c_str()); + LDBUG_LN("WARN:GKV:%s_%s:%s:%s:%s:%s[default]", __func__, callerName.c_str(), group.c_str(), to_str(keyParts).c_str(), the_type(defaultValue).c_str(), to_str(defaultValue).c_str()); return defaultValue; } From cdd91f5ad11b2ac5129c5732e071ba9b22e0ae15 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Wed, 15 May 2024 18:37:15 +0530 Subject: [PATCH 211/217] SimpCfg: Trap conversion error and raise appropriate exception --- common/simpcfg.hpp | 6 ++++++ tests/test-chaton-simpcfg.cpp | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index 0fca387d79a84..dfaf16c07e863 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -73,6 +73,9 @@ inline size_t wcs_to_mbs(std::string &sDest, const std::wstring &wSrc) { std::mbstate_t mbState = std::mbstate_t(); const wchar_t *wSrcP = wSrc.c_str(); auto reqLen = std::wcsrtombs(nullptr, &wSrcP, 0, &mbState); + if (reqLen == static_cast(-1)) { + throw std::runtime_error("ERRR:WCS2MBS:Failed probing of size..."); + } sDest.resize(reqLen); return std::wcsrtombs(sDest.data(), &wSrcP, sDest.length(), &mbState); } @@ -81,6 +84,9 @@ inline size_t mbs_to_wcs(std::wstring &wDest, const std::string &sSrc) { std::mbstate_t mbState = std::mbstate_t(); const char *sSrcP = sSrc.c_str(); auto reqLen = std::mbsrtowcs(nullptr, &sSrcP, 0, &mbState); + if (reqLen == static_cast(-1)) { + throw std::runtime_error("ERRR:MBS2WCS:Failed probing of size..."); + } wDest.resize(reqLen); return std::mbsrtowcs(wDest.data(), &sSrcP, wDest.length(), &mbState); } diff --git a/tests/test-chaton-simpcfg.cpp b/tests/test-chaton-simpcfg.cpp index 634e5d5c20387..3703a50a18015 100644 --- a/tests/test-chaton-simpcfg.cpp +++ b/tests/test-chaton-simpcfg.cpp @@ -96,7 +96,7 @@ static void check_strings() { SimpCfg::locale_prepare(sSavedLocale); check_string(); check_u8string(); - //check_wstring_wcout(); + check_wstring_wcout(); check_wstring_cout(); check_nonenglish(); SimpCfg::locale_restore(sSavedLocale); From bb3fe48c1649e46161d648d04f44010c9c7a45af Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Wed, 15 May 2024 19:20:38 +0530 Subject: [PATCH 212/217] SimpCfg+DataUtilsString: Move string helpers to its own file --- common/datautils_string.hpp | 220 ++++++++++++++++++++++++++++++++++++ common/simpcfg.hpp | 216 +---------------------------------- 2 files changed, 223 insertions(+), 213 deletions(-) create mode 100644 common/datautils_string.hpp diff --git a/common/datautils_string.hpp b/common/datautils_string.hpp new file mode 100644 index 0000000000000..b1488d40b879b --- /dev/null +++ b/common/datautils_string.hpp @@ -0,0 +1,220 @@ +#pragma once + +/** + * A bunch of helper routines to work with strings. + * by Humans for All + * + * ## Some notes for later + * + * NativeCharSize encoded char refers to chars which fit within the size of char type in a given + * type of c++ string or base bitsize of a encoding standard, like 1 byte in case of std::string, + * utf-8, ... + * * example english alphabets in utf-8 encoding space are 1byte chars, in its variable length + * encoding space. + * + * MultiNativeCharSize encoded char refers to chars which occupy multiple base-char-bit-size of + * a c++ string type or char encoding standard. + * * example indian scripts alphabets in utf-8 encoding space occupy multiple bytes in its variable + * length encoding space. + * + * Sane variable length encoding - refers to encoding where the values of NativeCharSized chars of + * a char encoding space cant overlap with values in NativeCharSize subparts of MultiNativeCharSized + * chars of the same char encoding standard. + * * utf-8 shows this behaviour + * * chances are utf-16 and utf-32 also show this behaviour (need to cross check once) + * +*/ + +#include + +#include "log.h" + + +#undef DUS_DEBUG_VERBOSE + +#undef DUS_STR_OVERSMART +#ifdef DUS_STR_OVERSMART +#define str_trim str_trim_oversmart +#else +#define str_trim str_trim_dumb +#endif + + +inline size_t wcs_to_mbs(std::string &sDest, const std::wstring &wSrc) { + std::mbstate_t mbState = std::mbstate_t(); + const wchar_t *wSrcP = wSrc.c_str(); + auto reqLen = std::wcsrtombs(nullptr, &wSrcP, 0, &mbState); + if (reqLen == static_cast(-1)) { + throw std::runtime_error("ERRR:WCS2MBS:Failed probing of size..."); + } + sDest.resize(reqLen); + return std::wcsrtombs(sDest.data(), &wSrcP, sDest.length(), &mbState); +} + +inline size_t mbs_to_wcs(std::wstring &wDest, const std::string &sSrc) { + std::mbstate_t mbState = std::mbstate_t(); + const char *sSrcP = sSrc.c_str(); + auto reqLen = std::mbsrtowcs(nullptr, &sSrcP, 0, &mbState); + if (reqLen == static_cast(-1)) { + throw std::runtime_error("ERRR:MBS2WCS:Failed probing of size..."); + } + wDest.resize(reqLen); + return std::mbsrtowcs(wDest.data(), &sSrcP, wDest.length(), &mbState); +} + +template +inline void dumphex_string(const TString &sIn, const std::string &msgTag){ + LDBUG("%s[ ", msgTag.c_str()); + for(auto c: sIn) { + auto cSize = sizeof(c); + if (cSize == 1) { + LDBUG("%02x, ", (uint8_t)c); + } else if (cSize == 2) { + LDBUG("%04x, ", (uint16_t)c); + } else if (cSize == 4) { + LDBUG("%08x, ", (uint32_t)c); + } else { + std::stringstream ss; + ss << "ERRR:" << __func__ << ":Unsupported char type with size [" << cSize << "]"; + throw std::runtime_error( ss.str().c_str() ); + } + } + LDBUG_LN(" ]"); +} + +// Remove chars from begin and end of the passed string, provided the char +// belongs to one of the chars in trimChars. +// +// NOTE: This will work perfectly provided the string being trimmed as well as +// chars being trimmed are made up of NativeCharSize chars from same encoded space. +// For utf-8, this means the ascii equivalent 1byteSized chars of utf8 and not +// variable length MultiNativeCharSize (ie multibye in case of utf-8) ones. +// NOTE: It will also work, if atleast either end of string as well as trimChars +// have NativeCharSize chars from their encoding space, rather than variable +// length MultiNativeCharSize based chars if any. There needs to be NativeCharSized +// chars beyond any chars that get trimmed, on either side. +// +// NOTE: Given the way UTF-8 char encoding is designed, where NativeCharSize 1byte +// encoded chars are fully unique and dont overlap with any bytes from any of the +// variable length MultiNativeCharSize encoded chars in the utf-8 space, so as long as +// the trimChars belong to NativeCharSize chars subset, the logic should work, even +// if string has a mixture of NativeCharSize and MultiNativeCharSize encoded chars. +// Chances are utf-16 and utf-32 also have similar characteristics wrt thier +// NativeCharSize encoded chars (ie those fully encoded within single 16bit and 32bit +// value respectively), and so equivalent semantic applies to them also. +// +// ALERT: Given that this simple minded logic, works at individual NativeCharSize level +// only, If trimChars involve variable length MultiNativeCharSize encoded chars, then +// * because different NativeCharSize subparts (bytes in case of utf-8) from different +// MultiNativeCharSize trim chars when clubbed together can map to some other new char +// in a variable length encoded char space, if there is that new char at either end +// of the string, it may get trimmed, because of the possibility of mix up mentioned. +// * given that different variable length MultiNativeCharSize encoded chars may have +// some common NativeCharSize subparts (bytes in case of utf-8) between them, if one +// of these chars is at either end of the string and another char is in trimChars, +// then string may get partially trimmed wrt such a char at either end. +// +template +inline TString str_trim_dumb(TString sin, const TString &trimChars=" \t\n") { +#ifdef DUS_DEBUG_VERBOSE + dumphex_string(sin, "DBUG:StrTrimDumb:Str:"); + dumphex_string(trimChars, "DBUG:StrTrimDumb:TrimChars:"); +#endif + sin.erase(sin.find_last_not_of(trimChars)+1); + sin.erase(0, sin.find_first_not_of(trimChars)); + return sin; +} + +// Remove chars from begin and end of the passed string, provided the char belongs +// to one of the chars in trimChars. +// NOTE: Internally converts to wchar/wstring to try and support proper trimming, +// wrt possibly more languages, to some extent. IE even if the passed string +// contains multibyte encoded characters in it in utf-8 space (ie MultiNativeCharSize), +// it may get converted to NativeCharSize chars in the expanded wchar_t encoding space, +// thus leading to fixed NativeCharSize driven logic itself handling things sufficiently. +// Look at str_trim_dumb comments for additional aspects. +inline std::string str_trim_oversmart(std::string sIn, const std::string &trimChars=" \t\n") { + std::wstring wIn; + mbs_to_wcs(wIn, sIn); + std::wstring wTrimChars; + mbs_to_wcs(wTrimChars, trimChars); + auto wOut = str_trim_dumb(wIn, wTrimChars); + std::string sOut; + wcs_to_mbs(sOut, wOut); + return sOut; +} + +// Remove atmost 1 char at the begin and 1 char at the end of the passed string, +// provided the char belongs to one of the chars in trimChars. +// +// NOTE: Chars being trimmed (ie in trimChars) needs to be part of NativeCharSize +// subset of the string's encoded char space, to avoid mix up when working with +// strings which can be utf-8/utf-16/utf-32/sane-variable-length encoded strings. +// +// NOTE:UTF8: This will work provided the string being trimmed as well the chars +// being trimmed are made up of 1byte encoded chars in case of utf8 encoding space. +// If the string being trimmed includes multibyte (ie MultiNativeCharSize) encoded +// characters at either end, then trimming can mess things up, if you have multibyte +// encoded utf-8 chars in the trimChars set. +// +// Currently given that SimpCfg only uses this with NativeCharSize chars in the +// trimChars and most of the platforms are likely to be using utf-8 based char +// space (which is a realtively sane variable length char encoding from this +// logics perspective), so not providing oversmart variant. +// +template +inline TString str_trim_single(TString sin, const TString& trimChars=" \t\n") { + if (sin.empty()) return sin; + for(auto c: trimChars) { + if (c == sin.front()) { + sin = sin.substr(1, TString::npos); + break; + } + } + if (sin.empty()) return sin; + for(auto c: trimChars) { + if (c == sin.back()) { + sin = sin.substr(0, sin.length()-1); + break; + } + } + return sin; +} + +// Convert to lower case, if language has upper and lower case semantic +// +// This works for fixed size encoded char spaces. +// +// For variable length encoded char spaces, it can work +// * if one is doing the conversion for languages which fit into NativeCharSized chars in it +// * AND if one is working with a sane variable length encoding standard +// * ex: this will work if trying to do the conversion for english language within utf-8 +// +template +inline TString str_tolower(const TString &sin) { + TString sout; + sout.resize(sin.size()); + std::transform(sin.begin(), sin.end(), sout.begin(), [](auto c)->auto {return std::tolower(c);}); +#ifdef DUS_DEBUG_VERBOSE + dumphex_string(sin, "DBUG:StrToLower:in:"); + dumphex_string(sout, "DBUG:StrToLower:out:"); +#endif + return sout; +} + +inline void str_compare_dump(const std::string &s1, const std::string &s2) { + LDBUG_LN("DBUG:%s:%s:Len:%zu", __func__, s1.c_str(), s1.length()); + LDBUG_LN("DBUG:%s:%s:Len:%zu", __func__, s2.c_str(), s2.length()); + int minLen = s1.length() < s2.length() ? s1.length() : s2.length(); + for(int i=0; i +std::string str(TypeWithStrSupp value) { + std::stringstream ss; + ss << value; + return ss.str(); +} diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp index dfaf16c07e863..ffcadc3f473e9 100644 --- a/common/simpcfg.hpp +++ b/common/simpcfg.hpp @@ -4,6 +4,8 @@ * Provides a simple direct 1-level only config file logic * by Humans for All * + * This builds on the GroupKV class. + * * ## File format * * It can consist of multiple config groups. @@ -24,24 +26,6 @@ * It tries to provide a crude expanded form of array wrt any of the above supported types. * For this one needs to define keys using the pattern TheKeyName-0, TheKeyName-1, .... * - * ## Additional notes - * - * NativeCharSize encoded char refers to chars which fit within the size of char type in a given - * type of c++ string or base bitsize of a encoding standard, like 1 byte in case of std::string, - * utf-8, ... - * * example english alphabets in utf-8 encoding space are 1byte chars, in its variable length - * encoding space. - * - * MultiNativeCharSize encoded char refers to chars which occupy multiple base-char-bit-size of - * a c++ string type or char encoding standard. - * * example indian scripts alphabets in utf-8 encoding space occupy multiple bytes in its variable - * length encoding space. - * - * Sane variable length encoding - refers to encoding where the values of NativeCharSized chars of - * a char encoding space cant overlap with values in NativeCharSize subparts of MultiNativeCharSized - * chars of the same char encoding standard. - * * utf-8 shows this behaviour - * * chances are utf-16 and utf-32 also show this behaviour (need to cross check once) */ #include @@ -54,203 +38,9 @@ #include #include "groupkv.hpp" +#include "datautils_string.hpp" -#undef SC_DEBUG_VERBOSE - -#undef SC_STR_OVERSMART -#ifdef SC_STR_OVERSMART -#define str_trim str_trim_oversmart -#else -#define str_trim str_trim_dumb -#endif - - -// **** **** **** String related helpers **** **** **** // - - -inline size_t wcs_to_mbs(std::string &sDest, const std::wstring &wSrc) { - std::mbstate_t mbState = std::mbstate_t(); - const wchar_t *wSrcP = wSrc.c_str(); - auto reqLen = std::wcsrtombs(nullptr, &wSrcP, 0, &mbState); - if (reqLen == static_cast(-1)) { - throw std::runtime_error("ERRR:WCS2MBS:Failed probing of size..."); - } - sDest.resize(reqLen); - return std::wcsrtombs(sDest.data(), &wSrcP, sDest.length(), &mbState); -} - -inline size_t mbs_to_wcs(std::wstring &wDest, const std::string &sSrc) { - std::mbstate_t mbState = std::mbstate_t(); - const char *sSrcP = sSrc.c_str(); - auto reqLen = std::mbsrtowcs(nullptr, &sSrcP, 0, &mbState); - if (reqLen == static_cast(-1)) { - throw std::runtime_error("ERRR:MBS2WCS:Failed probing of size..."); - } - wDest.resize(reqLen); - return std::mbsrtowcs(wDest.data(), &sSrcP, wDest.length(), &mbState); -} - -template -inline void dumphex_string(const TString &sIn, const std::string &msgTag){ - LDBUG("%s[ ", msgTag.c_str()); - for(auto c: sIn) { - auto cSize = sizeof(c); - if (cSize == 1) { - LDBUG("%02x, ", (uint8_t)c); - } else if (cSize == 2) { - LDBUG("%04x, ", (uint16_t)c); - } else if (cSize == 4) { - LDBUG("%08x, ", (uint32_t)c); - } else { - std::stringstream ss; - ss << "ERRR:" << __func__ << ":Unsupported char type with size [" << cSize << "]"; - throw std::runtime_error( ss.str().c_str() ); - } - } - LDBUG_LN(" ]"); -} - -// Remove chars from begin and end of the passed string, provided the char -// belongs to one of the chars in trimChars. -// -// NOTE: This will work perfectly provided the string being trimmed as well as -// chars being trimmed are made up of NativeCharSize chars from same encoded space. -// For utf-8, this means the ascii equivalent 1byteSized chars of utf8 and not -// variable length MultiNativeCharSize (ie multibye in case of utf-8) ones. -// NOTE: It will also work, if atleast either end of string as well as trimChars -// have NativeCharSize chars from their encoding space, rather than variable -// length MultiNativeCharSize based chars if any. There needs to be NativeCharSized -// chars beyond any chars that get trimmed, on either side. -// -// NOTE: Given the way UTF-8 char encoding is designed, where NativeCharSize 1byte -// encoded chars are fully unique and dont overlap with any bytes from any of the -// variable length MultiNativeCharSize encoded chars in the utf-8 space, so as long as -// the trimChars belong to NativeCharSize chars subset, the logic should work, even -// if string has a mixture of NativeCharSize and MultiNativeCharSize encoded chars. -// Chances are utf-16 and utf-32 also have similar characteristics wrt thier -// NativeCharSize encoded chars (ie those fully encoded within single 16bit and 32bit -// value respectively), and so equivalent semantic applies to them also. -// -// ALERT: Given that this simple minded logic, works at individual NativeCharSize level -// only, If trimChars involve variable length MultiNativeCharSize encoded chars, then -// * because different NativeCharSize subparts (bytes in case of utf-8) from different -// MultiNativeCharSize trim chars when clubbed together can map to some other new char -// in a variable length encoded char space, if there is that new char at either end -// of the string, it may get trimmed, because of the possibility of mix up mentioned. -// * given that different variable length MultiNativeCharSize encoded chars may have -// some common NativeCharSize subparts (bytes in case of utf-8) between them, if one -// of these chars is at either end of the string and another char is in trimChars, -// then string may get partially trimmed wrt such a char at either end. -// -template -inline TString str_trim_dumb(TString sin, const TString &trimChars=" \t\n") { -#ifdef SC_DEBUG_VERBOSE - dumphex_string(sin, "DBUG:StrTrimDumb:Str:"); - dumphex_string(trimChars, "DBUG:StrTrimDumb:TrimChars:"); -#endif - sin.erase(sin.find_last_not_of(trimChars)+1); - sin.erase(0, sin.find_first_not_of(trimChars)); - return sin; -} - -// Remove chars from begin and end of the passed string, provided the char belongs -// to one of the chars in trimChars. -// NOTE: Internally converts to wchar/wstring to try and support proper trimming, -// wrt possibly more languages, to some extent. IE even if the passed string -// contains multibyte encoded characters in it in utf-8 space (ie MultiNativeCharSize), -// it may get converted to NativeCharSize chars in the expanded wchar_t encoding space, -// thus leading to fixed NativeCharSize driven logic itself handling things sufficiently. -// Look at str_trim_dumb comments for additional aspects. -inline std::string str_trim_oversmart(std::string sIn, const std::string &trimChars=" \t\n") { - std::wstring wIn; - mbs_to_wcs(wIn, sIn); - std::wstring wTrimChars; - mbs_to_wcs(wTrimChars, trimChars); - auto wOut = str_trim_dumb(wIn, wTrimChars); - std::string sOut; - wcs_to_mbs(sOut, wOut); - return sOut; -} - -// Remove atmost 1 char at the begin and 1 char at the end of the passed string, -// provided the char belongs to one of the chars in trimChars. -// -// NOTE: Chars being trimmed (ie in trimChars) needs to be part of NativeCharSize -// subset of the string's encoded char space, to avoid mix up when working with -// strings which can be utf-8/utf-16/utf-32/sane-variable-length encoded strings. -// -// NOTE:UTF8: This will work provided the string being trimmed as well the chars -// being trimmed are made up of 1byte encoded chars in case of utf8 encoding space. -// If the string being trimmed includes multibyte (ie MultiNativeCharSize) encoded -// characters at either end, then trimming can mess things up, if you have multibyte -// encoded utf-8 chars in the trimChars set. -// -// Currently given that SimpCfg only uses this with NativeCharSize chars in the -// trimChars and most of the platforms are likely to be using utf-8 based char -// space (which is a realtively sane variable length char encoding from this -// logics perspective), so not providing oversmart variant. -// -template -inline TString str_trim_single(TString sin, const TString& trimChars=" \t\n") { - if (sin.empty()) return sin; - for(auto c: trimChars) { - if (c == sin.front()) { - sin = sin.substr(1, TString::npos); - break; - } - } - if (sin.empty()) return sin; - for(auto c: trimChars) { - if (c == sin.back()) { - sin = sin.substr(0, sin.length()-1); - break; - } - } - return sin; -} - -// Convert to lower case, if language has upper and lower case semantic -// -// This works for fixed size encoded char spaces. -// -// For variable length encoded char spaces, it can work -// * if one is doing the conversion for languages which fit into NativeCharSized chars in it -// * AND if one is working with a sane variable length encoding standard -// * ex: this will work if trying to do the conversion for english language within utf-8 -// -template -inline TString str_tolower(const TString &sin) { - TString sout; - sout.resize(sin.size()); - std::transform(sin.begin(), sin.end(), sout.begin(), [](auto c)->auto {return std::tolower(c);}); -#ifdef SC_DEBUG_VERBOSE - dumphex_string(sin, "DBUG:StrToLower:in:"); - dumphex_string(sout, "DBUG:StrToLower:out:"); -#endif - return sout; -} - -inline void str_compare_dump(const std::string &s1, const std::string &s2) { - LDBUG_LN("DBUG:%s:%s:Len:%zu", __func__, s1.c_str(), s1.length()); - LDBUG_LN("DBUG:%s:%s:Len:%zu", __func__, s2.c_str(), s2.length()); - int minLen = s1.length() < s2.length() ? s1.length() : s2.length(); - for(int i=0; i -std::string str(TypeWithStrSupp value) { - std::stringstream ss; - ss << value; - return ss.str(); -} - - -// **** **** **** the SimpCfg **** **** **** // - class SimpCfg : public GroupKV { From 397249df6107c6aa6f7e408b3f693e9cb4f531f1 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Wed, 15 May 2024 19:44:07 +0530 Subject: [PATCH 213/217] DataUtilsString: string_as_hex and use direct log helpers --- common/datautils_string.hpp | 39 ++++++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/common/datautils_string.hpp b/common/datautils_string.hpp index b1488d40b879b..f790d48b738b9 100644 --- a/common/datautils_string.hpp +++ b/common/datautils_string.hpp @@ -26,6 +26,7 @@ */ #include +#include #include "log.h" @@ -62,24 +63,34 @@ inline size_t mbs_to_wcs(std::wstring &wDest, const std::string &sSrc) { return std::mbsrtowcs(wDest.data(), &sSrcP, wDest.length(), &mbState); } +inline std::string uint8_as_hex(uint8_t c) { + char hex[16] = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' }; + std::string out = "00"; + out[0] = hex[((c & 0xf0) >> 4)]; + out[1] = hex[(c & 0x0f)]; + return out; +} + template -inline void dumphex_string(const TString &sIn, const std::string &msgTag){ - LDBUG("%s[ ", msgTag.c_str()); +inline std::string string_as_hex(const TString &sIn){ + std::stringstream ssout; + ssout << "[ "; for(auto c: sIn) { auto cSize = sizeof(c); if (cSize == 1) { - LDBUG("%02x, ", (uint8_t)c); + ssout << uint8_as_hex(c) << ", "; } else if (cSize == 2) { - LDBUG("%04x, ", (uint16_t)c); + ssout << std::setfill('0') << std::setw(cSize*2) << std::hex << static_cast(c) << ", "; } else if (cSize == 4) { - LDBUG("%08x, ", (uint32_t)c); + ssout << std::setfill('0') << std::setw(cSize*2) << std::hex << static_cast(c) << ", "; } else { std::stringstream ss; ss << "ERRR:" << __func__ << ":Unsupported char type with size [" << cSize << "]"; throw std::runtime_error( ss.str().c_str() ); } } - LDBUG_LN(" ]"); + ssout << " ]"; + return ssout.str(); } // Remove chars from begin and end of the passed string, provided the char @@ -117,8 +128,8 @@ inline void dumphex_string(const TString &sIn, const std::string &msgTag){ template inline TString str_trim_dumb(TString sin, const TString &trimChars=" \t\n") { #ifdef DUS_DEBUG_VERBOSE - dumphex_string(sin, "DBUG:StrTrimDumb:Str:"); - dumphex_string(trimChars, "DBUG:StrTrimDumb:TrimChars:"); + LOG_TEELN("DBUG:StrTrimDumb:Str:%s", string_as_hex(sin).c_str()); + LOG_TEELN("DBUG:StrTrimDumb:TrimChars:%s", string_as_hex(trimChars).c_str()); #endif sin.erase(sin.find_last_not_of(trimChars)+1); sin.erase(0, sin.find_first_not_of(trimChars)); @@ -196,24 +207,24 @@ inline TString str_tolower(const TString &sin) { sout.resize(sin.size()); std::transform(sin.begin(), sin.end(), sout.begin(), [](auto c)->auto {return std::tolower(c);}); #ifdef DUS_DEBUG_VERBOSE - dumphex_string(sin, "DBUG:StrToLower:in:"); - dumphex_string(sout, "DBUG:StrToLower:out:"); + LOG_TEELN("DBUG:StrToLower:in:%s", string_as_hex(sin).c_str()); + LOG_TEELN("DBUG:StrToLower:out:%s", string_as_hex(sout).c_str()); #endif return sout; } inline void str_compare_dump(const std::string &s1, const std::string &s2) { - LDBUG_LN("DBUG:%s:%s:Len:%zu", __func__, s1.c_str(), s1.length()); - LDBUG_LN("DBUG:%s:%s:Len:%zu", __func__, s2.c_str(), s2.length()); + LOG_TEELN("DBUG:%s:%s:Len:%zu", __func__, s1.c_str(), s1.length()); + LOG_TEELN("DBUG:%s:%s:Len:%zu", __func__, s2.c_str(), s2.length()); int minLen = s1.length() < s2.length() ? s1.length() : s2.length(); for(int i=0; i -std::string str(TypeWithStrSupp value) { +std::string as_str(TypeWithStrSupp value) { std::stringstream ss; ss << value; return ss.str(); From 239b5be2197a3ca8a2c52f3f6b06ad9ee0d694f9 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Thu, 16 May 2024 11:50:57 +0530 Subject: [PATCH 214/217] ChatON+: Cleanup integration with CMake Rename chaton-meta hpp to cpp and include this cpp file which brings in the compile time built-in global chaton configurable template data into the common library, and avoid the nop hpp file references. Update chaton.hpp to not include the meta-cpp, instead just make a reference to the global ChatTemplates instance, so that the hpp can be used as a header file proper. Avoid pragma once in the chaton-meta.cpp, including the script, which helps create it. --- common/CMakeLists.txt | 3 +-- common/chaton.hpp | 5 ++--- common/{chaton_meta.hpp => chaton_meta.cpp} | 1 - scripts/chaton-meta-json-to-hpp.py | 4 ++-- 4 files changed, 5 insertions(+), 8 deletions(-) rename common/{chaton_meta.hpp => chaton_meta.cpp} (99%) diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index a07566c0e13ad..f34249532f92d 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -65,8 +65,7 @@ add_library(${TARGET} STATIC train.cpp ngram-cache.h ngram-cache.cpp - chaton.hpp - groupkv.hpp + chaton_meta.cpp ) if (BUILD_SHARED_LIBS) diff --git a/common/chaton.hpp b/common/chaton.hpp index 17bd59be73661..b2046d198a600 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -537,9 +537,8 @@ class ChatTemplates : public GroupKV { }; -// The compiled-in configurable template data (the meta) -#include "chaton_meta.hpp" -//ChatTemplates gCT = {{}}; +// The compiled-in configurable template data (the meta) from chaton_meta.cpp +extern ChatTemplates gCT; inline bool chaton_tmpl_exists(const std::string &tmpl) { diff --git a/common/chaton_meta.hpp b/common/chaton_meta.cpp similarity index 99% rename from common/chaton_meta.hpp rename to common/chaton_meta.cpp index 3516d5bb91e80..d0013b519d052 100644 --- a/common/chaton_meta.hpp +++ b/common/chaton_meta.cpp @@ -1,5 +1,4 @@ //This is auto created/converted from chaton-meta-json file -#pragma once #include "chaton.hpp" diff --git a/scripts/chaton-meta-json-to-hpp.py b/scripts/chaton-meta-json-to-hpp.py index d72c4cdf5858b..3329b9b7f43ed 100755 --- a/scripts/chaton-meta-json-to-hpp.py +++ b/scripts/chaton-meta-json-to-hpp.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Convert chaton meta json file to c++ hpp file +# Convert chaton meta json file to equivalent c++ cpp format # by Humans for All import sys @@ -19,7 +19,7 @@ def kv_bool(j, tmpl, k1, comma): fp=open(sys.argv[1]) j=json.load(fp) print("//This is auto created/converted from chaton-meta-json file") -print("#pragma once\n\n#include \"chaton.hpp\"\n\nChatTemplates gCT = {{") +print("\n\n#include \"chaton.hpp\"\n\nChatTemplates gCT = {{") for tmpl in j: print("\t{{ \"{}\", {{".format(tmpl)) From 1a0df950ebb9b763b3d132fb60f7f2e918adffe3 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Thu, 16 May 2024 14:46:18 +0530 Subject: [PATCH 215/217] C++17: Use and limit C++17 to common library for now C++17 provides a good enough variant as a standard feature, and chaton uses the same at its core, instead of rolling out its own struct of union based variant. And given that currently chaton is part of common library and not the base llama library, so limit the use of c++17 to common library. Initially while experimenting, had set the flag for full llama, limitting it for now. Also by now most embedded targets should be potentially having c++ compilers and libraries with support for c++17 features. So chances are it is a ok enough path to take. --- CMakeLists.txt | 2 -- common/CMakeLists.txt | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6840b23ec6cab..feb6f39d09662 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -849,8 +849,6 @@ function(get_flags CCID CCVER) set(GF_CXX_FLAGS ${CXX_FLAGS} PARENT_SCOPE) endfunction() -list(APPEND CXX_FLAGS -std=c++17) - if (LLAMA_FATAL_WARNINGS) if (CMAKE_CXX_COMPILER_ID MATCHES "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") list(APPEND C_FLAGS -Werror) diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index f34249532f92d..8a6dbcd8214fb 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -84,5 +84,5 @@ if (LLAMA_CURL) endif () target_include_directories(${TARGET} PUBLIC .) -target_compile_features(${TARGET} PUBLIC cxx_std_11) +target_compile_features(${TARGET} PUBLIC cxx_std_17) target_link_libraries(${TARGET} PRIVATE ${LLAMA_COMMON_EXTRA_LIBS} PUBLIC llama) From 0cbfd40f18809e34df9a4ab89eb77234c3fec1ba Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Thu, 16 May 2024 23:27:34 +0530 Subject: [PATCH 216/217] ChatON: Option for a fallback tmpl to use wrt chat-tmpl-apply-ex --- common/chaton.hpp | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index b2046d198a600..9842ff39357a3 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -326,6 +326,8 @@ class ChatTemplates : public GroupKV { public: + std::string tmplFallback; + ChatTemplates(GroupKVMapMapVariant defaultMap) : GroupKV(defaultMap) {} /** @@ -450,7 +452,7 @@ class ChatTemplates : public GroupKV { * cntUserMsgCnt arguments. */ bool chaton_tmpl_apply_ex( - const std::string &tmpl, + const std::string &tmplId, const std::vector &msgs, bool alertAssistantAtEnd, bool applyGlobalIfAny, @@ -460,8 +462,13 @@ class ChatTemplates : public GroupKV { int curSystemMsgCnt = 0, int curUserMsgCnt = 0 ) { + std::string tmpl = tmplId; if (!tmpl_exists(tmpl)) { - return false; + tmpl = tmplFallback; + LDBUG_LN("DBUG:%s:Tmpl [%s] missing, fallback to [%s]", __func__, tmplId, tmpl); + if (!tmpl_exists(tmpl)) { + return false; + } } ChatParts cp = {}; if (applyGlobalIfAny) { @@ -772,6 +779,13 @@ inline std::vector chaton_llama_tokenize_ex( } +inline void chaton_meta_fallback(std::string &tmpl, ChatTemplates *ct=nullptr) { + if (ct == nullptr) { + ct = &gCT; + } + ct->tmplFallback = tmpl; +} + /** * Validate specified chaton-template-id and inturn dump the contents related to that * specific chat-handshake-template-standard, wrt the specified ChatTemplates. From 999bd396d05e1029d45aa779b773de06b6064a60 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Thu, 16 May 2024 23:49:38 +0530 Subject: [PATCH 217/217] ChatON: forgot to get c string format --- common/chaton.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/chaton.hpp b/common/chaton.hpp index 9842ff39357a3..2ea9fed7a6efd 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -465,7 +465,7 @@ class ChatTemplates : public GroupKV { std::string tmpl = tmplId; if (!tmpl_exists(tmpl)) { tmpl = tmplFallback; - LDBUG_LN("DBUG:%s:Tmpl [%s] missing, fallback to [%s]", __func__, tmplId, tmpl); + LDBUG_LN("DBUG:%s:Tmpl [%s] missing, fallback to [%s]", __func__, tmplId.c_str(), tmpl.c_str()); if (!tmpl_exists(tmpl)) { return false; }