tool-call: add support for tool-calls using Model Context Protocol

CMakeLists.txt

@@ -84,6 +84,9 @@ option(LLAMA_BUILD_SERVER   "llama: build server example" ${LLAMA_STANDALONE})
 option(LLAMA_CURL "llama: use libcurl to download model from an URL" OFF)
 option(LLAMA_LLGUIDANCE "llama-common: include LLGuidance library for structured output in common utils" OFF)
 
+# Toolcall support - needs LLAMA_CURL support to connect with SSE endpoints
+option(LLAMA_TOOLCALL "llama: add toolcall support via Model Context Protocol" ON)
+
 # Required for relocatable CMake package
 include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/build-info.cmake)
 include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/common.cmake)
@@ -168,6 +171,10 @@ add_subdirectory(src)
 # utils, programs, examples and tests
 #
 
+if (LLAMA_TOOLCALL)
+    add_subdirectory(toolcall)
+endif()
+
 if (LLAMA_BUILD_COMMON)
     add_subdirectory(common)
 endif()

common/arg.cpp

@@ -2740,6 +2740,24 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
                 std::back_inserter(params.chat_template));
         }
     ).set_examples({LLAMA_EXAMPLE_MAIN, LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_CHAT_TEMPLATE_FILE"));
+
+    add_opt(common_arg(
+        {"--tools"}, "JINJA_TOOLS",
+        "set to URI of a Model Context Protocol server, or "
+        "a JSON array containing tool definitions (requires --jinja)",
+        [](common_params &params, const std::string & value) {
+            params.toolcall.tools = value;
+
+        }).set_examples({LLAMA_EXAMPLE_MAIN}));
+
+    add_opt(common_arg(
+        {"--tool-choice"}, "JINJA_TOOL_CHOICE",
+        "set to \"auto\", \"required\", or \"none\" (default: \"auto\")",
+        [](common_params &params, const std::string & value) {
+            params.toolcall.choice = value;
+
+        }).set_examples({LLAMA_EXAMPLE_MAIN}));
+
     add_opt(common_arg(
         {"-sps", "--slot-prompt-similarity"}, "SIMILARITY",
         string_format("how much the prompt of a request must match the prompt of a slot in order to use that slot (default: %.2f, 0.0 = disabled)\n", params.slot_prompt_similarity),

common/common.cpp

@@ -8,6 +8,7 @@
 #include "common.h"
 #include "log.h"
 #include "llama.h"
+#include "chat.h"
 
 #include <algorithm>
 #include <cinttypes>
@@ -1291,6 +1292,67 @@ std::string common_detokenize(const struct llama_vocab * vocab, const std::vecto
     return text;
 }
 
+void common_chat_grammar_to_sampler(const common_chat_params * src,
+                                    const llama_vocab * vocab,
+                                    common_params_sampling * sparams)
+{
+    GGML_ASSERT(src && vocab && sparams);
+
+    auto & dst = *sparams;
+
+    dst.grammar      = src->grammar;
+    dst.grammar_lazy = src->grammar_lazy;
+
+    for (const auto & preserved : src->preserved_tokens) {
+        auto ids = common_tokenize(vocab, preserved, false, true);
+        if (ids.size() == 1) {
+            LOG_DBG("Preserved token: %d\n", ids[0]);
+            dst.preserved_tokens.insert(ids[0]);
+
+        } else {
+            // This may happen when using a tool call style meant for a model
+            // with special tokens to preserve on a model without said tokens.
+            LOG_WRN("Not preserved because more than 1 token (wrong chat template override?): %s\n",
+                    preserved.c_str());
+        }
+    }
+
+    for (const auto & trigger : src->grammar_triggers) {
+        if (trigger.type == COMMON_GRAMMAR_TRIGGER_TYPE_WORD) {
+            const auto & word = trigger.value;
+            auto ids = common_tokenize(vocab, word, /* add_special= */ false, /* parse_special= */ true);
+
+            if (ids.size() == 1) {
+                auto token = ids[0];
+                auto found = std::find(dst.preserved_tokens.begin(), dst.preserved_tokens.end(),
+                                       (llama_token) token);
+
+                if (found == dst.preserved_tokens.end()) {
+                    throw std::runtime_error("Grammar trigger word should be marked as preserved token: " + word);
+                }
+
+                LOG_DBG("Grammar trigger token: %d (`%s`)\n", token, word.c_str());
+                common_grammar_trigger trigger;
+                trigger.type = COMMON_GRAMMAR_TRIGGER_TYPE_TOKEN;
+                trigger.value = word;
+                trigger.token = token;
+                dst.grammar_triggers.push_back(std::move(trigger));
+
+            } else {
+                LOG_DBG("Grammar trigger word: `%s`\n", word.c_str());
+                dst.grammar_triggers.push_back({COMMON_GRAMMAR_TRIGGER_TYPE_WORD, word});
+            }
+
+        } else {
+            dst.grammar_triggers.push_back(trigger);
+        }
+    }
+    if (dst.grammar_lazy && dst.grammar_triggers.empty()) {
+        throw std::runtime_error("Error: no triggers set for lazy grammar!");
+    }
+}
+
+
 //
 // KV cache utils
 //

common/common.h

@@ -216,6 +216,11 @@ enum common_reasoning_format {
     COMMON_REASONING_FORMAT_DEEPSEEK, // Extract thinking tag contents and return as `message.reasoning_content`
 };
 
+struct common_toolcall_params {
+    std::string tools  = "";
+    std::string choice = "auto";
+};
+
 struct common_params {
     int32_t n_predict             =    -1; // new tokens to predict
     int32_t n_ctx                 =  4096; // context size
@@ -363,6 +368,9 @@ struct common_params {
     std::string chat_template = "";                                                                         // NOLINT
     bool use_jinja = false;                                                                                 // NOLINT
     bool enable_chat_template = true;
+
+    struct common_toolcall_params toolcall;
+
     common_reasoning_format reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK;
 
     std::vector<std::string> api_keys;
@@ -609,6 +617,12 @@ std::string common_detokenize(
         const std::vector<llama_token> & tokens,
                                   bool   special = true);
 
+struct common_chat_params;
+void common_chat_grammar_to_sampler(const common_chat_params * src,
+                                    const llama_vocab * vocab,
+                                    common_params_sampling * sparams);
+
+
 //
 // KV cache utils
 //

examples/main/CMakeLists.txt

@@ -2,4 +2,9 @@ set(TARGET llama-cli)
 add_executable(${TARGET} main.cpp)
 install(TARGETS ${TARGET} RUNTIME)
 target_link_libraries(${TARGET} PRIVATE common llama ${CMAKE_THREAD_LIBS_INIT})
+
+if (LLAMA_TOOLCALL)
+    target_link_libraries(${TARGET} PRIVATE toolcall)
+endif()
+
 target_compile_features(${TARGET} PRIVATE cxx_std_17)

examples/main/main.cpp

@@ -5,6 +5,7 @@
 #include "sampling.h"
 #include "llama.h"
 #include "chat.h"
+#include <json.hpp>
 
 #include <cstdio>
 #include <cstring>
@@ -15,6 +16,10 @@
 #include <string>
 #include <vector>
 
+#ifdef LLAMA_USE_TOOLCALL
+#    include "toolcall-client.h"
+#endif
+
 #if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__))
 #include <signal.h>
 #include <unistd.h>
@@ -83,6 +88,99 @@ static void sigint_handler(int signo) {
 }
 #endif
 
+class chat_formatter {
+public:
+
+    struct result {
+        std::string formatted;
+        bool tool_was_called;
+    };
+
+    chat_formatter(common_params & params,
+                   std::vector<common_chat_msg> & chat_msgs,
+                   struct common_chat_templates * chat_templates)
+
+        : params_(params), chat_msgs_(chat_msgs), chat_templates_(chat_templates) {}
+
+#ifdef LLAMA_USE_TOOLCALL
+    chat_formatter(common_params & params,
+                   std::vector<common_chat_msg> & chat_msgs,
+                   struct common_chat_templates * chat_templates,
+                   const llama_vocab * vocab,
+                   toolcall::client::ptr tc_client)
+
+        : params_(params), chat_msgs_(chat_msgs), chat_templates_(chat_templates),
+          vocab_(vocab), tc_client_(tc_client),
+          chat_format_(COMMON_CHAT_FORMAT_CONTENT_ONLY),
+          formatted_() {}
+#endif
+
+    chat_formatter::result operator() (const std::string & role, const std::string & content) {
+
+        common_chat_msg new_msg = common_chat_parse(content, chat_format_);
+        new_msg.role = role;
+
+        common_chat_templates_inputs cinputs;
+        cinputs.use_jinja = params_.use_jinja;
+        cinputs.add_generation_prompt = (role == "user");
+#ifdef LLAMA_USE_TOOLCALL
+        if (tc_client_ != nullptr) {
+            cinputs.tool_choice = common_chat_tool_choice_parse_oaicompat(tc_client_->tool_choice());
+            cinputs.tools = common_chat_tools_parse_oaicompat(tc_client_->tool_list());
+        }
+#endif
+        cinputs.messages.assign(chat_msgs_.cbegin(), chat_msgs_.cend());
+        cinputs.messages.push_back(new_msg);
+        chat_msgs_.push_back(new_msg);
+
+        bool tool_was_called = false;
+        if (! new_msg.tool_calls.empty()) { // Call tool and re-prompt
+            nlohmann::json result_array = nlohmann::json::array();
+            for (const auto & tc : new_msg.tool_calls) {
+                toolcall::result_set res = tc_client_->call(tc.name, tc.arguments, tc.id);
+                if (! res.empty()) {
+                    for (const auto & r : res) {
+                        result_array.push_back(r.data);
+                    }
+                }
+            }
+            common_chat_msg toolcall_msg;
+            toolcall_msg.role = "tool";
+            toolcall_msg.content = result_array.dump(-1);
+
+            cinputs.add_generation_prompt = true;
+            cinputs.messages.push_back(toolcall_msg);
+            chat_msgs_.push_back(toolcall_msg);
+
+            tool_was_called = true;
+        }
+
+        common_chat_params cparams = common_chat_templates_apply(chat_templates_, cinputs);
+        std::string formatted = cparams.prompt.substr(formatted_.size(), cparams.prompt.size());
+        formatted_ = cparams.prompt;
+
+        LOG_DBG("formatted: '%s'\n", formatted.c_str());
+
+#ifdef LLAMA_USE_TOOLCALL
+        chat_format_ = cparams.format;
+        common_chat_grammar_to_sampler(&cparams, vocab_, &params_.sampling);
+#endif
+        return chat_formatter::result{std::move(formatted), tool_was_called};
+    }
+
+private:
+    common_params & params_;
+    std::vector<common_chat_msg> & chat_msgs_;
+    struct common_chat_templates * chat_templates_;
+
+#ifdef LLAMA_USE_TOOLCALL
+    const llama_vocab * vocab_;
+    toolcall::client::ptr tc_client_;
+    common_chat_format chat_format_;
+    std::string formatted_;
+#endif
+};
+
 int main(int argc, char ** argv) {
     common_params params;
     g_params = &params;
@@ -94,6 +192,11 @@ int main(int argc, char ** argv) {
 
     auto & sparams = params.sampling;
 
+#ifdef LLAMA_USE_TOOLCALL
+    // Ensure parameters are validated before the model loads
+    toolcall::params tc_params(params.toolcall.tools, params.toolcall.choice);
+#endif
+
     // save choice to use color for later
     // (note for later: this is a slightly awkward choice)
     console::init(params.simple_io, params.use_color);
@@ -266,15 +369,16 @@ int main(int argc, char ** argv) {
     std::vector<llama_token> embd_inp;
 
     bool waiting_for_first_input = false;
-    auto chat_add_and_format = [&chat_msgs, &chat_templates](const std::string & role, const std::string & content) {
-        common_chat_msg new_msg;
-        new_msg.role = role;
-        new_msg.content = content;
-        auto formatted = common_chat_format_single(chat_templates.get(), chat_msgs, new_msg, role == "user", g_params->use_jinja);
-        chat_msgs.push_back(new_msg);
-        LOG_DBG("formatted: '%s'\n", formatted.c_str());
-        return formatted;
-    };
+
+#ifdef LLAMA_USE_TOOLCALL
+    auto tc_client = toolcall::create_client(tc_params);
+    if (tc_client) {
+        tc_client->initialize();
+    }
+    chat_formatter chat_add_and_format(params, chat_msgs, chat_templates.get(), vocab, tc_client);
+#else
+    chat_formatter chat_add_and_format(params, chat_msgs, chat_templates.get());
+#endif
 
     std::string prompt;
     {
@@ -296,6 +400,12 @@ int main(int argc, char ** argv) {
                 inputs.messages = chat_msgs;
                 inputs.add_generation_prompt = !params.prompt.empty();
 
+#ifdef LLAMA_USE_TOOLCALL
+                if (tc_client != nullptr) {
+                    inputs.tool_choice = common_chat_tool_choice_parse_oaicompat(tc_client->tool_choice());
+                    inputs.tools = common_chat_tools_parse_oaicompat(tc_client->tool_list());
+                }
+#endif
                 prompt = common_chat_templates_apply(chat_templates.get(), inputs).prompt;
             }
         } else {
@@ -814,10 +924,17 @@ int main(int argc, char ** argv) {
                     }
 
                     if (params.enable_chat_template) {
-                        chat_add_and_format("assistant", assistant_ss.str());
+                        auto format_res = chat_add_and_format("assistant", assistant_ss.str());
+                        if (format_res.tool_was_called) {
+                            auto format_res_tok = common_tokenize(ctx, format_res.formatted, false, true);
+                            embd_inp.insert(embd_inp.end(), format_res_tok.begin(), format_res_tok.end());
+                            assistant_ss.str("");
+
+                        } else {
+                            is_interacting = true;
+                            LOG("\n");
+                        }
                     }
-                    is_interacting = true;
-                    LOG("\n");
                 }
             }
 
@@ -884,7 +1001,7 @@ int main(int argc, char ** argv) {
 
                     bool format_chat = params.conversation_mode && params.enable_chat_template;
                     std::string user_inp = format_chat
-                        ? chat_add_and_format("user", std::move(buffer))
+                        ? chat_add_and_format("user", std::move(buffer)).formatted
                         : std::move(buffer);
                     // TODO: one inconvenient of current chat template implementation is that we can't distinguish between user input and special tokens (prefix/postfix)
                     const auto line_pfx = common_tokenize(ctx, params.input_prefix, false, true);