diff --git a/.gitignore b/.gitignore index 1092d097a7542..a72bb1381d049 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,11 @@ # Extensions +*package-lock.json +*package.json +*node_modules +*.ipynb +.*.json + *.a *.bat *.bin diff --git a/Makefile b/Makefile index 719f45d167463..9df8d51b417bc 100644 --- a/Makefile +++ b/Makefile @@ -1453,6 +1453,8 @@ endif # GGML_RPC llama-server: \ examples/server/server.cpp \ examples/server/utils.hpp \ + examples/server/function-call-parser.hpp \ + examples/server/function-call.hpp \ examples/server/httplib.h \ examples/server/colorthemes.css.hpp \ examples/server/style.css.hpp \ diff --git a/README.md b/README.md index 8fe1f4b4b6a7a..d4b1c06d55756 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,124 @@ -# llama.cpp +# tools.cpp +tools.cpp is Rubra's fork of llama.cpp, offering inference of Rubra's function calling models (and others) in pure C/C++. -![llama](https://user-images.githubusercontent.com/1991296/230134379-7181e485-c521-4d23-a0d6-f7b3b61ba524.png) +## tools.cpp quickstart +1. build from source: + +- Mac user +``` +make +``` + +- Nvidia-Cuda user: +``` +make LLAMA_CUDA=1 +``` + +2. Install a helper package that fixes some rare edgecases: +``` +npm install jsonrepair +``` -[![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](https://opensource.org/licenses/MIT) -[![Server](https://github.com/ggerganov/llama.cpp/actions/workflows/server.yml/badge.svg)](https://github.com/ggerganov/llama.cpp/actions/workflows/server.yml) -[![Conan Center](https://shields.io/conan/v/llama-cpp)](https://conan.io/center/llama-cpp) +3. Download a compatible Rubra GGUF model: +For example: +``` +wget https://huggingface.co/rubra-ai/Meta-Llama-3-8B-Instruct-GGUF/resolve/main/rubra-meta-llama-3-8b-instruct.Q6_K.gguf +``` + +For large multi-part model files, such as [rubra-meta-llama-3-70b-instruct_Q6_K-0000*-of-00003.gguf](https://huggingface.co/rubra-ai/Meta-Llama-3-70B-Instruct-GGUF/tree/main), use the following command to merge them before proceeding to the next step: +``` +./llama-gguf-split --merge rubra-meta-llama-3-70b-instruct_Q6_K-0000*-of-00003.gguf rubra-meta-llama-3-70b-instruct_Q6_K.gguf +``` +This will merge multi-part model files to one gguf file `rubra-meta-llama-3-70b-instruct_Q6_K.gguf`. + +4. start openai compatible server: +``` +./llama-server -ngl 37 -m rubra-meta-llama-3-8b-instruct.Q6_K.gguf --port 1234 --host 0.0.0.0 -c 8000 --chat-template llama3 +``` -[Roadmap](https://github.com/users/ggerganov/projects/7) / [Project status](https://github.com/ggerganov/llama.cpp/discussions/3471) / [Manifesto](https://github.com/ggerganov/llama.cpp/discussions/205) / [ggml](https://github.com/ggerganov/ggml) +5. Test the server, ensure it is available: +```bash +curl localhost:1234/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer tokenabc-123" \ + -d '{ + "model": "rubra-model", + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant." + }, + { + "role": "user", + "content": "hello" + } + ] + }' +``` + +6. Try a python function calling example: +```python +# if openai not installed, do `pip install openai` +from openai import OpenAI +client = OpenAI(api_key="123", base_url = "http://localhost:1234/v1/") + +tools = [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, + }, + "required": ["location"], + }, + } + } +] +messages = [{"role": "user", "content": "What's the weather like in Boston today?"}] +completion = client.chat.completions.create( + model="rubra-model", + messages=messages, + tools=tools, + tool_choice="auto" +) + +print(completion) +``` + +The output should look like this: +``` +ChatCompletion(id='chatcmpl-EmHd8kai4DVwBUOyim054GmfcyUbjiLf', choices=[Choice(finish_reason='tool_calls', index=0, logprobs=None, message=ChatCompletionMessage(content=None, role='assistant', function_call=None, tool_calls=[ChatCompletionMessageToolCall(id='e885974b', function=Function(arguments='{"location":"Boston"}', name='get_current_weather'), type='function')]))], created=1719528056, model='rubra-model', object='chat.completion', system_fingerprint=None, usage=CompletionUsage(completion_tokens=29, prompt_tokens=241, total_tokens=270)) +``` + +That's it! MAKE SURE you turn `stream` OFF when making api calls to the server, as the streaming feature is not supported yet. And we will support streaming too soon. + +For more function calling examples, you can checkout `test_llamacpp.ipynb` notebook. + +### Choosing a Chat Template for Different Models + +| Model | Chat Template | +|---------|:-------------:| +| Llama3 | llama3 | +| Mistral | llama2 | +| Phi3 | phi3 | +| Gemma | gemma | +| Qwen2 | chatml | + +For example, to run [Rubra's enhanced Phi3 model](https://huggingface.co/rubra-ai/Phi-3-mini-128k-instruct-function-calling-alpha-v1-GGUF), use the following command: + +```bash +./llama-server -ngl 37 -m phi-3-mini-128k-instruct-function-calling-alpha-v1.Q8_0.gguf --port 1234 --host 0.0.0.0 -c 32000 --chat-template phi3 +``` -Inference of Meta's [LLaMA](https://arxiv.org/abs/2302.13971) model (and others) in pure C/C++ +============================================================================== ## Recent API changes diff --git a/examples/server/function-call-parser.hpp b/examples/server/function-call-parser.hpp new file mode 100644 index 0000000000000..8d894749f97ab --- /dev/null +++ b/examples/server/function-call-parser.hpp @@ -0,0 +1,148 @@ +#include +#include +#include "json.hpp" +#include +#include + +using json = nlohmann::ordered_json; + +std::string generate_uuid() { + static std::random_device rd; + static std::mt19937 generator(rd()); + static std::uniform_int_distribution distribution(0, 15); + + const char *v = "0123456789abcdef"; + std::stringstream uuid; + + for (int i = 0; i < 8; ++i) { + uuid << v[distribution(generator)]; + } + return uuid.str(); +} + + +std::string jsonrepair(const std::string value) { + std::array buffer; + std::string result; + // Ensure the command passed to popen() is null-terminated + std::string tmpfile_name = "." + generate_uuid() + ".json"; + std::ofstream outfile(tmpfile_name); + outfile << value; // Assuming jsonStr contains your JSON string + outfile.close(); + std::string command = "node jsonrepair.ts " + tmpfile_name; + std::unique_ptr pipe(popen(command.c_str(), "r"), pclose); + if (!pipe) { + throw std::runtime_error("popen() failed!"); + } + while (fgets(buffer.data(), buffer.size(), pipe.get()) != nullptr) { + result += buffer.data(); + } + return result; +} + + +json parse_if_json(const std::string& value) { + try { + // json repair here + return json::parse(jsonrepair(value)); + } catch (const json::parse_error&) { + return value; // Return the original string if parsing fails + } +} + + +std::string clean_command_string(const std::string& command_str) { + std::string cleaned_command = std::regex_replace(command_str, std::regex(R"(\\(?!["\\/bfnrt]|u[a-fA-F0-9]{4}))"), ""); + cleaned_command = std::regex_replace(cleaned_command, std::regex(R"(\\")"), "\""); + + if (cleaned_command.front() == '"' && cleaned_command.back() == '"') { + cleaned_command = cleaned_command.substr(1, cleaned_command.size() - 2); + } + return cleaned_command; +} + + +json clean_json_strings(const std::string& input_str) { + try { + // json repair here + std::string fixed_str = jsonrepair(input_str); + json data = json::parse(fixed_str); + for (auto& [key, value] : data.items()) { + if (value.is_string()) { + std::string val = value.get(); + if (val.front() == '{' || val.front() == '[') { + data[key] = parse_if_json(val); + } else { + data[key] = clean_command_string(val); + } + } else if (value.is_object()) { + for (auto& [k, v] : value.items()) { + if (v.is_string()) { + v = clean_command_string(v.get()); + } + + } + } + } + return data; + } catch (const json::parse_error& e) { + std::cerr << "Error decoding JSON: " << e.what() << std::endl; + return nullptr; + } +} + + + + +std::vector rubra_fc_json_tool_extractor(const std::string& output_str) { + std::vector result; + std::cout << "Output to Parse : " << output_str.c_str() << std::endl; + if (output_str.find("endtoolcall") == std::string::npos) { + return result; + } + + std::vector listOfStrToParse; + size_t start = 0, end = 0; + + // Iterate until all instances of "endtoolcall" are processed + while ((end = output_str.find("endtoolcall", start)) != std::string::npos) { + std::string segment = output_str.substr(start, end - start); + size_t pos = segment.find("starttoolcall"); + if (pos != std::string::npos) { + // Extract substring after "toolcall" + std::string ss = segment.substr(pos + std::string("starttoolcall").length()); + listOfStrToParse.push_back(ss); + } + start = end + std::string("endtoolcall").length(); // Move past the "endtoolcall" + } + + std::vector function_call_json; + + try { + for (const auto & line : listOfStrToParse) { + // json fc = json::parse(line); + + json fc = clean_json_strings(line); + if (!fc["arguments"].is_string()) { + fc["arguments"] = fc["arguments"].dump(); + } + if (!fc.is_null()) { + function_call_json.push_back(fc); + } + + } + } catch (const std::exception& e) { + std::cerr << "Error: " << e.what() << std::endl; + } + + for (const auto& fc : function_call_json) { + json func_call; + func_call["id"] = generate_uuid(); + func_call["name"] = fc["name"]; + func_call["kwargs"] = fc["arguments"]; + func_call["type"] = "function"; + result.push_back(func_call); + } + + return result; +} \ No newline at end of file diff --git a/examples/server/function-call.hpp b/examples/server/function-call.hpp new file mode 100644 index 0000000000000..d0ec214acae94 --- /dev/null +++ b/examples/server/function-call.hpp @@ -0,0 +1,337 @@ +#include +#include "json.hpp" +#include +#include + +using json = nlohmann::ordered_json; + + +static std::string join(const std::vector& vec, const std::string& delimiter) { + std::string result; + for (size_t i = 0; i < vec.size(); i++) { + result += vec[i]; + if (i < vec.size() - 1) { + result += delimiter; + } + } + return result; +} + +static std::string capitalize(const std::string& str) { + std::string capitalized = str; + if (!capitalized.empty()) { + capitalized[0] = toupper(capitalized[0]); + for (size_t i = 1; i < capitalized.length(); i++) { + capitalized[i] = tolower(capitalized[i]); + } + } + return capitalized; +} + +static std::string json_schema_to_typescript_type(const json& schema, const std::string& param_name, std::string& enum_comment, std::string& integer_comment, std::string& description_comment); + +static std::pair generate_typescript_interface(const json& schema, const std::string& interface_name); + +static std::string generate_typescript_function(const json& function_schema); + +// Main functions +static std::string json_schema_to_typescript_type(const json& schema, const std::string& param_name, std::string& enum_comment, std::string& integer_comment, std::string& description_comment) { + std::string ts_type = "any"; // Default type + enum_comment = ""; + integer_comment = ""; + description_comment = ""; + + if (schema.contains("type")) { + std::string json_type = schema["type"]; + if (json_type == "array") { + std::string item_type = "any"; + if (schema.contains("items")) { + item_type = json_schema_to_typescript_type(schema["items"], param_name, enum_comment, integer_comment, description_comment); + } + ts_type = item_type + "[]"; + } else if (json_type == "number") { + ts_type = "number"; + } else if (json_type == "integer") { + ts_type = "number"; + integer_comment = " * @param " + param_name + " - Integer"; + } else if (json_type == "object") { + auto [interface_type, _] = generate_typescript_interface(schema, param_name); + ts_type = interface_type; + } else if (json_type == "boolean") { + ts_type = "boolean"; + } else if (json_type == "null") { + ts_type = "null"; + } else if (json_type == "string") { + ts_type = "string"; + } + } + + if (schema.contains("enum")) { + std::vector enum_values; + for (const auto& val : schema["enum"]) { + enum_values.push_back("\"" + val.get() + "\""); + } + enum_comment = " * @enum " + param_name + " - Possible values: " + join(enum_values, ", "); + ts_type = "string"; + } + if (schema.contains("description")) { + description_comment = " * @param " + param_name + " - " + schema["description"].get(); + } + + return ts_type; +} + +static std::pair generate_typescript_interface(const json& schema, const std::string& interface_name) { + json properties = schema.contains("properties") && !schema["properties"].is_null() + ? schema["properties"] + : json::object(); + std::vector required = schema.value("required", std::vector()); + + std::vector interface_body; + std::vector descriptions; + for (auto& [prop_name, prop_schema] : properties.items()) { + std::string enum_comment, integer_comment, description_comment; + std::string prop_type = json_schema_to_typescript_type(prop_schema, prop_name, enum_comment, integer_comment, description_comment); + bool is_optional = find(required.begin(), required.end(), prop_name) == required.end(); + interface_body.push_back(" " + prop_name + (is_optional ? "?" : "") + ": " + prop_type + ";"); + if (!description_comment.empty()) { + descriptions.push_back(description_comment); + } + if (!enum_comment.empty()) { + descriptions.push_back(enum_comment); + } + if (!integer_comment.empty()) { + descriptions.push_back(integer_comment); + } + } + + std::string comments = join(descriptions, "\n"); + std::string interface_definition = "interface " + interface_name + " {\n" + join(interface_body, "\n") + "\n}"; + return {interface_definition, comments}; +} + + +bool starts_with(const std::string& fullString, const std::string& prefix) { + return fullString.find(prefix) == 0; +} + +static std::string generate_typescript_function(const json& function_schema) { + std::string func_name = function_schema["name"]; + std::string description = function_schema.value("description", ""); + json parameters_schema = function_schema.contains("parameters") && !function_schema["parameters"].is_null() + ? function_schema["parameters"] + : json::object(); + std::vector required_params = parameters_schema.value("required", std::vector()); + + std::vector args_list; + std::vector comments_list; + std::vector interfaces; + + if (parameters_schema.contains("properties") && parameters_schema["properties"].is_object()){ + for (auto& [param_name, param_schema] : parameters_schema["properties"].items()) { + std::string enum_comment, integer_comment, description_comment; + std::string ts_type = json_schema_to_typescript_type(param_schema, param_name, enum_comment, integer_comment, description_comment); + if (starts_with(ts_type, "interface")) { + auto [interface_definition, nested_comments] = generate_typescript_interface(param_schema, func_name + "_" + capitalize(param_name) + "Params"); + interfaces.push_back(interface_definition); + comments_list.push_back(nested_comments); + ts_type = func_name + "_" + capitalize(param_name) + "Params"; + } else { + if (!description_comment.empty()) { + comments_list.push_back(description_comment); + } + if (!enum_comment.empty()) { + comments_list.push_back(enum_comment); + } + if (!integer_comment.empty()) { + comments_list.push_back(integer_comment); + } + } + bool is_optional = find(required_params.begin(), required_params.end(), param_name) == required_params.end(); + args_list.push_back(param_name + (is_optional ? "?" : "") + ": " + ts_type); + } + } + + + std::string args_str = join(args_list, ", "); + std::string comments_str = join(comments_list, "\n"); + std::string interfaces_str = join(interfaces, "\n\n"); + + std::string description_comment = (!description.empty()) ? " * " + description + "\n" : ""; + std::string typescript_func_declaration = + "/**\n" + + description_comment + + (comments_str.empty() ? "" : (comments_str + "\n")) + + " */\n" + + (interfaces_str.empty() ? "" : (interfaces_str + "\n\n")) + + "function " + func_name + "(" + args_str + "): any {};"; + + return typescript_func_declaration; +} + + + +std::string rubra_format_typescript_function_call_str(const std::vector &functions, json &tool_name_map) { + std::string final_str = "You have access to the following tools:\n"; + std::vector function_definitions; + for (auto& function : functions) { + // If function is directly the object or nested under "function" + json spec = function.contains("function") ? function["function"] : function; + + // Making a modifiable copy of spec + json spec_copy = spec; + + std::string func_name = spec_copy.value("name", ""); + + if (func_name.find('-') != std::string::npos) { + const std::string origin_func_name = func_name; + std::replace(func_name.begin(), func_name.end(), '-', '_'); // replace "-" with "_" because - is invalid in typescript function name + tool_name_map[func_name] = origin_func_name; + spec_copy["name"] = func_name; // Modify the name in the copied json object + } + + std::string res_string = generate_typescript_function(spec_copy); // generate TypeScript function + function_definitions.push_back(res_string); + } + + + for (const auto& def : function_definitions) { + final_str += def + "\n\n"; + } + final_str += "You can choose to respond with one or more tool calls at once, or with a chat message back to the user. Ensure you have all necessary details before making tool calls. If additional information is needed, ask the user appropriately. Any tool call you make must correspond to the functions listed above. If you decide to call a tool, format it like this: starttoolcall{\"name\": \"\", \"arguments\": {\"\": \"\", \"\": \"\", ...}}endtoolcall where the JSON wrapped between starttoolcall and endtoolcall represents the function call.\n"; + return final_str; + +} + + +std::string construct_json_tool_call_str(const json& tool_calls, nlohmann::ordered_map & func_observation_map) { + std::string tool_call_str; + bool first = true; + for (const auto& tool_call : tool_calls) { + std::string tool_call_id = tool_call["id"]; + func_observation_map[tool_call_id] = ""; // Initialize with empty value, updated later from the message with tool role + + if (!first) { + tool_call_str += "\n"; + } + json tc = tool_call["function"]; + if (tc["arguments"].is_string()) { + tc["arguments"] = json::parse(tc["arguments"].get()); + } + tool_call_str += std::string("starttoolcall") + tc.dump() + std::string("endtoolcall"); + first = false; + } + + return tool_call_str; +} + + + + + +const std::vector expand_messages(const json & body, json &tool_name_map) { + std::string function_str = ""; + if (body.contains("tools") && !body["tools"].empty()) { + function_str = rubra_format_typescript_function_call_str(body["tools"], tool_name_map); + } + // If 'tool' is not set or empty, check 'functions' + else if (body.contains("functions") && !body["functions"].empty()) { + function_str = rubra_format_typescript_function_call_str(body["functions"], tool_name_map); + } + + if (function_str != "") { + const std::vector expanded_messages = [&]() { + std::vector temp_vec; + nlohmann::ordered_map func_observation_map; + for (size_t i = 0; i < body["messages"].size(); ++i) { + if (body["messages"][i]["role"] != "tool" and func_observation_map.size() > 0) { + // insert the observation from the tool call before the next message + json func_json_array = json::array(); + for (const auto& [key, value] : func_observation_map) { + func_json_array.push_back(value); + } + std::string observation_str = "start observation " + func_json_array.dump() + " end observation"; + json observation_call; + observation_call["role"] = "user"; + observation_call["content"] = observation_str; + temp_vec.push_back(observation_call); + func_observation_map.clear(); + } + + if (i == 0){ + if (body["messages"][0]["role"] == "system") { + std::string old_content = body["messages"][0]["content"]; + json function_call; + function_call["role"] = "system"; + function_call["content"] = old_content + "\n" + function_str; + temp_vec.push_back(function_call); + } + else { // insert a system message of tool definition before the first message + json function_call; + function_call["role"] = "system"; + function_call["content"] = "You are a helpful assistant.\n" + function_str; + temp_vec.push_back(function_call); + temp_vec.push_back(body["messages"][0]); + } + } + // else if (body["messages"][i]["role"] == "assistant" and (body["messages"][i]["content"].is_null() or body["messages"][i]["content"]=="") and !body["messages"][i]["tool_calls"].is_null() and !body["messages"][i]["tool_calls"].empty()){ + else if (body["messages"][i]["role"] == "assistant" and (body["messages"][i].contains("tool_calls") or body["messages"][i].contains("function_call"))){ + // convert OpenAI function call format to Rubra format + std::string tool_call_str; + if (body["messages"][i].contains("tool_calls")) { + tool_call_str = construct_json_tool_call_str(body["messages"][i]["tool_calls"], func_observation_map); + } + else { + tool_call_str = std::string("starttoolcall") + body["messages"][i]["function_call"].dump() + std::string("endtoolcall"); + } + json function_call; + function_call["role"] = "assistant"; + function_call["content"] = tool_call_str; + temp_vec.push_back(function_call); + } + else if (body["messages"][i]["role"] == "tool") { + std::string tool_call_id = body["messages"][i]["tool_call_id"].get(); + if (func_observation_map.find(tool_call_id) != func_observation_map.end()) { + func_observation_map[tool_call_id] = body["messages"][i]["content"].get(); + } else { + std::cerr << "Tool call id not found in the map :" << tool_call_id.c_str() << std::endl; + // TODO: the input is not valid in this case, should return an error + } + + } + else if (body["messages"][i]["role"] == "function") { + json func_json_array = json::array(); + func_json_array.push_back(body["messages"][i]["content"]); + std::string observation_str = std::string("start observation ") + func_json_array.dump() + std::string(" end observation"); + json observation_call; + observation_call["role"] = "user"; + observation_call["content"] = observation_str; + temp_vec.push_back(observation_call); + } + else { + temp_vec.push_back(body["messages"][i]); + } + + } + if (func_observation_map.size() > 0) { + // insert the observation from the tool call before the next message + json func_json_array = json::array(); + for (const auto& [key, value] : func_observation_map) { + func_json_array.push_back(value); + } + std::string observation_str = "start observation " + func_json_array.dump() + " end observation"; + json observation_call; + observation_call["role"] = "user"; + observation_call["content"] = observation_str; + temp_vec.push_back(observation_call); + func_observation_map.clear(); + } + return temp_vec; + }(); + return expanded_messages; + } + else { + return body["messages"]; + } + +} \ No newline at end of file diff --git a/examples/server/utils.hpp b/examples/server/utils.hpp index 69519ef95b2d9..cee160c080e3f 100644 --- a/examples/server/utils.hpp +++ b/examples/server/utils.hpp @@ -15,11 +15,14 @@ // Change JSON_ASSERT from assert() to GGML_ASSERT: #define JSON_ASSERT GGML_ASSERT #include "json.hpp" +#include "function-call.hpp" +#include "function-call-parser.hpp" #include #include #include #include +#include #define DEFAULT_OAICOMPAT_MODEL "gpt-3.5-turbo-0613" @@ -86,6 +89,7 @@ inline std::string format_chat(const struct llama_model * model, const std::stri const auto formatted_chat = common_chat_apply_template(model, tmpl, chat, true); LOG_DBG("formatted_chat: '%s'\n", formatted_chat.c_str()); + printf("formatted_chat: %s\n", formatted_chat.c_str()); return formatted_chat; } @@ -327,9 +331,20 @@ static json oaicompat_completion_params_parse( json llama_params; llama_params["__oaicompat"] = true; + json tool_name_map; + const std::vector expanded_messages = expand_messages(body, tool_name_map); + llama_params["tool_field"] = "tool_calls"; + if (body.contains("tools") && !body["tools"].empty()) { + llama_params["tool_field"] = "tool_calls"; + } + else if (body.contains("functions") && !body["functions"].empty()) { + llama_params["tool_field"] = "function_call"; + } + llama_params["prompt"] = format_chat(model, chat_template, expanded_messages); + llama_params["tool_name_map"] = tool_name_map; // Apply chat template to the list of messages - llama_params["prompt"] = format_chat(model, chat_template, body.at("messages")); + // llama_params["prompt"] = format_chat(model, chat_template, body.at("messages")); // Handle "stop" field if (body.contains("stop") && body.at("stop").is_string()) { @@ -367,12 +382,12 @@ static json oaicompat_completion_params_parse( } // Params supported by OAI but unsupported by llama.cpp - static const std::vector unsupported_params { "tools", "tool_choice" }; - for (const auto & param : unsupported_params) { - if (body.contains(param)) { - throw std::runtime_error("Unsupported param: " + param); - } - } + // static const std::vector unsupported_params { "tools", "tool_choice" }; + // for (const auto & param : unsupported_params) { + // if (body.contains(param)) { + // throw std::runtime_error("Unsupported param: " + param); + // } + // } // Copy remaining properties to llama_params // This allows user to use llama.cpp-specific params like "mirostat", "tfs_z",... via OAI endpoint. @@ -394,19 +409,45 @@ static json format_final_response_oaicompat(const json & request, const json & r int num_prompt_tokens = json_value(result, "tokens_evaluated", 0); std::string content = json_value(result, "content", std::string("")); + std::vector parsed_content = rubra_fc_json_tool_extractor(content); std::string finish_reason = "length"; if (stopped_word || stopped_eos) { finish_reason = "stop"; } - json choices = - streaming ? json::array({json{{"finish_reason", finish_reason}, - {"index", 0}, - {"delta", json::object()}}}) - : json::array({json{{"finish_reason", finish_reason}, + json choices; + + if (streaming) { + choices = json::array({json{{"finish_reason", finish_reason}, + {"index", 0}, + {"delta", json::object()}}}); + } else { + if (parsed_content.empty()) { + choices = json::array({json{{"finish_reason", finish_reason}, {"index", 0}, {"message", json{{"content", content}, {"role", "assistant"}}}}}); + } else { + std::vector oai_format_tool_calls; + for (size_t i = 0; i < parsed_content.size(); ++i) { + const auto &pc = parsed_content[i]; + // Use 'pc' and 'i' as needed + json tool_call; + tool_call["id"] = pc["id"]; + tool_call["type"] = "function"; + + tool_call["function"] = json{ + {"name" , pc["name"]}, + {"arguments" , pc["kwargs"]}, + }; + oai_format_tool_calls.push_back(tool_call); + } + choices = json::array({json{{"finish_reason", "tool_calls"}, + {"index", 0}, + {"message", json{{"tool_calls", oai_format_tool_calls}, + {"role", "assistant"}}}}}); + } + } std::time_t t = std::time(0); @@ -425,6 +466,7 @@ static json format_final_response_oaicompat(const json & request, const json & r }; // extra fields for debugging purposes + printf("final_oai_response %s", res.dump().c_str()); if (verbose) { res["__verbose"] = result; } @@ -449,6 +491,8 @@ static std::vector format_partial_response_oaicompat(const json & result, bool stopped_eos = json_value(result, "stopped_eos", false); bool stopped_limit = json_value(result, "stopped_limit", false); std::string content = json_value(result, "content", std::string("")); + std::vector parsed_content = rubra_fc_json_tool_extractor(content); + std::string tool_field = json_value(result, "tool_field", std::string("tool_calls")); std::string finish_reason; if (stopped_word || stopped_eos) { @@ -485,18 +529,58 @@ static std::vector format_partial_response_oaicompat(const json & result, {"model", modelname}, {"object", "chat.completion.chunk"}}; - json second_ret = json{ - {"choices", json::array({json{{"finish_reason", nullptr}, - {"index", 0}, - {"delta", json{ - {"content", content}}} - }})}, - {"created", t}, - {"id", completion_id}, - {"model", modelname}, - {"object", "chat.completion.chunk"}}; + if (parsed_content.empty()) { + json second_ret = json{ + {"choices", json::array({json{{"finish_reason", nullptr}, + {"index", 0}, + {"delta", json{ + {"content", content}}} + }})}, + {"created", t}, + {"id", completion_id}, + {"model", modelname}, + {"object", "chat.completion.chunk"}}; + + return std::vector({initial_ret, second_ret}); + } + else { + std::vector oai_format_tool_calls; + for (size_t i = 0; i < parsed_content.size(); ++i) { + const auto &pc = parsed_content[i]; + // Use 'pc' and 'i' as needed + json tool_call; + tool_call["id"] = pc["id"]; + tool_call["type"] = "function"; + + tool_call["function"] = json{ + {"name" , pc["name"]}, + {"arguments" , pc["kwargs"]}, + }; + oai_format_tool_calls.push_back(tool_call); + } + if (tool_field == "tool_calls") { + choices = json::array({json{{"finish_reason", nullptr}, + {"index", 0}, + {"delta", json{{tool_field, oai_format_tool_calls}, + {"role", "assistant"}}}}}); + } + else { + choices = json::array({json{{"finish_reason", nullptr}, + {"index", 0}, + {"delta", json{{tool_field, oai_format_tool_calls[0]["function"]}, + {"role", "assistant"}}}}}); + } + + json second_ret = json{ + {"choices", choices}, + {"created", t}, + {"id", completion_id}, + {"model", modelname}, + {"object", "chat.completion.chunk"}}; + + return std::vector({initial_ret, second_ret}); + } - return std::vector({initial_ret, second_ret}); } } else { // Some idiosyncrasy in task processing logic makes several trailing calls @@ -505,14 +589,46 @@ static std::vector format_partial_response_oaicompat(const json & result, return std::vector({json::object()}); } - choices = json::array({json{ - {"finish_reason", nullptr}, - {"index", 0}, - {"delta", - json{ - {"content", content}, - }}, - }}); + if (parsed_content.empty()) { + choices = json::array({json{ + {"finish_reason", nullptr}, + {"index", 0}, + {"delta", + json{ + {"content", content}, + }}, + }}); + } + else { + std::vector oai_format_tool_calls; + for (size_t i = 0; i < parsed_content.size(); ++i) { + const auto &pc = parsed_content[i]; + // Use 'pc' and 'i' as needed + json tool_call; + tool_call["id"] = pc["id"]; + tool_call["type"] = "function"; + tool_call["index"] = i; + + tool_call["function"] = json{ + {"name" , pc["name"]}, + {"arguments" , pc["kwargs"]}, + }; + oai_format_tool_calls.push_back(tool_call); + } + if (tool_field == "tool_calls") { + choices = json::array({json{{"finish_reason", nullptr}, + {"index", 0}, + {"delta", json{{tool_field, oai_format_tool_calls}, + {"role", "assistant"}}}}}); + } + else { + choices = json::array({json{{"finish_reason", nullptr}, + {"index", 0}, + {"delta", json{{tool_field, oai_format_tool_calls[0]["function"]}, + {"role", "assistant"}}}}}); + } + } + } } diff --git a/jsonrepair.ts b/jsonrepair.ts new file mode 100644 index 0000000000000..a9c2899de8adc --- /dev/null +++ b/jsonrepair.ts @@ -0,0 +1,23 @@ +const fs = require('fs'); +const { jsonrepair } = require('jsonrepair'); + +// This script processes command-line arguments +const filename = process.argv[2]; // Skip the first two elements +// Simple processing: join arguments into a stri + +function processFile(filePath) { + try { + const data = fs.readFileSync(filePath, { encoding: 'utf8' }); + const repairData = jsonrepair(data); + console.log(repairData); + fs.unlinkSync(filePath); + return repairData; + } catch (error) { + console.error('Error reading file:', error); + return ''; + } +} + + +processFile(filename); +// Output the result diff --git a/test_llamacpp.ipynb b/test_llamacpp.ipynb new file mode 100644 index 0000000000000..3f36314589d4e --- /dev/null +++ b/test_llamacpp.ipynb @@ -0,0 +1,792 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Function Definitions, and mock function call results" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "import uuid\n", + "from functools import partial\n", + "\n", + "\n", + "def add(args: str):\n", + " args = json.loads(args)\n", + " return str(float(args[\"a\"]) + float(args[\"b\"]))\n", + "\n", + "\n", + "def sub(args: str):\n", + " args = json.loads(args)\n", + " return str(float(args[\"a\"]) - float(args[\"b\"]))\n", + "\n", + "\n", + "def mult(args: str):\n", + " args = json.loads(args)\n", + " return str(float(args[\"a\"]) * float(args[\"b\"]))\n", + "\n", + "\n", + "def div(args: str):\n", + " args = json.loads(args)\n", + " return str(float(args[\"a\"]) / float(args[\"b\"]))\n", + "\n", + "\n", + "def get_oai_response(model, functions, msgs, api_key, base_url):\n", + " import openai\n", + " openai.api_key = api_key \n", + " openai.base_url = base_url\n", + " \n", + " try:\n", + " completion = openai.chat.completions.create(\n", + " model=model,\n", + " temperature=0.1,\n", + " messages=msgs,\n", + " tools=functions,\n", + " tool_choice=\"auto\",\n", + " stream=False,\n", + " )\n", + " return completion.choices[0]\n", + " except Exception as e:\n", + " print(e)\n", + "\n", + "\n", + "def insert_tool_response(res, msgs):\n", + " assistant_message = res.message\n", + " tool_calls = []\n", + " for tool_call in assistant_message.tool_calls:\n", + " tool_calls.append( {\n", + " \"id\": tool_call.id,\n", + " \"function\": {\"name\": tool_call.function.name,\n", + " \"arguments\": tool_call.function.arguments},\n", + " \"type\": \"function\",\n", + " })\n", + " msgs.append({\"role\": \"assistant\", \"tool_calls\": tool_calls})\n", + " \n", + " for i, tool_call in enumerate(assistant_message.tool_calls):\n", + " if tool_call.function.name == \"getCurrentWeather\":\n", + " print()\n", + " l = len((json.loads(assistant_message.tool_calls[i].function.arguments))[\"location\"])\n", + " msgs.append({\"role\": \"tool\", \"tool_call_id\": str(assistant_message.tool_calls[i].id), \"name\": assistant_message.tool_calls[i].function.name, \"content\": f\"temprature is {(i+1) * 50 + l } degree\"})\n", + " elif tool_call.function.name == \"calculate_distance\":\n", + " msgs.append({\"role\": \"tool\", \"tool_call_id\": str(assistant_message.tool_calls[i].id), \"name\": assistant_message.tool_calls[i].function.name, \"content\": f\"Distance is {(i+1) * 50} miles.\"})\n", + " elif tool_call.function.name == \"generate_password\":\n", + " msgs.append({\"role\": \"tool\", \"tool_call_id\": str(assistant_message.tool_calls[i].id), \"name\": assistant_message.tool_calls[i].function.name, \"content\": f\"Password generated: {uuid.uuid4().hex[:8]}\"})\n", + " elif tool_call.function.name == \"orderUmbrella\":\n", + " msgs.append({\"role\": \"tool\", \"tool_call_id\": str(assistant_message.tool_calls[i].id), \"name\": assistant_message.tool_calls[i].function.name, \"content\": f\"Order placed. the price is {(i+1) * 10} dollars.\"})\n", + " elif tool_call.function.name == \"list_files\":\n", + " msgs.append({\"role\": \"tool\", \"tool_call_id\": str(assistant_message.tool_calls[i].id), \"name\": assistant_message.tool_calls[i].function.name, \"content\": f\"File list:\\nreport.docx\\ntask.txt\\nnotes.txt\"})\n", + " elif tool_call.function.name == \"get_file_size\":\n", + " msgs.append({\"role\": \"tool\", \"tool_call_id\": str(assistant_message.tool_calls[i].id), \"name\": assistant_message.tool_calls[i].function.name, \"content\": f\"the size is {(i+1) * 100} bytes.\"})\n", + " elif tool_call.function.name == \"addition\":\n", + " msgs.append({\n", + " \"role\": \"tool\",\n", + " \"name\": \"addition\",\n", + " \"content\": add(tool_call.function.arguments),\n", + " \"tool_call_id\": tool_call.id\n", + " })\n", + " elif tool_call.function.name == \"subtraction\":\n", + " msgs.append({\n", + " \"role\": \"tool\",\n", + " \"name\": \"subtraction\",\n", + " \"content\": sub(tool_call.function.arguments),\n", + " \"tool_call_id\": tool_call.id\n", + " })\n", + " elif tool_call.function.name == \"multiplication\":\n", + " msgs.append({\n", + " \"role\": \"tool\",\n", + " \"name\": \"multiplication\",\n", + " \"content\": mult(tool_call.function.arguments),\n", + " \"tool_call_id\": tool_call.id\n", + " })\n", + " elif tool_call.function.name == \"division\":\n", + " msgs.append({\n", + " \"role\": \"tool\",\n", + " \"name\": \"division\",\n", + " \"content\": div(tool_call.function.arguments),\n", + " \"tool_call_id\": tool_call.id\n", + " })\n", + " elif tool_call.function.name == \"create_3d_model\":\n", + " msgs.append({\n", + " \"role\": \"tool\",\n", + " \"name\": \"division\",\n", + " \"content\": \"created.\",\n", + " \"tool_call_id\": tool_call.id\n", + " })\n", + " print(f\"Observation: {msgs[-1]['content']}\")\n", + " \n", + " return msgs\n", + "\n", + "def run_completion(chat_method, user_query, msgs=[], model=\"gpt-4-0125-preview\"):\n", + " system_prompt = \"You are a helpful assistant.\"\n", + " functions = [\n", + " {\n", + " \"type\": \"function\",\n", + " \"function\": {\n", + " \"name\": \"list_files\",\n", + " \"description\": \"List all files in a directory\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"directory\": {\n", + " \"type\": \"string\",\n", + " \"description\": \"the directory to list files from\"\n", + " }\n", + " },\n", + " \"required\": [\n", + " \"directory\"\n", + " ]\n", + " }\n", + " }\n", + " },\n", + " {\n", + " \"type\": \"function\",\n", + " \"function\": {\n", + " \"description\": \"Create a 3D model of an object with specified dimensions\",\n", + " \"name\": \"create_3d_model\",\n", + " \"parameters\": {\n", + " \"properties\": {\n", + " \"object_name\": {\n", + " \"description\": \"Name of the object to be modeled\",\n", + " \"type\": \"string\"\n", + " },\n", + " \"dimensions\": {\n", + " \"description\": \"Dimensions of the 3D object (length, width, height)\",\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"length\": {\n", + " \"type\": \"number\"\n", + " },\n", + " \"width\": {\n", + " \"type\": \"number\"\n", + " },\n", + " \"height\": {\n", + " \"type\": \"number\"\n", + " }\n", + " },\n", + " \"required\": [\n", + " \"length\",\n", + " \"width\",\n", + " \"height\"\n", + " ]\n", + " }\n", + " },\n", + " \"required\": [\n", + " \"object_name\",\n", + " \"dimensions\"\n", + " ],\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " },\n", + " {\n", + " \"type\": \"function\",\n", + " \"function\": {\n", + " \"description\": \"Get the latest insurance premium from a list of premiums.\",\n", + " \"name\": \"latest_insurance_premium\",\n", + " \"parameters\": {\n", + " \"properties\": {\n", + " \"premiums\": {\n", + " \"description\": \"List of insurance premiums\",\n", + " \"type\": \"array\",\n", + " \"items\": {\n", + " \"type\": \"number\"\n", + " }\n", + " }\n", + " },\n", + " \"required\": [\n", + " \"premiums\"\n", + " ],\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " },\n", + " {\n", + " \"type\": \"function\",\n", + " \"function\": {\n", + " \"description\": \"Calculate insurance premium based on age and coverage\",\n", + " \"name\": \"calculate_insurance_premium\",\n", + " \"parameters\": {\n", + " \"properties\": {\n", + " \"age\": {\n", + " \"description\": \"Age of the person applying for insurance\",\n", + " \"type\": \"integer\"\n", + " },\n", + " \"coverage_type\": {\n", + " \"description\": \"Type of insurance coverage\",\n", + " \"type\": \"string\",\n", + " \"enum\": [\n", + " \"basic\",\n", + " \"standard\",\n", + " \"premium\"\n", + " ]\n", + " }\n", + " },\n", + " \"required\": [\n", + " \"age\",\n", + " \"coverage_type\"\n", + " ],\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " },\n", + " {\n", + " \"type\": \"function\",\n", + " \"function\": {\n", + " \"name\": \"get_file_size\",\n", + " \"description\": \"Get the size of a file in bytes\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"filename\": {\n", + " \"type\": \"string\",\n", + " \"description\": \"the name of the file to get its size\"\n", + " }\n", + " },\n", + " \"required\": [\n", + " \"filename\"\n", + " ]\n", + " }\n", + " }\n", + " },\n", + " {\n", + " 'type': 'function',\n", + " 'function': {\n", + " 'name': 'addition',\n", + " 'description': \"Adds two numbers together\",\n", + " 'parameters': {\n", + " 'type': 'object',\n", + " 'properties': {\n", + " 'a': {\n", + " 'description': 'First number to add',\n", + " 'type': 'string'\n", + " },\n", + " 'b': {\n", + " 'description': 'Second number to add',\n", + " 'type': 'string'\n", + " }\n", + " },\n", + " 'required': []\n", + " }\n", + " }\n", + " },\n", + " {\n", + " 'type': 'function',\n", + " 'function': {\n", + " 'name': 'subtraction',\n", + " 'description': \"Subtracts two numbers\",\n", + " 'parameters': {\n", + " 'type': 'object',\n", + " 'properties': {\n", + " 'a': {\n", + " 'description': 'First number to be subtracted from',\n", + " 'type': 'string'\n", + " },\n", + " 'b': {\n", + " 'description': 'Number to subtract',\n", + " 'type': 'string'\n", + " }\n", + " },\n", + " 'required': []\n", + " }\n", + " }\n", + " },\n", + " {\n", + " 'type': 'function',\n", + " 'function': {\n", + " 'name': 'multiplication',\n", + " 'description': \"Multiply two numbers together\",\n", + " 'parameters': {\n", + " 'type': 'object',\n", + " 'properties': {\n", + " 'a': {\n", + " 'description': 'First number to multiply',\n", + " 'type': 'string'\n", + " },\n", + " 'b': {\n", + " 'description': 'Second number to multiply',\n", + " 'type': 'string'\n", + " }\n", + " },\n", + " 'required': []\n", + " }\n", + " }\n", + " },\n", + " {\n", + " 'type': 'function',\n", + " 'function': {\n", + " 'name': 'division',\n", + " 'description': \"Divide two numbers\",\n", + " 'parameters': {\n", + " 'type': 'object',\n", + " 'properties': {\n", + " 'a': {\n", + " 'description': 'First number to use as the dividend',\n", + " 'type': 'string'\n", + " },\n", + " 'b': {\n", + " 'description': 'Second number to use as the divisor',\n", + " 'type': 'string'\n", + " }\n", + " },\n", + " 'required': []\n", + " }\n", + " }\n", + " },\n", + " {\n", + " \"type\": \"function\",\n", + " \"function\": {\n", + " \"name\": \"getCurrentWeather\",\n", + " \"description\": \"Get the weather in location\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"location\": {\"type\": \"string\", \"description\": \"The city and state e.g. San Francisco, CA\"},\n", + " \"unit\": {\"type\": \"string\", \"enum\": [\"c\", \"f\"]}\n", + " },\n", + " \"required\": [\"location\"]\n", + " }\n", + " }\n", + " },\n", + " { \"type\": \"function\",\n", + " \"function\":\n", + " {\n", + " \"name\": \"orderUmbrella\",\n", + " \"description\": \"Do this to help user to order an umbrella online\", \n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"number_to_buy\": {\n", + " \"type\": \"integer\",\n", + " \"description\": \"the amount of umbrellas to buy\"\n", + " }\n", + " },\n", + " \"required\": [\n", + " \"number_to_buy\"\n", + " ]\n", + " }\n", + " }},\n", + " {\"type\": \"function\",\"function\":{\"name\":\"calculate_distance\",\"description\":\"Calculate the distance between two locations\",\"parameters\":{\"type\":\"object\",\"properties\":{\"origin\":{\"type\":\"string\",\"description\":\"The starting location\"},\"destination\":{\"type\":\"string\",\"description\":\"The destination location\"},\"mode\":{\"type\":\"string\",\"description\":\"The mode of transportation\"}},\"required\":[\"origin\",\"destination\",\"mode\"]}}},{\"type\": \"function\",\"function\":{\"name\":\"generate_password\",\"description\":\"Generate a random password\",\"parameters\":{\"type\":\"object\",\"properties\":{\"length\":{\"type\":\"integer\",\"description\":\"The length of the password\"}},\"required\":[\"length\"]}}}\n", + " ]\n", + " if not msgs or len(msgs) == 0:\n", + " msgs = [{\"role\": \"system\", \"content\":system_prompt} ,{\"role\": \"user\", \"content\": user_query}]\n", + " else:\n", + " msgs.append({\"role\": \"user\", \"content\": user_query})\n", + "\n", + " l = 0\n", + " print(f\":\")\n", + " res = chat_method(model=model, functions=functions, msgs=msgs)\n", + " res_next = res\n", + " if res_next.message.content and len(res_next.message.content) > 0:\n", + " print(\"[AI response]:\\n\", res_next.message.content)\n", + " else:\n", + " print(\"[AI calling functions]:\")\n", + " for tool_call in res_next.message.tool_calls:\n", + " print(f\"Tool Call: {tool_call.function}\")\n", + " \n", + " while res_next.message.tool_calls and len(res_next.message.tool_calls) > 0:\n", + " l += 1\n", + " msgs = insert_tool_response(res_next, msgs)\n", + " print(f\"\\n:\")\n", + " res_next = chat_method(model=model, functions=functions, msgs=msgs)\n", + " \n", + " if res_next.message.content and len(res_next.message.content) > 0:\n", + " print(\"[AI response]:\\n\", res_next.message.content)\n", + " else:\n", + " print(\"[AI calling functions]:\")\n", + " for tool_call in res_next.message.tool_calls:\n", + " print(f\"Tool Call: {tool_call.function}\")\n", + " \n", + " msgs.append({\"role\": \"assistant\", \"content\": res_next.message.content})\n", + " return msgs\n", + " \n", + " \n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "import openai\n", + "local_api_key = \"sk-\"\n", + "local_base_url = \"http://localhost:1234/v1/\"\n", + "get_rubra_response = partial(get_oai_response, api_key=local_api_key, base_url=local_base_url)\n", + "model = \"Llama-3-8b-function-calling-alpha-v1.gguf\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Chat" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + ":\n", + "[AI response]:\n", + " I'm just a computer program, so I don't have feelings, but I'm here and ready to help you with any questions or tasks you have! How can I assist you today?\n" + ] + } + ], + "source": [ + "user_query = \"how are you?\"\n", + "msgs = run_completion(get_rubra_response, user_query, model=model)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Multi + Parallel Function Call Test examples" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + ":\n", + "[AI calling functions]:\n", + "Tool Call: Function(arguments='{\"origin\":\"San Francisco\",\"destination\":\"Cupertino\",\"mode\":\"driving\"}', name='calculate_distance')\n", + "Observation: Distance is 50 miles.\n", + "\n", + ":\n", + "[AI calling functions]:\n", + "Tool Call: Function(arguments='{\"origin\":\"Cupertino\",\"destination\":\"San Francisco\",\"mode\":\"driving\"}', name='calculate_distance')\n", + "Observation: Distance is 50 miles.\n", + "\n", + ":\n", + "[AI calling functions]:\n", + "Tool Call: Function(arguments='{\"origin\":\"San Francisco\",\"destination\":\"Cupertino\",\"mode\":\"air\"}', name='calculate_distance')\n", + "Observation: Distance is 50 miles.\n", + "\n", + ":\n", + "[AI calling functions]:\n", + "Tool Call: Function(arguments='{\"origin\":\"Cupertino\",\"destination\":\"San Francisco\",\"mode\":\"air\"}', name='calculate_distance')\n", + "Observation: Distance is 50 miles.\n", + "\n", + ":\n", + "[AI response]:\n", + " The distance between San Francisco and Cupertino by driving and by air is 50 miles in both directions.\n" + ] + } + ], + "source": [ + "\n", + "user_query = \"What is the distance between San Francisco and Cupertino by driving and by air from both directions?\"\n", + "msgs = run_completion(get_rubra_response, user_query, model=model)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + ":\n", + "[AI calling functions]:\n", + "Tool Call: Function(arguments='{\"origin\":\"San Francisco\",\"destination\":\"New York City\",\"mode\":\"airplane\"}', name='calculate_distance')\n", + "Observation: Distance is 50 miles.\n", + "\n", + ":\n", + "[AI calling functions]:\n", + "Tool Call: Function(arguments='{\"a\":\"50\",\"b\":\"8\"}', name='multiplication')\n", + "Observation: 400.0\n", + "\n", + ":\n", + "[AI calling functions]:\n", + "Tool Call: Function(arguments='{\"a\":\"400.0\",\"b\":\"2\"}', name='division')\n", + "Observation: 200.0\n", + "\n", + ":\n", + "[AI calling functions]:\n", + "Tool Call: Function(arguments='{\"a\":\"200.0\",\"b\":\"30\"}', name='subtraction')\n", + "Observation: 170.0\n", + "\n", + ":\n", + "[AI response]:\n", + " The final result after performing the operations is 170.0 miles.\n" + ] + } + ], + "source": [ + "user_query1 = \"what's the distance between SF and NYC? Use the result value to multiply by 8, and then divide by 2, and then minus 30\"\n", + "msgs = run_completion(get_rubra_response, user_query1, model=model)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + ":\n", + "[AI calling functions]:\n", + "Tool Call: Function(arguments='{\"number_to_buy\":3}', name='orderUmbrella')\n", + "Observation: Order placed. the price is 10 dollars.\n", + "\n", + ":\n", + "[AI calling functions]:\n", + "Tool Call: Function(arguments='{\"length\":8}', name='generate_password')\n", + "Observation: Password generated: 44e7bc52\n", + "\n", + ":\n", + "[AI response]:\n", + " Your order for 3 umbrellas has been placed, and the price is $10 each. A random password '44e7bc52' of length 8 has also been generated for you. Is there anything else you need assistance with?\n" + ] + } + ], + "source": [ + "user_query2 = \"now order 3 umbrellas for me and generate a password of length 8\"\n", + "msgs = run_completion(get_rubra_response, user_query2, msgs, model=model)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Simple Math Chaining" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + ":\n", + "[AI calling functions]:\n", + "Tool Call: Function(arguments='{\"a\":\"4\",\"b\":\"6\"}', name='addition')\n", + "Observation: 10.0\n", + "\n", + ":\n", + "[AI calling functions]:\n", + "Tool Call: Function(arguments='{\"a\":\"10.0\",\"b\":\"2\"}', name='subtraction')\n", + "Observation: 8.0\n", + "\n", + ":\n", + "[AI calling functions]:\n", + "Tool Call: Function(arguments='{\"a\":\"8.0\",\"b\":\"5\"}', name='multiplication')\n", + "Observation: 40.0\n", + "\n", + ":\n", + "[AI calling functions]:\n", + "Tool Call: Function(arguments='{\"a\":\"40.0\",\"b\":\"2\"}', name='division')\n", + "Observation: 20.0\n", + "\n", + ":\n", + "[AI response]:\n", + " The result of subtracting 2 from the sum of 4 and 6, then multiplying by 5, and finally dividing by 2 is 20.0.\n" + ] + } + ], + "source": [ + "user_query3 = \"What is four plus six? What is the result of that minus 2? Take the result and multiply by 5 and then divide by two\"\n", + "\n", + "\n", + "msgs = run_completion(get_rubra_response, user_query3, model=model)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Condition" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + ":\n", + "[AI calling functions]:\n", + "Tool Call: Function(arguments='{\"location\":\"Boston\",\"unit\":\"f\"}', name='getCurrentWeather')\n", + "\n", + "Observation: temprature is 56 degree\n", + "\n", + ":\n", + "[AI calling functions]:\n", + "Tool Call: Function(arguments='{\"origin\":\"Boston\",\"destination\":\"NYC\",\"mode\":\"car\"}', name='calculate_distance')\n", + "Observation: Distance is 50 miles.\n", + "\n", + ":\n", + "[AI response]:\n", + " The temperature in Boston is 56 degrees Fahrenheit, which is below 100 degrees. The distance from Boston to New York City is approximately 50 miles.\n" + ] + } + ], + "source": [ + "user_query4 = \"check the weather in boston, if it's less than 100 degrees Fahrenheit, calculate the distance from boston to NYC\"\n", + "msgs = run_completion(get_rubra_response, user_query4, model=model)" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + ":\n", + "[AI calling functions]:\n", + "Tool Call: Function(arguments='{\"location\":\"Boston\",\"unit\":\"f\"}', name='getCurrentWeather')\n", + "\n", + "Observation: temprature is 56 degree\n", + "\n", + ":\n", + "[AI response]:\n", + " The temperature in Boston is 56 degrees Fahrenheit, which is not greater than 100 degrees. Therefore, I will not calculate the distance from Boston to NYC as the condition was not met. Is there anything else I can assist you with?\n" + ] + } + ], + "source": [ + "user_query5 = \"check the weather in boston, if it's greater than 100 degrees Fahrenheit, calculate the distance from boston to NYC\"\n", + "msgs = run_completion(get_rubra_response, user_query5, model=model)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# dependency" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + ":\n", + "[AI calling functions]:\n", + "Tool Call: Function(arguments='{\"directory\":\"documents\"}', name='list_files')\n", + "Observation: File list:\n", + "report.docx\n", + "task.txt\n", + "notes.txt\n", + "\n", + ":\n", + "[AI calling functions]:\n", + "Tool Call: Function(arguments='{\"filename\":\"report.docx\"}', name='get_file_size')\n", + "Observation: the size is 100 bytes.\n", + "\n", + ":\n", + "[AI calling functions]:\n", + "Tool Call: Function(arguments='{\"filename\":\"task.txt\"}', name='get_file_size')\n", + "Observation: the size is 100 bytes.\n", + "\n", + ":\n", + "[AI calling functions]:\n", + "Tool Call: Function(arguments='{\"filename\":\"notes.txt\"}', name='get_file_size')\n", + "Observation: the size is 100 bytes.\n", + "\n", + ":\n", + "[AI response]:\n", + " The sizes of the files in the 'documents' directory are as follows:\n", + "- 'report.docx' is 100 bytes.\n", + "- 'task.txt' is 100 bytes.\n", + "- 'notes.txt' is 100 bytes.\n", + "Is there anything else you would like to know or another task you need assistance with?\n" + ] + } + ], + "source": [ + "user_query6 = \"check the size of all files in the 'documents' directory.\"\n", + "msgs = run_completion(get_rubra_response, user_query6, model=model)" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + ":\n", + "[AI response]:\n", + " It seems you've provided the dimensions for the 3D model, but you didn't specify the name of the object. Please provide the object name along with the dimensions to proceed with creating the 3D model.\n", + ":\n", + "[AI calling functions]:\n", + "Tool Call: Function(arguments='{\"object_name\":\"my_test_box\",\"dimensions\":{\"length\":12,\"width\":32,\"height\":45}}', name='create_3d_model')\n", + "Observation: created.\n", + "\n", + ":\n", + "[AI response]:\n", + " The 3D model named 'my_test_box' with dimensions 12x32x45 has been successfully created.\n" + ] + } + ], + "source": [ + "\n", + "user_query0 = \"create a 3d model with length 12, width 32, and height 45\"\n", + "msgs = run_completion(get_rubra_response, user_query0, model=model)\n", + "user_query0_next = \"name is my_test_box\"\n", + "msgs = run_completion(get_rubra_response, user_query0_next, msgs=msgs, model=model)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "py310", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/test_llamacpp_streaming.ipynb b/test_llamacpp_streaming.ipynb new file mode 100644 index 0000000000000..d3013fcc576f5 --- /dev/null +++ b/test_llamacpp_streaming.ipynb @@ -0,0 +1,818 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Function Definitions, and mock function call results" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "import uuid\n", + "from functools import partial\n", + "\n", + "\n", + "def add(args: str):\n", + " args = json.loads(args)\n", + " return str(float(args[\"a\"]) + float(args[\"b\"]))\n", + "\n", + "\n", + "def sub(args: str):\n", + " args = json.loads(args)\n", + " return str(float(args[\"a\"]) - float(args[\"b\"]))\n", + "\n", + "\n", + "def mult(args: str):\n", + " args = json.loads(args)\n", + " return str(float(args[\"a\"]) * float(args[\"b\"]))\n", + "\n", + "\n", + "def div(args: str):\n", + " args = json.loads(args)\n", + " return str(float(args[\"a\"]) / float(args[\"b\"]))\n", + "\n", + "\n", + "def get_oai_response(model, functions, msgs, api_key, base_url):\n", + " import openai\n", + " openai.api_key = api_key \n", + " openai.base_url = base_url\n", + " \n", + " try:\n", + " completion = openai.chat.completions.create(\n", + " model=model,\n", + " temperature=0.1,\n", + " messages=msgs,\n", + " tools=functions,\n", + " tool_choice=\"auto\",\n", + " stream=True,\n", + " )\n", + " return completion\n", + " except Exception as e:\n", + " print(e)\n", + "\n", + "\n", + "def insert_tool_response(res, msgs):\n", + " assistant_message = res.delta\n", + " tool_calls = []\n", + " for tool_call in assistant_message.tool_calls:\n", + " tool_calls.append( {\n", + " \"id\": tool_call.id,\n", + " \"function\": {\"name\": tool_call.function.name,\n", + " \"arguments\": tool_call.function.arguments},\n", + " \"type\": \"function\",\n", + " })\n", + " msgs.append({\"role\": \"assistant\", \"tool_calls\": tool_calls})\n", + " \n", + " for i, tool_call in enumerate(assistant_message.tool_calls):\n", + " if tool_call.function.name == \"getCurrentWeather\":\n", + " print()\n", + " l = len((json.loads(assistant_message.tool_calls[i].function.arguments))[\"location\"])\n", + " msgs.append({\"role\": \"tool\", \"tool_call_id\": str(assistant_message.tool_calls[i].id), \"name\": assistant_message.tool_calls[i].function.name, \"content\": f\"temprature is {(i+1) * 50 + l } degree\"})\n", + " elif tool_call.function.name == \"calculate_distance\":\n", + " msgs.append({\"role\": \"tool\", \"tool_call_id\": str(assistant_message.tool_calls[i].id), \"name\": assistant_message.tool_calls[i].function.name, \"content\": f\"Distance is {(i+1) * 50} miles.\"})\n", + " elif tool_call.function.name == \"generate_password\":\n", + " msgs.append({\"role\": \"tool\", \"tool_call_id\": str(assistant_message.tool_calls[i].id), \"name\": assistant_message.tool_calls[i].function.name, \"content\": f\"Password generated: {uuid.uuid4().hex[:8]}\"})\n", + " elif tool_call.function.name == \"orderUmbrella\":\n", + " msgs.append({\"role\": \"tool\", \"tool_call_id\": str(assistant_message.tool_calls[i].id), \"name\": assistant_message.tool_calls[i].function.name, \"content\": f\"Order placed. the price is {(i+1) * 10} dollars.\"})\n", + " elif tool_call.function.name == \"list_files\":\n", + " msgs.append({\"role\": \"tool\", \"tool_call_id\": str(assistant_message.tool_calls[i].id), \"name\": assistant_message.tool_calls[i].function.name, \"content\": f\"File list:\\nreport.docx\\ntask.txt\\nnotes.txt\"})\n", + " elif tool_call.function.name == \"get_file_size\":\n", + " msgs.append({\"role\": \"tool\", \"tool_call_id\": str(assistant_message.tool_calls[i].id), \"name\": assistant_message.tool_calls[i].function.name, \"content\": f\"the size is {(i+1) * 100} bytes.\"})\n", + " elif tool_call.function.name == \"addition\":\n", + " msgs.append({\n", + " \"role\": \"tool\",\n", + " \"name\": \"addition\",\n", + " \"content\": add(tool_call.function.arguments),\n", + " \"tool_call_id\": tool_call.id\n", + " })\n", + " elif tool_call.function.name == \"subtraction\":\n", + " msgs.append({\n", + " \"role\": \"tool\",\n", + " \"name\": \"subtraction\",\n", + " \"content\": sub(tool_call.function.arguments),\n", + " \"tool_call_id\": tool_call.id\n", + " })\n", + " elif tool_call.function.name == \"multiplication\":\n", + " msgs.append({\n", + " \"role\": \"tool\",\n", + " \"name\": \"multiplication\",\n", + " \"content\": mult(tool_call.function.arguments),\n", + " \"tool_call_id\": tool_call.id\n", + " })\n", + " elif tool_call.function.name == \"division\":\n", + " msgs.append({\n", + " \"role\": \"tool\",\n", + " \"name\": \"division\",\n", + " \"content\": div(tool_call.function.arguments),\n", + " \"tool_call_id\": tool_call.id\n", + " })\n", + " elif tool_call.function.name == \"create_3d_model\":\n", + " msgs.append({\n", + " \"role\": \"tool\",\n", + " \"name\": \"division\",\n", + " \"content\": \"created.\",\n", + " \"tool_call_id\": tool_call.id\n", + " })\n", + " print(f\"Observation: {msgs[-1]['content']}\")\n", + " \n", + " return msgs\n", + "\n", + "def run_completion(chat_method, user_query, msgs=[], model=\"gpt-4-0125-preview\"):\n", + " system_prompt = \"You are a helpful assistant.\"\n", + " functions = [\n", + " {\n", + " \"type\": \"function\",\n", + " \"function\": {\n", + " \"name\": \"list_files\",\n", + " \"description\": \"List all files in a directory\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"directory\": {\n", + " \"type\": \"string\",\n", + " \"description\": \"the directory to list files from\"\n", + " }\n", + " },\n", + " \"required\": [\n", + " \"directory\"\n", + " ]\n", + " }\n", + " }\n", + " },\n", + " {\n", + " \"type\": \"function\",\n", + " \"function\": {\n", + " \"description\": \"Create a 3D model of an object with specified dimensions\",\n", + " \"name\": \"create_3d_model\",\n", + " \"parameters\": {\n", + " \"properties\": {\n", + " \"object_name\": {\n", + " \"description\": \"Name of the object to be modeled\",\n", + " \"type\": \"string\"\n", + " },\n", + " \"dimensions\": {\n", + " \"description\": \"Dimensions of the 3D object (length, width, height)\",\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"length\": {\n", + " \"type\": \"number\"\n", + " },\n", + " \"width\": {\n", + " \"type\": \"number\"\n", + " },\n", + " \"height\": {\n", + " \"type\": \"number\"\n", + " }\n", + " },\n", + " \"required\": [\n", + " \"length\",\n", + " \"width\",\n", + " \"height\"\n", + " ]\n", + " }\n", + " },\n", + " \"required\": [\n", + " \"object_name\",\n", + " \"dimensions\"\n", + " ],\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " },\n", + " {\n", + " \"type\": \"function\",\n", + " \"function\": {\n", + " \"description\": \"Get the latest insurance premium from a list of premiums.\",\n", + " \"name\": \"latest_insurance_premium\",\n", + " \"parameters\": {\n", + " \"properties\": {\n", + " \"premiums\": {\n", + " \"description\": \"List of insurance premiums\",\n", + " \"type\": \"array\",\n", + " \"items\": {\n", + " \"type\": \"number\"\n", + " }\n", + " }\n", + " },\n", + " \"required\": [\n", + " \"premiums\"\n", + " ],\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " },\n", + " {\n", + " \"type\": \"function\",\n", + " \"function\": {\n", + " \"description\": \"Calculate insurance premium based on age and coverage\",\n", + " \"name\": \"calculate_insurance_premium\",\n", + " \"parameters\": {\n", + " \"properties\": {\n", + " \"age\": {\n", + " \"description\": \"Age of the person applying for insurance\",\n", + " \"type\": \"integer\"\n", + " },\n", + " \"coverage_type\": {\n", + " \"description\": \"Type of insurance coverage\",\n", + " \"type\": \"string\",\n", + " \"enum\": [\n", + " \"basic\",\n", + " \"standard\",\n", + " \"premium\"\n", + " ]\n", + " }\n", + " },\n", + " \"required\": [\n", + " \"age\",\n", + " \"coverage_type\"\n", + " ],\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " },\n", + " {\n", + " \"type\": \"function\",\n", + " \"function\": {\n", + " \"name\": \"get_file_size\",\n", + " \"description\": \"Get the size of a file in bytes\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"filename\": {\n", + " \"type\": \"string\",\n", + " \"description\": \"the name of the file to get its size\"\n", + " }\n", + " },\n", + " \"required\": [\n", + " \"filename\"\n", + " ]\n", + " }\n", + " }\n", + " },\n", + " {\n", + " 'type': 'function',\n", + " 'function': {\n", + " 'name': 'addition',\n", + " 'description': \"Adds two numbers together\",\n", + " 'parameters': {\n", + " 'type': 'object',\n", + " 'properties': {\n", + " 'a': {\n", + " 'description': 'First number to add',\n", + " 'type': 'string'\n", + " },\n", + " 'b': {\n", + " 'description': 'Second number to add',\n", + " 'type': 'string'\n", + " }\n", + " },\n", + " 'required': []\n", + " }\n", + " }\n", + " },\n", + " {\n", + " 'type': 'function',\n", + " 'function': {\n", + " 'name': 'subtraction',\n", + " 'description': \"Subtracts two numbers\",\n", + " 'parameters': {\n", + " 'type': 'object',\n", + " 'properties': {\n", + " 'a': {\n", + " 'description': 'First number to be subtracted from',\n", + " 'type': 'string'\n", + " },\n", + " 'b': {\n", + " 'description': 'Number to subtract',\n", + " 'type': 'string'\n", + " }\n", + " },\n", + " 'required': []\n", + " }\n", + " }\n", + " },\n", + " {\n", + " 'type': 'function',\n", + " 'function': {\n", + " 'name': 'multiplication',\n", + " 'description': \"Multiply two numbers together\",\n", + " 'parameters': {\n", + " 'type': 'object',\n", + " 'properties': {\n", + " 'a': {\n", + " 'description': 'First number to multiply',\n", + " 'type': 'string'\n", + " },\n", + " 'b': {\n", + " 'description': 'Second number to multiply',\n", + " 'type': 'string'\n", + " }\n", + " },\n", + " 'required': []\n", + " }\n", + " }\n", + " },\n", + " {\n", + " 'type': 'function',\n", + " 'function': {\n", + " 'name': 'division',\n", + " 'description': \"Divide two numbers\",\n", + " 'parameters': {\n", + " 'type': 'object',\n", + " 'properties': {\n", + " 'a': {\n", + " 'description': 'First number to use as the dividend',\n", + " 'type': 'string'\n", + " },\n", + " 'b': {\n", + " 'description': 'Second number to use as the divisor',\n", + " 'type': 'string'\n", + " }\n", + " },\n", + " 'required': []\n", + " }\n", + " }\n", + " },\n", + " {\n", + " \"type\": \"function\",\n", + " \"function\": {\n", + " \"name\": \"getCurrentWeather\",\n", + " \"description\": \"Get the weather in location\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"location\": {\"type\": \"string\", \"description\": \"The city and state e.g. San Francisco, CA\"},\n", + " \"unit\": {\"type\": \"string\", \"enum\": [\"c\", \"f\"]}\n", + " },\n", + " \"required\": [\"location\"]\n", + " }\n", + " }\n", + " },\n", + " { \"type\": \"function\",\n", + " \"function\":\n", + " {\n", + " \"name\": \"orderUmbrella\",\n", + " \"description\": \"Do this to help user to order an umbrella online\", \n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"number_to_buy\": {\n", + " \"type\": \"integer\",\n", + " \"description\": \"the amount of umbrellas to buy\"\n", + " }\n", + " },\n", + " \"required\": [\n", + " \"number_to_buy\"\n", + " ]\n", + " }\n", + " }},\n", + " {\"type\": \"function\",\"function\":{\"name\":\"calculate_distance\",\"description\":\"Calculate the distance between two locations\",\"parameters\":{\"type\":\"object\",\"properties\":{\"origin\":{\"type\":\"string\",\"description\":\"The starting location\"},\"destination\":{\"type\":\"string\",\"description\":\"The destination location\"},\"mode\":{\"type\":\"string\",\"description\":\"The mode of transportation\"}},\"required\":[\"origin\",\"destination\",\"mode\"]}}},{\"type\": \"function\",\"function\":{\"name\":\"generate_password\",\"description\":\"Generate a random password\",\"parameters\":{\"type\":\"object\",\"properties\":{\"length\":{\"type\":\"integer\",\"description\":\"The length of the password\"}},\"required\":[\"length\"]}}}\n", + " ]\n", + " if not msgs or len(msgs) == 0:\n", + " msgs = [{\"role\": \"system\", \"content\":system_prompt} ,{\"role\": \"user\", \"content\": user_query}]\n", + " else:\n", + " msgs.append({\"role\": \"user\", \"content\": user_query})\n", + "\n", + " res = chat_method(model=model, functions=functions, msgs=msgs)\n", + " \n", + " next_content = \"\"\n", + " l = 0\n", + " print(f\":\")\n", + " print(\"[AI response]:\")\n", + " for chunk in res:\n", + " if chunk.choices[0].delta.content and len(chunk.choices[0].delta.content) > 0:\n", + " next_content += chunk.choices[0].delta.content\n", + " print(chunk.choices[0].delta.content, end=\"\")\n", + " elif chunk.choices[0].delta.tool_calls:\n", + " # Just for testing rubra tools.cpp purpose, won't work for gpt4.\n", + " res_next = chunk.choices[0]\n", + " print(\"[AI calling functions]:\")\n", + " for tool_call in res_next.delta.tool_calls:\n", + " print(f\"Tool Call: {tool_call.function}\")\n", + " break\n", + " \n", + " \n", + " while not next_content:\n", + " l += 1\n", + " msgs = insert_tool_response(res_next, msgs)\n", + " print(f\"\\n:\")\n", + " res_next = chat_method(model=model, functions=functions, msgs=msgs)\n", + " next_content = \"\"\n", + " print(\"[AI response]:\")\n", + " for chunk in res_next:\n", + " if chunk.choices[0].delta.content and len(chunk.choices[0].delta.content) > 0:\n", + " next_content += chunk.choices[0].delta.content\n", + " print(chunk.choices[0].delta.content, end=\"\")\n", + " elif chunk.choices[0].delta.tool_calls:\n", + " res_next = chunk.choices[0]\n", + " print(\"[AI calling functions]:\")\n", + " for tool_call in res_next.delta.tool_calls:\n", + " print(f\"Tool Call: {tool_call.function}\")\n", + " break\n", + " \n", + " msgs.append({\"role\": \"assistant\", \"content\": next_content})\n", + " return msgs\n", + " \n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import openai\n", + "local_api_key = \"sk-\"\n", + "local_base_url = \"http://localhost:1234/v1/\"\n", + "get_rubra_response = partial(get_oai_response, api_key=local_api_key, base_url=local_base_url)\n", + "model = \"Llama-3-8b-function-calling-alpha-v1.gguf\"\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Chat" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + ":\n", + "[AI response]:\n", + "I'm just a computer program, so I don't have feelings, but I'm here and ready to help you with any questions or tasks you have! How can I assist you today?" + ] + } + ], + "source": [ + "user_query = \"how are you?\"\n", + "msgs = run_completion(get_rubra_response, user_query, model=model)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Multi + Parallel Function Call Test examples" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + ":\n", + "[AI response]:\n", + "[AI calling functions]:\n", + "Tool Call: ChoiceDeltaToolCallFunction(arguments='{\"origin\":\"San Francisco\",\"destination\":\"Cupertino\",\"mode\":\"driving\"}', name='calculate_distance')\n", + "Observation: Distance is 50 miles.\n", + "\n", + ":\n", + "[AI response]:\n", + "[AI calling functions]:\n", + "Tool Call: ChoiceDeltaToolCallFunction(arguments='{\"origin\":\"Cupertino\",\"destination\":\"San Francisco\",\"mode\":\"driving\"}', name='calculate_distance')\n", + "Observation: Distance is 50 miles.\n", + "\n", + ":\n", + "[AI response]:\n", + "[AI calling functions]:\n", + "Tool Call: ChoiceDeltaToolCallFunction(arguments='{\"origin\":\"San Francisco\",\"destination\":\"Cupertino\",\"mode\":\"air\"}', name='calculate_distance')\n", + "Observation: Distance is 50 miles.\n", + "\n", + ":\n", + "[AI response]:\n", + "[AI calling functions]:\n", + "Tool Call: ChoiceDeltaToolCallFunction(arguments='{\"origin\":\"Cupertino\",\"destination\":\"San Francisco\",\"mode\":\"air\"}', name='calculate_distance')\n", + "Observation: Distance is 50 miles.\n", + "\n", + ":\n", + "[AI response]:\n", + "The distance between San Francisco and Cupertino by driving and by air is 50 miles in both directions." + ] + } + ], + "source": [ + "user_query = \"What is the distance between San Francisco and Cupertino by driving and by air from both directions?\"\n", + "msgs = run_completion(get_rubra_response, user_query, model=model)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + ":\n", + "[AI response]:\n", + "[AI calling functions]:\n", + "Tool Call: ChoiceDeltaToolCallFunction(arguments='{\"origin\":\"San Francisco\",\"destination\":\"New York City\",\"mode\":\"airplane\"}', name='calculate_distance')\n", + "Observation: Distance is 50 miles.\n", + "\n", + ":\n", + "[AI response]:\n", + "[AI calling functions]:\n", + "Tool Call: ChoiceDeltaToolCallFunction(arguments='{\"a\":\"50\",\"b\":\"8\"}', name='multiplication')\n", + "Observation: 400.0\n", + "\n", + ":\n", + "[AI response]:\n", + "[AI calling functions]:\n", + "Tool Call: ChoiceDeltaToolCallFunction(arguments='{\"a\":\"400.0\",\"b\":\"2\"}', name='division')\n", + "Observation: 200.0\n", + "\n", + ":\n", + "[AI response]:\n", + "[AI calling functions]:\n", + "Tool Call: ChoiceDeltaToolCallFunction(arguments='{\"a\":\"200.0\",\"b\":\"30\"}', name='subtraction')\n", + "Observation: 170.0\n", + "\n", + ":\n", + "[AI response]:\n", + "The final result after performing the operations is 170.0 miles." + ] + } + ], + "source": [ + "user_query1 = \"what's the distance between SF and NYC? Use the result value to multiply by 8, and then divide by 2, and then minus 30\"\n", + "msgs = run_completion(get_rubra_response, user_query1, model=model)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + ":\n", + "[AI response]:\n", + "[AI calling functions]:\n", + "Tool Call: ChoiceDeltaToolCallFunction(arguments='{\"number_to_buy\":3}', name='orderUmbrella')\n", + "Observation: Order placed. the price is 10 dollars.\n", + "\n", + ":\n", + "[AI response]:\n", + "[AI calling functions]:\n", + "Tool Call: ChoiceDeltaToolCallFunction(arguments='{\"length\":8}', name='generate_password')\n", + "Observation: Password generated: 0cc31a6f\n", + "\n", + ":\n", + "[AI response]:\n", + "Your order for 3 umbrellas has been placed, and the price is $10 per umbrella. A random password of length 8 has also been generated: 0cc31a6f." + ] + } + ], + "source": [ + "user_query2 = \"now order 3 umbrellas for me and generate a password of length 8\"\n", + "msgs = run_completion(get_rubra_response, user_query2, msgs, model=model)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Simple Math Chaining" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + ":\n", + "[AI response]:\n", + "[AI calling functions]:\n", + "Tool Call: ChoiceDeltaToolCallFunction(arguments='{\"a\":\"4\",\"b\":\"6\"}', name='addition')\n", + "Observation: 10.0\n", + "\n", + ":\n", + "[AI response]:\n", + "[AI calling functions]:\n", + "Tool Call: ChoiceDeltaToolCallFunction(arguments='{\"a\":\"10.0\",\"b\":\"2\"}', name='addition')\n", + "Observation: 12.0\n", + "\n", + ":\n", + "[AI response]:\n", + "[AI calling functions]:\n", + "Tool Call: ChoiceDeltaToolCallFunction(arguments='{\"a\":\"12.0\",\"b\":\"5\"}', name='multiplication')\n", + "Observation: 60.0\n", + "\n", + ":\n", + "[AI response]:\n", + "[AI calling functions]:\n", + "Tool Call: ChoiceDeltaToolCallFunction(arguments='{\"a\":\"60.0\",\"b\":\"2\"}', name='division')\n", + "Observation: 30.0\n", + "\n", + ":\n", + "[AI response]:\n", + "The result of four plus six is ten. Adding two to that gives twelve. Multiplying twelve by five gives sixty, and dividing sixty by two results in thirty." + ] + } + ], + "source": [ + "user_query3 = \"What is four plus six? What is the result of that plus 2? Take the result and multiply by 5 and then divide by two\"\n", + "\n", + "\n", + "msgs = run_completion(get_rubra_response, user_query3, model=model)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Condition" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + ":\n", + "[AI response]:\n", + "[AI calling functions]:\n", + "Tool Call: ChoiceDeltaToolCallFunction(arguments='{\"location\":\"Boston\",\"unit\":\"f\"}', name='getCurrentWeather')\n", + "\n", + "Observation: temprature is 56 degree\n", + "\n", + ":\n", + "[AI response]:\n", + "[AI calling functions]:\n", + "Tool Call: ChoiceDeltaToolCallFunction(arguments='{\"origin\":\"Boston\",\"destination\":\"NYC\",\"mode\":\"car\"}', name='calculate_distance')\n", + "Observation: Distance is 50 miles.\n", + "\n", + ":\n", + "[AI response]:\n", + "The temperature in Boston is 56 degrees Fahrenheit, which is below 100 degrees. The distance from Boston to New York City is approximately 50 miles." + ] + } + ], + "source": [ + "user_query4 = \"check the weather in boston, if it's less than 100 degrees Fahrenheit, calculate the distance from boston to NYC\"\n", + "msgs = run_completion(get_rubra_response, user_query4, model=model)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + ":\n", + "[AI response]:\n", + "[AI calling functions]:\n", + "Tool Call: ChoiceDeltaToolCallFunction(arguments='{\"location\":\"Boston\",\"unit\":\"f\"}', name='getCurrentWeather')\n", + "\n", + "Observation: temprature is 56 degree\n", + "\n", + ":\n", + "[AI response]:\n", + "The temperature in Boston is 56 degrees Fahrenheit, which is not greater than 100 degrees. Therefore, I will not calculate the distance from Boston to NYC as the condition was not met. Is there anything else I can assist you with?" + ] + } + ], + "source": [ + "user_query5 = \"check the weather in boston, if it's greater than 100 degrees Fahrenheit, calculate the distance from boston to NYC\"\n", + "msgs = run_completion(get_rubra_response, user_query5, model=model)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# dependency" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + ":\n", + "[AI response]:\n", + "[AI calling functions]:\n", + "Tool Call: ChoiceDeltaToolCallFunction(arguments='{\"directory\":\"documents\"}', name='list_files')\n", + "Observation: File list:\n", + "report.docx\n", + "task.txt\n", + "notes.txt\n", + "\n", + ":\n", + "[AI response]:\n", + "[AI calling functions]:\n", + "Tool Call: ChoiceDeltaToolCallFunction(arguments='{\"filename\":\"report.docx\"}', name='get_file_size')\n", + "Observation: the size is 100 bytes.\n", + "\n", + ":\n", + "[AI response]:\n", + "[AI calling functions]:\n", + "Tool Call: ChoiceDeltaToolCallFunction(arguments='{\"filename\":\"task.txt\"}', name='get_file_size')\n", + "Observation: the size is 100 bytes.\n", + "\n", + ":\n", + "[AI response]:\n", + "[AI calling functions]:\n", + "Tool Call: ChoiceDeltaToolCallFunction(arguments='{\"filename\":\"notes.txt\"}', name='get_file_size')\n", + "Observation: the size is 100 bytes.\n", + "\n", + ":\n", + "[AI response]:\n", + "The sizes of the files in the 'documents' directory are as follows:\n", + "- 'report.docx' is 100 bytes.\n", + "- 'task.txt' is 100 bytes.\n", + "- 'notes.txt' is 100 bytes.\n", + "Is there anything else you would like to know or another task you need assistance with?" + ] + } + ], + "source": [ + "user_query6 = \"check the size of all files in the 'documents' directory.\"\n", + "msgs = run_completion(get_rubra_response, user_query6, model=model)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + ":\n", + "[AI response]:\n", + "[AI calling functions]:\n", + "Tool Call: ChoiceDeltaToolCallFunction(arguments='{\"object_name\":\"rectangle\",\"dimensions\":{\"length\":12,\"width\":21,\"height\":6}}', name='create_3d_model')\n", + "Observation: created.\n", + "\n", + ":\n", + "[AI response]:\n", + "The 3D model of the rectangle with dimensions 12x21x6 has been successfully created." + ] + } + ], + "source": [ + "user_query0 = \"create a 3d model with length 12, width 21, and height 6\"\n", + "msgs = run_completion(get_rubra_response, user_query0, model=model)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "py310", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}