diff --git a/backend/cpp/llama-cpp/grpc-server.cpp b/backend/cpp/llama-cpp/grpc-server.cpp index 1009d36fd7df..24d2f7ed2925 100644 --- a/backend/cpp/llama-cpp/grpc-server.cpp +++ b/backend/cpp/llama-cpp/grpc-server.cpp @@ -293,6 +293,8 @@ json parse_options(bool streaming, const backend::PredictOptions* predict, const return data; } +// Sanitize tools JSON to remove null values from tool.parameters.properties +// This prevents Jinja template errors when processing tools with malformed parameter schemas const std::vector kv_cache_types = { GGML_TYPE_F32, diff --git a/core/http/endpoints/openai/chat.go b/core/http/endpoints/openai/chat.go index dd27edcb9ac7..3af9ebd2d830 100644 --- a/core/http/endpoints/openai/chat.go +++ b/core/http/endpoints/openai/chat.go @@ -622,7 +622,9 @@ func handleQuestion(config *config.ModelConfig, cl *config.ModelConfigLoader, in // Serialize tools and tool_choice to JSON strings toolsJSON := "" if len(input.Tools) > 0 { - toolsBytes, err := json.Marshal(input.Tools) + // Sanitize tools to remove null values from parameters.properties + sanitizedTools := functions.SanitizeTools(input.Tools) + toolsBytes, err := json.Marshal(sanitizedTools) if err == nil { toolsJSON = string(toolsBytes) } diff --git a/core/http/endpoints/openai/inference.go b/core/http/endpoints/openai/inference.go index 37b14c98bcfa..7db7707ed0a2 100644 --- a/core/http/endpoints/openai/inference.go +++ b/core/http/endpoints/openai/inference.go @@ -7,6 +7,7 @@ import ( "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/schema" + "github.com/mudler/LocalAI/pkg/functions" model "github.com/mudler/LocalAI/pkg/model" ) @@ -42,7 +43,9 @@ func ComputeChoices( // Serialize tools and tool_choice to JSON strings toolsJSON := "" if len(req.Tools) > 0 { - toolsBytes, err := json.Marshal(req.Tools) + // Sanitize tools to remove null values from parameters.properties + sanitizedTools := functions.SanitizeTools(req.Tools) + toolsBytes, err := json.Marshal(sanitizedTools) if err == nil { toolsJSON = string(toolsBytes) } diff --git a/pkg/functions/functions.go b/pkg/functions/functions.go index b76d1d0b06c4..9152d4517046 100644 --- a/pkg/functions/functions.go +++ b/pkg/functions/functions.go @@ -2,6 +2,7 @@ package functions import ( "encoding/json" + "fmt" "github.com/mudler/xlog" ) @@ -102,3 +103,91 @@ func (f Functions) Select(name string) Functions { return funcs } + +// sanitizeValue recursively sanitizes null values in a JSON structure, converting them to empty objects. +// It handles maps, slices, and nested structures. +func sanitizeValue(value interface{}, path string) interface{} { + if value == nil { + // Convert null to empty object + xlog.Debug("SanitizeTools: found null value, converting to empty object", "path", path) + return map[string]interface{}{} + } + + switch v := value.(type) { + case map[string]interface{}: + // Recursively sanitize map values + sanitized := make(map[string]interface{}) + for key, val := range v { + newPath := path + if newPath != "" { + newPath += "." + } + newPath += key + sanitized[key] = sanitizeValue(val, newPath) + } + return sanitized + + case []interface{}: + // Recursively sanitize slice elements + sanitized := make([]interface{}, len(v)) + for i, val := range v { + newPath := fmt.Sprintf("%s[%d]", path, i) + sanitized[i] = sanitizeValue(val, newPath) + } + return sanitized + + default: + // For primitive types (string, number, bool), return as-is + return value + } +} + +// SanitizeTools removes null values from tool.parameters.properties and converts them to empty objects. +// This prevents Jinja template errors when processing tools with malformed parameter schemas. +// It works by marshaling to JSON, recursively sanitizing the JSON structure, and unmarshaling back. +func SanitizeTools(tools Tools) Tools { + if len(tools) == 0 { + return tools + } + + xlog.Debug("SanitizeTools: processing tools", "count", len(tools)) + + // Marshal to JSON to work with the actual JSON representation + toolsJSON, err := json.Marshal(tools) + if err != nil { + xlog.Warn("SanitizeTools: failed to marshal tools to JSON", "error", err) + return tools + } + + // Parse JSON into a generic structure + var toolsData []map[string]interface{} + if err := json.Unmarshal(toolsJSON, &toolsData); err != nil { + xlog.Warn("SanitizeTools: failed to unmarshal tools JSON", "error", err) + return tools + } + + // Recursively sanitize the JSON structure + for i, tool := range toolsData { + if function, ok := tool["function"].(map[string]interface{}); ok { + // Recursively sanitize the entire tool structure + tool["function"] = sanitizeValue(function, fmt.Sprintf("tools[%d].function", i)) + } + toolsData[i] = tool + } + + // Marshal back to JSON + sanitizedJSON, err := json.Marshal(toolsData) + if err != nil { + xlog.Warn("SanitizeTools: failed to marshal sanitized tools", "error", err) + return tools + } + + // Unmarshal back into Tools structure + var sanitized Tools + if err := json.Unmarshal(sanitizedJSON, &sanitized); err != nil { + xlog.Warn("SanitizeTools: failed to unmarshal sanitized tools", "error", err) + return tools + } + + return sanitized +} diff --git a/pkg/functions/functions_test.go b/pkg/functions/functions_test.go index 2eb0946a6e8e..c82271bc9c02 100644 --- a/pkg/functions/functions_test.go +++ b/pkg/functions/functions_test.go @@ -82,4 +82,202 @@ var _ = Describe("LocalAI grammar functions", func() { Expect(functions[0].Name).To(Equal("create_event")) }) }) + Context("SanitizeTools()", func() { + It("returns empty slice when input is empty", func() { + tools := Tools{} + sanitized := SanitizeTools(tools) + Expect(len(sanitized)).To(Equal(0)) + }) + + It("converts null values in parameters.properties to empty objects", func() { + tools := Tools{ + { + Type: "function", + Function: Function{ + Name: "test_function", + Description: "A test function", + Parameters: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "valid_param": map[string]interface{}{ + "type": "string", + }, + "null_param": nil, + "another_valid": map[string]interface{}{ + "type": "integer", + }, + }, + }, + }, + }, + } + + sanitized := SanitizeTools(tools) + Expect(len(sanitized)).To(Equal(1)) + Expect(sanitized[0].Function.Name).To(Equal("test_function")) + + properties := sanitized[0].Function.Parameters["properties"].(map[string]interface{}) + Expect(properties["valid_param"]).NotTo(BeNil()) + Expect(properties["null_param"]).NotTo(BeNil()) + Expect(properties["null_param"]).To(Equal(map[string]interface{}{})) + Expect(properties["another_valid"]).NotTo(BeNil()) + }) + + It("preserves valid parameter structures unchanged", func() { + tools := Tools{ + { + Type: "function", + Function: Function{ + Name: "valid_function", + Description: "A function with valid parameters", + Parameters: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "param1": map[string]interface{}{ + "type": "string", + "description": "First parameter", + }, + "param2": map[string]interface{}{ + "type": "integer", + }, + }, + }, + }, + }, + } + + sanitized := SanitizeTools(tools) + Expect(len(sanitized)).To(Equal(1)) + Expect(sanitized[0].Function.Name).To(Equal("valid_function")) + + properties := sanitized[0].Function.Parameters["properties"].(map[string]interface{}) + Expect(properties["param1"].(map[string]interface{})["type"]).To(Equal("string")) + Expect(properties["param1"].(map[string]interface{})["description"]).To(Equal("First parameter")) + Expect(properties["param2"].(map[string]interface{})["type"]).To(Equal("integer")) + }) + + It("handles tools without parameters field", func() { + tools := Tools{ + { + Type: "function", + Function: Function{ + Name: "no_params_function", + Description: "A function without parameters", + }, + }, + } + + sanitized := SanitizeTools(tools) + Expect(len(sanitized)).To(Equal(1)) + Expect(sanitized[0].Function.Name).To(Equal("no_params_function")) + Expect(sanitized[0].Function.Parameters).To(BeNil()) + }) + + It("handles tools without properties field", func() { + tools := Tools{ + { + Type: "function", + Function: Function{ + Name: "no_properties_function", + Description: "A function without properties", + Parameters: map[string]interface{}{ + "type": "object", + }, + }, + }, + } + + sanitized := SanitizeTools(tools) + Expect(len(sanitized)).To(Equal(1)) + Expect(sanitized[0].Function.Name).To(Equal("no_properties_function")) + Expect(sanitized[0].Function.Parameters["type"]).To(Equal("object")) + }) + + It("handles multiple tools with mixed valid and null values", func() { + tools := Tools{ + { + Type: "function", + Function: Function{ + Name: "function_with_nulls", + Parameters: map[string]interface{}{ + "properties": map[string]interface{}{ + "valid": map[string]interface{}{ + "type": "string", + }, + "null1": nil, + "null2": nil, + }, + }, + }, + }, + { + Type: "function", + Function: Function{ + Name: "function_all_valid", + Parameters: map[string]interface{}{ + "properties": map[string]interface{}{ + "param1": map[string]interface{}{ + "type": "string", + }, + "param2": map[string]interface{}{ + "type": "integer", + }, + }, + }, + }, + }, + { + Type: "function", + Function: Function{ + Name: "function_no_params", + }, + }, + } + + sanitized := SanitizeTools(tools) + Expect(len(sanitized)).To(Equal(3)) + + // First tool should have nulls converted to empty objects + props1 := sanitized[0].Function.Parameters["properties"].(map[string]interface{}) + Expect(props1["valid"]).NotTo(BeNil()) + Expect(props1["null1"]).To(Equal(map[string]interface{}{})) + Expect(props1["null2"]).To(Equal(map[string]interface{}{})) + + // Second tool should remain unchanged + props2 := sanitized[1].Function.Parameters["properties"].(map[string]interface{}) + Expect(props2["param1"].(map[string]interface{})["type"]).To(Equal("string")) + Expect(props2["param2"].(map[string]interface{})["type"]).To(Equal("integer")) + + // Third tool should remain unchanged + Expect(sanitized[2].Function.Parameters).To(BeNil()) + }) + + It("does not modify the original tools slice", func() { + tools := Tools{ + { + Type: "function", + Function: Function{ + Name: "test_function", + Parameters: map[string]interface{}{ + "properties": map[string]interface{}{ + "null_param": nil, + }, + }, + }, + }, + } + + originalProperties := tools[0].Function.Parameters["properties"].(map[string]interface{}) + originalNullValue := originalProperties["null_param"] + + sanitized := SanitizeTools(tools) + + // Original should still have nil + Expect(originalNullValue).To(BeNil()) + + // Sanitized should have empty object + sanitizedProperties := sanitized[0].Function.Parameters["properties"].(map[string]interface{}) + Expect(sanitizedProperties["null_param"]).To(Equal(map[string]interface{}{})) + }) + }) })