diff --git a/changelog/unreleased/kong/analytics-for-anthropic.yml b/changelog/unreleased/kong/analytics-for-anthropic.yml new file mode 100644 index 000000000000..04de991390df --- /dev/null +++ b/changelog/unreleased/kong/analytics-for-anthropic.yml @@ -0,0 +1,4 @@ +message: | + **AI-proxy-plugin**: Fix the bug that the route_type `/llm/v1/chat` does not include the analytics in the responses. +scope: Plugin +type: bugfix diff --git a/kong/llm/drivers/anthropic.lua b/kong/llm/drivers/anthropic.lua index 9a88415364e6..18c3f2bce5b5 100644 --- a/kong/llm/drivers/anthropic.lua +++ b/kong/llm/drivers/anthropic.lua @@ -148,6 +148,12 @@ local transformers_from = { finish_reason = response_table.stop_reason, }, }, + usage = { + prompt_tokens = response_table.usage.input_tokens or 0, + completion_tokens = response_table.usage.output_tokens or 0, + total_tokens = response_table.usage.input_tokens and response_table.usage.output_tokens and + response_table.usage.input_tokens + response_table.usage.output_tokens or 0, + }, model = response_table.model, object = "chat.content", } diff --git a/kong/llm/init.lua b/kong/llm/init.lua index 489af760ccea..6b973ae262e0 100644 --- a/kong/llm/init.lua +++ b/kong/llm/init.lua @@ -131,7 +131,7 @@ local logging_schema = { description = "If enabled and supported by the driver, " .. "will add model usage and token metrics into the Kong log plugin(s) output.", required = true, - default = true }}, + default = false }}, { log_payloads = { type = "boolean", description = "If enabled, will log the request and response body into the Kong log plugin(s) output.", @@ -139,6 +139,10 @@ local logging_schema = { } } +local UNSUPPORTED_LOG_STATISTICS = { + ["llm/v1/completions"] = { ["anthropic"] = true }, +} + _M.config_schema = { type = "record", fields = { @@ -201,6 +205,23 @@ _M.config_schema = { if_match = { one_of = { "mistral", "llama2" } }, then_at_least_one_of = { "model.options.upstream_url" }, then_err = "must set %s for self-hosted providers/models" }}, + + { + custom_entity_check = { + field_sources = { "route_type", "model", "logging" }, + fn = function(entity) + -- print(cjson.encode(entity)) + if entity.logging.log_statistics and UNSUPPORTED_LOG_STATISTICS[entity.route_type] + and UNSUPPORTED_LOG_STATISTICS[entity.route_type][entity.model.provider] then + return nil, fmt("%s does not support statistics when route_type is %s", + entity.model.provider, entity.route_type) + + else + return true + end + end, + } + }, }, } diff --git a/spec/03-plugins/38-ai-proxy/00-config_spec.lua b/spec/03-plugins/38-ai-proxy/00-config_spec.lua index 296ecc8c47bc..bbd495918bbb 100644 --- a/spec/03-plugins/38-ai-proxy/00-config_spec.lua +++ b/spec/03-plugins/38-ai-proxy/00-config_spec.lua @@ -115,6 +115,34 @@ describe(PLUGIN_NAME .. ": (schema)", function() assert.is_falsy(ok) end) + it("do not support log statistics when /chat route_type is used for anthropic provider", function() + local config = { + route_type = "llm/v1/completions", + auth = { + header_name = "x-api-key", + header_value = "anthropic_key", + }, + model = { + name = "anthropic-chat", + provider = "anthropic", + options = { + max_tokens = 256, + temperature = 1.0, + anthropic_version = "2021-09-01", + }, + }, + logging = { + log_statistics = true, + }, + } + + local ok, err = validate(config) + assert.is_falsy(ok) + assert.not_nil(err["config"]["@entity"]) + assert.not_nil(err["config"]["@entity"][1]) + assert.not_nil(err["config"]["@entity"][1], "anthropic does not support statistics when route_type is llm/v1/completions") + end) + it("requires [azure_instance] field when azure provider is used", function() local config = { route_type = "llm/v1/chat", diff --git a/spec/03-plugins/38-ai-proxy/03-anthropic_integration_spec.lua b/spec/03-plugins/38-ai-proxy/03-anthropic_integration_spec.lua index 13d9ec937e6b..d5b36fce7b22 100644 --- a/spec/03-plugins/38-ai-proxy/03-anthropic_integration_spec.lua +++ b/spec/03-plugins/38-ai-proxy/03-anthropic_integration_spec.lua @@ -225,6 +225,9 @@ for _, strategy in helpers.all_strategies() do if strategy ~= "cassandra" then anthropic_version = "2023-06-01", }, }, + logging = { + log_statistics = false, -- anthropic does not support statistics + }, }, } -- @@ -315,6 +318,9 @@ for _, strategy in helpers.all_strategies() do if strategy ~= "cassandra" then anthropic_version = "2023-06-01", }, }, + logging = { + log_statistics = false, -- anthropic does not support statistics + }, }, } -- @@ -363,7 +369,7 @@ for _, strategy in helpers.all_strategies() do if strategy ~= "cassandra" then declarative_config = strategy == "off" and helpers.make_yaml_file() or nil, }, nil, nil, fixtures)) end) - + lazy_teardown(function() helpers.stop_kong() end) @@ -434,7 +440,7 @@ for _, strategy in helpers.all_strategies() do if strategy ~= "cassandra" then }, body = pl_file.read("spec/fixtures/ai-proxy/anthropic/llm-v1-chat/requests/good.json"), }) - + local body = assert.res_status(200 , r) local json = cjson.decode(body) @@ -481,6 +487,7 @@ for _, strategy in helpers.all_strategies() do if strategy ~= "cassandra" then -- check this is in the 'kong' response format assert.equals(json.error.message, "request format not recognised") end) + end) describe("anthropic llm/v1/completions", function() it("good request", function() @@ -521,6 +528,5 @@ for _, strategy in helpers.all_strategies() do if strategy ~= "cassandra" then end) end) end) -end) end end diff --git a/spec/fixtures/ai-proxy/anthropic/llm-v1-chat/responses/good.json b/spec/fixtures/ai-proxy/anthropic/llm-v1-chat/responses/good.json index 34aafac88752..174220a4a210 100644 --- a/spec/fixtures/ai-proxy/anthropic/llm-v1-chat/responses/good.json +++ b/spec/fixtures/ai-proxy/anthropic/llm-v1-chat/responses/good.json @@ -5,6 +5,11 @@ "type": "text" } ], - "stop_reason": "stop_sequence", - "model": "claude-2.1" + "model": "claude-2.1", + "stop_reason": "end_turn", + "stop_sequence": "string", + "usage": { + "input_tokens": 0, + "output_tokens": 0 + } } diff --git a/spec/fixtures/ai-proxy/unit/real-responses/anthropic/llm-v1-chat.json b/spec/fixtures/ai-proxy/unit/real-responses/anthropic/llm-v1-chat.json index a6afa37fca8e..624214cff5d0 100644 --- a/spec/fixtures/ai-proxy/unit/real-responses/anthropic/llm-v1-chat.json +++ b/spec/fixtures/ai-proxy/unit/real-responses/anthropic/llm-v1-chat.json @@ -4,5 +4,9 @@ "type": "text" }], "stop_reason": "stop_sequence", - "model": "claude-2.1" + "model": "claude-2.1", + "usage": { + "input_tokens": 0, + "output_tokens": 0 + } } \ No newline at end of file