fix(ai-proxy): add analytics for chat route_type of anthropic (#12781)

* fix(AI-Proxy): add a validator for checking if the ai provider supports
log statistics, and fix a bug that Anthropic don't provide log
statistics in `chat` route_type.

FTI-5769

* 1. add changelog

2. remain completions of anthropic only in the unsupported api

* Update analytics-for-anthropic.yml

* update default of statistics and update mock response of anthropic for /chat

* update test case

changelog/unreleased/kong/analytics-for-anthropic.yml

@@ -0,0 +1,4 @@
+message: |
+  **AI-proxy-plugin**: Fix the bug that the route_type `/llm/v1/chat` does not include the analytics in the responses.
+scope: Plugin
+type: bugfix

kong/llm/drivers/anthropic.lua

@@ -148,6 +148,12 @@ local transformers_from = {
             finish_reason = response_table.stop_reason,
           },
         },
+        usage = {
+          prompt_tokens = response_table.usage.input_tokens or 0,
+          completion_tokens = response_table.usage.output_tokens or 0,
+          total_tokens = response_table.usage.input_tokens and response_table.usage.output_tokens and
+            response_table.usage.input_tokens + response_table.usage.output_tokens or 0,
+        },
         model = response_table.model,
         object = "chat.content",
       }

kong/llm/init.lua

@@ -131,14 +131,18 @@ local logging_schema = {
         description = "If enabled and supported by the driver, "
                    .. "will add model usage and token metrics into the Kong log plugin(s) output.",
                    required = true,
-                   default = true }},
+                   default = false }},
     { log_payloads = {
         type = "boolean", 
         description = "If enabled, will log the request and response body into the Kong log plugin(s) output.",
         required = true, default = false }},
   }
 }
 
+local UNSUPPORTED_LOG_STATISTICS = {
+  ["llm/v1/completions"] = { ["anthropic"] = true },
+}
+
 _M.config_schema = {
   type = "record",
   fields = {
@@ -201,6 +205,23 @@ _M.config_schema = {
                                       if_match = { one_of = { "mistral", "llama2" } },
                                       then_at_least_one_of = { "model.options.upstream_url" },
                                       then_err = "must set %s for self-hosted providers/models" }},
+
+    {
+      custom_entity_check = {
+        field_sources = { "route_type", "model", "logging" },
+        fn = function(entity)
+          -- print(cjson.encode(entity))
+          if entity.logging.log_statistics and UNSUPPORTED_LOG_STATISTICS[entity.route_type]
+            and UNSUPPORTED_LOG_STATISTICS[entity.route_type][entity.model.provider] then
+              return nil, fmt("%s does not support statistics when route_type is %s",
+                               entity.model.provider, entity.route_type)
+
+          else
+            return true
+          end
+        end,
+      }
+    },
   },
 }
 

spec/03-plugins/38-ai-proxy/00-config_spec.lua

@@ -115,6 +115,34 @@ describe(PLUGIN_NAME .. ": (schema)", function()
     assert.is_falsy(ok)
   end)
 
+  it("do not support log statistics when /chat route_type is used for anthropic provider", function()
+    local config = {
+      route_type = "llm/v1/completions",
+      auth = {
+        header_name = "x-api-key",
+        header_value = "anthropic_key",
+      },
+      model = {
+        name = "anthropic-chat",
+        provider = "anthropic",
+        options = {
+          max_tokens = 256,
+          temperature = 1.0,
+          anthropic_version = "2021-09-01",
+        },
+      },
+      logging = {
+        log_statistics = true,
+      },
+    }
+
+    local ok, err = validate(config)
+    assert.is_falsy(ok)
+    assert.not_nil(err["config"]["@entity"])
+    assert.not_nil(err["config"]["@entity"][1])
+    assert.not_nil(err["config"]["@entity"][1], "anthropic does not support statistics when route_type is llm/v1/completions")
+  end)
+
   it("requires [azure_instance] field when azure provider is used", function()
     local config = {
       route_type = "llm/v1/chat",

spec/03-plugins/38-ai-proxy/03-anthropic_integration_spec.lua

@@ -225,6 +225,9 @@ for _, strategy in helpers.all_strategies() do if strategy ~= "cassandra" then
               anthropic_version = "2023-06-01",
             },
           },
+          logging = {
+            log_statistics = false,  -- anthropic does not support statistics
+          },
         },
       }
       --
@@ -315,6 +318,9 @@ for _, strategy in helpers.all_strategies() do if strategy ~= "cassandra" then
               anthropic_version = "2023-06-01",
             },
           },
+          logging = {
+            log_statistics = false,  -- anthropic does not support statistics
+          },
         },
       }
       --
@@ -363,7 +369,7 @@ for _, strategy in helpers.all_strategies() do if strategy ~= "cassandra" then
         declarative_config = strategy == "off" and helpers.make_yaml_file() or nil,
       }, nil, nil, fixtures))
     end)
-    
+
     lazy_teardown(function()
       helpers.stop_kong()
     end)
@@ -434,7 +440,7 @@ for _, strategy in helpers.all_strategies() do if strategy ~= "cassandra" then
           },
           body = pl_file.read("spec/fixtures/ai-proxy/anthropic/llm-v1-chat/requests/good.json"),
         })
-        
+
         local body = assert.res_status(200 , r)
         local json = cjson.decode(body)
 
@@ -481,6 +487,7 @@ for _, strategy in helpers.all_strategies() do if strategy ~= "cassandra" then
         -- check this is in the 'kong' response format
         assert.equals(json.error.message, "request format not recognised")
       end)
+    end)
 
     describe("anthropic llm/v1/completions", function()
       it("good request", function()
@@ -521,6 +528,5 @@ for _, strategy in helpers.all_strategies() do if strategy ~= "cassandra" then
       end)
     end)
   end)
-end)
 
 end end

spec/fixtures/ai-proxy/anthropic/llm-v1-chat/responses/good.json

@@ -5,6 +5,11 @@
             "type": "text"
         }
     ],
-    "stop_reason": "stop_sequence",
-    "model": "claude-2.1"
+    "model": "claude-2.1",
+    "stop_reason": "end_turn",
+    "stop_sequence": "string",
+    "usage": {
+        "input_tokens": 0,
+        "output_tokens": 0
+    }
 }

spec/fixtures/ai-proxy/unit/real-responses/anthropic/llm-v1-chat.json

@@ -4,5 +4,9 @@
     "type": "text"
   }],
   "stop_reason": "stop_sequence",
-  "model": "claude-2.1"
+  "model": "claude-2.1",
+  "usage": {
+    "input_tokens": 0,
+    "output_tokens": 0
+  }
 }