update default of statistics and update mock response of anthropic fo…

…r /chat
Kong · Apr 11, 2024 · bc0beaf · bc0beaf
1 parent f8d6837
commit bc0beaf
Show file tree

Hide file tree

Showing 4 changed files with 26 additions and 9 deletions.
diff --git a/kong/llm/init.lua b/kong/llm/init.lua
@@ -131,7 +131,7 @@ local logging_schema = {
         description = "If enabled and supported by the driver, "
                    .. "will add model usage and token metrics into the Kong log plugin(s) output.",
                    required = true,
-                   default = true }},
+                   default = false }},
     { log_payloads = {
         type = "boolean", 
         description = "If enabled, will log the request and response body into the Kong log plugin(s) output.",
@@ -140,7 +140,7 @@ local logging_schema = {
 }
 
 local UNSUPPORTED_LOG_STATISTICS = {
-  ["llm/v1/completions"] = { "anthropic" },
+  ["llm/v1/completions"] = { ["anthropic"] = true },
 }
 
 _M.config_schema = {
@@ -210,8 +210,10 @@ _M.config_schema = {
       custom_entity_check = {
         field_sources = { "route_type", "model", "logging" },
         fn = function(entity)
-          if entity.logging.log_statistics and UNSUPPORTED_LOG_STATISTICS[entity.route_type][entity.model.provider] then
-              return nil, fmt("cannot log statistics for %s provider when route_type is %s",
+          -- print(cjson.encode(entity))
+          if entity.logging.log_statistics and UNSUPPORTED_LOG_STATISTICS[entity.route_type]
+            and UNSUPPORTED_LOG_STATISTICS[entity.route_type][entity.model.provider] then
+              return nil, fmt("%s does not support statistics when route_type is %s",
                                entity.model.provider, entity.route_type)
 
           else

diff --git a/spec/03-plugins/38-ai-proxy/03-anthropic_integration_spec.lua b/spec/03-plugins/38-ai-proxy/03-anthropic_integration_spec.lua
@@ -225,6 +225,9 @@ for _, strategy in helpers.all_strategies() do if strategy ~= "cassandra" then
               anthropic_version = "2023-06-01",
             },
           },
+          logging = {
+            log_statistics = false,  -- anthropic does not support statistics
+          },
         },
       }
       --
@@ -315,6 +318,9 @@ for _, strategy in helpers.all_strategies() do if strategy ~= "cassandra" then
               anthropic_version = "2023-06-01",
             },
           },
+          logging = {
+            log_statistics = false,  -- anthropic does not support statistics
+          },
         },
       }
       --
@@ -363,7 +369,7 @@ for _, strategy in helpers.all_strategies() do if strategy ~= "cassandra" then
         declarative_config = strategy == "off" and helpers.make_yaml_file() or nil,
       }, nil, nil, fixtures))
     end)
-    
+
     lazy_teardown(function()
       helpers.stop_kong()
     end)
@@ -434,7 +440,7 @@ for _, strategy in helpers.all_strategies() do if strategy ~= "cassandra" then
           },
           body = pl_file.read("spec/fixtures/ai-proxy/anthropic/llm-v1-chat/requests/good.json"),
         })
-        
+
         local body = assert.res_status(200 , r)
         local json = cjson.decode(body)
 

diff --git a/spec/fixtures/ai-proxy/anthropic/llm-v1-chat/responses/good.json b/spec/fixtures/ai-proxy/anthropic/llm-v1-chat/responses/good.json
@@ -5,6 +5,11 @@
             "type": "text"
         }
     ],
-    "stop_reason": "stop_sequence",
-    "model": "claude-2.1"
+    "model": "claude-2.1",
+    "stop_reason": "end_turn",
+    "stop_sequence": "string",
+    "usage": {
+        "input_tokens": 0,
+        "output_tokens": 0
+    }
 }
diff --git a/spec/fixtures/ai-proxy/unit/real-responses/anthropic/llm-v1-chat.json b/spec/fixtures/ai-proxy/unit/real-responses/anthropic/llm-v1-chat.json
@@ -4,5 +4,9 @@
     "type": "text"
   }],
   "stop_reason": "stop_sequence",
-  "model": "claude-2.1"
+  "model": "claude-2.1",
+  "usage": {
+    "input_tokens": 0,
+    "output_tokens": 0
+  }
 }