From eb545002a868772df1c9d7805cf69b4832daa07a Mon Sep 17 00:00:00 2001 From: Qi Date: Mon, 19 Feb 2024 04:59:08 +0000 Subject: [PATCH 1/4] tests: start mocking server with random port instead of fixed port (cherry picked from commit 91ca2cfdda11ef7f9f34ac74b266a803a2da7639) --- spec/02-integration/07-sdk/03-cluster_spec.lua | 10 ++++++---- .../38-ai-proxy/02-openai_integration_spec.lua | 2 +- .../38-ai-proxy/03-anthropic_integration_spec.lua | 2 +- .../38-ai-proxy/04-cohere_integration_spec.lua | 2 +- .../38-ai-proxy/05-azure_integration_spec.lua | 2 +- .../38-ai-proxy/06-mistral_integration_spec.lua | 2 +- .../38-ai-proxy/07-llama2_integration_spec.lua | 2 +- .../38-ai-proxy/08-encoding_integration_spec.lua | 2 +- .../39-ai-request-transformer/01-transformer_spec.lua | 2 +- .../39-ai-request-transformer/02-integration_spec.lua | 2 +- .../40-ai-response-transformer/02-integration_spec.lua | 2 +- 11 files changed, 16 insertions(+), 14 deletions(-) diff --git a/spec/02-integration/07-sdk/03-cluster_spec.lua b/spec/02-integration/07-sdk/03-cluster_spec.lua index 5f592dd8272c..b7af4481cf53 100644 --- a/spec/02-integration/07-sdk/03-cluster_spec.lua +++ b/spec/02-integration/07-sdk/03-cluster_spec.lua @@ -1,4 +1,6 @@ local helpers = require("spec.helpers") +local CP_MOCK_PORT = helpers.get_available_port() +local DP_MOCK_PORT = helpers.get_available_port() local uuid_pattern = "^" .. ("%x"):rep(8) .. "%-" .. ("%x"):rep(4) .. "%-" .. ("%x"):rep(4) .. "%-" .. ("%x"):rep(4) .. "%-" @@ -10,7 +12,7 @@ local fixtures_dp = { fixtures_dp.http_mock.my_server_block = [[ server { server_name my_server; - listen 62349; + listen ]] .. DP_MOCK_PORT .. [[; location = "/hello" { content_by_lua_block { @@ -28,7 +30,7 @@ local fixtures_cp = { fixtures_cp.http_mock.my_server_block = [[ server { server_name my_server; - listen 62350; + listen ]] .. CP_MOCK_PORT .. [[; location = "/hello" { content_by_lua_block { @@ -83,7 +85,7 @@ for _, strategy in helpers.each_strategy() do end) it("kong.cluster.get_id() in Hybrid mode", function() - proxy_client = helpers.http_client(helpers.get_proxy_ip(false), 62350) + proxy_client = helpers.http_client(helpers.get_proxy_ip(false), CP_MOCK_PORT) local res = proxy_client:get("/hello") local cp_cluster_id = assert.response(res).has_status(200) @@ -93,7 +95,7 @@ for _, strategy in helpers.each_strategy() do proxy_client:close() helpers.wait_until(function() - proxy_client = helpers.http_client(helpers.get_proxy_ip(false), 62349) + proxy_client = helpers.http_client(helpers.get_proxy_ip(false), DP_MOCK_PORT) local res = proxy_client:get("/hello") local body = assert.response(res).has_status(200) proxy_client:close() diff --git a/spec/03-plugins/38-ai-proxy/02-openai_integration_spec.lua b/spec/03-plugins/38-ai-proxy/02-openai_integration_spec.lua index 914bfc9a52be..409ed8096ab9 100644 --- a/spec/03-plugins/38-ai-proxy/02-openai_integration_spec.lua +++ b/spec/03-plugins/38-ai-proxy/02-openai_integration_spec.lua @@ -3,7 +3,7 @@ local cjson = require "cjson" local pl_file = require "pl.file" local PLUGIN_NAME = "ai-proxy" -local MOCK_PORT = 62349 +local MOCK_PORT = helpers.get_available_port() for _, strategy in helpers.all_strategies() do if strategy ~= "cassandra" then describe(PLUGIN_NAME .. ": (access) [#" .. strategy .. "]", function() diff --git a/spec/03-plugins/38-ai-proxy/03-anthropic_integration_spec.lua b/spec/03-plugins/38-ai-proxy/03-anthropic_integration_spec.lua index a02d77463b39..a9feb38baecc 100644 --- a/spec/03-plugins/38-ai-proxy/03-anthropic_integration_spec.lua +++ b/spec/03-plugins/38-ai-proxy/03-anthropic_integration_spec.lua @@ -3,7 +3,7 @@ local cjson = require "cjson" local pl_file = require "pl.file" local PLUGIN_NAME = "ai-proxy" -local MOCK_PORT = 62349 +local MOCK_PORT = helpers.get_available_port() for _, strategy in helpers.all_strategies() do if strategy ~= "cassandra" then describe(PLUGIN_NAME .. ": (access) [#" .. strategy .. "]", function() diff --git a/spec/03-plugins/38-ai-proxy/04-cohere_integration_spec.lua b/spec/03-plugins/38-ai-proxy/04-cohere_integration_spec.lua index cf473505a65e..621fbcd786b6 100644 --- a/spec/03-plugins/38-ai-proxy/04-cohere_integration_spec.lua +++ b/spec/03-plugins/38-ai-proxy/04-cohere_integration_spec.lua @@ -3,7 +3,7 @@ local cjson = require "cjson" local pl_file = require "pl.file" local PLUGIN_NAME = "ai-proxy" -local MOCK_PORT = 62349 +local MOCK_PORT = helpers.get_available_port() for _, strategy in helpers.all_strategies() do if strategy ~= "cassandra" then describe(PLUGIN_NAME .. ": (access) [#" .. strategy .. "]", function() diff --git a/spec/03-plugins/38-ai-proxy/05-azure_integration_spec.lua b/spec/03-plugins/38-ai-proxy/05-azure_integration_spec.lua index f6aa33efd7a8..d976689f92aa 100644 --- a/spec/03-plugins/38-ai-proxy/05-azure_integration_spec.lua +++ b/spec/03-plugins/38-ai-proxy/05-azure_integration_spec.lua @@ -3,7 +3,7 @@ local cjson = require "cjson" local pl_file = require "pl.file" local PLUGIN_NAME = "ai-proxy" -local MOCK_PORT = 62349 +local MOCK_PORT = helpers.get_available_port() for _, strategy in helpers.all_strategies() do if strategy ~= "cassandra" then describe(PLUGIN_NAME .. ": (access) [#" .. strategy .. "]", function() diff --git a/spec/03-plugins/38-ai-proxy/06-mistral_integration_spec.lua b/spec/03-plugins/38-ai-proxy/06-mistral_integration_spec.lua index 7a82c7614fc0..16bcea29ecd5 100644 --- a/spec/03-plugins/38-ai-proxy/06-mistral_integration_spec.lua +++ b/spec/03-plugins/38-ai-proxy/06-mistral_integration_spec.lua @@ -3,7 +3,7 @@ local cjson = require "cjson" local pl_file = require "pl.file" local PLUGIN_NAME = "ai-proxy" -local MOCK_PORT = 62349 +local MOCK_PORT = helpers.get_available_port() for _, strategy in helpers.all_strategies() do if strategy ~= "cassandra" then describe(PLUGIN_NAME .. ": (access) [#" .. strategy .. "]", function() diff --git a/spec/03-plugins/38-ai-proxy/07-llama2_integration_spec.lua b/spec/03-plugins/38-ai-proxy/07-llama2_integration_spec.lua index ef0f01729766..b41aaa6e11a5 100644 --- a/spec/03-plugins/38-ai-proxy/07-llama2_integration_spec.lua +++ b/spec/03-plugins/38-ai-proxy/07-llama2_integration_spec.lua @@ -3,7 +3,7 @@ local cjson = require "cjson" local pl_file = require "pl.file" local PLUGIN_NAME = "ai-proxy" -local MOCK_PORT = 62349 +local MOCK_PORT = helpers.get_available_port() for _, strategy in helpers.all_strategies() do if strategy ~= "cassandra" then describe(PLUGIN_NAME .. ": (access) [#" .. strategy .. "]", function() diff --git a/spec/03-plugins/38-ai-proxy/08-encoding_integration_spec.lua b/spec/03-plugins/38-ai-proxy/08-encoding_integration_spec.lua index 371f99b11f2a..b11c16a973fb 100644 --- a/spec/03-plugins/38-ai-proxy/08-encoding_integration_spec.lua +++ b/spec/03-plugins/38-ai-proxy/08-encoding_integration_spec.lua @@ -3,7 +3,7 @@ local cjson = require "cjson" local inflate_gzip = require("kong.tools.gzip").inflate_gzip local PLUGIN_NAME = "ai-proxy" -local MOCK_PORT = 62349 +local MOCK_PORT = helpers.get_available_port() local openai_driver = require("kong.llm.drivers.openai") diff --git a/spec/03-plugins/39-ai-request-transformer/01-transformer_spec.lua b/spec/03-plugins/39-ai-request-transformer/01-transformer_spec.lua index 5f4bd4cdc5db..de6b0d254167 100644 --- a/spec/03-plugins/39-ai-request-transformer/01-transformer_spec.lua +++ b/spec/03-plugins/39-ai-request-transformer/01-transformer_spec.lua @@ -2,7 +2,7 @@ local llm_class = require("kong.llm") local helpers = require "spec.helpers" local cjson = require "cjson" -local MOCK_PORT = 62349 +local MOCK_PORT = helpers.get_available_port() local PLUGIN_NAME = "ai-request-transformer" local FORMATS = { diff --git a/spec/03-plugins/39-ai-request-transformer/02-integration_spec.lua b/spec/03-plugins/39-ai-request-transformer/02-integration_spec.lua index 1d0ff2a00ba7..7ddedad91fb6 100644 --- a/spec/03-plugins/39-ai-request-transformer/02-integration_spec.lua +++ b/spec/03-plugins/39-ai-request-transformer/02-integration_spec.lua @@ -1,7 +1,7 @@ local helpers = require "spec.helpers" local cjson = require "cjson" -local MOCK_PORT = 62349 +local MOCK_PORT = helpers.get_available_port() local PLUGIN_NAME = "ai-request-transformer" local OPENAI_FLAT_RESPONSE = { diff --git a/spec/03-plugins/40-ai-response-transformer/02-integration_spec.lua b/spec/03-plugins/40-ai-response-transformer/02-integration_spec.lua index 9f724629da95..40c55add51db 100644 --- a/spec/03-plugins/40-ai-response-transformer/02-integration_spec.lua +++ b/spec/03-plugins/40-ai-response-transformer/02-integration_spec.lua @@ -1,7 +1,7 @@ local helpers = require "spec.helpers" local cjson = require "cjson" -local MOCK_PORT = 62349 +local MOCK_PORT = helpers.get_available_port() local PLUGIN_NAME = "ai-response-transformer" local OPENAI_INSTRUCTIONAL_RESPONSE = { From 06bdee0d1203bbfb8c9a515ac5e3ac28ee9edb96 Mon Sep 17 00:00:00 2001 From: Qi Date: Mon, 19 Feb 2024 12:16:41 +0800 Subject: [PATCH 2/4] tests(plugin/ai-response-transformer): replace mocking server by http_mock module (cherry picked from commit 18c1b40970e4bb76b2fcf2c4b156fd13edd663a5) --- .../01-transformer_spec.lua | 155 +++++++----------- 1 file changed, 63 insertions(+), 92 deletions(-) diff --git a/spec/03-plugins/40-ai-response-transformer/01-transformer_spec.lua b/spec/03-plugins/40-ai-response-transformer/01-transformer_spec.lua index c13f9dc27eda..6409fbcafefa 100644 --- a/spec/03-plugins/40-ai-response-transformer/01-transformer_spec.lua +++ b/spec/03-plugins/40-ai-response-transformer/01-transformer_spec.lua @@ -1,8 +1,10 @@ local llm_class = require("kong.llm") local helpers = require "spec.helpers" local cjson = require "cjson" +local http_mock = require "spec.helpers.http_mock" +local pl_path = require "pl.path" -local MOCK_PORT = 62349 +local MOCK_PORT = helpers.get_available_port() local PLUGIN_NAME = "ai-response-transformer" local OPENAI_INSTRUCTIONAL_RESPONSE = { @@ -13,7 +15,7 @@ local OPENAI_INSTRUCTIONAL_RESPONSE = { options = { max_tokens = 512, temperature = 0.5, - upstream_url = "http://"..helpers.mock_upstream_host..":"..MOCK_PORT.."/instructions" + upstream_url = "http://" .. helpers.mock_upstream_host .. ":" .. MOCK_PORT .. "/instructions" }, }, auth = { @@ -55,98 +57,67 @@ local EXPECTED_RESULT = { } local SYSTEM_PROMPT = "You are a mathematician. " - .. "Multiply all numbers in my JSON request, by 2. Return me this message: " - .. "{\"status\": 400, \"headers: {\"content-type\": \"application/xml\"}, \"body\": \"OUTPUT\"} " - .. "where 'OUTPUT' is the result but transformed into XML format." - - -local client - - -for _, strategy in helpers.all_strategies() do if strategy ~= "cassandra" then - - describe(PLUGIN_NAME .. ": (unit)", function() - - lazy_setup(function() - -- set up provider fixtures - local fixtures = { - http_mock = {}, - } - - fixtures.http_mock.openai = [[ - server { - server_name llm; - listen ]]..MOCK_PORT..[[; - - default_type 'application/json'; - - location ~/instructions { - content_by_lua_block { - local pl_file = require "pl.file" - ngx.print(pl_file.read("spec/fixtures/ai-proxy/openai/request-transformer/response-with-instructions.json")) - } - } - } - ]] - - -- start kong - assert(helpers.start_kong({ - -- set the strategy - database = strategy, - -- use the custom test template to create a local mock server - nginx_conf = "spec/fixtures/custom_nginx.template", - -- make sure our plugin gets loaded - plugins = "bundled," .. PLUGIN_NAME, - -- write & load declarative config, only if 'strategy=off' - declarative_config = strategy == "off" and helpers.make_yaml_file() or nil, - }, nil, nil, fixtures)) - end) - - lazy_teardown(function() - helpers.stop_kong(nil, true) - end) - - before_each(function() - client = helpers.proxy_client() - end) + .. "Multiply all numbers in my JSON request, by 2. Return me this message: " + .. "{\"status\": 400, \"headers: {\"content-type\": \"application/xml\"}, \"body\": \"OUTPUT\"} " + .. "where 'OUTPUT' is the result but transformed into XML format." + + +describe(PLUGIN_NAME .. ": (unit)", function() + local mock + local mock_response_file = pl_path.abspath( + "spec/fixtures/ai-proxy/openai/request-transformer/response-with-instructions.json") + + lazy_setup(function() + mock = http_mock.new(tostring(MOCK_PORT), { + ["/instructions"] = { + content = string.format([[ + local pl_file = require "pl.file" + ngx.header["Content-Type"] = "application/json" + ngx.say(pl_file.read("%s")) + ]], mock_response_file), + }, + }, { + hostname = "llm", + }) - after_each(function() - if client then client:close() end - end) + assert(mock:start()) + end) - describe("openai transformer tests, specific response", function() - it("transforms request based on LLM instructions, with response transformation instructions format", function() - local llm = llm_class:new(OPENAI_INSTRUCTIONAL_RESPONSE, {}) - assert.truthy(llm) - - local result, err = llm:ai_introspect_body( - REQUEST_BODY, -- request body - SYSTEM_PROMPT, -- conf.prompt - {}, -- http opts - nil -- transformation extraction pattern (loose json) - ) - - assert.is_nil(err) - - local table_result, err = cjson.decode(result) - assert.is_nil(err) - assert.same(EXPECTED_RESULT, table_result) - - -- parse in response string format - local headers, body, status, err = llm:parse_json_instructions(result) - assert.is_nil(err) - assert.same({ ["content-type"] = "application/xml"}, headers) - assert.same(209, status) - assert.same(EXPECTED_RESULT.body, body) - - -- parse in response table format - headers, body, status, err = llm:parse_json_instructions(table_result) - assert.is_nil(err) - assert.same({ ["content-type"] = "application/xml"}, headers) - assert.same(209, status) - assert.same(EXPECTED_RESULT.body, body) - end) + lazy_teardown(function() + assert(mock:stop()) + end) + describe("openai transformer tests, specific response", function() + it("transforms request based on LLM instructions, with response transformation instructions format", function() + local llm = llm_class:new(OPENAI_INSTRUCTIONAL_RESPONSE, {}) + assert.truthy(llm) + + local result, err = llm:ai_introspect_body( + REQUEST_BODY, -- request body + SYSTEM_PROMPT, -- conf.prompt + {}, -- http opts + nil -- transformation extraction pattern (loose json) + ) + + assert.is_nil(err) + + local table_result, err = cjson.decode(result) + assert.is_nil(err) + assert.same(EXPECTED_RESULT, table_result) + + -- parse in response string format + local headers, body, status, err = llm:parse_json_instructions(result) + assert.is_nil(err) + assert.same({ ["content-type"] = "application/xml" }, headers) + assert.same(209, status) + assert.same(EXPECTED_RESULT.body, body) + + -- parse in response table format + headers, body, status, err = llm:parse_json_instructions(table_result) + assert.is_nil(err) + assert.same({ ["content-type"] = "application/xml" }, headers) + assert.same(209, status) + assert.same(EXPECTED_RESULT.body, body) end) end) -end end +end) From 1179fc46c2b2814e0ea29fd796073194734d6525 Mon Sep 17 00:00:00 2001 From: Qi Date: Mon, 19 Feb 2024 05:29:18 +0000 Subject: [PATCH 3/4] tests(hybrid): reset and bootstrap DB before starting CP Some tests might change the DB in front of this test, which causes incompatible data to prevent the CP from starting up. (cherry picked from commit df48729b257e4ccb15d84b4fc428cf7cec38e40d) --- spec/02-integration/09-hybrid_mode/02-start_stop_spec.lua | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/spec/02-integration/09-hybrid_mode/02-start_stop_spec.lua b/spec/02-integration/09-hybrid_mode/02-start_stop_spec.lua index 91ec0eb72a79..43cbf6ec988e 100644 --- a/spec/02-integration/09-hybrid_mode/02-start_stop_spec.lua +++ b/spec/02-integration/09-hybrid_mode/02-start_stop_spec.lua @@ -154,7 +154,10 @@ end) describe("when CP exits before DP", function() local need_exit = true - setup(function() + lazy_setup(function() + -- reset and bootstrap DB before starting CP + helpers.get_db_utils(nil) + assert(helpers.start_kong({ role = "control_plane", prefix = "servroot1", @@ -179,7 +182,7 @@ describe("when CP exits before DP", function() })) end) - teardown(function() + lazy_teardown(function() if need_exit then helpers.stop_kong("servroot1") end From 0f23b74b8e04a656ab1bfa4bc90efc17f0579583 Mon Sep 17 00:00:00 2001 From: Qi Date: Mon, 19 Feb 2024 06:28:43 +0000 Subject: [PATCH 4/4] tests(plugin/ai-request-transformer): replace mocking server by http_mock module (cherry picked from commit acffb9d52ec1ec25a11b80f7e4887b06e8fb38f6) --- .../01-transformer_spec.lua | 221 ++++++++---------- 1 file changed, 95 insertions(+), 126 deletions(-) diff --git a/spec/03-plugins/39-ai-request-transformer/01-transformer_spec.lua b/spec/03-plugins/39-ai-request-transformer/01-transformer_spec.lua index de6b0d254167..db1aef512b0c 100644 --- a/spec/03-plugins/39-ai-request-transformer/01-transformer_spec.lua +++ b/spec/03-plugins/39-ai-request-transformer/01-transformer_spec.lua @@ -1,6 +1,8 @@ local llm_class = require("kong.llm") local helpers = require "spec.helpers" local cjson = require "cjson" +local http_mock = require "spec.helpers.http_mock" +local pl_path = require "pl.path" local MOCK_PORT = helpers.get_available_port() local PLUGIN_NAME = "ai-request-transformer" @@ -14,7 +16,7 @@ local FORMATS = { options = { max_tokens = 512, temperature = 0.5, - upstream_url = "http://"..helpers.mock_upstream_host..":"..MOCK_PORT.."/chat/openai" + upstream_url = "http://" .. helpers.mock_upstream_host .. ":" .. MOCK_PORT .. "/chat/openai" }, }, auth = { @@ -30,7 +32,7 @@ local FORMATS = { options = { max_tokens = 512, temperature = 0.5, - upstream_url = "http://"..helpers.mock_upstream_host..":"..MOCK_PORT.."/chat/cohere" + upstream_url = "http://" .. helpers.mock_upstream_host .. ":" .. MOCK_PORT .. "/chat/cohere" }, }, auth = { @@ -46,7 +48,7 @@ local FORMATS = { options = { max_tokens = 512, temperature = 0.5, - upstream_url = "http://"..helpers.mock_upstream_host..":"..MOCK_PORT.."/chat/anthropic" + upstream_url = "http://" .. helpers.mock_upstream_host .. ":" .. MOCK_PORT .. "/chat/anthropic" }, }, auth = { @@ -62,7 +64,7 @@ local FORMATS = { options = { max_tokens = 512, temperature = 0.5, - upstream_url = "http://"..helpers.mock_upstream_host..":"..MOCK_PORT.."/chat/azure" + upstream_url = "http://" .. helpers.mock_upstream_host .. ":" .. MOCK_PORT .. "/chat/azure" }, }, auth = { @@ -78,7 +80,7 @@ local FORMATS = { options = { max_tokens = 512, temperature = 0.5, - upstream_url = "http://"..helpers.mock_upstream_host..":"..MOCK_PORT.."/chat/llama2", + upstream_url = "http://" .. helpers.mock_upstream_host .. ":" .. MOCK_PORT .. "/chat/llama2", llama2_format = "raw", }, }, @@ -95,7 +97,7 @@ local FORMATS = { options = { max_tokens = 512, temperature = 0.5, - upstream_url = "http://"..helpers.mock_upstream_host..":"..MOCK_PORT.."/chat/mistral", + upstream_url = "http://" .. helpers.mock_upstream_host .. ":" .. MOCK_PORT .. "/chat/mistral", mistral_format = "ollama", }, }, @@ -114,7 +116,7 @@ local OPENAI_NOT_JSON = { options = { max_tokens = 512, temperature = 0.5, - upstream_url = "http://"..helpers.mock_upstream_host..":"..MOCK_PORT.."/not-json" + upstream_url = "http://" .. helpers.mock_upstream_host .. ":" .. MOCK_PORT .. "/not-json" }, }, auth = { @@ -152,131 +154,77 @@ local EXPECTED_RESULT = { } local SYSTEM_PROMPT = "You are a mathematician. " - .. "Multiply all numbers in my JSON request, by 2. Return me the JSON output only" + .. "Multiply all numbers in my JSON request, by 2. Return me the JSON output only" -local client +describe(PLUGIN_NAME .. ": (unit)", function() + local mock + local ai_proxy_fixtures_dir = pl_path.abspath("spec/fixtures/ai-proxy/") + lazy_setup(function() + mock = http_mock.new(MOCK_PORT, { + ["~/chat/(?[a-z0-9]+)"] = { + content = string.format([[ + local base_dir = "%s/" + ngx.header["Content-Type"] = "application/json" -for _, strategy in helpers.all_strategies() do if strategy ~= "cassandra" then + local pl_file = require "pl.file" + local json = require("cjson.safe") + ngx.req.read_body() + local body, err = ngx.req.get_body_data() + body, err = json.decode(body) - describe(PLUGIN_NAME .. ": (unit)", function() - - lazy_setup(function() - -- set up provider fixtures - local fixtures = { - http_mock = {}, - } - - fixtures.http_mock.openai = [[ - server { - server_name llm; - listen ]]..MOCK_PORT..[[; - - default_type 'application/json'; - - location ~/chat/(?[a-z0-9]+) { - content_by_lua_block { - local pl_file = require "pl.file" - local json = require("cjson.safe") + local token = ngx.req.get_headers()["authorization"] + local token_query = ngx.req.get_uri_args()["apikey"] + if token == "Bearer " .. ngx.var.provider .. "-key" or token_query == "$1-key" or body.apikey == "$1-key" then ngx.req.read_body() local body, err = ngx.req.get_body_data() body, err = json.decode(body) - local token = ngx.req.get_headers()["authorization"] - local token_query = ngx.req.get_uri_args()["apikey"] + if err or (body.messages == ngx.null) then + ngx.status = 400 + ngx.say(pl_file.read(base_dir .. ngx.var.provider .. "/llm-v1-chat/responses/bad_request.json")) - if token == "Bearer " .. ngx.var.provider .. "-key" or token_query == "$1-key" or body.apikey == "$1-key" then - ngx.req.read_body() - local body, err = ngx.req.get_body_data() - body, err = json.decode(body) - - if err or (body.messages == ngx.null) then - ngx.status = 400 - ngx.print(pl_file.read("spec/fixtures/ai-proxy/" .. ngx.var.provider .. "/llm-v1-chat/responses/bad_request.json")) - else - ngx.status = 200 - ngx.print(pl_file.read("spec/fixtures/ai-proxy/" .. ngx.var.provider .. "/request-transformer/response-in-json.json")) - end else - ngx.status = 401 - ngx.print(pl_file.read("spec/fixtures/ai-proxy/" .. ngx.var.provider .. "/llm-v1-chat/responses/unauthorized.json")) + ngx.status = 200 + ngx.say(pl_file.read(base_dir .. ngx.var.provider .. "/request-transformer/response-in-json.json")) end - } - } - - location ~/not-json { - content_by_lua_block { - local pl_file = require "pl.file" - ngx.print(pl_file.read("spec/fixtures/ai-proxy/openai/request-transformer/response-not-json.json")) - } - } - } - ]] - - -- start kong - assert(helpers.start_kong({ - -- set the strategy - database = strategy, - -- use the custom test template to create a local mock server - nginx_conf = "spec/fixtures/custom_nginx.template", - -- make sure our plugin gets loaded - plugins = "bundled," .. PLUGIN_NAME, - -- write & load declarative config, only if 'strategy=off' - declarative_config = strategy == "off" and helpers.make_yaml_file() or nil, - }, nil, nil, fixtures)) - end) - - lazy_teardown(function() - helpers.stop_kong(nil, true) - end) - - before_each(function() - client = helpers.proxy_client() - end) - - after_each(function() - if client then client:close() end - end) - - for name, format_options in pairs(FORMATS) do - - describe(name .. " transformer tests, exact json response", function() - - it("transforms request based on LLM instructions", function() - local llm = llm_class:new(format_options, {}) - assert.truthy(llm) - local result, err = llm:ai_introspect_body( - REQUEST_BODY, -- request body - SYSTEM_PROMPT, -- conf.prompt - {}, -- http opts - nil -- transformation extraction pattern - ) - - assert.is_nil(err) - - result, err = cjson.decode(result) - assert.is_nil(err) + else + ngx.status = 401 + ngx.say(pl_file.read(base_dir .. ngx.var.provider .. "/llm-v1-chat/responses/unauthorized.json")) + end + ]], ai_proxy_fixtures_dir), + }, + ["~/not-json"] = { + content = string.format([[ + local base_dir = "%s/" + local pl_file = require "pl.file" + ngx.header["Content-Type"] = "application/json" + ngx.print(pl_file.read(base_dir .. "openai/request-transformer/response-not-json.json")) + ]], ai_proxy_fixtures_dir), + }, + }) - assert.same(EXPECTED_RESULT, result) - end) - end) + assert(mock:start()) + end) - - end + lazy_teardown(function() + assert(mock:stop()) + end) - describe("openai transformer tests, pattern matchers", function() - it("transforms request based on LLM instructions, with json extraction pattern", function() - local llm = llm_class:new(OPENAI_NOT_JSON, {}) + for name, format_options in pairs(FORMATS) do + describe(name .. " transformer tests, exact json response", function() + it("transforms request based on LLM instructions", function() + local llm = llm_class:new(format_options, {}) assert.truthy(llm) local result, err = llm:ai_introspect_body( - REQUEST_BODY, -- request body - SYSTEM_PROMPT, -- conf.prompt - {}, -- http opts - "\\{((.|\n)*)\\}" -- transformation extraction pattern (loose json) + REQUEST_BODY, -- request body + SYSTEM_PROMPT, -- conf.prompt + {}, -- http opts + nil -- transformation extraction pattern ) assert.is_nil(err) @@ -286,22 +234,43 @@ for _, strategy in helpers.all_strategies() do if strategy ~= "cassandra" then assert.same(EXPECTED_RESULT, result) end) + end) + end - it("transforms request based on LLM instructions, but fails to match pattern", function() - local llm = llm_class:new(OPENAI_NOT_JSON, {}) - assert.truthy(llm) + describe("openai transformer tests, pattern matchers", function() + it("transforms request based on LLM instructions, with json extraction pattern", function() + local llm = llm_class:new(OPENAI_NOT_JSON, {}) + assert.truthy(llm) - local result, err = llm:ai_introspect_body( - REQUEST_BODY, -- request body - SYSTEM_PROMPT, -- conf.prompt - {}, -- http opts - "\\#*\\=" -- transformation extraction pattern (loose json) - ) + local result, err = llm:ai_introspect_body( + REQUEST_BODY, -- request body + SYSTEM_PROMPT, -- conf.prompt + {}, -- http opts + "\\{((.|\n)*)\\}" -- transformation extraction pattern (loose json) + ) - assert.is_nil(result) - assert.is_not_nil(err) - assert.same("AI response did not match specified regular expression", err) - end) + assert.is_nil(err) + + result, err = cjson.decode(result) + assert.is_nil(err) + + assert.same(EXPECTED_RESULT, result) end) + + it("transforms request based on LLM instructions, but fails to match pattern", function() + local llm = llm_class:new(OPENAI_NOT_JSON, {}) + assert.truthy(llm) + + local result, err = llm:ai_introspect_body( + REQUEST_BODY, -- request body + SYSTEM_PROMPT, -- conf.prompt + {}, -- http opts + "\\#*\\=" -- transformation extraction pattern (loose json) + ) + + assert.is_nil(result) + assert.is_not_nil(err) + assert.same("AI response did not match specified regular expression", err) + end) -- it end) -end end +end)