From 01a30455410184dc629fa9243ed66143c7c07f59 Mon Sep 17 00:00:00 2001 From: Michael Martin <3277009+flrgh@users.noreply.github.com> Date: Thu, 19 Jan 2023 17:35:12 -0800 Subject: [PATCH] feat(dbless): improve validation errors from /config --- kong/api/routes/config.lua | 37 +- kong/db/declarative/init.lua | 16 +- kong/db/errors.lua | 417 ++++++ kong/db/schema/others/declarative_config.lua | 14 +- .../04-admin_api/15-off_spec.lua | 1282 ++++++++++++++++- 5 files changed, 1753 insertions(+), 13 deletions(-) diff --git a/kong/api/routes/config.lua b/kong/api/routes/config.lua index e84d1c648fa6..49347a93a569 100644 --- a/kong/api/routes/config.lua +++ b/kong/api/routes/config.lua @@ -24,6 +24,16 @@ local function reports_timer(premature) end +local function truthy(val) + return val == true + or val == 1 + or val == "true" + or val == "1" + or val == "on" + or val == "yes" +end + + return { ["/config"] = { GET = function(self, db) @@ -57,6 +67,19 @@ return { }) end + local opts + local flatten_errors = truthy(self.params.flatten_errors) + self.params.flatten_errors = nil + local processed_input + if flatten_errors then + opts = { + handle_error = function(input, err_t) + processed_input = input + return nil, err_t + end, + } + end + local check_hash, old_hash if tostring(self.params.check_hash) == "1" then check_hash = true @@ -68,7 +91,7 @@ return { local entities, _, err_t, meta, new_hash if self.params._format_version then - entities, _, err_t, meta, new_hash = dc:parse_table(self.params) + entities, _, err_t, meta, new_hash = dc:parse_table(self.params, nil, opts) else local config = self.params.config if not config then @@ -82,14 +105,22 @@ return { end end entities, _, err_t, meta, new_hash = - dc:parse_string(config, nil, old_hash) + dc:parse_string(config, nil, old_hash, opts) end if not entities then if check_hash and err_t and err_t.error == "configuration is identical" then return kong.response.exit(304) end - return kong.response.exit(400, errors:declarative_config(err_t)) + + local res + if flatten_errors then + res = errors:declarative_config_flattened(err_t, processed_input) + else + res = errors:declarative_config(err_t) + end + + return kong.response.exit(400, res) end local ok, err, ttl = declarative.load_into_cache_with_events(entities, meta, new_hash) diff --git a/kong/db/declarative/init.lua b/kong/db/declarative/init.lua index 19c496e09fc2..1ca18fc140a7 100644 --- a/kong/db/declarative/init.lua +++ b/kong/db/declarative/init.lua @@ -74,6 +74,8 @@ local function pretty_print_error(err_t, item, indent) end + +-- @tparam table|nil opts options to pass to the parser -- @treturn table|nil a table with the following format: -- { -- services: { @@ -89,7 +91,7 @@ end -- _format_version: "2.1", -- _transform: true, -- } -function _M:parse_file(filename, old_hash) +function _M:parse_file(filename, old_hash, opts) if type(filename) ~= "string" then error("filename must be a string", 2) end @@ -99,7 +101,7 @@ function _M:parse_file(filename, old_hash) return nil, err end - return self:parse_string(contents, filename, old_hash) + return self:parse_string(contents, filename, old_hash, opts) end @@ -114,6 +116,7 @@ end -- @tparam string contents the json/yml/lua being parsed -- @tparam string|nil filename. If nil, json will be tried first, then yaml -- @tparam string|nil old_hash used to avoid loading the same content more than once, if present +-- @tparam table|nil opts options to pass to the parser -- @treturn nil|string error message, only if error happened -- @treturn nil|table err_t, only if error happened -- @treturn table|nil a table with the following format: @@ -121,7 +124,7 @@ end -- _format_version: "2.1", -- _transform: true, -- } -function _M:parse_string(contents, filename, old_hash) +function _M:parse_string(contents, filename, old_hash, opts) -- we don't care about the strength of the hash -- because declarative config is only loaded by Kong administrators, -- not outside actors that could exploit it for collisions @@ -171,7 +174,7 @@ function _M:parse_string(contents, filename, old_hash) return nil, err, { error = err } end - return self:parse_table(dc_table, new_hash) + return self:parse_table(dc_table, new_hash, opts) end @@ -185,6 +188,7 @@ end -- }, -- } -- This table is not flattened: entities can exist inside other entities +-- @tparam table|nil opts options to pass to the parser -- @treturn table|nil A table with the following format: -- { -- services: { @@ -202,12 +206,12 @@ end -- } -- @treturn string|nil given hash if everything went well, -- new hash if everything went well and no given hash, -function _M:parse_table(dc_table, hash) +function _M:parse_table(dc_table, hash, opts) if type(dc_table) ~= "table" then error("expected a table as input", 2) end - local entities, err_t, meta = self.schema:flatten(dc_table) + local entities, err_t, meta = self.schema:flatten(dc_table, opts) if err_t then return nil, pretty_print_error(err_t), err_t end diff --git a/kong/db/errors.lua b/kong/db/errors.lua index 3dc9a7d0dbad..65f336d55e08 100644 --- a/kong/db/errors.lua +++ b/kong/db/errors.lua @@ -1,9 +1,14 @@ local pl_pretty = require("pl.pretty").write local pl_keys = require("pl.tablex").keys +local nkeys = require("table.nkeys") +local is_array = require("table.isarray") +local utils = require("kong.tools.utils") local type = type local null = ngx.null +local log = ngx.log +local WARN = ngx.WARN local error = error local upper = string.upper local fmt = string.format @@ -14,6 +19,7 @@ local setmetatable = setmetatable local getmetatable = getmetatable local concat = table.concat local sort = table.sort +local insert = table.insert local sorted_keys = function(tbl) @@ -510,4 +516,415 @@ function _M:invalid_unique_global(name) end +local flatten_errors +do + local function singular(noun) + if noun:sub(-1) == "s" then + return noun:sub(1, -2) + end + return noun + end + + + local function join(ns, field) + if type(ns) == "string" and ns ~= "" then + return ns .. "." .. field + end + return field + end + + + local each_foreign_field + do + ---@type table + local relationships + + -- for each known entity, build a table of other entities which may + -- reference it via a foreign key relationship as well as any of its + -- own foreign key relationships. + local function build_relationships() + relationships = setmetatable({}, { + __index = function(self, k) + local t = {} + rawset(self, k, t) + return t + end, + }) + + for entity, dao in pairs(kong.db.daos) do + for fname, field in dao.schema:each_field() do + if field.type == "foreign" then + insert(relationships[entity], { + field = fname, + entity = entity, + reference = field.reference, + }) + + -- create a backref for entities that may be nested under their + -- foreign key reference entity (one-to-many relationships) + -- + -- example: services and routes + -- + -- route.service = { type = "foreign", reference = "services" } + -- + -- insert(relationships.services, { + -- field = "service", + -- entity = "routes", + -- reference = "services", + -- }) + -- + insert(relationships[field.reference], { + field = fname, + entity = entity, + reference = field.reference, + }) + end + end + end + end + + local empty = function() end + + ---@param entity_type string + ---@return fun():{ field:string, entity:string, reference:string }? iterator + function each_foreign_field(entity_type) + -- this module is require()-ed before the kong global is initialized, so + -- the lookup table of relationships needs to be built lazily + if not relationships then + build_relationships() + end + + local fields = relationships[entity_type] + + if not fields then + return empty + end + + local i = 0 + return function() + i = i + 1 + return fields[i] + end + end + end + + + ---@param err table|string + ---@param flattened table + local function add_entity_error(err, flattened) + if type(err) == "table" then + for _, message in ipairs(err) do + add_entity_error(message, flattened) + end + + else + insert(flattened, { + type = "entity", + message = err, + }) + end + end + + + ---@param field string + ---@param err table|string + ---@param flattened table + local function add_field_error(field, err, flattened) + if type(err) == "table" then + for _, message in ipairs(err) do + add_field_error(field, message, flattened) + end + + else + insert(flattened, { + type = "field", + field = field, + message = err, + }) + end + end + + + ---@param errs table + ---@param ns? string + ---@param flattened? table + local function categorize_errors(errs, ns, flattened) + flattened = flattened or {} + + for field, err in pairs(errs) do + local errtype = type(err) + + if field == "@entity" then + add_entity_error(err, flattened) + + elseif errtype == "string" then + add_field_error(join(ns, field), err, flattened) + + elseif errtype == "table" then + categorize_errors(err, join(ns, field), flattened) + + else + log(WARN, "unknown error type: ", errtype, " at key: ", field) + end + end + + return flattened + end + + + ---@param name any + ---@return string|nil + local function validate_name(name) + return (type(name) == "string" + and name:len() > 0 + and name) + or nil + end + + + ---@param id any + ---@return string|nil + local function validate_id(id) + return (type(id) == "string" + and utils.is_valid_uuid(id) + and id) + or nil + end + + + ---@param tags any + ---@return string[]|nil + local function validate_tags(tags) + if type(tags) == "table" and is_array(tags) then + for i = 1, #tags do + if type(tags[i]) ~= "string" then + return + end + end + + return tags + end + end + + + ---@param entity_type string + ---@param entity table + ---@param err_t table + ---@param flattened table + local function add_entity_errors(entity_type, entity, err_t, flattened) + if type(err_t) ~= "table" or nkeys(err_t) == 0 then + return + end + + if is_array(entity) then + for i = 1, #entity do + add_entity_errors(entity_type, entity[i], err_t[i], flattened) + end + return + end + + -- promote errors for foreign key relationships up to the top level + -- array of errors and recursively flatten any of their validation + -- errors + for ref in each_foreign_field(entity_type) do + local field_name + local field_entity_type + + -- owned one-to-one relationship (e.g. service->client_certificate) + if ref.entity == entity_type then + field_name = ref.field + field_entity_type = ref.reference + + -- foreign one-to-many relationship (e.g. service->routes) + else + field_name = ref.entity + field_entity_type = field_name + end + + local field_value = entity[field_name] + local field_err_t = err_t[field_name] + + err_t[field_name] = nil + entity[field_name] = nil + + if type(field_value) == "table" and field_value.id then + entity[field_name] = { id = field_value.id } + end + + if type(field_value) == "table" and type(field_err_t) == "table" then + add_entity_errors(field_entity_type, field_value, field_err_t, flattened) + end + end + + -- all of our errors were related to foreign relationships + if nkeys(err_t) == 0 then + return + end + + local entity_errors = categorize_errors(err_t) + if #entity_errors > 0 then + insert(flattened, { + -- entity_id, entity_name, and entity_tags must be validated to ensure + -- that the response is well-formed. They are also optional, so we will + -- simply leave them out if they are invalid. + -- + -- The nested entity object itself will retain the original, untouched + -- values for these fields. + entity_name = validate_name(entity.name), + entity_id = validate_id(entity.id), + entity_tags = validate_tags(entity.tags), + entity_type = singular(entity_type), + entity = entity, + errors = entity_errors, + }) + else + log(WARN, "failed to categorize errors for ", entity_type, + ", ", entity.name or entity.id) + end + end + + + -- traverse declarative schema validation errors and correlate them with + -- objects/entities from the original user input + -- + -- Produces a list of errors with the following format: + -- + -- ```lua + -- { + -- entity_type = "service", -- service, route, plugin, etc + -- entity_id = "", -- useful to correlate errors across fk relationships + -- entity_name = "my-service", -- may be nil + -- entity_tags = { "my-tag" }, + -- entity = { -- the full entity object + -- name = "my-service", + -- id = "", + -- tags = { "my-tag" }, + -- host = "127.0.0.1", + -- protocol = "tcp", + -- path = "/path", + -- }, + -- errors = { + -- { + -- type = "entity" + -- message = "failed conditional validation given value of field 'protocol'", + -- }, + -- { + -- type = "field" + -- field = "path", + -- message = "value must be null", + -- } + -- } + -- } + -- ``` + -- + -- Nested foreign relationships are hoisted up to the top level, so + -- given the following input: + -- + -- ```lua + -- { + -- services = { + -- name = "matthew", + -- url = "http:/127.0.0.1:80/", + -- routes = { + -- { + -- name = "joshua", + -- protocols = { "nope" }, -- invalid protocol + -- } + -- }, + -- plugins = { + -- { + -- name = "i-am-not-a-real-plugin", -- nonexistent plugin + -- config = { + -- foo = "bar", + -- }, + -- }, + -- { + -- name = "http-log", + -- config = {}, -- missing required field(s) + -- }, + -- }, + -- } + -- } + -- ``` + -- ... the output error array will have three entries, one for the route, + -- and one for each of the plugins. + -- + -- Errors for record fields and nested schema properties are rolled up and + -- added to their parent entity, with the full path to the property + -- represented as a period-delimited string: + -- + -- ```lua + -- { + -- entity_type = "plugin", + -- entity_name = "http-log", + -- entity = { + -- name = "http-log", + -- config = { + -- -- empty + -- }, + -- }, + -- errors = { + -- { + -- field = "config.http_endpoint", + -- message = "missing host in url", + -- type = "field" + -- } + -- }, + -- } + -- ``` + function flatten_errors(input, err_t) + local flattened = {} + + for entity_type, section_errors in pairs(err_t) do + if type(section_errors) ~= "table" then + log(WARN, "failed to resolve errors for ", entity_type) + goto next_section + end + + local entities = input[entity_type] + + if type(entities) ~= "table" then + log(WARN, "failed to resolve errors for ", entity_type) + goto next_section + end + + for idx, errs in pairs(section_errors) do + local entity = entities[idx] + + if type(entity) == "table" then + add_entity_errors(entity_type, entity, errs, flattened) + + else + log(WARN, "failed to resolve errors for ", entity_type, " at ", + "index '", idx, "'") + end + end + + ::next_section:: + end + + return flattened + end +end + + +function _M:declarative_config_flattened(err_t, input) + if type(err_t) ~= "table" then + error("err_t must be a table", 2) + end + + if type(input) ~= "table" then + error("err_t input is nil or not a table", 2) + end + + local flattened = flatten_errors(input, err_t) + + err_t = self:declarative_config(err_t) + + err_t.flattened = flattened + + return err_t +end + + return _M diff --git a/kong/db/schema/others/declarative_config.lua b/kong/db/schema/others/declarative_config.lua index 4470b1516e77..39828a8908ab 100644 --- a/kong/db/schema/others/declarative_config.lua +++ b/kong/db/schema/others/declarative_config.lua @@ -672,7 +672,15 @@ local function insert_default_workspace_if_not_given(_, entities) end -local function flatten(self, input) +local function new_error(input, err_t, opts) + if opts and opts.handle_error then + return opts.handle_error(input, err_t) + end + return nil, err_t +end + + +local function flatten(self, input, opts) -- manually set transform here -- we can't do this in the schema with a `default` because validate -- needs to happen before process_auto_fields, which @@ -697,7 +705,7 @@ local function flatten(self, input) local ok2, err2 = self.full_schema:validate(input_copy) if not ok2 then local err3 = utils.deep_merge(err2, extract_null_errors(err)) - return nil, err3 + return new_error(input_copy, err3, opts) end yield() @@ -713,7 +721,7 @@ local function flatten(self, input) local by_id, by_key = validate_references(self, processed) if not by_id then - return nil, by_key + return new_error(processed, by_key, opts) end yield() diff --git a/spec/02-integration/04-admin_api/15-off_spec.lua b/spec/02-integration/04-admin_api/15-off_spec.lua index c70323a67916..ea73c6287dd2 100644 --- a/spec/02-integration/04-admin_api/15-off_spec.lua +++ b/spec/02-integration/04-admin_api/15-off_spec.lua @@ -5,7 +5,9 @@ local pl_utils = require "pl.utils" local helpers = require "spec.helpers" local Errors = require "kong.db.errors" local mocker = require("spec.fixtures.mocker") - +local deepcompare = require("pl.tablex").deepcompare +local inspect = require "inspect" +local nkeys = require "table.nkeys" local WORKER_SYNC_TIMEOUT = 10 local LMDB_MAP_SIZE = "10m" @@ -861,6 +863,1284 @@ describe("Admin API #off", function() end) end) + +describe("Admin API #off /config [flattened errors]", function() + local client + local tags + + local function make_tag_t(name) + return setmetatable({ + name = name, + count = 0, + last = nil, + }, { + __index = function(self, k) + if k == "next" then + self.count = self.count + 1 + local tag = ("%s-%02d"):format(self.name, self.count) + self.last = tag + return tag + else + error("unknown key: " .. k) + end + end, + }) + end + + lazy_setup(function() + assert(helpers.start_kong({ + database = "off", + lmdb_map_size = LMDB_MAP_SIZE, + stream_listen = "127.0.0.1:9011", + nginx_conf = "spec/fixtures/custom_nginx.template", + plugins = "bundled", + vaults = "bundled", + })) + end) + + + lazy_teardown(function() + helpers.stop_kong() + end) + + before_each(function() + client = assert(helpers.admin_client()) + helpers.clean_logfile() + + tags = setmetatable({}, { + __index = function(self, k) + self[k] = make_tag_t(k) + return self[k] + end, + }) + end) + + after_each(function() + if client then + client:close() + end + end) + + local function sort_errors(t) + if type(t) ~= "table" then + return + end + table.sort(t, function(a, b) + if a.type ~= b.type then + return a.type < b.type + end + + if a.field ~= b.field then + return a.field < b.field + end + + return a.message < b.message + end) + end + + + local function is_fk(value) + return type(value) == "table" + and nkeys(value) == 1 + and value.id ~= nil + end + + local compare_entity + + local function compare_field(field, exp, got, diffs) + -- Entity IDs are a special case + -- + -- In general, we don't want to bother comparing them because they + -- are going to be auto-generated at random for each test run. The + -- exception to this rule is that when the expected data explicitly + -- specifies an ID, we want to compare it. + if field == "entity_id" or field == "id" or is_fk(got) then + if exp == nil then + got = nil + + elseif exp == ngx.null then + exp = nil + end + + elseif field == "entity" then + return compare_entity(exp, got, diffs) + + -- sort the errors array; its order is not guaranteed and does not + -- really matter, so sorting is just for ease of deep comparison + elseif field == "errors" then + sort_errors(exp) + sort_errors(got) + end + + if not deepcompare(exp, got) then + if diffs then + table.insert(diffs, field) + end + return false + end + + return true + end + + function compare_entity(exp, got, diffs) + local seen = {} + + for field in pairs(exp) do + if not compare_field(field, exp[field], got[field]) + then + table.insert(diffs, "entity." .. field) + end + seen[field] = true + end + + for field in pairs(got) do + -- NOTE: certain fields may be present in the actual response + -- but missing from the expected response (e.g. `id`) + if not seen[field] and + not compare_field(field, exp[field], got[field]) + then + table.insert(diffs, "entity." .. field) + end + end + end + + local function compare(exp, got, diffs) + if type(exp) ~= "table" or type(got) ~= "table" then + return exp == got + end + + local seen = {} + + for field in pairs(exp) do + seen[field] = true + compare_field(field, exp[field], got[field], diffs) + end + + for field in pairs(got) do + if not seen[field] then + compare_field(field, exp[field], got[field], diffs) + end + end + + return #diffs == 0 + end + + local function get_by_tag(tag, haystack) + if type(tag) == "table" then + tag = tag[1] + end + + for i = 1, #haystack do + local item = haystack[i] + if item.entity.tags and + item.entity.tags[1] == tag + then + return table.remove(haystack, i) + end + end + end + + local function find(needle, haystack) + local tag = needle.entity + and needle.entity.tags + and needle.entity.tags[1] + if not tag then + return + end + + return get_by_tag(tag, haystack) + end + + + local function post_config(config, debug) + config._format_version = config._format_version or "3.0" + + local res = client:post("/config?flatten_errors=1", { + body = config, + headers = { + ["Content-Type"] = "application/json" + }, + }) + + assert.response(res).has.status(400) + local body = assert.response(res).has.jsonbody() + + local flattened = body.flattened + + assert.not_nil(flattened, "response.flattened is missing") + assert.is_table(flattened, "response.flattened is not a table") + + if debug then + helpers.intercept(flattened) + end + return flattened + end + + + -- Testing Methodology: + -- + -- 1. Iterate through each array (expected, received) + -- 2. Correlate expected and received entries by comparing the first + -- entity tag of each + -- 3. Compare the two entries + + local function validate(expected, received) + local errors = {} + + while #expected > 0 do + local exp = table.remove(expected) + local got = find(exp, received) + local diffs = {} + if not compare(exp, got, diffs) then + table.insert(errors, { exp = exp, got = got, diffs = diffs }) + end + end + + -- everything left in flattened is an unexpected, extra entry + for _, got in ipairs(received) do + assert.is_nil(find(got, expected)) + table.insert(errors, { got = got }) + end + + if #errors > 0 then + local msg = {} + + for i, err in ipairs(errors) do + local exp, got = err.exp, err.got + + table.insert(msg, ("\n======== Error #%00d ========\n"):format(i)) + + if not exp then + table.insert(msg, "Unexpected entry:\n") + table.insert(msg, inspect(got)) + table.insert(msg, "\n") + + elseif not got then + table.insert(msg, "Missing entry:\n") + table.insert(msg, inspect(exp)) + table.insert(msg, "\n") + + else + table.insert(msg, "Expected:\n\n") + table.insert(msg, inspect(exp)) + table.insert(msg, "\n\n") + table.insert(msg, "Got:\n\n") + table.insert(msg, inspect(got)) + table.insert(msg, "\n\n") + + table.insert(msg, "Unmatched Fields:\n") + for _, field in ipairs(err.diffs) do + table.insert(msg, (" - %s\n"):format(field)) + end + end + + table.insert(msg, "\n") + end + + assert.equals(0, #errors, table.concat(msg)) + end + end + + + + it("sanity", function() + -- Test Cases + -- + -- The first tag string in the entity tags table is a unique ID for + -- that entity. This allows the test code to locate and correlate + -- each item in the actual response to one in the expected response + -- when deepcompare() will not consider the entries to be equivalent. + -- + -- Use the tag helper table to generate this tag for each entity you + -- add to the input (`tag.ENTITY_NAME.next`): + -- + -- tags = { tags.consumer.next } -> { "consumer-01" } + -- tags = { tags.consumer.next } -> { "consumer-02" } + -- + -- You can use `tag.ENTITY_NAME.last` if you want to refer to the last + -- ID that was generated for an entity type. This has no special + -- meaning in the tests, but it can be helpful in correlating an entity + -- with its parent when debugging: + -- + -- services = { + -- { + -- name = "foo", + -- tags = { tags.service.next }, -- > "service-01", + -- routes = { + -- tags = { + -- tags.route_service.next, -- > "route_service-01", + -- tags.service.last -- > "service-01", + -- }, + -- } + -- } + -- } + -- + -- Additional tags can be added after the first one, and they will be + -- deepcompare()-ed when error-checking is done. + local input = { + consumers = { + { username = "valid_user", + tags = { tags.consumer.next }, + }, + + { username = "bobby_in_json_body", + not_allowed = true, + tags = { tags.consumer.next }, + }, + + { username = "super_valid_user", + tags = { tags.consumer.next }, + }, + + { username = "credentials", + tags = { tags.consumer.next }, + basicauth_credentials = { + { username = "superduper", + password = "hard2guess", + tags = { tags.basicauth_credentials.next, tags.consumer.last }, + }, + + { username = "dont-add-extra-fields-yo", + password = "12354", + extra_field = "NO!", + tags = { tags.basicauth_credentials.next, tags.consumer.last }, + }, + }, + }, + }, + + plugins = { + { name = "http-log", + config = { http_endpoint = "invalid::#//url", }, + tags = { tags.global_plugin.next }, + }, + }, + + certificates = { + { + cert = [[-----BEGIN CERTIFICATE----- +MIICIzCCAYSgAwIBAgIUUMiD8e3GDZ+vs7XBmdXzMxARUrgwCgYIKoZIzj0EAwIw +IzENMAsGA1UECgwES29uZzESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTIyMTIzMDA0 +MDcwOFoXDTQyMTIyNTA0MDcwOFowIzENMAsGA1UECgwES29uZzESMBAGA1UEAwwJ +bG9jYWxob3N0MIGbMBAGByqGSM49AgEGBSuBBAAjA4GGAAQBxSldGzzRAtjt825q +Uwl+BNgxecswnvbQFLiUDqJjVjCfs/B53xQfV97ddxsRymES2viC2kjAm1Ete4TH +CQmVltUBItHzI77HB+UsfqHoUdjl3lC/HC1yDSPBp5wd9eRRSagdl0eiJwnB9lof +MEnmOQLg177trb/YPz1vcCCZj7ikhzCjUzBRMB0GA1UdDgQWBBSUI6+CKqKFz/Te +ZJppMNl/Dh6d9DAfBgNVHSMEGDAWgBSUI6+CKqKFz/TeZJppMNl/Dh6d9DAPBgNV +HRMBAf8EBTADAQH/MAoGCCqGSM49BAMCA4GMADCBiAJCAZL3qX21MnGtQcl9yOMr +hNR54VrDKgqLR+ChU7/358n/sK/sVOjmrwVyQ52oUyqaQlfBQS2EufQVO/01+2sx +86gzAkIB/4Ilf4RluN2/gqHYlVEDRZzsqbwVJBHLeNKsZBSJkhNNpJBwa2Ndl9/i +u2tDk0KZFSAvRnqRAo9iDBUkIUI1ahA= +-----END CERTIFICATE-----]], + key = [[-----BEGIN EC PRIVATE KEY----- +MIHcAgEBBEIARPKnAYLB54bxBvkDfqV4NfZ+Mxl79rlaYRB6vbWVwFpy+E2pSZBR +doCy1tHAB/uPo+QJyjIK82Zwa3Kq0i1D2QigBwYFK4EEACOhgYkDgYYABAHFKV0b +PNEC2O3zbmpTCX4E2DF5yzCe9tAUuJQOomNWMJ+z8HnfFB9X3t13GxHKYRLa+ILa +SMCbUS17hMcJCZWW1QEi0fMjvscH5Sx+oehR2OXeUL8cLXINI8GnnB315FFJqB2X +R6InCcH2Wh8wSeY5AuDXvu2tv9g/PW9wIJmPuKSHMA== +-----END EC PRIVATE KEY-----]], + tags = { tags.certificate.next }, + }, + + { + cert = [[-----BEGIN CERTIFICATE----- +MIICIzCCAYSgAwIBAgIUUMiD8e3GDZ+vs7XBmdXzMxARUrgwCgYIKoZIzj0EAwIw +IzENMAsGA1UECgwES29uZzESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTIyMTIzMDA0 +MDcwOFoXDTQyohnoooooooooooooooooooooooooooooooooooooooooooasdfa +Uwl+BNgxecswnvbQFLiUDqJjVjCfs/B53xQfV97ddxsRymES2viC2kjAm1Ete4TH +CQmVltUBItHzI77AAAAAAAAAAAAAAAC/HC1yDSBBBBBBBBBBBBBdl0eiJwnB9lof +MEnmOQLg177trb/AAAAAAAAAAAAAAACjUzBRMBBBBBBBBBBBBBBUI6+CKqKFz/Te +ZJppMNl/Dh6d9DAAAAAAAAAAAAAAAASUI6+CKqBBBBBBBBBBBBB/Dh6d9DAPBgNV +HRMBAf8EBTADAQHAAAAAAAAAAAAAAAMCA4GMADBBBBBBBBBBBBB1MnGtQcl9yOMr +hNR54VrDKgqLR+CAAAAAAAAAAAAAAAjmrwVyQ5BBBBBBBBBBBBBEufQVO/01+2sx +86gzAkIB/4Ilf4RluN2/gqHYlVEDRZzsqbwVJBHLeNKsZBSJkhNNpJBwa2Ndl9/i +u2tDk0KZFSAvRnqRAo9iDBUkIUI1ahA= +-----END CERTIFICATE-----]], + key = [[-----BEGIN EC PRIVATE KEY----- +MIHcAgEBBEIARPKnAYLB54bxBvkDfqV4NfZ+Mxl79rlaYRB6vbWVwFpy+E2pSZBR +doCy1tHAB/uPo+QJyjIK82Zwa3Kq0i1D2QigBwYFK4EEACOhgYkDgYYABAHFKV0b +PNEC2O3zbmpTCX4E2DF5yzCe9tAUuJQOomNWMJ+z8HnfFB9X3t13GxHKYRLa+ILa +SMCbUS17hMcJCZWW1QEi0fMjvscH5Sx+oehR2OXeUL8cLXINI8GnnB315FFJqB2X +R6InCcH2Wh8wSeY5AuDXvu2tv9g/PW9wIJmPuKSHMA== +-----END EC PRIVATE KEY-----]], + tags = { tags.certificate.next }, + }, + + }, + + services = { + { name = "nope", + host = "localhost", + port = 1234, + protocol = "nope", + tags = { tags.service.next }, + -- this ID is hard-coded so that we can validate its + -- presence in the invalid route's service.id foreign key + id = "0175e0e8-3de9-56b4-96f1-b12dcb4b6691", + + routes = { + { name = "valid.route", + protocols = { "http", "https" }, + methods = { "GET" }, + hosts = { "test" }, + tags = { tags.route_service.next, tags.service.last }, + }, + + { name = "nope.route", + protocols = { "tcp" }, + tags = { tags.route_service.next, tags.service.last }, + } + }, + }, + + { name = "mis-matched", + host = "localhost", + protocol = "tcp", + path = "/path", + tags = { tags.service.next }, + + routes = { + { name = "invalid", + protocols = { "http", "https" }, + hosts = { "test" }, + methods = { "GET" }, + tags = { tags.route_service.next, tags.service.last }, + }, + }, + }, + + { name = "okay", + url = "http://localhost:1234", + tags = { tags.service.next }, + routes = { + { name = "probably-valid", + protocols = { "http", "https" }, + methods = { "GET" }, + hosts = { "test" }, + tags = { tags.route_service.next, tags.service.last }, + -- explicitly setting this ID to assert that it is preserved + -- in the nested plugin's route.id + id = "bbcceba9-4399-512f-8549-c2d74452b453", + plugins = { + { name = "http-log", + config = { not_endpoint = "anything" }, + tags = { tags.route_service_plugin.next, + tags.route_service.last, + tags.service.last, }, + }, + }, + }, + }, + }, + + { name = "bad-service-plugins", + url = "http://localhost:1234", + tags = { tags.service.next }, + plugins = { + { name = "i-dont-exist", + config = {}, + tags = { tags.service_plugin.next, tags.service.last }, + }, + + { name = "tcp-log", + config = { + deeply = { nested = { undefined = true } }, + port = 1234, + }, + tags = { tags.service_plugin.next, tags.service.last }, + }, + }, + }, + + { name = "bad-client-cert", + url = "https://localhost:1234", + tags = { tags.service.next }, + client_certificate = { + cert = "", + key = "", + tags = { tags.service_client_certificate.next, + tags.service.last, }, + }, + }, + + { + name = "invalid-id", + id = 123456, + url = "https://localhost:1234", + tags = { tags.service.next, "invalid-id" }, + }, + + { + name = "invalid-tags", + url = "https://localhost:1234", + tags = { tags.service.next, "invalid-tags", {1,2,3}, true }, + }, + + { + name = "", + url = "https://localhost:1234", + tags = { tags.service.next, tags.invalid_service_name.next }, + }, + + { + name = 1234, + url = "https://localhost:1234", + tags = { tags.service.next, tags.invalid_service_name.next }, + }, + + + }, + + upstreams = { + { name = "ok", + tags = { tags.upstream.next }, + hash_on = "ip", + }, + + { name = "bad", + tags = { tags.upstream.next }, + hash_on = "ip", + healthchecks = { + active = { + type = "http", + http_path = "/", + https_verify_certificate = true, + https_sni = "example.com", + timeout = 1, + concurrency = -1, + healthy = { + interval = 0, + successes = 0, + }, + unhealthy = { + interval = 0, + http_failures = 0, + }, + }, + }, + host_header = 123, + }, + + { + name = "ok-bad-targets", + tags = { tags.upstream.next }, + targets = { + { target = "127.0.0.1:99", + tags = { tags.upstream_target.next, + tags.upstream.last, }, + }, + { target = "hostname:1.0", + tags = { tags.upstream_target.next, + tags.upstream.last, }, + }, + }, + } + }, + + vaults = { + { + name = "env", + prefix = "test", + config = { prefix = "SSL_" }, + tags = { tags.vault.next }, + }, + + { + name = "vault-not-installed", + prefix = "env", + config = { prefix = "SSL_" }, + tags = { tags.vault.next, "vault-not-installed" }, + }, + + }, + } + + local expect = { + { + entity = { + extra_field = "NO!", + password = "12354", + tags = { "basicauth_credentials-02", "consumer-04", }, + username = "dont-add-extra-fields-yo", + }, + entity_tags = { "basicauth_credentials-02", "consumer-04", }, + entity_type = "basicauth_credential", + errors = { { + field = "extra_field", + message = "unknown field", + type = "field" + } } + }, + + { + entity = { + config = { + prefix = "SSL_" + }, + name = "vault-not-installed", + prefix = "env", + tags = { "vault-02", "vault-not-installed" } + }, + entity_name = "vault-not-installed", + entity_tags = { "vault-02", "vault-not-installed" }, + entity_type = "vault", + errors = { { + field = "name", + message = "vault 'vault-not-installed' is not installed", + type = "field" + } } + }, + + { + -- note entity_name is nil, but entity.name is not + entity_name = nil, + entity = { + name = "", + tags = { "service-08", "invalid_service_name-01" }, + url = "https://localhost:1234" + }, + entity_tags = { "service-08", "invalid_service_name-01" }, + entity_type = "service", + errors = { { + field = "name", + message = "length must be at least 1", + type = "field" + } } + }, + + { + -- note entity_name is nil, but entity.name is not + entity_name = nil, + entity = { + name = 1234, + tags = { "service-09", "invalid_service_name-02" }, + url = "https://localhost:1234" + }, + entity_tags = { "service-09", "invalid_service_name-02" }, + entity_type = "service", + errors = { { + field = "name", + message = "expected a string", + type = "field" + } } + }, + + { + -- note entity_tags is nil, but entity.tags is not + entity_tags = nil, + entity = { + name = "invalid-tags", + tags = { "service-07", "invalid-tags", { 1, 2, 3 }, true }, + url = "https://localhost:1234" + }, + entity_name = "invalid-tags", + entity_type = "service", + errors = { { + field = "tags.3", + message = "expected a string", + type = "field" + }, { + field = "tags.4", + message = "expected a string", + type = "field" + } } + }, + + { + entity_id = ngx.null, + entity = { + name = "invalid-id", + id = 123456, + tags = { "service-06", "invalid-id" }, + url = "https://localhost:1234" + }, + entity_name = "invalid-id", + entity_tags = { "service-06", "invalid-id" }, + entity_type = "service", + errors = { { + field = "id", + message = "expected a string", + type = "field" + } } + }, + + { + entity = { + cert = "-----BEGIN CERTIFICATE-----\nMIICIzCCAYSgAwIBAgIUUMiD8e3GDZ+vs7XBmdXzMxARUrgwCgYIKoZIzj0EAwIw\nIzENMAsGA1UECgwES29uZzESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTIyMTIzMDA0\nMDcwOFoXDTQyohnoooooooooooooooooooooooooooooooooooooooooooasdfa\nUwl+BNgxecswnvbQFLiUDqJjVjCfs/B53xQfV97ddxsRymES2viC2kjAm1Ete4TH\nCQmVltUBItHzI77AAAAAAAAAAAAAAAC/HC1yDSBBBBBBBBBBBBBdl0eiJwnB9lof\nMEnmOQLg177trb/AAAAAAAAAAAAAAACjUzBRMBBBBBBBBBBBBBBUI6+CKqKFz/Te\nZJppMNl/Dh6d9DAAAAAAAAAAAAAAAASUI6+CKqBBBBBBBBBBBBB/Dh6d9DAPBgNV\nHRMBAf8EBTADAQHAAAAAAAAAAAAAAAMCA4GMADBBBBBBBBBBBBB1MnGtQcl9yOMr\nhNR54VrDKgqLR+CAAAAAAAAAAAAAAAjmrwVyQ5BBBBBBBBBBBBBEufQVO/01+2sx\n86gzAkIB/4Ilf4RluN2/gqHYlVEDRZzsqbwVJBHLeNKsZBSJkhNNpJBwa2Ndl9/i\nu2tDk0KZFSAvRnqRAo9iDBUkIUI1ahA=\n-----END CERTIFICATE-----", + key = "-----BEGIN EC PRIVATE KEY-----\nMIHcAgEBBEIARPKnAYLB54bxBvkDfqV4NfZ+Mxl79rlaYRB6vbWVwFpy+E2pSZBR\ndoCy1tHAB/uPo+QJyjIK82Zwa3Kq0i1D2QigBwYFK4EEACOhgYkDgYYABAHFKV0b\nPNEC2O3zbmpTCX4E2DF5yzCe9tAUuJQOomNWMJ+z8HnfFB9X3t13GxHKYRLa+ILa\nSMCbUS17hMcJCZWW1QEi0fMjvscH5Sx+oehR2OXeUL8cLXINI8GnnB315FFJqB2X\nR6InCcH2Wh8wSeY5AuDXvu2tv9g/PW9wIJmPuKSHMA==\n-----END EC PRIVATE KEY-----", + tags = { "certificate-02", } + }, + entity_tags = { "certificate-02", }, + entity_type = "certificate", + errors = { { + field = "cert", + message = "invalid certificate: x509.new: asn1/tasn_dec.c:309:error:0D07803A:asn1 encoding routines:asn1_item_embed_d2i:nested asn1 error", + type = "field" + } } + }, + + { + entity = { + hash_on = "ip", + healthchecks = { + active = { + concurrency = -1, + healthy = { + interval = 0, + successes = 0 + }, + http_path = "/", + https_sni = "example.com", + https_verify_certificate = true, + timeout = 1, + type = "http", + unhealthy = { + http_failures = 0, + interval = 0 + } + } + }, + host_header = 123, + name = "bad", + tags = { + "upstream-02" + } + }, + entity_name = "bad", + entity_tags = { + "upstream-02" + }, + entity_type = "upstream", + errors = { + { + field = "host_header", + message = "expected a string", + type = "field" + }, + { + field = "healthchecks.active.concurrency", + message = "value should be between 1 and 2147483648", + type = "field" + }, + } + }, + + { + entity = { + config = { + http_endpoint = "invalid::#//url" + }, + name = "http-log", + tags = { + "global_plugin-01", + } + }, + entity_name = "http-log", + entity_tags = { + "global_plugin-01", + }, + entity_type = "plugin", + errors = { + { + field = "config.http_endpoint", + message = "missing host in url", + type = "field" + } + } + }, + + { + entity = { + not_allowed = true, + tags = { + "consumer-02" + }, + username = "bobby_in_json_body" + }, + entity_tags = { + "consumer-02" + }, + entity_type = "consumer", + errors = { + { + field = "not_allowed", + message = "unknown field", + type = "field" + } + } + }, + + { + entity = { + name = "nope.route", + protocols = { + "tcp" + }, + service = { + -- note that foreign keys are retained + id = "0175e0e8-3de9-56b4-96f1-b12dcb4b6691", + }, + tags = { + "route_service-02", + "service-01", + } + }, + entity_name = "nope.route", + entity_tags = { + "route_service-02", + "service-01", + }, + entity_type = "route", + errors = { + { + message = "must set one of 'sources', 'destinations', 'snis' when 'protocols' is 'tcp', 'tls' or 'udp'", + type = "entity" + } + } + }, + + { + entity = { + host = "localhost", + id = "0175e0e8-3de9-56b4-96f1-b12dcb4b6691", + name = "nope", + port = 1234, + protocol = "nope", + tags = { + "service-01" + } + }, + entity_id = "0175e0e8-3de9-56b4-96f1-b12dcb4b6691", + entity_name = "nope", + entity_tags = { + "service-01" + }, + entity_type = "service", + errors = { + { + field = "protocol", + message = "expected one of: grpc, grpcs, http, https, tcp, tls, tls_passthrough, udp", + type = "field" + } + } + }, + + { + entity = { + host = "localhost", + name = "mis-matched", + path = "/path", + protocol = "tcp", + tags = { + "service-02" + } + }, + entity_name = "mis-matched", + entity_tags = { + "service-02" + }, + entity_type = "service", + errors = { + { + field = "path", + message = "value must be null", + type = "field" + }, + { + message = "failed conditional validation given value of field 'protocol'", + type = "entity" + } + } + }, + + { + entity = { + config = { + not_endpoint = "anything" + }, + name = "http-log", + route = { + -- this FK is explicitly set for tests + id = "bbcceba9-4399-512f-8549-c2d74452b453" + }, + tags = { + "route_service_plugin-01", + "route_service-04", + "service-03", + } + }, + entity_name = "http-log", + entity_tags = { + "route_service_plugin-01", + "route_service-04", + "service-03", + }, + entity_type = "plugin", + errors = { + { + field = "config.not_endpoint", + message = "unknown field", + type = "field" + }, + { + field = "config.http_endpoint", + message = "required field missing", + type = "field" + } + } + }, + + { + entity = { + config = {}, + name = "i-dont-exist", + tags = { + "service_plugin-01", + "service-04", + } + }, + entity_name = "i-dont-exist", + entity_tags = { + "service_plugin-01", + "service-04", + }, + entity_type = "plugin", + errors = { + { + field = "name", + message = "plugin 'i-dont-exist' not enabled; add it to the 'plugins' configuration property", + type = "field" + } + } + }, + + { + entity = { + config = { + deeply = { + nested = { + undefined = true + } + }, + port = 1234 + }, + name = "tcp-log", + tags = { + "service_plugin-02", + "service-04", + } + }, + entity_name = "tcp-log", + entity_tags = { + "service_plugin-02", + "service-04", + }, + entity_type = "plugin", + errors = { + { + field = "config.deeply", + message = "unknown field", + type = "field" + }, + { + field = "config.host", + message = "required field missing", + type = "field" + } + } + }, + + { + entity = { + cert = "", + key = "", + tags = { + "service_client_certificate-01", + "service-05", + } + }, + entity_tags = { + "service_client_certificate-01", + "service-05", + }, + entity_type = "certificate", + errors = { + { + field = "key", + message = "length must be at least 1", + type = "field" + }, + { + field = "cert", + message = "length must be at least 1", + type = "field" + }, + }, + }, + + { + entity = { + tags = { + "upstream_target-02", + "upstream-03", + }, + target = "hostname:1.0" + }, + entity_tags = { + "upstream_target-02", + "upstream-03", + }, + entity_type = "target", + errors = { { + field = "target", + message = "Invalid target ('hostname:1.0'); not a valid hostname or ip address", + type = "field" + } } + }, + + } + + validate(expect, post_config(input)) + end) + + it("flattens nested, non-entity field errors", function() + local upstream = { + name = "bad", + tags = { tags.upstream.next }, + hash_on = "ip", + healthchecks = { + active = { + type = "http", + http_path = "/", + https_verify_certificate = true, + https_sni = "example.com", + timeout = 1, + concurrency = -1, + healthy = { + interval = 0, + successes = 0, + }, + unhealthy = { + interval = 0, + http_failures = 0, + }, + }, + }, + host_header = 123, + } + + validate({ + { + entity_type = "upstream", + entity_name = "bad", + entity_tags = { tags.upstream.last }, + entity = upstream, + errors = { + { + field = "healthchecks.active.concurrency", + message = "value should be between 1 and 2147483648", + type = "field" + }, + { + field = "host_header", + message = "expected a string", + type = "field" + }, + }, + }, + }, post_config({ upstreams = { upstream } })) + end) + + it("flattens nested, entity field errors", function() + local input = { + services = { + { name = "bad-client-cert", + url = "https://localhost:1234", + tags = { tags.service.next }, + -- error + client_certificate = { + cert = "", + key = "", + tags = { tags.service_client_certificate.next, + tags.service.last, }, + }, + + routes = { + { hosts = { "test" }, + paths = { "/" }, + protocols = { "http" }, + tags = { tags.service_route.next }, + plugins = { + -- error + { + name = "http-log", + config = { a = { b = { c = "def" } } }, + tags = { tags.route_service_plugin.next }, + }, + }, + }, + + -- error + { hosts = { "invalid" }, + paths = { "/" }, + protocols = { "nope" }, + tags = { tags.service_route.next }, + }, + }, + + plugins = { + -- error + { + name = "i-do-not-exist", + config = {}, + tags = { tags.service_plugin.next }, + }, + }, + }, + } + } + + validate({ + { + entity = { + cert = "", + key = "", + tags = { "service_client_certificate-01", "service-01" } + }, + entity_tags = { "service_client_certificate-01", "service-01" }, + entity_type = "certificate", + errors = { { + field = "cert", + message = "length must be at least 1", + type = "field" + }, { + field = "key", + message = "length must be at least 1", + type = "field" + } } + }, + + { + entity = { + hosts = { "invalid" }, + paths = { "/" }, + protocols = { "nope" }, + tags = { "service_route-02" } + }, + entity_tags = { "service_route-02" }, + entity_type = "route", + errors = { { + field = "protocols", + message = "unknown type: nope", + type = "field" + } } + }, + + { + entity = { + config = { a = { b = { c = "def" } } }, + name = "http-log", + tags = { "route_service_plugin-01" }, + }, + entity_name = "http-log", + entity_type = "plugin", + entity_tags = { "route_service_plugin-01" }, + errors = { { + field = "config.a", + message = "unknown field", + type = "field" + }, { + field = "config.http_endpoint", + message = "required field missing", + type = "field" + } } + + }, + + { + entity = { + config = {}, + name = "i-do-not-exist", + tags = { "service_plugin-01" } + }, + entity_name = "i-do-not-exist", + entity_tags = { "service_plugin-01" }, + entity_type = "plugin", + errors = { { + field = "name", + message = "plugin 'i-do-not-exist' not enabled; add it to the 'plugins' configuration property", + type = "field" + } } + }, + }, post_config(input)) + end) + + it("preserves IDs from the input", function() + local id = "0175e0e8-3de9-56b4-96f1-b12dcb4b6691" + local service = { + id = id, + name = "nope", + host = "localhost", + port = 1234, + protocol = "nope", + tags = { tags.service.next }, + } + + local flattened = post_config({ services = { service } }) + local got = get_by_tag(tags.service.last, flattened) + assert.not_nil(got) + + assert.equals(id, got.entity_id) + assert.equals(id, got.entity.id) + end) + + it("preserves foreign keys from nested entity collections", function() + local id = "cb019421-62c2-47a8-b714-d7567b114037" + + local service = { + id = id, + name = "test", + host = "localhost", + port = 1234, + protocol = "nope", + tags = { tags.service.next }, + routes = { + { + super_duper_invalid = true, + tags = { tags.route.next }, + } + }, + } + + local flattened = post_config({ services = { service } }) + local got = get_by_tag(tags.route.last, flattened) + assert.not_nil(got) + assert.equals(id, got.entity.service.id) + end) + + it("omits top-level entity_* fields if they are invalid", function() + local service = { + id = 1234, + name = false, + tags = { tags.service.next, { 1.5 }, }, + url = "http://localhost:1234", + } + + local flattened = post_config({ services = { service } }) + local got = get_by_tag(tags.service.last, flattened) + assert.not_nil(got) + + assert.is_nil(got.entity_id) + assert.is_nil(got.entity_name) + assert.is_nil(got.entity_tags) + + assert.equals(1234, got.entity.id) + assert.equals(false, got.entity.name) + assert.same({ tags.service.last, { 1.5 }, }, got.entity.tags) + end) +end) + + describe("Admin API (concurrency tests) #off", function() local client