From e3069f041cdce400ad2d08d1daa44df59c04b279 Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Fri, 25 Nov 2022 16:07:11 +0800 Subject: [PATCH 01/17] feat: support global data encryption of secret information --- apisix/admin/consumers.lua | 19 +++++ apisix/consumer.lua | 19 +++++ apisix/plugins/basic-auth.lua | 3 +- apisix/ssl.lua | 101 +++++++++++++++++------- conf/config-default.yaml | 4 + t/admin/global_data_encrypt.t | 143 ++++++++++++++++++++++++++++++++++ 6 files changed, 259 insertions(+), 30 deletions(-) create mode 100644 t/admin/global_data_encrypt.t diff --git a/apisix/admin/consumers.lua b/apisix/admin/consumers.lua index 7ab0ec1e2ad2..00cacab8654b 100644 --- a/apisix/admin/consumers.lua +++ b/apisix/admin/consumers.lua @@ -18,6 +18,7 @@ local core = require("apisix.core") local plugins = require("apisix.admin.plugins") local utils = require("apisix.admin.utils") local plugin = require("apisix.plugin") +local apisix_ssl = require("apisix.ssl") local pairs = pairs local _M = { @@ -26,6 +27,22 @@ local _M = { } +local function encrypt_items(name, conf) + local schema = plugin.get(name) + local consumer_schema = schema.consumer_schema + if not consumer_schema then + return + end + + for key, props in pairs(consumer_schema.properties) do + if props.type == "string" and props.encrypted then + local encrypted = apisix_ssl.aes_encrypt_pkey(conf[key], "global_data_encrypt") + conf[key] = encrypted + end + end +end + + local function check_conf(username, conf) -- core.log.error(core.json.encode(conf)) if not conf then @@ -55,6 +72,8 @@ local function check_conf(username, conf) if plugin_obj.type == 'auth' then count_auth_plugin = count_auth_plugin + 1 end + + encrypt_items(name, conf) end if count_auth_plugin == 0 then diff --git a/apisix/consumer.lua b/apisix/consumer.lua index 32ff69275809..180c8fd244db 100644 --- a/apisix/consumer.lua +++ b/apisix/consumer.lua @@ -17,6 +17,7 @@ local core = require("apisix.core") local plugin = require("apisix.plugin") local plugin_checker = require("apisix.plugin").plugin_checker +local apisix_ssl = require("apisix.ssl") local error = error local ipairs = ipairs local pairs = pairs @@ -32,6 +33,23 @@ local lrucache = core.lrucache.new({ ttl = 300, count = 512 }) + +local function decrypt_items(name, conf) + local schema = plugin.get(name) + local consumer_schema = schema.consumer_schema + if not consumer_schema then + return + end + + for key, props in pairs(consumer_schema.properties) do + if props.type == "string" and props.encrypted then + local encrypted = apisix_ssl.aes_decrypt_pkey(conf[key], "global_data_encrypt") + conf[key] = encrypted + end + end +end + + local function plugin_consumer() local plugins = {} @@ -60,6 +78,7 @@ local function plugin_consumer() new_consumer.consumer_name = new_consumer.id new_consumer.auth_conf = config new_consumer.modifiedIndex = consumer.modifiedIndex + decrypt_items(name, new_consumer.auth_conf) core.log.info("consumer:", core.json.delay_encode(new_consumer)) core.table.insert(plugins[name].nodes, new_consumer) end diff --git a/apisix/plugins/basic-auth.lua b/apisix/plugins/basic-auth.lua index 5f984ed66a09..65d96e5c78d4 100644 --- a/apisix/plugins/basic-auth.lua +++ b/apisix/plugins/basic-auth.lua @@ -18,7 +18,6 @@ local core = require("apisix.core") local ngx = ngx local ngx_re = require("ngx.re") local consumer = require("apisix.consumer") - local lrucache = core.lrucache.new({ ttl = 300, count = 512 }) @@ -39,7 +38,7 @@ local consumer_schema = { title = "work with consumer object", properties = { username = { type = "string" }, - password = { type = "string" }, + password = { type = "string", encrypted = true }, }, required = {"username", "password"}, } diff --git a/apisix/ssl.lua b/apisix/ssl.lua index 26fa5c6ef359..7e1d0058e805 100644 --- a/apisix/ssl.lua +++ b/apisix/ssl.lua @@ -56,52 +56,96 @@ function _M.server_name() end -local _aes_128_cbc_with_iv_tbl -local function get_aes_128_cbc_with_iv() - if _aes_128_cbc_with_iv_tbl == nil then - _aes_128_cbc_with_iv_tbl = core.table.new(2, 0) - local local_conf = core.config.local_conf() - local ivs = core.table.try_read_attr(local_conf, "apisix", "ssl", "key_encrypt_salt") - local type_ivs = type(ivs) +local function init_iv_tbl(ivs) + local _aes_128_cbc_with_iv_tbl = core.table.new(2, 0) + local type_ivs = type(ivs) - if type_ivs == "table" then - for _, iv in ipairs(ivs) do - local aes_with_iv = assert(aes:new(iv, nil, aes.cipher(128, "cbc"), {iv = iv})) - core.table.insert(_aes_128_cbc_with_iv_tbl, aes_with_iv) - end - elseif type_ivs == "string" then - local aes_with_iv = assert(aes:new(ivs, nil, aes.cipher(128, "cbc"), {iv = ivs})) + if type_ivs == "table" then + for _, iv in ipairs(ivs) do + local aes_with_iv = assert(aes:new(iv, nil, aes.cipher(128, "cbc"), {iv = iv})) core.table.insert(_aes_128_cbc_with_iv_tbl, aes_with_iv) end + elseif type_ivs == "string" then + local aes_with_iv = assert(aes:new(ivs, nil, aes.cipher(128, "cbc"), {iv = ivs})) + core.table.insert(_aes_128_cbc_with_iv_tbl, aes_with_iv) end return _aes_128_cbc_with_iv_tbl end -function _M.aes_encrypt_pkey(origin) - local aes_128_cbc_with_iv_tbl = get_aes_128_cbc_with_iv() - local aes_128_cbc_with_iv = aes_128_cbc_with_iv_tbl[1] - if aes_128_cbc_with_iv ~= nil and core.string.has_prefix(origin, "---") then - local encrypted = aes_128_cbc_with_iv:encrypt(origin) - if encrypted == nil then - core.log.error("failed to encrypt key[", origin, "] ") - return origin - end +local _aes_128_cbc_with_iv_tbl_ssl +local function get_aes_128_cbc_with_iv_ssl(local_conf) + if _aes_128_cbc_with_iv_tbl_ssl == nil then + local ivs = core.table.try_read_attr(local_conf, "apisix", "ssl", "key_encrypt_salt") + _aes_128_cbc_with_iv_tbl_ssl = init_iv_tbl(ivs) + end - return ngx_encode_base64(encrypted) + return _aes_128_cbc_with_iv_tbl_ssl +end + + +local _aes_128_cbc_with_iv_tbl_gde +local function get_aes_128_cbc_with_iv_gde(local_conf) + if _aes_128_cbc_with_iv_tbl_gde == nil then + local ivs = core.table.try_read_attr(local_conf, "apisix", "data_encryption", "keyring") + _aes_128_cbc_with_iv_tbl_gde = init_iv_tbl(ivs) end - return origin + return _aes_128_cbc_with_iv_tbl_gde end -local function aes_decrypt_pkey(origin) - if core.string.has_prefix(origin, "---") then + +local function encrypt(aes_128_cbc_with_iv, origin) + local encrypted = aes_128_cbc_with_iv:encrypt(origin) + if encrypted == nil then + core.log.error("failed to encrypt key[", origin, "] ") return origin end - local aes_128_cbc_with_iv_tbl = get_aes_128_cbc_with_iv() + return ngx_encode_base64(encrypted) +end + +function _M.aes_encrypt_pkey(origin, field) + local local_conf = core.config.local_conf() + + if not field then + -- default used by ssl + local aes_128_cbc_with_iv_tbl_ssl = get_aes_128_cbc_with_iv_ssl(local_conf) + local aes_128_cbc_with_iv_ssl = aes_128_cbc_with_iv_tbl_ssl[1] + if aes_128_cbc_with_iv_ssl ~= nil and core.string.has_prefix(origin, "---") then + return encrypt(aes_128_cbc_with_iv_ssl, origin) + end + else + if field == "global_data_encrypt" then + local aes_128_cbc_with_iv_tbl_gde = get_aes_128_cbc_with_iv_gde(local_conf) + local aes_128_cbc_with_iv_gde = aes_128_cbc_with_iv_tbl_gde[1] + if aes_128_cbc_with_iv_gde ~= nil then + return encrypt(aes_128_cbc_with_iv_gde, origin) + end + end + end + + return origin +end + + +local function aes_decrypt_pkey(origin, field) + local local_conf = core.config.local_conf() + local aes_128_cbc_with_iv_tbl + + if not field then + if core.string.has_prefix(origin, "---") then + return origin + end + aes_128_cbc_with_iv_tbl = get_aes_128_cbc_with_iv_ssl(local_conf) + else + if field == "global_data_encrypt" then + aes_128_cbc_with_iv_tbl = get_aes_128_cbc_with_iv_gde(local_conf) + end + end + if #aes_128_cbc_with_iv_tbl == 0 then return origin end @@ -123,6 +167,7 @@ local function aes_decrypt_pkey(origin) return nil end +_M.aes_decrypt_pkey = aes_decrypt_pkey local function validate(cert, key) diff --git a/conf/config-default.yaml b/conf/config-default.yaml index 6f9de38bd812..ae7f83f6119c 100755 --- a/conf/config-default.yaml +++ b/conf/config-default.yaml @@ -123,6 +123,10 @@ apisix: # ip: 127.0.0.1 # port: 9090 disable_sync_configuration_during_start: false # safe exit. Remove this once the feature is stable + data_encryption: + enable: false + keyring: + - edd1c9f0985e76a2 nginx_config: # config for render the template to generate nginx.conf #user: root # specifies the execution user of the worker process. diff --git a/t/admin/global_data_encrypt.t b/t/admin/global_data_encrypt.t new file mode 100644 index 000000000000..5c78c87bd4f0 --- /dev/null +++ b/t/admin/global_data_encrypt.t @@ -0,0 +1,143 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +use t::APISIX 'no_plan'; + +repeat_each(1); +no_long_string(); +no_root_location(); +no_shuffle(); +log_level("info"); + +add_block_preprocessor(sub { + my ($block) = @_; + + if (!$block->request) { + $block->set_value("request", "GET /t"); + } + + if (!$block->no_error_log) { + $block->set_value("no_error_log", "[error]\n[alert]"); + } +}); + +run_tests; + +__DATA__ + +=== TEST 1: sanity +--- yaml_config +apisix: + data_encryption: + enable: true + keyring: + - edd1c9f0985e76a2 +--- config + location /t { + content_by_lua_block { + local json = require("toolkit.json") + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/consumers', + ngx.HTTP_PUT, + [[{ + "username": "foo", + "plugins": { + "basic-auth": { + "username": "foo", + "password": "bar" + }, + "key-auth": { + "key": "auth-one" + } + } + }]] + ) + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + + ngx.sleep(0.1) + local code, message, res = t('/apisix/admin/consumers/foo', + ngx.HTTP_GET + ) + res = json.decode(res) + if code >= 300 then + ngx.status = code + ngx.say(message) + return + end + + ngx.say(res.value.plugins["basic-auth"].password) + } + } +--- response_body +77+NmbYqNfN+oLm0aX5akg== + + + +=== TEST 2: enable basic auth plugin using admin api +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "basic-auth": {}, + "key-auth": {} + }, + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- request +GET /t +--- response_body +passed + + + +=== TEST 3: verify +--- yaml_config +apisix: + data_encryption: + enable: true + keyring: + - edd1c9f0985e76a2 +--- request +GET /hello +--- more_headers +apikey: auth-one +Authorization: Basic Zm9vOmJhcg== +--- response_body +hello world +--- error_log +find consumer foo From 8c17481401d9c5eea9d764b439af65c7e278c77a Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Mon, 28 Nov 2022 19:04:02 +0800 Subject: [PATCH 02/17] improvements --- apisix/admin/consumers.lua | 26 +- apisix/admin/routes.lua | 11 + apisix/admin/utils.lua | 32 ++ apisix/consumer.lua | 18 - apisix/plugin.lua | 71 +++- apisix/plugins/clickhouse-logger.lua | 2 +- apisix/plugins/key-auth.lua | 2 +- apisix/ssl.lua | 18 +- conf/config-default.yaml | 9 +- t/admin/global_data_encrypt.t | 143 ------- t/admin/plugins.t | 2 +- t/node/consumer-group.t | 137 +++++++ t/node/data_encrypt.t | 581 +++++++++++++++++++++++++++ t/node/global-rule.t | 109 +++++ t/plugin/clickhouse-logger.t | 232 +++++++++++ 15 files changed, 1200 insertions(+), 193 deletions(-) delete mode 100644 t/admin/global_data_encrypt.t create mode 100644 t/node/data_encrypt.t diff --git a/apisix/admin/consumers.lua b/apisix/admin/consumers.lua index 00cacab8654b..a0224005167b 100644 --- a/apisix/admin/consumers.lua +++ b/apisix/admin/consumers.lua @@ -18,7 +18,6 @@ local core = require("apisix.core") local plugins = require("apisix.admin.plugins") local utils = require("apisix.admin.utils") local plugin = require("apisix.plugin") -local apisix_ssl = require("apisix.ssl") local pairs = pairs local _M = { @@ -27,22 +26,6 @@ local _M = { } -local function encrypt_items(name, conf) - local schema = plugin.get(name) - local consumer_schema = schema.consumer_schema - if not consumer_schema then - return - end - - for key, props in pairs(consumer_schema.properties) do - if props.type == "string" and props.encrypted then - local encrypted = apisix_ssl.aes_encrypt_pkey(conf[key], "global_data_encrypt") - conf[key] = encrypted - end - end -end - - local function check_conf(username, conf) -- core.log.error(core.json.encode(conf)) if not conf then @@ -73,7 +56,7 @@ local function check_conf(username, conf) count_auth_plugin = count_auth_plugin + 1 end - encrypt_items(name, conf) + plugin.encrypt_conf(name, conf, core.schema.TYPE_CONSUMER) end if count_auth_plugin == 0 then @@ -138,6 +121,13 @@ function _M.get(consumer_name) end utils.fix_count(res.body, consumer_name) + + -- decrypt the conf + if res.body then + utils.decrypt_conf(plugin.decrypt_conf, res.body, core.schema.TYPE_CONSUMER) + end + + return res.status, res.body end diff --git a/apisix/admin/routes.lua b/apisix/admin/routes.lua index 4cd36b385146..49b5955d5ef0 100644 --- a/apisix/admin/routes.lua +++ b/apisix/admin/routes.lua @@ -19,6 +19,8 @@ local core = require("apisix.core") local apisix_upstream = require("apisix.upstream") local schema_plugin = require("apisix.admin.plugins").check_schema local utils = require("apisix.admin.utils") +local plugin = require("apisix.plugin") +local pairs = pairs local tostring = tostring local type = type local loadstring = loadstring @@ -131,6 +133,9 @@ local function check_conf(id, conf, need_id) if not ok then return nil, {error_msg = err} end + for name, conf in pairs(conf.plugins) do + plugin.encrypt_conf(name, conf) + end end if conf.vars then @@ -204,6 +209,12 @@ function _M.get(id) end utils.fix_count(res.body, id) + + -- decrypt the conf + if res.body then + utils.decrypt_conf(plugin.decrypt_conf, res.body) + end + return res.status, res.body end diff --git a/apisix/admin/utils.lua b/apisix/admin/utils.lua index db73dda6751f..ac0f05c4348b 100644 --- a/apisix/admin/utils.lua +++ b/apisix/admin/utils.lua @@ -17,6 +17,8 @@ local core = require("apisix.core") local ngx_time = ngx.time local tonumber = tonumber +local ipairs = ipairs +local pairs = pairs local _M = {} @@ -78,4 +80,34 @@ function _M.fix_count(body, id) end +function _M.decrypt_conf(decrypt_func, body, schema_type) + local local_conf = core.config.local_conf() + local enable = core.table.try_read_attr(local_conf, "apisix", "data_encryption", "enable") + if not enable then + return + end + + -- list + if body.list and #body.list > 0 then + for _, route in ipairs(body.list) do + if route.value and route.value.plugins and core.table.nkeys(route.value.plugins) > 0 then + for name, conf in pairs(route.value.plugins) do + decrypt_func(name, conf, schema_type) + end + end + end + return + end + + -- node + local plugins = body.node and body.node.value + and body.node.value.plugins + + if plugins then + for name, conf in pairs(plugins) do + decrypt_func(name, conf, schema_type) + end + end +end + return _M diff --git a/apisix/consumer.lua b/apisix/consumer.lua index 180c8fd244db..1302e048d282 100644 --- a/apisix/consumer.lua +++ b/apisix/consumer.lua @@ -17,7 +17,6 @@ local core = require("apisix.core") local plugin = require("apisix.plugin") local plugin_checker = require("apisix.plugin").plugin_checker -local apisix_ssl = require("apisix.ssl") local error = error local ipairs = ipairs local pairs = pairs @@ -34,22 +33,6 @@ local lrucache = core.lrucache.new({ }) -local function decrypt_items(name, conf) - local schema = plugin.get(name) - local consumer_schema = schema.consumer_schema - if not consumer_schema then - return - end - - for key, props in pairs(consumer_schema.properties) do - if props.type == "string" and props.encrypted then - local encrypted = apisix_ssl.aes_decrypt_pkey(conf[key], "global_data_encrypt") - conf[key] = encrypted - end - end -end - - local function plugin_consumer() local plugins = {} @@ -78,7 +61,6 @@ local function plugin_consumer() new_consumer.consumer_name = new_consumer.id new_consumer.auth_conf = config new_consumer.modifiedIndex = consumer.modifiedIndex - decrypt_items(name, new_consumer.auth_conf) core.log.info("consumer:", core.json.delay_encode(new_consumer)) core.table.insert(plugins[name].nodes, new_consumer) end diff --git a/apisix/plugin.lua b/apisix/plugin.lua index 4221b7387348..9d9755f140dc 100644 --- a/apisix/plugin.lua +++ b/apisix/plugin.lua @@ -20,6 +20,7 @@ local config_util = require("apisix.core.config_util") local enable_debug = require("apisix.debug").enable_debug local wasm = require("apisix.wasm") local expr = require("resty.expr.v1") +local apisix_ssl = require("apisix.ssl") local ngx = ngx local crc32 = ngx.crc32_short local ngx_exit = ngx.exit @@ -849,6 +850,66 @@ check_plugin_metadata = function(item) end +local function decrypt_conf(name, conf, schema_type) + local enable = core.table.try_read_attr(local_conf, "apisix", "data_encryption", "enable") + if not enable then + return conf + end + + local plugin_schema = local_plugins_hash and local_plugins_hash[name] + local schema + if schema_type == core.schema.TYPE_CONSUMER then + schema = plugin_schema.consumer_schema + else + schema = plugin_schema.schema + end + + if not schema then + return + end + + for key, props in pairs(schema.properties) do + if props.type == "string" and props.encrypted and conf[key] then + local encrypted, err = apisix_ssl.aes_decrypt_pkey(conf[key], "data_encrypt") + if not encrypted then + core.log.warn("failed to decrypt the conf of plugin [", name, + "] key [", key, "], err: ", err) + else + conf[key] = encrypted + end + end + end +end +_M.decrypt_conf = decrypt_conf + + +local function encrypt_conf(name, conf, schema_type) + local enable = core.table.try_read_attr(local_conf, "apisix", "data_encryption", "enable") + if not enable then + return conf + end + + local plugin_schema = local_plugins_hash and local_plugins_hash[name] + local schema + if schema_type == core.schema.TYPE_CONSUMER then + schema = plugin_schema.consumer_schema + else + schema = plugin_schema.schema + end + + if not schema then + return + end + + for key, props in pairs(schema.properties) do + if props.type == "string" and props.encrypted and conf[key] then + local encrypted = apisix_ssl.aes_encrypt_pkey(conf[key], "data_encrypt") + conf[key] = encrypted + end + end +end +_M.encrypt_conf = encrypt_conf + local function check_schema(plugins_conf, schema_type, skip_disabled_plugin) for name, plugin_conf in pairs(plugins_conf) do @@ -901,7 +962,15 @@ _M.stream_check_schema = stream_check_schema function _M.plugin_checker(item, schema_type) if item.plugins then - return check_schema(item.plugins, schema_type, true) + local ok, err = check_schema(item.plugins, schema_type, true) + + if ok then + -- decrypt conf + for name, conf in pairs(item.plugins) do + decrypt_conf(name, conf, schema_type) + end + end + return ok, err end return true diff --git a/apisix/plugins/clickhouse-logger.lua b/apisix/plugins/clickhouse-logger.lua index 026f0cfa93da..28404f9ae8ad 100644 --- a/apisix/plugins/clickhouse-logger.lua +++ b/apisix/plugins/clickhouse-logger.lua @@ -36,7 +36,7 @@ local schema = { endpoint_addr = core.schema.uri_def, endpoint_addrs = {items = core.schema.uri_def, type = "array", minItems = 1}, user = {type = "string", default = ""}, - password = {type = "string", default = ""}, + password = {type = "string", default = "", encrypted = true}, database = {type = "string", default = ""}, logtable = {type = "string", default = ""}, timeout = {type = "integer", minimum = 1, default = 3}, diff --git a/apisix/plugins/key-auth.lua b/apisix/plugins/key-auth.lua index 5806025a6c7f..63e41a084d34 100644 --- a/apisix/plugins/key-auth.lua +++ b/apisix/plugins/key-auth.lua @@ -40,7 +40,7 @@ local schema = { local consumer_schema = { type = "object", properties = { - key = {type = "string"}, + key = { type = "string", encrypted = true }, }, required = {"key"}, } diff --git a/apisix/ssl.lua b/apisix/ssl.lua index 7e1d0058e805..4ffb3a0f468a 100644 --- a/apisix/ssl.lua +++ b/apisix/ssl.lua @@ -118,7 +118,7 @@ function _M.aes_encrypt_pkey(origin, field) return encrypt(aes_128_cbc_with_iv_ssl, origin) end else - if field == "global_data_encrypt" then + if field == "data_encrypt" then local aes_128_cbc_with_iv_tbl_gde = get_aes_128_cbc_with_iv_gde(local_conf) local aes_128_cbc_with_iv_gde = aes_128_cbc_with_iv_tbl_gde[1] if aes_128_cbc_with_iv_gde ~= nil then @@ -141,7 +141,7 @@ local function aes_decrypt_pkey(origin, field) end aes_128_cbc_with_iv_tbl = get_aes_128_cbc_with_iv_ssl(local_conf) else - if field == "global_data_encrypt" then + if field == "data_encrypt" then aes_128_cbc_with_iv_tbl = get_aes_128_cbc_with_iv_gde(local_conf) end end @@ -163,9 +163,7 @@ local function aes_decrypt_pkey(origin, field) end end - core.log.error("decrypt ssl key failed") - - return nil + return nil, "decrypt ssl key failed" end _M.aes_decrypt_pkey = aes_decrypt_pkey @@ -181,8 +179,10 @@ local function validate(cert, key) return true end - key = aes_decrypt_pkey(key) + local err + key, err = aes_decrypt_pkey(key) if not key then + core.log.error(err) return nil, "failed to decrypt previous encrypted key" end @@ -218,7 +218,11 @@ end local function parse_pem_priv_key(sni, pkey) core.log.debug("parsing priv key for sni: ", sni) - local parsed, err = ngx_ssl.parse_pem_priv_key(aes_decrypt_pkey(pkey)) + local key, err = aes_decrypt_pkey(pkey) + if not key then + core.log.error(err) + end + local parsed, err = ngx_ssl.parse_pem_priv_key(key) return parsed, err end diff --git a/conf/config-default.yaml b/conf/config-default.yaml index ae7f83f6119c..620fad427a48 100755 --- a/conf/config-default.yaml +++ b/conf/config-default.yaml @@ -123,10 +123,13 @@ apisix: # ip: 127.0.0.1 # port: 9090 disable_sync_configuration_during_start: false # safe exit. Remove this once the feature is stable - data_encryption: - enable: false + data_encryption: # use `encrypted = {fields}` in plugin schema to enable encryption + enable: false # if not set, the default value is `false`. keyring: - - edd1c9f0985e76a2 + - edd1c9f0985e76a2 # If not set, will save origin value into etcd. + # If set this, the keyring should be an array whose elements are string, and the size is also 16, and it will encrypt fields with AES-128-CBC + # !!! So do not change it after encryption, it can't decrypt the fields have be saved if you change !! + # Only use the first key to encrypt, and decrypt in the order of the array. nginx_config: # config for render the template to generate nginx.conf #user: root # specifies the execution user of the worker process. diff --git a/t/admin/global_data_encrypt.t b/t/admin/global_data_encrypt.t deleted file mode 100644 index 5c78c87bd4f0..000000000000 --- a/t/admin/global_data_encrypt.t +++ /dev/null @@ -1,143 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -use t::APISIX 'no_plan'; - -repeat_each(1); -no_long_string(); -no_root_location(); -no_shuffle(); -log_level("info"); - -add_block_preprocessor(sub { - my ($block) = @_; - - if (!$block->request) { - $block->set_value("request", "GET /t"); - } - - if (!$block->no_error_log) { - $block->set_value("no_error_log", "[error]\n[alert]"); - } -}); - -run_tests; - -__DATA__ - -=== TEST 1: sanity ---- yaml_config -apisix: - data_encryption: - enable: true - keyring: - - edd1c9f0985e76a2 ---- config - location /t { - content_by_lua_block { - local json = require("toolkit.json") - local t = require("lib.test_admin").test - local code, body = t('/apisix/admin/consumers', - ngx.HTTP_PUT, - [[{ - "username": "foo", - "plugins": { - "basic-auth": { - "username": "foo", - "password": "bar" - }, - "key-auth": { - "key": "auth-one" - } - } - }]] - ) - if code >= 300 then - ngx.status = code - ngx.say(body) - return - end - - ngx.sleep(0.1) - local code, message, res = t('/apisix/admin/consumers/foo', - ngx.HTTP_GET - ) - res = json.decode(res) - if code >= 300 then - ngx.status = code - ngx.say(message) - return - end - - ngx.say(res.value.plugins["basic-auth"].password) - } - } ---- response_body -77+NmbYqNfN+oLm0aX5akg== - - - -=== TEST 2: enable basic auth plugin using admin api ---- config - location /t { - content_by_lua_block { - local t = require("lib.test_admin").test - local code, body = t('/apisix/admin/routes/1', - ngx.HTTP_PUT, - [[{ - "plugins": { - "basic-auth": {}, - "key-auth": {} - }, - "upstream": { - "nodes": { - "127.0.0.1:1980": 1 - }, - "type": "roundrobin" - }, - "uri": "/hello" - }]] - ) - - if code >= 300 then - ngx.status = code - end - ngx.say(body) - } - } ---- request -GET /t ---- response_body -passed - - - -=== TEST 3: verify ---- yaml_config -apisix: - data_encryption: - enable: true - keyring: - - edd1c9f0985e76a2 ---- request -GET /hello ---- more_headers -apikey: auth-one -Authorization: Basic Zm9vOmJhcg== ---- response_body -hello world ---- error_log -find consumer foo diff --git a/t/admin/plugins.t b/t/admin/plugins.t index 2ab993152fcd..00a500e08e96 100644 --- a/t/admin/plugins.t +++ b/t/admin/plugins.t @@ -338,7 +338,7 @@ qr/\[\{"name":"wolf-rbac","priority":2555\},\{"name":"ldap-auth","priority":2540 } } --- response_body eval -qr/\{"properties":\{"password":\{"type":"string"\},"username":\{"type":"string"\}\},"required":\["username","password"\],"title":"work with consumer object","type":"object"\}/ +qr/\{"properties":\{"password":\{"encrypted":true,"type":"string"\},"username":\{"type":"string"\}\},"required":\["username","password"\],"title":"work with consumer object","type":"object"\}/ diff --git a/t/node/consumer-group.t b/t/node/consumer-group.t index 5f863ad003e2..ef16bbf7fdcb 100644 --- a/t/node/consumer-group.t +++ b/t/node/consumer-group.t @@ -310,3 +310,140 @@ world } --- response_body bar + + + +=== TEST 5: data encryption +--- yaml_config +apisix: + data_encryption: + enable: true + keyring: + - edd1c9f0985e76a2 +--- config + location /t { + content_by_lua_block { + local json = require("toolkit.json") + local t = require("lib.test_admin").test + local etcd = require("apisix.core.etcd") + local code, body = t('/apisix/admin/consumer_groups/company_a', + ngx.HTTP_PUT, + [[{ + "plugins": { + "limit-count": { + "count": 2, + "time_window": 60, + "rejected_code": 503, + "key": "remote_addr" + } + } + }]] + ) + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + + ngx.sleep(0.1) + + local code, body = t('/apisix/admin/consumers/foobar', + ngx.HTTP_PUT, + [[{ + "username": "foobar", + "plugins": { + "key-auth": { + "key": "auth-two" + } + }, + "group_id": "company_a" + }]] + ) + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + ngx.sleep(0.1) + + -- get plugin conf from admin api, key is decrypted + local code, message, res = t('/apisix/admin/consumers/foobar', + ngx.HTTP_GET + ) + res = json.decode(res) + if code >= 300 then + ngx.status = code + ngx.say(message) + return + end + + ngx.say(res.value.plugins["key-auth"].key) + + -- get plugin conf from etcd, key is encrypted + local etcd = require("apisix.core.etcd") + local res = assert(etcd.get('/consumers/foobar')) + ngx.say(res.body.node.value.plugins["key-auth"].key) + } + } +--- response_body +auth-two +vU/ZHVJw7b0XscDJ1Fhtig== + + + +=== TEST 6: verify data encryption +--- yaml_config +apisix: + data_encryption: + enable: true + keyring: + - edd1c9f0985e76a2 +--- config + location /t { + content_by_lua_block { + local json = require "t.toolkit.json" + local t = require("lib.test_admin").test + local code, err = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "uri": "/hello", + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + }, + "plugins": { + "key-auth": {} + } + }]] + ) + if code > 300 then + ngx.log(ngx.ERR, err) + return + end + ngx.sleep(0.1) + + local http = require "resty.http" + local uri = "http://127.0.0.1:" .. ngx.var.server_port + .. "/hello" + local ress = {} + for i = 1, 3 do + local httpc = http.new() + local res, err = httpc:request_uri(uri, { + method = "GET", + headers = { + ["apikey"] = "auth-two" + } + }) + if not res then + ngx.say(err) + return + end + table.insert(ress, res.status) + end + ngx.say(json.encode(ress)) + } + } +--- response_body +[200,200,503] diff --git a/t/node/data_encrypt.t b/t/node/data_encrypt.t new file mode 100644 index 000000000000..f6026df7f867 --- /dev/null +++ b/t/node/data_encrypt.t @@ -0,0 +1,581 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +use t::APISIX 'no_plan'; + +repeat_each(1); +no_long_string(); +no_root_location(); +no_shuffle(); +log_level("info"); + +add_block_preprocessor(sub { + my ($block) = @_; + + if (!$block->request) { + $block->set_value("request", "GET /t"); + } + + if (!$block->no_error_log) { + $block->set_value("no_error_log", "[error]\n[alert]"); + } +}); + +run_tests; + +__DATA__ + +=== TEST 1: sanity +# the sensitive data is encrypted in etcd, and it is normal to read it from the admin API +--- yaml_config +apisix: + data_encryption: + enable: true + keyring: + - edd1c9f0985e76a2 +--- config + location /t { + content_by_lua_block { + local json = require("toolkit.json") + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/consumers', + ngx.HTTP_PUT, + [[{ + "username": "foo", + "plugins": { + "basic-auth": { + "username": "foo", + "password": "bar" + } + } + }]] + ) + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + + ngx.sleep(0.1) + + -- get plugin conf from admin api, password is decrypted + local code, message, res = t('/apisix/admin/consumers/foo', + ngx.HTTP_GET + ) + res = json.decode(res) + if code >= 300 then + ngx.status = code + ngx.say(message) + return + end + + ngx.say(res.value.plugins["basic-auth"].password) + + -- get plugin conf from etcd, password is encrypted + local etcd = require("apisix.core.etcd") + local res = assert(etcd.get('/consumers/foo')) + ngx.say(res.body.node.value.plugins["basic-auth"].password) + + } + } +--- response_body +bar +77+NmbYqNfN+oLm0aX5akg== + + + +=== TEST 2: enable basic auth plugin +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "basic-auth": {} + }, + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- request +GET /t +--- response_body +passed + + + +=== TEST 3: verify +--- yaml_config +apisix: + data_encryption: + enable: true + keyring: + - edd1c9f0985e76a2 +--- request +GET /hello +--- more_headers +Authorization: Basic Zm9vOmJhcg== +--- response_body +hello world + + + +=== TEST 4: multiple auth plugins work well +--- yaml_config +apisix: + data_encryption: + enable: true + keyring: + - edd1c9f0985e76a2 +--- config + location /t { + content_by_lua_block { + local json = require("toolkit.json") + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/consumers', + ngx.HTTP_PUT, + [[{ + "username": "foo", + "plugins": { + "basic-auth": { + "username": "foo", + "password": "bar" + }, + "key-auth": { + "key": "auth-one" + } + } + }]] + ) + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + + ngx.sleep(0.1) + local code, message, res = t('/apisix/admin/consumers/foo', + ngx.HTTP_GET + ) + + if code >= 300 then + ngx.status = code + ngx.say(message) + return + end + + ngx.say("done") + } + } +--- response_body +done + + + +=== TEST 5: enable multiple auth plugins on route +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "basic-auth": {}, + "key-auth": {} + }, + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- request +GET /t +--- response_body +passed + + + +=== TEST 6: verify +--- yaml_config +apisix: + data_encryption: + enable: true + keyring: + - edd1c9f0985e76a2 +--- request +GET /hello +--- more_headers +apikey: auth-one +Authorization: Basic Zm9vOmJhcg== +--- response_body +hello world + + + +=== TEST 7: disable data_encryption +--- yaml_config +apisix: + data_encryption: + enable: false + keyring: + - edd1c9f0985e76a2 +--- config + location /t { + content_by_lua_block { + local json = require("toolkit.json") + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/consumers', + ngx.HTTP_PUT, + [[{ + "username": "foo", + "plugins": { + "basic-auth": { + "username": "foo", + "password": "bar" + } + } + }]] + ) + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + + ngx.sleep(0.1) + + -- get plugin conf from etcd, password is encrypted + local etcd = require("apisix.core.etcd") + local res = assert(etcd.get('/consumers/foo')) + ngx.say(res.body.node.value.plugins["basic-auth"].password) + + } + } +--- response_body +bar + + + +=== TEST 8: etcd store unencrypted password, enable data_encryption, decryption fails, use original password +--- yaml_config +apisix: + data_encryption: + enable: true + keyring: + - edd1c9f0985e76a2 +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local core = require("apisix.core") + -- get plugin conf from etcd, password is encrypted + local etcd = require("apisix.core.etcd") + local res, err = core.etcd.set("/consumers/foo2", core.json.decode([[{ + "username":"foo2", + "plugins":{ + "basic-auth":{ + "username":"foo2", + "password":"bar" + } + } + }]])) + + -- get plugin conf from admin api, password is decrypted + local code, message, res = t('/apisix/admin/consumers/foo2', + ngx.HTTP_GET + ) + res = core.json.decode(res) + if code >= 300 then + ngx.status = code + ngx.say(message) + return + end + + ngx.say(res.value.plugins["basic-auth"].password) + + -- get plugin conf from etcd, password is encrypted + local etcd = require("apisix.core.etcd") + local res = assert(etcd.get('/consumers/foo2')) + ngx.say(res.body.node.value.plugins["basic-auth"].password) + } + } +--- response_body +bar +bar +--- error_log +failed to decrypt the conf of plugin [basic-auth] key [password], err: decrypt ssl key failed + + + +=== TEST 9: etcd stores both encrypted and unencrypted data +# enable data_encryption, decryption of encrypted data succeeds +# decryption of unencrypted data fails, make sure it works well +--- yaml_config +apisix: + data_encryption: + enable: true + keyring: + - edd1c9f0985e76a2 +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local core = require("apisix.core") + -- get plugin conf from etcd, password is encrypted + local etcd = require("apisix.core.etcd") + local res, err = core.etcd.set("/consumers/foo2", core.json.decode([[{ + "username":"foo2", + "plugins":{ + "basic-auth":{ + "username":"foo2", + "password":"bar" + }, + "key-auth": { + "key": "vU/ZHVJw7b0XscDJ1Fhtig==" + } + } + }]])) + + -- get plugin conf from admin api, password is decrypted + local code, message, res = t('/apisix/admin/consumers/foo2', + ngx.HTTP_GET + ) + res = core.json.decode(res) + if code >= 300 then + ngx.status = code + ngx.say(message) + return + end + + ngx.say(res.value.plugins["basic-auth"].password) + ngx.say(res.value.plugins["key-auth"].key) + + -- get plugin conf from etcd, password is encrypted + local etcd = require("apisix.core.etcd") + local res = assert(etcd.get('/consumers/foo2')) + ngx.say(res.body.node.value.plugins["basic-auth"].password) + ngx.say(res.body.node.value.plugins["key-auth"].key) + } + } +--- response_body +bar +auth-two +bar +vU/ZHVJw7b0XscDJ1Fhtig== +--- error_log +failed to decrypt the conf of plugin [basic-auth] key [password], err: decrypt ssl key failed + + + +=== TEST 10: verify, use the foo2 consumer +--- yaml_config +apisix: + data_encryption: + enable: true + keyring: + - edd1c9f0985e76a2 +--- request +GET /hello +--- more_headers +apikey: auth-two +Authorization: Basic Zm9vMjpiYXI= +--- response_body +hello world + + + +=== TEST 11: keyring rotate, encrypt with edd1c9f0985e76a2 +--- yaml_config +apisix: + data_encryption: + enable: true + keyring: + - edd1c9f0985e76a2 +--- config + location /t { + content_by_lua_block { + local json = require("toolkit.json") + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/consumers', + ngx.HTTP_PUT, + [[{ + "username": "foo", + "plugins": { + "basic-auth": { + "username": "foo", + "password": "bar" + } + } + }]] + ) + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + + ngx.sleep(0.1) + + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "basic-auth": {} + }, + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 12: keyring rotate, decrypt with qeddd145sfvddff3 would fail, but encrypt with edd1c9f0985e76a2 would success +--- yaml_config +apisix: + data_encryption: + enable: true + keyring: + - qeddd145sfvddff3 + - edd1c9f0985e76a2 +--- request +GET /hello +--- more_headers +Authorization: Basic Zm9vOmJhcg== +--- response_body +hello world + + + +=== TEST 13: search consumer list +--- yaml_config +apisix: + data_encryption: + enable: true + keyring: + - edd1c9f0985e76a2 +--- config + location /t { + content_by_lua_block { + local json = require("toolkit.json") + local t = require("lib.test_admin").test + + -- dletet exist consumers + t('/apisix/admin/consumers/foo', ngx.HTTP_DELETE) + t('/apisix/admin/consumers/foo2', ngx.HTTP_DELETE) + + local code, body = t('/apisix/admin/consumers', + ngx.HTTP_PUT, + [[{ + "username": "foo", + "plugins": { + "basic-auth": { + "username": "foo", + "password": "bar" + } + } + }]] + ) + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + + ngx.sleep(0.1) + + local code, body = t('/apisix/admin/consumers', + ngx.HTTP_PUT, + [[{ + "username": "test", + "plugins": { + "basic-auth": { + "username": "test", + "password": "test" + } + } + }]] + ) + + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + + ngx.sleep(0.1) + + -- get plugin conf from admin api, password is decrypted + local code, message, res = t('/apisix/admin/consumers', + ngx.HTTP_GET + ) + res = json.decode(res) + if code >= 300 then + ngx.status = code + ngx.say(message) + return + end + + local pwds = {} + table.insert(pwds, res.list[1].value.plugins["basic-auth"].password) + table.insert(pwds, res.list[2].value.plugins["basic-auth"].password) + + ngx.say(json.encode(pwds)) + } + } +--- request +GET /t +--- response_body +["bar","test"] diff --git a/t/node/global-rule.t b/t/node/global-rule.t index 0e61d68d873c..6bd885dd2bf5 100644 --- a/t/node/global-rule.t +++ b/t/node/global-rule.t @@ -417,3 +417,112 @@ changed GET /t --- response_body passed + + + +=== TEST 15: data encription work well with global rule +--- yaml_config +apisix: + data_encryption: + enable: true + keyring: + - edd1c9f0985e76a2 +--- config + location /t { + content_by_lua_block { + local json = require("toolkit.json") + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/consumers', + ngx.HTTP_PUT, + [[{ + "username": "test", + "plugins": { + "basic-auth": { + "username": "test", + "password": "test" + } + }, + "desc": "test description" + }]] + ) + + if code >= 300 then + ngx.status = code + return + end + + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "uri": "/hello", + "upstream": { + "type": "roundrobin", + "nodes": { + "127.0.0.1:1980": 1 + } + } + }]] + ) + + if code >= 300 then + ngx.status = code + return + end + + local code, body = t('/apisix/admin/global_rules/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "basic-auth": {} + } + }]] + ) + + if code >= 300 then + ngx.status = code + return + end + + -- sleep for data sync + ngx.sleep(0.5) + + -- get plugin conf from admin api, password is decrypted + local code, message, res = t('/apisix/admin/consumers/test', + ngx.HTTP_GET + ) + res = json.decode(res) + if code >= 300 then + ngx.status = code + ngx.say(message) + return + end + + ngx.say(res.value.plugins["basic-auth"].password) + + -- get plugin conf from etcd, password is encrypted + local etcd = require("apisix.core.etcd") + local res = assert(etcd.get('/consumers/test')) + ngx.say(res.body.node.value.plugins["basic-auth"].password) + + -- hit the route with authorization + local code, body = t('/hello', + ngx.HTTP_PUT, + nil, + nil, + {Authorization = "Basic dGVzdDp0ZXN0"} + ) + + if code ~= 200 then + ngx.status = code + return + end + + ngx.say(body) + } + } +--- request +GET /t +--- response_body +test +9QKrmTT3TkWGvjlIoe5XXw== +passed diff --git a/t/plugin/clickhouse-logger.t b/t/plugin/clickhouse-logger.t index bcb874768202..1cf619c1853c 100644 --- a/t/plugin/clickhouse-logger.t +++ b/t/plugin/clickhouse-logger.t @@ -236,3 +236,235 @@ clickhouse headers: x-clickhouse-key:a clickhouse headers: x-clickhouse-user:default clickhouse headers: x-clickhouse-database:default --- wait: 5 + + + +=== TEST 7: password use data encryption +--- yaml_config +apisix: + data_encryption: + enable: true + keyring: + - edd1c9f0985e76a2 +--- config + location /t { + content_by_lua_block { + local json = require("toolkit.json") + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "clickhouse-logger": { + "user": "default", + "password": "abc123", + "database": "default", + "logtable": "t", + "endpoint_addr": "http://127.0.0.1:10420/clickhouse-logger/test", + "batch_max_size":1, + "inactive_timeout":1 + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/opentracing" + }]] + ) + + ngx.sleep(0.5) + + -- get plugin conf from admin api, password is decrypted + local code, message, res = t('/apisix/admin/routes/1', + ngx.HTTP_GET + ) + res = json.decode(res) + if code >= 300 then + ngx.status = code + ngx.say(message) + return + end + + ngx.say(res.value.plugins["clickhouse-logger"].password) + + -- get plugin conf from etcd, password is encrypted + local etcd = require("apisix.core.etcd") + local res = assert(etcd.get('/routes/1')) + ngx.say(res.body.node.value.plugins["clickhouse-logger"].password) + } + } +--- response_body +abc123 +7ipXoKyiZZUAgf3WWNPI5A== +--- error_log +base64 decode ssl key failed. key[a] + + + +=== TEST 8: verify +--- yaml_config +apisix: + data_encryption: + enable: true + keyring: + - edd1c9f0985e76a2 +--- request +GET /opentracing +--- response_body +opentracing +--- error_log +clickhouse body: INSERT INTO t FORMAT JSONEachRow +clickhouse headers: x-clickhouse-key:abc123 +clickhouse headers: x-clickhouse-user:default +clickhouse headers: x-clickhouse-database:default +--- wait: 5 + + + +=== TEST 9: POST and get lis +--- yaml_config +apisix: + data_encryption: + enable: true + keyring: + - edd1c9f0985e76a2 +--- config + location /t { + content_by_lua_block { + local json = require("toolkit.json") + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes', + ngx.HTTP_POST, + [[{ + "plugins": { + "clickhouse-logger": { + "user": "default", + "password": "abc123", + "database": "default", + "logtable": "t", + "endpoint_addr": "http://127.0.0.1:10420/clickhouse-logger/test", + "batch_max_size":1, + "inactive_timeout":1 + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/opentracing" + }]] + ) + + ngx.sleep(0.1) + + -- get plugin conf from admin api, password is decrypted + local code, message, res = t('/apisix/admin/routes', + ngx.HTTP_GET + ) + res = json.decode(res) + + + if code >= 300 then + ngx.status = code + ngx.say(message) + return + end + + ngx.say(res.list[1].value.plugins["clickhouse-logger"].password) + + -- get plugin conf from etcd, password is encrypted + local etcd = require("apisix.core.etcd") + local id = res.list[1].value.id + local key = "/routes/" .. id + local res = assert(etcd.get(key)) + + ngx.say(res.body.node.value.plugins["clickhouse-logger"].password) + } + } +--- response_body +abc123 +7ipXoKyiZZUAgf3WWNPI5A== + + + +=== TEST 10: PATCH +--- yaml_config +apisix: + data_encryption: + enable: true + keyring: + - edd1c9f0985e76a2 +--- config + location /t { + content_by_lua_block { + local json = require("toolkit.json") + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "clickhouse-logger": { + "user": "default", + "password": "abc123", + "database": "default", + "logtable": "t", + "endpoint_addr": "http://127.0.0.1:10420/clickhouse-logger/test", + "batch_max_size":1, + "inactive_timeout":1 + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/opentracing" + }]] + ) + + ngx.sleep(0.1) + + local code, body = t('/apisix/admin/routes/1/plugins', + ngx.HTTP_PATCH, + [[{ + "clickhouse-logger": { + "user": "default", + "password": "def456", + "database": "default", + "logtable": "t", + "endpoint_addr": "http://127.0.0.1:10420/clickhouse-logger/test", + "batch_max_size":1, + "inactive_timeout":1 + } + }]] + ) + + ngx.sleep(0.1) + -- get plugin conf from admin api, password is decrypted + local code, message, res = t('/apisix/admin/routes/1', + ngx.HTTP_GET + ) + res = json.decode(res) + if code >= 300 then + ngx.status = code + ngx.say(message) + return + end + + ngx.say(res.value.plugins["clickhouse-logger"].password) + + -- get plugin conf from etcd, password is encrypted + local etcd = require("apisix.core.etcd") + local res = assert(etcd.get('/routes/1')) + ngx.say(res.body.node.value.plugins["clickhouse-logger"].password) + } + } +--- response_body +def456 +3hlZu5mwUbqROm+cy0Vi9A== From 3fa3c00f17eeba7315ab3c3ff90ebe63c9a3e469 Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Mon, 28 Nov 2022 19:22:31 +0800 Subject: [PATCH 03/17] add docs --- docs/en/latest/plugin-develop.md | 33 +++++++++++++++++++++++++++++++- docs/zh/latest/plugin-develop.md | 33 +++++++++++++++++++++++++++++++- 2 files changed, 64 insertions(+), 2 deletions(-) diff --git a/docs/en/latest/plugin-develop.md b/docs/en/latest/plugin-develop.md index b47d9dfdf940..520398b6fff3 100644 --- a/docs/en/latest/plugin-develop.md +++ b/docs/en/latest/plugin-develop.md @@ -216,7 +216,7 @@ end Note: the project has provided the public method "__core.schema.check__", which can be used directly to complete JSON verification. -In addition, if the plugin needs to use some metadata, we can define the plugin `metadata_schema`, and then we can dynamically manage these metadata through the `admin api`. Example: +In addition, if the plugin needs to use some metadata, we can define the plugin `metadata_schema`, and then we can dynamically manage these metadata through the `Admin API`. Example: ```lua local metadata_schema = { @@ -293,6 +293,37 @@ function _M.check_schema(conf, schema_type) end ``` +Specify the parameters to be stored encrypted(Requires APISIX version > 3.0.1) + +Some plugins require parameters to be stored encrypted, such as the `password` parameter of the `basic-auth` plugin. This plugin needs to specify in the `schema` which parameters need to be stored encrypted. + +```lua +password = { type = "string", encrypted = true }, +``` + +Parameters can be stored encrypted by specifying `encrypted = true` in the `schema`. APISIX will provide the following functionality. + +- When adding and updating resources via the `Admin API`, APISIX automatically encrypts parameters with `encrypted = true` and stores them in etcd +- When fetching resources via the `Admin API`, APISIX automatically decrypts the `encrypted = true` parameter and returns it +- APISIX automatically decrypts the `encrypted = true` parameter when it is used + +How to enable this feature? + +Enable `data_encryption` in `config.yaml`. + +```yaml +apisix: + data_encryption: + enable: true + keyring: + - edd1c9f0985e76a2 + - qeddd145sfvddff4 +``` + +APISIX will try to decrypt the data with keys in the order of the keys in the keyring (only for parameters declared `encrypted = true`). If the decryption fails, the next key will be tried until the decryption succeeds. + +If none of the keys in `keyring` can decrypt the data, the original data is used. + ## choose phase to run Determine which phase to run, generally access or rewrite. If you don't know the [OpenResty lifecycle](https://github.com/openresty/lua-nginx-module/blob/master/README.markdown#directives), it's diff --git a/docs/zh/latest/plugin-develop.md b/docs/zh/latest/plugin-develop.md index d9e5705156cd..b46945c0f58d 100644 --- a/docs/zh/latest/plugin-develop.md +++ b/docs/zh/latest/plugin-develop.md @@ -196,7 +196,7 @@ end 注：项目已经提供了 __core.schema.check__ 公共方法，直接使用即可完成配置参数校验。 -另外，如果插件需要使用一些元数据，可以定义插件的 `metadata_schema` ，然后就可以通过 `admin api` 动态的管理这些元数据了。如： +另外，如果插件需要使用一些元数据，可以定义插件的 `metadata_schema` ，然后就可以通过 `Admin API` 动态的管理这些元数据了。如： ```lua local metadata_schema = { @@ -273,6 +273,37 @@ function _M.check_schema(conf, schema_type) end ``` +指定参数需要被加密存储（需要 APISIX 版本大于 3.0.1） + +有些插件需要将参数加密存储，比如 `basic-auth` 插件的 `password` 参数。这个插件需要在 `schema` 中指定哪些参数需要被加密存储。 + +```lua +password = { type = "string", encrypted = true }, +``` + +通过在 `schema` 中指定 `encrypted = true`，可以将参数加密存储。APISIX 将提供以下功能： + +- 通过 `Admin API` 来新增和更新资源时，对于 `encrypted = true` 的参数，APISIX 会自动加密存储在 etcd 中 +- 通过 `Admin API` 来获取资源时，对于 `encrypted = true` 的参数，APISIX 会自动解密返回 +- APISIX 在使用 `encrypted = true` 的参数时，会自动解密 + +如何开启该功能？ + +在 `config.yaml` 中开启 `data_encryption`： + +```yaml +apisix: + data_encryption: + enable: true + keyring: + - edd1c9f0985e76a2 + - qeddd145sfvddff4 +``` + +`keyring` 是一个数组，可以指定多个 key，APISIX 会按照 keyring 中 key 的顺序，依次尝试用 key 来解密数据（只对声明 `encrypted = true` 的参数）。如果解密失败，会尝试下一个 key，直到解密成功。 + +如果 `keyring` 中的 key 都无法解密数据，则使用原始数据。 + ## 确定执行阶段 根据业务功能，确定你的插件需要在哪个阶段执行。 key-auth 是一个认证插件，所以需要在 rewrite 阶段执行。在 APISIX，只有认证逻辑可以在 rewrite 阶段里面完成，其他需要在代理到上游之前执行的逻辑都是在 access 阶段完成的。 From 5b94212cce046549c44cee09a2f518779acaccd3 Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Mon, 28 Nov 2022 19:28:29 +0800 Subject: [PATCH 04/17] fix lint --- apisix/admin/utils.lua | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apisix/admin/utils.lua b/apisix/admin/utils.lua index ac0f05c4348b..f2402ef28a3f 100644 --- a/apisix/admin/utils.lua +++ b/apisix/admin/utils.lua @@ -90,7 +90,8 @@ function _M.decrypt_conf(decrypt_func, body, schema_type) -- list if body.list and #body.list > 0 then for _, route in ipairs(body.list) do - if route.value and route.value.plugins and core.table.nkeys(route.value.plugins) > 0 then + if route.value and route.value.plugins + and core.table.nkeys(route.value.plugins) > 0 then for name, conf in pairs(route.value.plugins) do decrypt_func(name, conf, schema_type) end From 6fd1700dd37fc68d4b057d302229c06bf342eb24 Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Mon, 28 Nov 2022 19:37:15 +0800 Subject: [PATCH 05/17] chore --- apisix/admin/utils.lua | 6 ------ apisix/plugin.lua | 27 +++++++++++++-------------- 2 files changed, 13 insertions(+), 20 deletions(-) diff --git a/apisix/admin/utils.lua b/apisix/admin/utils.lua index f2402ef28a3f..9b8010a9705f 100644 --- a/apisix/admin/utils.lua +++ b/apisix/admin/utils.lua @@ -81,12 +81,6 @@ end function _M.decrypt_conf(decrypt_func, body, schema_type) - local local_conf = core.config.local_conf() - local enable = core.table.try_read_attr(local_conf, "apisix", "data_encryption", "enable") - if not enable then - return - end - -- list if body.list and #body.list > 0 then for _, route in ipairs(body.list) do diff --git a/apisix/plugin.lua b/apisix/plugin.lua index 9d9755f140dc..c87cc3f40840 100644 --- a/apisix/plugin.lua +++ b/apisix/plugin.lua @@ -850,12 +850,7 @@ check_plugin_metadata = function(item) end -local function decrypt_conf(name, conf, schema_type) - local enable = core.table.try_read_attr(local_conf, "apisix", "data_encryption", "enable") - if not enable then - return conf - end - +local function check_enable_and_get_plugin_schema(name, schema_type) local plugin_schema = local_plugins_hash and local_plugins_hash[name] local schema if schema_type == core.schema.TYPE_CONSUMER then @@ -864,6 +859,17 @@ local function decrypt_conf(name, conf, schema_type) schema = plugin_schema.schema end + return schema +end + + +local function decrypt_conf(name, conf, schema_type) + local enable = core.table.try_read_attr(local_conf, "apisix", "data_encryption", "enable") + if not enable then + return conf + end + + local schema = check_enable_and_get_plugin_schema(name, schema_type) if not schema then return end @@ -889,14 +895,7 @@ local function encrypt_conf(name, conf, schema_type) return conf end - local plugin_schema = local_plugins_hash and local_plugins_hash[name] - local schema - if schema_type == core.schema.TYPE_CONSUMER then - schema = plugin_schema.consumer_schema - else - schema = plugin_schema.schema - end - + local schema = check_enable_and_get_plugin_schema(name, schema_type) if not schema then return end From 469e12930fa6f3fcd987bde78551c4e83a38a2b7 Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Tue, 29 Nov 2022 16:51:25 +0800 Subject: [PATCH 06/17] resolve comments --- apisix/admin/consumers.lua | 7 ++++--- apisix/admin/routes.lua | 6 ++++-- apisix/admin/utils.lua | 2 +- apisix/plugin.lua | 19 ++++++++++++++----- conf/config-default.yaml | 4 ++-- docs/en/latest/plugin-develop.md | 3 +-- docs/zh/latest/plugin-develop.md | 3 +-- t/plugin/clickhouse-logger.t | 2 +- 8 files changed, 28 insertions(+), 18 deletions(-) diff --git a/apisix/admin/consumers.lua b/apisix/admin/consumers.lua index a0224005167b..bd1c6e51631e 100644 --- a/apisix/admin/consumers.lua +++ b/apisix/admin/consumers.lua @@ -123,11 +123,12 @@ function _M.get(consumer_name) utils.fix_count(res.body, consumer_name) -- decrypt the conf - if res.body then - utils.decrypt_conf(plugin.decrypt_conf, res.body, core.schema.TYPE_CONSUMER) + if plugin.enable_data_encryption then + if res.body then + utils.decrypt_params(plugin.decrypt_conf, res.body, core.schema.TYPE_CONSUMER) + end end - return res.status, res.body end diff --git a/apisix/admin/routes.lua b/apisix/admin/routes.lua index 49b5955d5ef0..476bde747609 100644 --- a/apisix/admin/routes.lua +++ b/apisix/admin/routes.lua @@ -211,8 +211,10 @@ function _M.get(id) utils.fix_count(res.body, id) -- decrypt the conf - if res.body then - utils.decrypt_conf(plugin.decrypt_conf, res.body) + if plugin.enable_data_encryption then + if res.body then + utils.decrypt_params(plugin.decrypt_conf, res.body) + end end return res.status, res.body diff --git a/apisix/admin/utils.lua b/apisix/admin/utils.lua index 9b8010a9705f..eec31da5cc2e 100644 --- a/apisix/admin/utils.lua +++ b/apisix/admin/utils.lua @@ -80,7 +80,7 @@ function _M.fix_count(body, id) end -function _M.decrypt_conf(decrypt_func, body, schema_type) +function _M.decrypt_params(decrypt_func, body, schema_type) -- list if body.list and #body.list > 0 then for _, route in ipairs(body.list) do diff --git a/apisix/plugin.lua b/apisix/plugin.lua index c87cc3f40840..591e14d4aa2e 100644 --- a/apisix/plugin.lua +++ b/apisix/plugin.lua @@ -863,9 +863,19 @@ local function check_enable_and_get_plugin_schema(name, schema_type) end +local enable_data_encryption +local function enable_gde() + if enable_data_encryption == nil then + enable_data_encryption = core.table.try_read_attr(local_conf, "apisix", "data_encryption", "enable") + _M.enable_data_encryption = enable_data_encryption + end + + return enable_data_encryption +end + + local function decrypt_conf(name, conf, schema_type) - local enable = core.table.try_read_attr(local_conf, "apisix", "data_encryption", "enable") - if not enable then + if not enable_gde() then return conf end @@ -890,8 +900,7 @@ _M.decrypt_conf = decrypt_conf local function encrypt_conf(name, conf, schema_type) - local enable = core.table.try_read_attr(local_conf, "apisix", "data_encryption", "enable") - if not enable then + if not enable_gde() then return conf end @@ -963,7 +972,7 @@ function _M.plugin_checker(item, schema_type) if item.plugins then local ok, err = check_schema(item.plugins, schema_type, true) - if ok then + if ok and enable_gde() then -- decrypt conf for name, conf in pairs(item.plugins) do decrypt_conf(name, conf, schema_type) diff --git a/conf/config-default.yaml b/conf/config-default.yaml index 620fad427a48..f14db8ef2323 100755 --- a/conf/config-default.yaml +++ b/conf/config-default.yaml @@ -123,10 +123,10 @@ apisix: # ip: 127.0.0.1 # port: 9090 disable_sync_configuration_during_start: false # safe exit. Remove this once the feature is stable - data_encryption: # use `encrypted = {fields}` in plugin schema to enable encryption + data_encryption: # add `encrypted = true` in plugin schema to enable encryption enable: false # if not set, the default value is `false`. keyring: - - edd1c9f0985e76a2 # If not set, will save origin value into etcd. + - qeddd145sfvddff3 # If not set, will save origin value into etcd. # If set this, the keyring should be an array whose elements are string, and the size is also 16, and it will encrypt fields with AES-128-CBC # !!! So do not change it after encryption, it can't decrypt the fields have be saved if you change !! # Only use the first key to encrypt, and decrypt in the order of the array. diff --git a/docs/en/latest/plugin-develop.md b/docs/en/latest/plugin-develop.md index 520398b6fff3..1509aa2f16b7 100644 --- a/docs/en/latest/plugin-develop.md +++ b/docs/en/latest/plugin-develop.md @@ -304,8 +304,7 @@ password = { type = "string", encrypted = true }, Parameters can be stored encrypted by specifying `encrypted = true` in the `schema`. APISIX will provide the following functionality. - When adding and updating resources via the `Admin API`, APISIX automatically encrypts parameters with `encrypted = true` and stores them in etcd -- When fetching resources via the `Admin API`, APISIX automatically decrypts the `encrypted = true` parameter and returns it -- APISIX automatically decrypts the `encrypted = true` parameter when it is used +- When fetching resources via the `Admin API` and when running the plugin, APISIX automatically decrypts the `encrypted = true` parameter How to enable this feature? diff --git a/docs/zh/latest/plugin-develop.md b/docs/zh/latest/plugin-develop.md index b46945c0f58d..ccd55dd634de 100644 --- a/docs/zh/latest/plugin-develop.md +++ b/docs/zh/latest/plugin-develop.md @@ -284,8 +284,7 @@ password = { type = "string", encrypted = true }, 通过在 `schema` 中指定 `encrypted = true`，可以将参数加密存储。APISIX 将提供以下功能： - 通过 `Admin API` 来新增和更新资源时，对于 `encrypted = true` 的参数，APISIX 会自动加密存储在 etcd 中 -- 通过 `Admin API` 来获取资源时，对于 `encrypted = true` 的参数，APISIX 会自动解密返回 -- APISIX 在使用 `encrypted = true` 的参数时，会自动解密 +- 通过 `Admin API` 来获取资源时，以及在运行插件时，对于 `encrypted = true` 的参数，APISIX 会自动解密 如何开启该功能？ diff --git a/t/plugin/clickhouse-logger.t b/t/plugin/clickhouse-logger.t index 1cf619c1853c..976f97267260 100644 --- a/t/plugin/clickhouse-logger.t +++ b/t/plugin/clickhouse-logger.t @@ -324,7 +324,7 @@ clickhouse headers: x-clickhouse-database:default -=== TEST 9: POST and get lis +=== TEST 9: POST and get list --- yaml_config apisix: data_encryption: From 773c00cab108d4b4196b3e11af2b7fb268abbef7 Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Tue, 29 Nov 2022 18:10:34 +0800 Subject: [PATCH 07/17] add test cases and docs --- apisix/admin/consumers.lua | 9 - apisix/admin/init.lua | 13 + apisix/admin/plugins.lua | 10 +- apisix/admin/routes.lua | 12 - apisix/plugin.lua | 3 +- docs/en/latest/plugin-develop.md | 2 + docs/en/latest/plugins/basic-auth.md | 2 + docs/en/latest/plugins/key-auth.md | 2 + docs/zh/latest/plugin-develop.md | 2 + docs/zh/latest/plugins/basic-auth.md | 2 + docs/zh/latest/plugins/clickhouse-logger.md | 2 + docs/zh/latest/plugins/key-auth.md | 2 + t/node/data_encrypt2.t | 594 ++++++++++++++++++++ t/node/global-rule.t | 109 ---- t/plugin/clickhouse-logger.t | 232 -------- 15 files changed, 632 insertions(+), 364 deletions(-) create mode 100644 t/node/data_encrypt2.t diff --git a/apisix/admin/consumers.lua b/apisix/admin/consumers.lua index bd1c6e51631e..99a5861b004c 100644 --- a/apisix/admin/consumers.lua +++ b/apisix/admin/consumers.lua @@ -55,8 +55,6 @@ local function check_conf(username, conf) if plugin_obj.type == 'auth' then count_auth_plugin = count_auth_plugin + 1 end - - plugin.encrypt_conf(name, conf, core.schema.TYPE_CONSUMER) end if count_auth_plugin == 0 then @@ -122,13 +120,6 @@ function _M.get(consumer_name) utils.fix_count(res.body, consumer_name) - -- decrypt the conf - if plugin.enable_data_encryption then - if res.body then - utils.decrypt_params(plugin.decrypt_conf, res.body, core.schema.TYPE_CONSUMER) - end - end - return res.status, res.body end diff --git a/apisix/admin/init.lua b/apisix/admin/init.lua index 5827f83978fb..b66bdd39098f 100644 --- a/apisix/admin/init.lua +++ b/apisix/admin/init.lua @@ -19,6 +19,7 @@ local core = require("apisix.core") local route = require("apisix.utils.router") local plugin = require("apisix.plugin") local v3_adapter = require("apisix.admin.v3_adapter") +local utils = require("apisix.admin.utils") local ngx = ngx local get_method = ngx.req.get_method local ngx_time = ngx.time @@ -188,6 +189,18 @@ local function run() local code, data = resource[method](seg_id, req_body, seg_sub_path, uri_args) if code then + if method == "get" then + if plugin.enable_data_encryption then + if data then + if seg_res == "consumers" then + utils.decrypt_params(plugin.decrypt_conf, data, core.schema.TYPE_CONSUMER) + else + utils.decrypt_params(plugin.decrypt_conf, data) + end + end + end + end + if v3_adapter.enable_v3() then core.response.set_header("X-API-VERSION", "v3") else diff --git a/apisix/admin/plugins.lua b/apisix/admin/plugins.lua index 380723163401..26dcab719332 100644 --- a/apisix/admin/plugins.lua +++ b/apisix/admin/plugins.lua @@ -23,12 +23,20 @@ local table_sort = table.sort local table_insert = table.insert local get_uri_args = ngx.req.get_uri_args local plugin_get_all = require("apisix.plugin").get_all +local encrypt_conf = require("apisix.plugin").encrypt_conf +local pairs = pairs local _M = {} function _M.check_schema(plugins_conf, schema_type) - return check_schema(plugins_conf, schema_type, false) + local ok, err = check_schema(plugins_conf, schema_type, false) + if ok then + for name, conf in pairs(plugins_conf) do + encrypt_conf(name, conf, schema_type) + end + end + return ok, err end diff --git a/apisix/admin/routes.lua b/apisix/admin/routes.lua index 476bde747609..cf9fe0043a3c 100644 --- a/apisix/admin/routes.lua +++ b/apisix/admin/routes.lua @@ -19,8 +19,6 @@ local core = require("apisix.core") local apisix_upstream = require("apisix.upstream") local schema_plugin = require("apisix.admin.plugins").check_schema local utils = require("apisix.admin.utils") -local plugin = require("apisix.plugin") -local pairs = pairs local tostring = tostring local type = type local loadstring = loadstring @@ -133,9 +131,6 @@ local function check_conf(id, conf, need_id) if not ok then return nil, {error_msg = err} end - for name, conf in pairs(conf.plugins) do - plugin.encrypt_conf(name, conf) - end end if conf.vars then @@ -210,13 +205,6 @@ function _M.get(id) utils.fix_count(res.body, id) - -- decrypt the conf - if plugin.enable_data_encryption then - if res.body then - utils.decrypt_params(plugin.decrypt_conf, res.body) - end - end - return res.status, res.body end diff --git a/apisix/plugin.lua b/apisix/plugin.lua index 591e14d4aa2e..674a72b6c482 100644 --- a/apisix/plugin.lua +++ b/apisix/plugin.lua @@ -866,7 +866,8 @@ end local enable_data_encryption local function enable_gde() if enable_data_encryption == nil then - enable_data_encryption = core.table.try_read_attr(local_conf, "apisix", "data_encryption", "enable") + enable_data_encryption = + core.table.try_read_attr(local_conf, "apisix", "data_encryption", "enable") _M.enable_data_encryption = enable_data_encryption end diff --git a/docs/en/latest/plugin-develop.md b/docs/en/latest/plugin-develop.md index 1509aa2f16b7..08c753a96af7 100644 --- a/docs/en/latest/plugin-develop.md +++ b/docs/en/latest/plugin-develop.md @@ -293,6 +293,8 @@ function _M.check_schema(conf, schema_type) end ``` +### encrypted storage fields + Specify the parameters to be stored encrypted(Requires APISIX version > 3.0.1) Some plugins require parameters to be stored encrypted, such as the `password` parameter of the `basic-auth` plugin. This plugin needs to specify in the `schema` which parameters need to be stored encrypted. diff --git a/docs/en/latest/plugins/basic-auth.md b/docs/en/latest/plugins/basic-auth.md index a55dd1467c53..25c574c9c88b 100644 --- a/docs/en/latest/plugins/basic-auth.md +++ b/docs/en/latest/plugins/basic-auth.md @@ -42,6 +42,8 @@ For Consumer: | username | string | True | Unique username for a Consumer. If multiple Consumers use the same `username`, a request matching exception is raised. | | password | string | True | Password of the user. | +NOTE: The schema for `password` also defines `encrypted = true`, which means that the field will be stored encrypted in etcd. See [encrypted storage fields](../plugin-develop.md#encrypted-storage-fields). + For Route: | Name | Type | Required | Default | Description | diff --git a/docs/en/latest/plugins/key-auth.md b/docs/en/latest/plugins/key-auth.md index ea3ee098b228..b188cbc27efa 100644 --- a/docs/en/latest/plugins/key-auth.md +++ b/docs/en/latest/plugins/key-auth.md @@ -41,6 +41,8 @@ For Consumer: |------|--------|-------------|----------------------------| | key | string | required | Unique key for a Consumer. | +NOTE: The schema for `key` also defines `encrypted = true`, which means that the field will be stored encrypted in etcd. See [encrypted storage fields](../plugin-develop.md#encrypted-storage-fields). + For Route: | Name | Type | Requirement | Default | Valid | Description | diff --git a/docs/zh/latest/plugin-develop.md b/docs/zh/latest/plugin-develop.md index ccd55dd634de..24ad81fba52a 100644 --- a/docs/zh/latest/plugin-develop.md +++ b/docs/zh/latest/plugin-develop.md @@ -273,6 +273,8 @@ function _M.check_schema(conf, schema_type) end ``` +### 加密存储字段 + 指定参数需要被加密存储（需要 APISIX 版本大于 3.0.1） 有些插件需要将参数加密存储，比如 `basic-auth` 插件的 `password` 参数。这个插件需要在 `schema` 中指定哪些参数需要被加密存储。 diff --git a/docs/zh/latest/plugins/basic-auth.md b/docs/zh/latest/plugins/basic-auth.md index dc2c597bb15d..f19dcab573a6 100644 --- a/docs/zh/latest/plugins/basic-auth.md +++ b/docs/zh/latest/plugins/basic-auth.md @@ -42,6 +42,8 @@ Consumer 端： | username | string | 是 | Consumer 的用户名并且该用户名是唯一，如果多个 Consumer 使用了相同的 `username`，将会出现请求匹配异常。| | password | string | 是 | 用户的密码。 | +注意：`password` 的 schema 中还定义了 `encrypted = true`，这意味着该字段将会被加密存储在 etcd 中。具体参考 [加密存储字段](../plugin-develop.md#加密存储字段)。 + Route 端： | 名称 | 类型 | 必选项 | 默认值 | 描述 | diff --git a/docs/zh/latest/plugins/clickhouse-logger.md b/docs/zh/latest/plugins/clickhouse-logger.md index 9b0ba11e7373..dc35db7e9d81 100644 --- a/docs/zh/latest/plugins/clickhouse-logger.md +++ b/docs/zh/latest/plugins/clickhouse-logger.md @@ -47,6 +47,8 @@ description: 本文介绍了 API 网关 Apache APISIX 如何使用 clickhouse-lo 该插件支持使用批处理器来聚合并批量处理条目（日志/数据）。这样可以避免插件频繁地提交数据，默认情况下批处理器每 `5` 秒钟或队列中的数据达到 `1000` 条时提交数据，如需了解批处理器相关参数设置，请参考 [Batch-Processor](../batch-processor.md#配置)。 +注意：`password` 的 schema 中还定义了 `encrypted = true`，这意味着该字段将会被加密存储在 etcd 中。具体参考 [加密存储字段](../plugin-develop.md#加密存储字段)。 + ## 配置插件元数据 `clickhouse-logger` 也支持自定义日志格式，与 [http-logger](./http-logger.md) 插件类似。 diff --git a/docs/zh/latest/plugins/key-auth.md b/docs/zh/latest/plugins/key-auth.md index 8b4ba2146e3b..8031854e7084 100644 --- a/docs/zh/latest/plugins/key-auth.md +++ b/docs/zh/latest/plugins/key-auth.md @@ -41,6 +41,8 @@ Consumer 端： | ---- | ------ | ------ | ------------------------------------------------------------------------------------------------------------- | | key | string | 是 | 不同的 Consumer 应有不同的 `key`，它应当是唯一的。如果多个 Consumer 使用了相同的 `key`，将会出现请求匹配异常。 | +注意：`key` 的 schema 中还定义了 `encrypted = true`，这意味着该字段将会被加密存储在 etcd 中。具体参考 [加密存储字段](../plugin-develop.md#加密存储字段)。 + Router 端： | 名称 | 类型 | 必选项 | 默认值 | 描述 | diff --git a/t/node/data_encrypt2.t b/t/node/data_encrypt2.t new file mode 100644 index 000000000000..434d631528d1 --- /dev/null +++ b/t/node/data_encrypt2.t @@ -0,0 +1,594 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +use t::APISIX 'no_plan'; + +repeat_each(1); +no_long_string(); +no_root_location(); +no_shuffle(); +log_level("info"); + +add_block_preprocessor(sub { + my ($block) = @_; + + if (!defined $block->request) { + $block->set_value("request", "GET /t"); + } + + my $http_config = $block->http_config // <<_EOC_; + server { + listen 10420; + location /clickhouse-logger/test { + content_by_lua_block { + ngx.req.read_body() + local data = ngx.req.get_body_data() + local headers = ngx.req.get_headers() + ngx.log(ngx.WARN, "clickhouse body: ", data) + for k, v in pairs(headers) do + ngx.log(ngx.WARN, "clickhouse headers: " .. k .. ":" .. v) + end + ngx.say("ok") + } + } + location /clickhouse-logger/test1 { + content_by_lua_block { + ngx.req.read_body() + local data = ngx.req.get_body_data() + local headers = ngx.req.get_headers() + ngx.log(ngx.WARN, "clickhouse body: ", data) + for k, v in pairs(headers) do + ngx.log(ngx.WARN, "clickhouse headers: " .. k .. ":" .. v) + end + ngx.say("ok") + } + } + } +_EOC_ + + $block->set_value("http_config", $http_config); +}); + +run_tests(); + +__DATA__ + +=== TEST 1: data encription work well with plugins that not the auth plugins +--- yaml_config +apisix: + data_encryption: + enable: true + keyring: + - edd1c9f0985e76a2 +--- config + location /t { + content_by_lua_block { + local json = require("toolkit.json") + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "clickhouse-logger": { + "user": "default", + "password": "abc123", + "database": "default", + "logtable": "t", + "endpoint_addr": "http://127.0.0.1:10420/clickhouse-logger/test", + "batch_max_size":1, + "inactive_timeout":1 + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/opentracing" + }]] + ) + + ngx.sleep(0.5) + + -- get plugin conf from admin api, password is decrypted + local code, message, res = t('/apisix/admin/routes/1', + ngx.HTTP_GET + ) + res = json.decode(res) + if code >= 300 then + ngx.status = code + ngx.say(message) + return + end + + ngx.say(res.value.plugins["clickhouse-logger"].password) + + -- get plugin conf from etcd, password is encrypted + local etcd = require("apisix.core.etcd") + local res = assert(etcd.get('/routes/1')) + ngx.say(res.body.node.value.plugins["clickhouse-logger"].password) + } + } +--- response_body +abc123 +7ipXoKyiZZUAgf3WWNPI5A== + + + +=== TEST 2: verify +--- yaml_config +apisix: + data_encryption: + enable: true + keyring: + - edd1c9f0985e76a2 +--- request +GET /opentracing +--- response_body +opentracing +--- error_log +clickhouse body: INSERT INTO t FORMAT JSONEachRow +clickhouse headers: x-clickhouse-key:abc123 +clickhouse headers: x-clickhouse-user:default +clickhouse headers: x-clickhouse-database:default +--- wait: 5 + + + +=== TEST 3: POST and get list +--- yaml_config +apisix: + data_encryption: + enable: true + keyring: + - edd1c9f0985e76a2 +--- config + location /t { + content_by_lua_block { + local json = require("toolkit.json") + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes', + ngx.HTTP_POST, + [[{ + "plugins": { + "clickhouse-logger": { + "user": "default", + "password": "abc123", + "database": "default", + "logtable": "t", + "endpoint_addr": "http://127.0.0.1:10420/clickhouse-logger/test", + "batch_max_size":1, + "inactive_timeout":1 + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/opentracing" + }]] + ) + + ngx.sleep(0.1) + + -- get plugin conf from admin api, password is decrypted + local code, message, res = t('/apisix/admin/routes', + ngx.HTTP_GET + ) + res = json.decode(res) + + + if code >= 300 then + ngx.status = code + ngx.say(message) + return + end + + ngx.say(res.list[1].value.plugins["clickhouse-logger"].password) + + -- get plugin conf from etcd, password is encrypted + local etcd = require("apisix.core.etcd") + local id = res.list[1].value.id + local key = "/routes/" .. id + local res = assert(etcd.get(key)) + + ngx.say(res.body.node.value.plugins["clickhouse-logger"].password) + } + } +--- response_body +abc123 +7ipXoKyiZZUAgf3WWNPI5A== + + + +=== TEST 4: PATCH +--- yaml_config +apisix: + data_encryption: + enable: true + keyring: + - edd1c9f0985e76a2 +--- config + location /t { + content_by_lua_block { + local json = require("toolkit.json") + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "clickhouse-logger": { + "user": "default", + "password": "abc123", + "database": "default", + "logtable": "t", + "endpoint_addr": "http://127.0.0.1:10420/clickhouse-logger/test", + "batch_max_size":1, + "inactive_timeout":1 + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/opentracing" + }]] + ) + + ngx.sleep(0.1) + + local code, body = t('/apisix/admin/routes/1/plugins', + ngx.HTTP_PATCH, + [[{ + "clickhouse-logger": { + "user": "default", + "password": "def456", + "database": "default", + "logtable": "t", + "endpoint_addr": "http://127.0.0.1:10420/clickhouse-logger/test", + "batch_max_size":1, + "inactive_timeout":1 + } + }]] + ) + + ngx.sleep(0.1) + -- get plugin conf from admin api, password is decrypted + local code, message, res = t('/apisix/admin/routes/1', + ngx.HTTP_GET + ) + res = json.decode(res) + if code >= 300 then + ngx.status = code + ngx.say(message) + return + end + + ngx.say(res.value.plugins["clickhouse-logger"].password) + + -- get plugin conf from etcd, password is encrypted + local etcd = require("apisix.core.etcd") + local res = assert(etcd.get('/routes/1')) + ngx.say(res.body.node.value.plugins["clickhouse-logger"].password) + } + } +--- response_body +def456 +3hlZu5mwUbqROm+cy0Vi9A== + + + +=== TEST 5: data encription work well with services +--- yaml_config +apisix: + data_encryption: + enable: true + keyring: + - edd1c9f0985e76a2 +--- config + location /t { + content_by_lua_block { + local json = require("toolkit.json") + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/services/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "clickhouse-logger": { + "user": "default", + "password": "abc123", + "database": "default", + "logtable": "t", + "endpoint_addr": "http://127.0.0.1:10420/clickhouse-logger/test", + "batch_max_size":1, + "inactive_timeout":1 + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + } + }]] + ) + if code >= 300 then + ngx.status = code + return + end + ngx.sleep(0.1) + + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "service_id": "1", + "uri": "/opentracing" + }]] + ) + if code >= 300 then + ngx.status = code + return + end + ngx.sleep(0.1) + + -- get plugin conf from admin api, password is decrypted + local code, message, res = t('/apisix/admin/services/1', + ngx.HTTP_GET + ) + res = json.decode(res) + if code >= 300 then + ngx.status = code + ngx.say(message) + return + end + ngx.say(res.value.plugins["clickhouse-logger"].password) + + -- get plugin conf from etcd, password is encrypted + local etcd = require("apisix.core.etcd") + local res = assert(etcd.get('/services/1')) + ngx.say(res.body.node.value.plugins["clickhouse-logger"].password) + } + } +--- response_body +abc123 +7ipXoKyiZZUAgf3WWNPI5A== + + + +=== TEST 6: verify +--- yaml_config +apisix: + data_encryption: + enable: true + keyring: + - edd1c9f0985e76a2 +--- request +GET /opentracing +--- response_body +opentracing +--- error_log +clickhouse body: INSERT INTO t FORMAT JSONEachRow +clickhouse headers: x-clickhouse-key:abc123 +clickhouse headers: x-clickhouse-user:default +clickhouse headers: x-clickhouse-database:default +--- wait: 5 + + + +=== TEST 7: data encription work well with plugin_configs +--- yaml_config +apisix: + data_encryption: + enable: true + keyring: + - edd1c9f0985e76a2 +--- config + location /t { + content_by_lua_block { + local json = require("toolkit.json") + local t = require("lib.test_admin").test + local code, err = t('/apisix/admin/plugin_configs/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "clickhouse-logger": { + "user": "default", + "password": "abc123", + "database": "default", + "logtable": "t", + "endpoint_addr": "http://127.0.0.1:10420/clickhouse-logger/test", + "batch_max_size":1, + "inactive_timeout":1 + } + } + }]] + ) + + if code >= 300 then + ngx.status = code + return + end + ngx.sleep(0.1) + + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugin_config_id": 1, + "uri": "/opentracing", + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + } + }]] + ) + if code >= 300 then + ngx.status = code + return + end + ngx.sleep(0.1) + + -- get plugin conf from admin api, password is decrypted + local code, message, res = t('/apisix/admin/plugin_configs/1', + ngx.HTTP_GET + ) + res = json.decode(res) + if code >= 300 then + ngx.status = code + ngx.say(message) + return + end + ngx.say(res.value.plugins["clickhouse-logger"].password) + + -- get plugin conf from etcd, password is encrypted + local etcd = require("apisix.core.etcd") + local res = assert(etcd.get('/plugin_configs/1')) + ngx.say(res.body.node.value.plugins["clickhouse-logger"].password) + } + } +--- response_body +abc123 +7ipXoKyiZZUAgf3WWNPI5A== + + + +=== TEST 8: verify +--- yaml_config +apisix: + data_encryption: + enable: true + keyring: + - edd1c9f0985e76a2 +--- request +GET /opentracing +--- response_body +opentracing +--- error_log +clickhouse body: INSERT INTO t FORMAT JSONEachRow +clickhouse headers: x-clickhouse-key:abc123 +clickhouse headers: x-clickhouse-user:default +clickhouse headers: x-clickhouse-database:default +--- wait: 5 + + + +=== TEST 9: data encription work well with global rule +--- yaml_config +apisix: + data_encryption: + enable: true + keyring: + - edd1c9f0985e76a2 +--- config + location /t { + content_by_lua_block { + local json = require("toolkit.json") + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/consumers', + ngx.HTTP_PUT, + [[{ + "username": "test", + "plugins": { + "basic-auth": { + "username": "test", + "password": "test" + } + }, + "desc": "test description" + }]] + ) + if code >= 300 then + ngx.status = code + return + end + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "uri": "/hello", + "upstream": { + "type": "roundrobin", + "nodes": { + "127.0.0.1:1980": 1 + } + } + }]] + ) + if code >= 300 then + ngx.status = code + return + end + local code, body = t('/apisix/admin/global_rules/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "basic-auth": {} + } + }]] + ) + if code >= 300 then + ngx.status = code + return + end + -- sleep for data sync + ngx.sleep(0.5) + -- get plugin conf from admin api, password is decrypted + local code, message, res = t('/apisix/admin/consumers/test', + ngx.HTTP_GET + ) + res = json.decode(res) + if code >= 300 then + ngx.status = code + ngx.say(message) + return + end + ngx.say(res.value.plugins["basic-auth"].password) + + -- get plugin conf from etcd, password is encrypted + local etcd = require("apisix.core.etcd") + local res = assert(etcd.get('/consumers/test')) + ngx.say(res.body.node.value.plugins["basic-auth"].password) + + -- hit the route with authorization + local code, body = t('/hello', + ngx.HTTP_PUT, + nil, + nil, + {Authorization = "Basic dGVzdDp0ZXN0"} + ) + if code ~= 200 then + ngx.status = code + return + end + + -- delete global rule + t('/apisix/admin/global_rules/1', + ngx.HTTP_DELETE + ) + ngx.say(body) + } + } +--- request +GET /t +--- response_body +test +9QKrmTT3TkWGvjlIoe5XXw== +passed diff --git a/t/node/global-rule.t b/t/node/global-rule.t index 6bd885dd2bf5..0e61d68d873c 100644 --- a/t/node/global-rule.t +++ b/t/node/global-rule.t @@ -417,112 +417,3 @@ changed GET /t --- response_body passed - - - -=== TEST 15: data encription work well with global rule ---- yaml_config -apisix: - data_encryption: - enable: true - keyring: - - edd1c9f0985e76a2 ---- config - location /t { - content_by_lua_block { - local json = require("toolkit.json") - local t = require("lib.test_admin").test - local code, body = t('/apisix/admin/consumers', - ngx.HTTP_PUT, - [[{ - "username": "test", - "plugins": { - "basic-auth": { - "username": "test", - "password": "test" - } - }, - "desc": "test description" - }]] - ) - - if code >= 300 then - ngx.status = code - return - end - - local code, body = t('/apisix/admin/routes/1', - ngx.HTTP_PUT, - [[{ - "uri": "/hello", - "upstream": { - "type": "roundrobin", - "nodes": { - "127.0.0.1:1980": 1 - } - } - }]] - ) - - if code >= 300 then - ngx.status = code - return - end - - local code, body = t('/apisix/admin/global_rules/1', - ngx.HTTP_PUT, - [[{ - "plugins": { - "basic-auth": {} - } - }]] - ) - - if code >= 300 then - ngx.status = code - return - end - - -- sleep for data sync - ngx.sleep(0.5) - - -- get plugin conf from admin api, password is decrypted - local code, message, res = t('/apisix/admin/consumers/test', - ngx.HTTP_GET - ) - res = json.decode(res) - if code >= 300 then - ngx.status = code - ngx.say(message) - return - end - - ngx.say(res.value.plugins["basic-auth"].password) - - -- get plugin conf from etcd, password is encrypted - local etcd = require("apisix.core.etcd") - local res = assert(etcd.get('/consumers/test')) - ngx.say(res.body.node.value.plugins["basic-auth"].password) - - -- hit the route with authorization - local code, body = t('/hello', - ngx.HTTP_PUT, - nil, - nil, - {Authorization = "Basic dGVzdDp0ZXN0"} - ) - - if code ~= 200 then - ngx.status = code - return - end - - ngx.say(body) - } - } ---- request -GET /t ---- response_body -test -9QKrmTT3TkWGvjlIoe5XXw== -passed diff --git a/t/plugin/clickhouse-logger.t b/t/plugin/clickhouse-logger.t index 976f97267260..bcb874768202 100644 --- a/t/plugin/clickhouse-logger.t +++ b/t/plugin/clickhouse-logger.t @@ -236,235 +236,3 @@ clickhouse headers: x-clickhouse-key:a clickhouse headers: x-clickhouse-user:default clickhouse headers: x-clickhouse-database:default --- wait: 5 - - - -=== TEST 7: password use data encryption ---- yaml_config -apisix: - data_encryption: - enable: true - keyring: - - edd1c9f0985e76a2 ---- config - location /t { - content_by_lua_block { - local json = require("toolkit.json") - local t = require("lib.test_admin").test - local code, body = t('/apisix/admin/routes/1', - ngx.HTTP_PUT, - [[{ - "plugins": { - "clickhouse-logger": { - "user": "default", - "password": "abc123", - "database": "default", - "logtable": "t", - "endpoint_addr": "http://127.0.0.1:10420/clickhouse-logger/test", - "batch_max_size":1, - "inactive_timeout":1 - } - }, - "upstream": { - "nodes": { - "127.0.0.1:1982": 1 - }, - "type": "roundrobin" - }, - "uri": "/opentracing" - }]] - ) - - ngx.sleep(0.5) - - -- get plugin conf from admin api, password is decrypted - local code, message, res = t('/apisix/admin/routes/1', - ngx.HTTP_GET - ) - res = json.decode(res) - if code >= 300 then - ngx.status = code - ngx.say(message) - return - end - - ngx.say(res.value.plugins["clickhouse-logger"].password) - - -- get plugin conf from etcd, password is encrypted - local etcd = require("apisix.core.etcd") - local res = assert(etcd.get('/routes/1')) - ngx.say(res.body.node.value.plugins["clickhouse-logger"].password) - } - } ---- response_body -abc123 -7ipXoKyiZZUAgf3WWNPI5A== ---- error_log -base64 decode ssl key failed. key[a] - - - -=== TEST 8: verify ---- yaml_config -apisix: - data_encryption: - enable: true - keyring: - - edd1c9f0985e76a2 ---- request -GET /opentracing ---- response_body -opentracing ---- error_log -clickhouse body: INSERT INTO t FORMAT JSONEachRow -clickhouse headers: x-clickhouse-key:abc123 -clickhouse headers: x-clickhouse-user:default -clickhouse headers: x-clickhouse-database:default ---- wait: 5 - - - -=== TEST 9: POST and get list ---- yaml_config -apisix: - data_encryption: - enable: true - keyring: - - edd1c9f0985e76a2 ---- config - location /t { - content_by_lua_block { - local json = require("toolkit.json") - local t = require("lib.test_admin").test - local code, body = t('/apisix/admin/routes', - ngx.HTTP_POST, - [[{ - "plugins": { - "clickhouse-logger": { - "user": "default", - "password": "abc123", - "database": "default", - "logtable": "t", - "endpoint_addr": "http://127.0.0.1:10420/clickhouse-logger/test", - "batch_max_size":1, - "inactive_timeout":1 - } - }, - "upstream": { - "nodes": { - "127.0.0.1:1982": 1 - }, - "type": "roundrobin" - }, - "uri": "/opentracing" - }]] - ) - - ngx.sleep(0.1) - - -- get plugin conf from admin api, password is decrypted - local code, message, res = t('/apisix/admin/routes', - ngx.HTTP_GET - ) - res = json.decode(res) - - - if code >= 300 then - ngx.status = code - ngx.say(message) - return - end - - ngx.say(res.list[1].value.plugins["clickhouse-logger"].password) - - -- get plugin conf from etcd, password is encrypted - local etcd = require("apisix.core.etcd") - local id = res.list[1].value.id - local key = "/routes/" .. id - local res = assert(etcd.get(key)) - - ngx.say(res.body.node.value.plugins["clickhouse-logger"].password) - } - } ---- response_body -abc123 -7ipXoKyiZZUAgf3WWNPI5A== - - - -=== TEST 10: PATCH ---- yaml_config -apisix: - data_encryption: - enable: true - keyring: - - edd1c9f0985e76a2 ---- config - location /t { - content_by_lua_block { - local json = require("toolkit.json") - local t = require("lib.test_admin").test - local code, body = t('/apisix/admin/routes/1', - ngx.HTTP_PUT, - [[{ - "plugins": { - "clickhouse-logger": { - "user": "default", - "password": "abc123", - "database": "default", - "logtable": "t", - "endpoint_addr": "http://127.0.0.1:10420/clickhouse-logger/test", - "batch_max_size":1, - "inactive_timeout":1 - } - }, - "upstream": { - "nodes": { - "127.0.0.1:1982": 1 - }, - "type": "roundrobin" - }, - "uri": "/opentracing" - }]] - ) - - ngx.sleep(0.1) - - local code, body = t('/apisix/admin/routes/1/plugins', - ngx.HTTP_PATCH, - [[{ - "clickhouse-logger": { - "user": "default", - "password": "def456", - "database": "default", - "logtable": "t", - "endpoint_addr": "http://127.0.0.1:10420/clickhouse-logger/test", - "batch_max_size":1, - "inactive_timeout":1 - } - }]] - ) - - ngx.sleep(0.1) - -- get plugin conf from admin api, password is decrypted - local code, message, res = t('/apisix/admin/routes/1', - ngx.HTTP_GET - ) - res = json.decode(res) - if code >= 300 then - ngx.status = code - ngx.say(message) - return - end - - ngx.say(res.value.plugins["clickhouse-logger"].password) - - -- get plugin conf from etcd, password is encrypted - local etcd = require("apisix.core.etcd") - local res = assert(etcd.get('/routes/1')) - ngx.say(res.body.node.value.plugins["clickhouse-logger"].password) - } - } ---- response_body -def456 -3hlZu5mwUbqROm+cy0Vi9A== From f871967dc53f3b80b611b45d2ff2f228a265ead5 Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Tue, 29 Nov 2022 18:14:18 +0800 Subject: [PATCH 08/17] chore --- apisix/consumer.lua | 1 - docs/en/latest/plugins/clickhouse-logger.md | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/apisix/consumer.lua b/apisix/consumer.lua index 1302e048d282..32ff69275809 100644 --- a/apisix/consumer.lua +++ b/apisix/consumer.lua @@ -32,7 +32,6 @@ local lrucache = core.lrucache.new({ ttl = 300, count = 512 }) - local function plugin_consumer() local plugins = {} diff --git a/docs/en/latest/plugins/clickhouse-logger.md b/docs/en/latest/plugins/clickhouse-logger.md index 22786f4b1e2a..9de95ccf1c9d 100644 --- a/docs/en/latest/plugins/clickhouse-logger.md +++ b/docs/en/latest/plugins/clickhouse-logger.md @@ -47,6 +47,8 @@ The `clickhouse-logger` Plugin is used to push logs to [ClickHouse](https://clic This Plugin supports using batch processors to aggregate and process entries (logs/data) in a batch. This avoids the need for frequently submitting the data. The batch processor submits data every `5` seconds or when the data in the queue reaches `1000`. See [Batch Processor](../batch-processor.md#configuration) for more information or setting your custom configuration. +NOTE: The schema for `password` also defines `encrypted = true`, which means that the field will be stored encrypted in etcd. See [encrypted storage fields](../plugin-develop.md#encrypted-storage-fields). + ## Metadata You can also set the format of the logs by configuring the Plugin metadata. The following configurations are available: From a8a8414cf29545151b43618a23711dece22198d0 Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Tue, 29 Nov 2022 18:20:18 +0800 Subject: [PATCH 09/17] chore test cases --- t/node/consumer-group.t | 137 ---------------------------------------- t/node/data_encrypt2.t | 101 +++++++++++++++++++++++++++++ 2 files changed, 101 insertions(+), 137 deletions(-) diff --git a/t/node/consumer-group.t b/t/node/consumer-group.t index ef16bbf7fdcb..5f863ad003e2 100644 --- a/t/node/consumer-group.t +++ b/t/node/consumer-group.t @@ -310,140 +310,3 @@ world } --- response_body bar - - - -=== TEST 5: data encryption ---- yaml_config -apisix: - data_encryption: - enable: true - keyring: - - edd1c9f0985e76a2 ---- config - location /t { - content_by_lua_block { - local json = require("toolkit.json") - local t = require("lib.test_admin").test - local etcd = require("apisix.core.etcd") - local code, body = t('/apisix/admin/consumer_groups/company_a', - ngx.HTTP_PUT, - [[{ - "plugins": { - "limit-count": { - "count": 2, - "time_window": 60, - "rejected_code": 503, - "key": "remote_addr" - } - } - }]] - ) - if code >= 300 then - ngx.status = code - ngx.say(body) - return - end - - ngx.sleep(0.1) - - local code, body = t('/apisix/admin/consumers/foobar', - ngx.HTTP_PUT, - [[{ - "username": "foobar", - "plugins": { - "key-auth": { - "key": "auth-two" - } - }, - "group_id": "company_a" - }]] - ) - if code >= 300 then - ngx.status = code - ngx.say(body) - return - end - ngx.sleep(0.1) - - -- get plugin conf from admin api, key is decrypted - local code, message, res = t('/apisix/admin/consumers/foobar', - ngx.HTTP_GET - ) - res = json.decode(res) - if code >= 300 then - ngx.status = code - ngx.say(message) - return - end - - ngx.say(res.value.plugins["key-auth"].key) - - -- get plugin conf from etcd, key is encrypted - local etcd = require("apisix.core.etcd") - local res = assert(etcd.get('/consumers/foobar')) - ngx.say(res.body.node.value.plugins["key-auth"].key) - } - } ---- response_body -auth-two -vU/ZHVJw7b0XscDJ1Fhtig== - - - -=== TEST 6: verify data encryption ---- yaml_config -apisix: - data_encryption: - enable: true - keyring: - - edd1c9f0985e76a2 ---- config - location /t { - content_by_lua_block { - local json = require "t.toolkit.json" - local t = require("lib.test_admin").test - local code, err = t('/apisix/admin/routes/1', - ngx.HTTP_PUT, - [[{ - "uri": "/hello", - "upstream": { - "nodes": { - "127.0.0.1:1980": 1 - }, - "type": "roundrobin" - }, - "plugins": { - "key-auth": {} - } - }]] - ) - if code > 300 then - ngx.log(ngx.ERR, err) - return - end - ngx.sleep(0.1) - - local http = require "resty.http" - local uri = "http://127.0.0.1:" .. ngx.var.server_port - .. "/hello" - local ress = {} - for i = 1, 3 do - local httpc = http.new() - local res, err = httpc:request_uri(uri, { - method = "GET", - headers = { - ["apikey"] = "auth-two" - } - }) - if not res then - ngx.say(err) - return - end - table.insert(ress, res.status) - end - ngx.say(json.encode(ress)) - } - } ---- response_body -[200,200,503] diff --git a/t/node/data_encrypt2.t b/t/node/data_encrypt2.t index 434d631528d1..49beb164fe40 100644 --- a/t/node/data_encrypt2.t +++ b/t/node/data_encrypt2.t @@ -592,3 +592,104 @@ GET /t test 9QKrmTT3TkWGvjlIoe5XXw== passed + + + +=== TEST 10: data encription work well with consumer groups +--- yaml_config +apisix: + data_encryption: + enable: true + keyring: + - edd1c9f0985e76a2 +--- config + location /t { + content_by_lua_block { + local json = require("toolkit.json") + local t = require("lib.test_admin").test + local etcd = require("apisix.core.etcd") + local code, body = t('/apisix/admin/consumer_groups/company_a', + ngx.HTTP_PUT, + [[{ + "plugins": { + "clickhouse-logger": { + "user": "default", + "password": "abc123", + "database": "default", + "logtable": "t", + "endpoint_addr": "http://127.0.0.1:10420/clickhouse-logger/test", + "batch_max_size":1, + "inactive_timeout":1 + } + } + }]] + ) + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + + ngx.sleep(0.1) + + local code, body = t('/apisix/admin/consumers/foobar', + ngx.HTTP_PUT, + [[{ + "username": "foobar", + "plugins": { + "key-auth": { + "key": "auth-two" + } + }, + "group_id": "company_a" + }]] + ) + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + ngx.sleep(0.1) + + -- get plugin conf from admin api, key is decrypted + local code, message, res = t('/apisix/admin/consumers/foobar', + ngx.HTTP_GET + ) + res = json.decode(res) + if code >= 300 then + ngx.status = code + ngx.say(message) + return + end + + ngx.say(res.value.plugins["key-auth"].key) + + -- get plugin conf from etcd, key is encrypted + local etcd = require("apisix.core.etcd") + local res = assert(etcd.get('/consumers/foobar')) + ngx.say(res.body.node.value.plugins["key-auth"].key) + } + } +--- response_body +auth-two +vU/ZHVJw7b0XscDJ1Fhtig== + + + +=== TEST 11: verify +--- yaml_config +apisix: + data_encryption: + enable: true + keyring: + - edd1c9f0985e76a2 +--- request +GET /opentracing +--- response_body +opentracing +--- error_log +clickhouse body: INSERT INTO t FORMAT JSONEachRow +clickhouse headers: x-clickhouse-key:abc123 +clickhouse headers: x-clickhouse-user:default +clickhouse headers: x-clickhouse-database:default +--- wait: 5 From 8c0501b4268ef679d009558f145dee3d349e9793 Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Tue, 29 Nov 2022 18:21:43 +0800 Subject: [PATCH 10/17] chore test cases --- t/node/data_encrypt2.t | 72 +++++++++++++++++++++++++++++++----------- 1 file changed, 54 insertions(+), 18 deletions(-) diff --git a/t/node/data_encrypt2.t b/t/node/data_encrypt2.t index 49beb164fe40..4c780bcce851 100644 --- a/t/node/data_encrypt2.t +++ b/t/node/data_encrypt2.t @@ -612,14 +612,11 @@ apisix: ngx.HTTP_PUT, [[{ "plugins": { - "clickhouse-logger": { - "user": "default", - "password": "abc123", - "database": "default", - "logtable": "t", - "endpoint_addr": "http://127.0.0.1:10420/clickhouse-logger/test", - "batch_max_size":1, - "inactive_timeout":1 + "limit-count": { + "count": 2, + "time_window": 60, + "rejected_code": 503, + "key": "remote_addr" } } }]] @@ -676,20 +673,59 @@ vU/ZHVJw7b0XscDJ1Fhtig== -=== TEST 11: verify +=== TEST 11: verify data encryption --- yaml_config apisix: data_encryption: enable: true keyring: - edd1c9f0985e76a2 ---- request -GET /opentracing +--- config + location /t { + content_by_lua_block { + local json = require "t.toolkit.json" + local t = require("lib.test_admin").test + local code, err = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "uri": "/hello", + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + }, + "plugins": { + "key-auth": {} + } + }]] + ) + if code > 300 then + ngx.log(ngx.ERR, err) + return + end + ngx.sleep(0.1) + + local http = require "resty.http" + local uri = "http://127.0.0.1:" .. ngx.var.server_port + .. "/hello" + local ress = {} + for i = 1, 3 do + local httpc = http.new() + local res, err = httpc:request_uri(uri, { + method = "GET", + headers = { + ["apikey"] = "auth-two" + } + }) + if not res then + ngx.say(err) + return + end + table.insert(ress, res.status) + end + ngx.say(json.encode(ress)) + } + } --- response_body -opentracing ---- error_log -clickhouse body: INSERT INTO t FORMAT JSONEachRow -clickhouse headers: x-clickhouse-key:abc123 -clickhouse headers: x-clickhouse-user:default -clickhouse headers: x-clickhouse-database:default ---- wait: 5 +[200,200,503] From 5581024a0be7ce7da35dff3b48a2e12fe7ccd210 Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Tue, 29 Nov 2022 18:24:00 +0800 Subject: [PATCH 11/17] chore --- apisix/admin/consumers.lua | 1 - apisix/admin/routes.lua | 1 - 2 files changed, 2 deletions(-) diff --git a/apisix/admin/consumers.lua b/apisix/admin/consumers.lua index 99a5861b004c..7ab0ec1e2ad2 100644 --- a/apisix/admin/consumers.lua +++ b/apisix/admin/consumers.lua @@ -119,7 +119,6 @@ function _M.get(consumer_name) end utils.fix_count(res.body, consumer_name) - return res.status, res.body end diff --git a/apisix/admin/routes.lua b/apisix/admin/routes.lua index cf9fe0043a3c..4cd36b385146 100644 --- a/apisix/admin/routes.lua +++ b/apisix/admin/routes.lua @@ -204,7 +204,6 @@ function _M.get(id) end utils.fix_count(res.body, id) - return res.status, res.body end From 67f43c4f57335e3b1d03c15c39e2f1f28faac787 Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Tue, 29 Nov 2022 18:26:31 +0800 Subject: [PATCH 12/17] fix typo --- t/node/data_encrypt2.t | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/t/node/data_encrypt2.t b/t/node/data_encrypt2.t index 4c780bcce851..a17e9084b226 100644 --- a/t/node/data_encrypt2.t +++ b/t/node/data_encrypt2.t @@ -66,7 +66,7 @@ run_tests(); __DATA__ -=== TEST 1: data encription work well with plugins that not the auth plugins +=== TEST 1: data encryption work well with plugins that not the auth plugins --- yaml_config apisix: data_encryption: @@ -296,7 +296,7 @@ def456 -=== TEST 5: data encription work well with services +=== TEST 5: data encryption work well with services --- yaml_config apisix: data_encryption: @@ -393,7 +393,7 @@ clickhouse headers: x-clickhouse-database:default -=== TEST 7: data encription work well with plugin_configs +=== TEST 7: data encryption work well with plugin_configs --- yaml_config apisix: data_encryption: @@ -491,7 +491,7 @@ clickhouse headers: x-clickhouse-database:default -=== TEST 9: data encription work well with global rule +=== TEST 9: data encryption work well with global rule --- yaml_config apisix: data_encryption: @@ -595,7 +595,7 @@ passed -=== TEST 10: data encription work well with consumer groups +=== TEST 10: data encryption work well with consumer groups --- yaml_config apisix: data_encryption: From 0666ad18a49720cb1cb22665a4f7f746fa5e23a4 Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Tue, 29 Nov 2022 18:34:35 +0800 Subject: [PATCH 13/17] optimize code structure --- apisix/plugin.lua | 38 +++++++++++++++++--------------------- 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/apisix/plugin.lua b/apisix/plugin.lua index 674a72b6c482..c8981565fabf 100644 --- a/apisix/plugin.lua +++ b/apisix/plugin.lua @@ -850,19 +850,6 @@ check_plugin_metadata = function(item) end -local function check_enable_and_get_plugin_schema(name, schema_type) - local plugin_schema = local_plugins_hash and local_plugins_hash[name] - local schema - if schema_type == core.schema.TYPE_CONSUMER then - schema = plugin_schema.consumer_schema - else - schema = plugin_schema.schema - end - - return schema -end - - local enable_data_encryption local function enable_gde() if enable_data_encryption == nil then @@ -875,12 +862,25 @@ local function enable_gde() end -local function decrypt_conf(name, conf, schema_type) +local function get_plugin_schema(name, schema_type) if not enable_gde() then - return conf + return false end - local schema = check_enable_and_get_plugin_schema(name, schema_type) + local plugin_schema = local_plugins_hash and local_plugins_hash[name] + local schema + if schema_type == core.schema.TYPE_CONSUMER then + schema = plugin_schema.consumer_schema + else + schema = plugin_schema.schema + end + + return schema +end + + +local function decrypt_conf(name, conf, schema_type) + local schema = get_plugin_schema(name, schema_type) if not schema then return end @@ -901,11 +901,7 @@ _M.decrypt_conf = decrypt_conf local function encrypt_conf(name, conf, schema_type) - if not enable_gde() then - return conf - end - - local schema = check_enable_and_get_plugin_schema(name, schema_type) + local schema = get_plugin_schema(name, schema_type) if not schema then return end From 72fb7b4fdcf58cc986669117d348927730e87eaa Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Tue, 29 Nov 2022 18:41:52 +0800 Subject: [PATCH 14/17] chore code --- apisix/admin/init.lua | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/apisix/admin/init.lua b/apisix/admin/init.lua index b66bdd39098f..013f2ccf85f7 100644 --- a/apisix/admin/init.lua +++ b/apisix/admin/init.lua @@ -189,15 +189,11 @@ local function run() local code, data = resource[method](seg_id, req_body, seg_sub_path, uri_args) if code then - if method == "get" then - if plugin.enable_data_encryption then - if data then - if seg_res == "consumers" then - utils.decrypt_params(plugin.decrypt_conf, data, core.schema.TYPE_CONSUMER) - else - utils.decrypt_params(plugin.decrypt_conf, data) - end - end + if method == "get" and plugin.enable_data_encryption then + if seg_res == "consumers" then + utils.decrypt_params(plugin.decrypt_conf, data, core.schema.TYPE_CONSUMER) + else + utils.decrypt_params(plugin.decrypt_conf, data) end end From a2b38bdfc6ef9ea616ceaba9b642804987fa185c Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Tue, 29 Nov 2022 20:22:27 +0800 Subject: [PATCH 15/17] resolve comments --- apisix/admin/utils.lua | 3 +-- apisix/plugin.lua | 8 ++++---- apisix/ssl.lua | 1 + docs/en/latest/plugin-develop.md | 2 +- docs/zh/latest/plugin-develop.md | 2 +- t/node/data_encrypt.t | 8 -------- 6 files changed, 8 insertions(+), 16 deletions(-) diff --git a/apisix/admin/utils.lua b/apisix/admin/utils.lua index eec31da5cc2e..9aa72e0efa4c 100644 --- a/apisix/admin/utils.lua +++ b/apisix/admin/utils.lua @@ -84,8 +84,7 @@ function _M.decrypt_params(decrypt_func, body, schema_type) -- list if body.list and #body.list > 0 then for _, route in ipairs(body.list) do - if route.value and route.value.plugins - and core.table.nkeys(route.value.plugins) > 0 then + if route.value and route.value.plugins then for name, conf in pairs(route.value.plugins) do decrypt_func(name, conf, schema_type) end diff --git a/apisix/plugin.lua b/apisix/plugin.lua index c8981565fabf..d305f1798ee6 100644 --- a/apisix/plugin.lua +++ b/apisix/plugin.lua @@ -862,9 +862,9 @@ local function enable_gde() end -local function get_plugin_schema(name, schema_type) +local function get_plugin_schema_for_gde(name, schema_type) if not enable_gde() then - return false + return nil end local plugin_schema = local_plugins_hash and local_plugins_hash[name] @@ -880,7 +880,7 @@ end local function decrypt_conf(name, conf, schema_type) - local schema = get_plugin_schema(name, schema_type) + local schema = get_plugin_schema_for_gde(name, schema_type) if not schema then return end @@ -901,7 +901,7 @@ _M.decrypt_conf = decrypt_conf local function encrypt_conf(name, conf, schema_type) - local schema = get_plugin_schema(name, schema_type) + local schema = get_plugin_schema_for_gde(name, schema_type) if not schema then return end diff --git a/apisix/ssl.lua b/apisix/ssl.lua index 4ffb3a0f468a..18898027aee5 100644 --- a/apisix/ssl.lua +++ b/apisix/ssl.lua @@ -221,6 +221,7 @@ local function parse_pem_priv_key(sni, pkey) local key, err = aes_decrypt_pkey(pkey) if not key then core.log.error(err) + return nil, err end local parsed, err = ngx_ssl.parse_pem_priv_key(key) return parsed, err diff --git a/docs/en/latest/plugin-develop.md b/docs/en/latest/plugin-develop.md index 08c753a96af7..c55c47bf1186 100644 --- a/docs/en/latest/plugin-develop.md +++ b/docs/en/latest/plugin-develop.md @@ -295,7 +295,7 @@ end ### encrypted storage fields -Specify the parameters to be stored encrypted(Requires APISIX version > 3.0.1) +Specify the parameters to be stored encrypted. (Requires APISIX version >= 3.1.0) Some plugins require parameters to be stored encrypted, such as the `password` parameter of the `basic-auth` plugin. This plugin needs to specify in the `schema` which parameters need to be stored encrypted. diff --git a/docs/zh/latest/plugin-develop.md b/docs/zh/latest/plugin-develop.md index 24ad81fba52a..6012c323c954 100644 --- a/docs/zh/latest/plugin-develop.md +++ b/docs/zh/latest/plugin-develop.md @@ -275,7 +275,7 @@ end ### 加密存储字段 -指定参数需要被加密存储（需要 APISIX 版本大于 3.0.1） +指定参数需要被加密存储（需要 APISIX 版本不小于 3.1） 有些插件需要将参数加密存储，比如 `basic-auth` 插件的 `password` 参数。这个插件需要在 `schema` 中指定哪些参数需要被加密存储。 diff --git a/t/node/data_encrypt.t b/t/node/data_encrypt.t index f6026df7f867..d47cf5875d15 100644 --- a/t/node/data_encrypt.t +++ b/t/node/data_encrypt.t @@ -28,10 +28,6 @@ add_block_preprocessor(sub { if (!$block->request) { $block->set_value("request", "GET /t"); } - - if (!$block->no_error_log) { - $block->set_value("no_error_log", "[error]\n[alert]"); - } }); run_tests; @@ -124,8 +120,6 @@ bar ngx.say(body) } } ---- request -GET /t --- response_body passed @@ -227,8 +221,6 @@ done ngx.say(body) } } ---- request -GET /t --- response_body passed From c635907c819f73ba0fb02961e691073a2e9efe07 Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Wed, 30 Nov 2022 11:34:24 +0800 Subject: [PATCH 16/17] resolve comments --- apisix/admin/utils.lua | 2 +- t/node/data_encrypt.t | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/apisix/admin/utils.lua b/apisix/admin/utils.lua index 9aa72e0efa4c..ee396d0d3670 100644 --- a/apisix/admin/utils.lua +++ b/apisix/admin/utils.lua @@ -82,7 +82,7 @@ end function _M.decrypt_params(decrypt_func, body, schema_type) -- list - if body.list and #body.list > 0 then + if body.list then for _, route in ipairs(body.list) do if route.value and route.value.plugins then for name, conf in pairs(route.value.plugins) do diff --git a/t/node/data_encrypt.t b/t/node/data_encrypt.t index d47cf5875d15..8746c88d50b0 100644 --- a/t/node/data_encrypt.t +++ b/t/node/data_encrypt.t @@ -567,7 +567,5 @@ apisix: ngx.say(json.encode(pwds)) } } ---- request -GET /t --- response_body ["bar","test"] From e806021326420817434bb234be1f7b464aa9134b Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Wed, 30 Nov 2022 12:14:11 +0800 Subject: [PATCH 17/17] resolve comments --- t/lib/server.lua | 13 ++++++++ t/node/data_encrypt2.t | 44 ++++---------------------- t/plugin/clickhouse-logger.t | 21 +++--------- t/plugin/error-log-logger-clickhouse.t | 34 ++++---------------- 4 files changed, 31 insertions(+), 81 deletions(-) diff --git a/t/lib/server.lua b/t/lib/server.lua index 72b1f30b1fbf..603ca2ae6b10 100644 --- a/t/lib/server.lua +++ b/t/lib/server.lua @@ -577,4 +577,17 @@ function _M.go() return _M[action]() end + +function _M.clickhouse_logger_server() + ngx.req.read_body() + local data = ngx.req.get_body_data() + local headers = ngx.req.get_headers() + ngx.log(ngx.WARN, "clickhouse body: ", data) + for k, v in pairs(headers) do + ngx.log(ngx.WARN, "clickhouse headers: " .. k .. ":" .. v) + end + ngx.say("ok") +end + + return _M diff --git a/t/node/data_encrypt2.t b/t/node/data_encrypt2.t index a17e9084b226..a1c73729e794 100644 --- a/t/node/data_encrypt2.t +++ b/t/node/data_encrypt2.t @@ -28,38 +28,6 @@ add_block_preprocessor(sub { if (!defined $block->request) { $block->set_value("request", "GET /t"); } - - my $http_config = $block->http_config // <<_EOC_; - server { - listen 10420; - location /clickhouse-logger/test { - content_by_lua_block { - ngx.req.read_body() - local data = ngx.req.get_body_data() - local headers = ngx.req.get_headers() - ngx.log(ngx.WARN, "clickhouse body: ", data) - for k, v in pairs(headers) do - ngx.log(ngx.WARN, "clickhouse headers: " .. k .. ":" .. v) - end - ngx.say("ok") - } - } - location /clickhouse-logger/test1 { - content_by_lua_block { - ngx.req.read_body() - local data = ngx.req.get_body_data() - local headers = ngx.req.get_headers() - ngx.log(ngx.WARN, "clickhouse body: ", data) - for k, v in pairs(headers) do - ngx.log(ngx.WARN, "clickhouse headers: " .. k .. ":" .. v) - end - ngx.say("ok") - } - } - } -_EOC_ - - $block->set_value("http_config", $http_config); }); run_tests(); @@ -87,7 +55,7 @@ apisix: "password": "abc123", "database": "default", "logtable": "t", - "endpoint_addr": "http://127.0.0.1:10420/clickhouse-logger/test", + "endpoint_addr": "http://127.0.0.1:1980/clickhouse_logger_server", "batch_max_size":1, "inactive_timeout":1 } @@ -170,7 +138,7 @@ apisix: "password": "abc123", "database": "default", "logtable": "t", - "endpoint_addr": "http://127.0.0.1:10420/clickhouse-logger/test", + "endpoint_addr": "http://127.0.0.1:1980/clickhouse_logger_server", "batch_max_size":1, "inactive_timeout":1 } @@ -238,7 +206,7 @@ apisix: "password": "abc123", "database": "default", "logtable": "t", - "endpoint_addr": "http://127.0.0.1:10420/clickhouse-logger/test", + "endpoint_addr": "http://127.0.0.1:1980/clickhouse_logger_server", "batch_max_size":1, "inactive_timeout":1 } @@ -263,7 +231,7 @@ apisix: "password": "def456", "database": "default", "logtable": "t", - "endpoint_addr": "http://127.0.0.1:10420/clickhouse-logger/test", + "endpoint_addr": "http://127.0.0.1:1980/clickhouse_logger_server", "batch_max_size":1, "inactive_timeout":1 } @@ -317,7 +285,7 @@ apisix: "password": "abc123", "database": "default", "logtable": "t", - "endpoint_addr": "http://127.0.0.1:10420/clickhouse-logger/test", + "endpoint_addr": "http://127.0.0.1:1980/clickhouse_logger_server", "batch_max_size":1, "inactive_timeout":1 } @@ -414,7 +382,7 @@ apisix: "password": "abc123", "database": "default", "logtable": "t", - "endpoint_addr": "http://127.0.0.1:10420/clickhouse-logger/test", + "endpoint_addr": "http://127.0.0.1:1980/clickhouse_logger_server", "batch_max_size":1, "inactive_timeout":1 } diff --git a/t/plugin/clickhouse-logger.t b/t/plugin/clickhouse-logger.t index bcb874768202..ed2f0dc29f70 100644 --- a/t/plugin/clickhouse-logger.t +++ b/t/plugin/clickhouse-logger.t @@ -32,18 +32,7 @@ add_block_preprocessor(sub { my $http_config = $block->http_config // <<_EOC_; server { listen 10420; - location /clickhouse-logger/test { - content_by_lua_block { - ngx.req.read_body() - local data = ngx.req.get_body_data() - local headers = ngx.req.get_headers() - ngx.log(ngx.WARN, "clickhouse body: ", data) - for k, v in pairs(headers) do - ngx.log(ngx.WARN, "clickhouse headers: " .. k .. ":" .. v) - end - ngx.say("ok") - } - } + location /clickhouse-logger/test1 { content_by_lua_block { ngx.req.read_body() @@ -78,7 +67,7 @@ __DATA__ password = "a", database = "default", logtable = "t", - endpoint_addr = "http://127.0.0.1:10420/clickhouse-logger/test", + endpoint_addr = "http://127.0.0.1:1980/clickhouse_logger_server", max_retry_count = 1, name = "clickhouse logger", ssl_verify = false @@ -105,7 +94,7 @@ passed password = "a", database = "default", logtable = "t", - endpoint_addr = "http://127.0.0.1:10420/clickhouse-logger/test" + endpoint_addr = "http://127.0.0.1:1980/clickhouse_logger_server" }) if not ok then @@ -157,7 +146,7 @@ value should match only one schema, but matches none "password": "a", "database": "default", "logtable": "t", - "endpoint_addr": "http://127.0.0.1:10420/clickhouse-logger/test", + "endpoint_addr": "http://127.0.0.1:1980/clickhouse_logger_server", "batch_max_size":1, "inactive_timeout":1 } @@ -197,7 +186,7 @@ passed "password": "a", "database": "default", "logtable": "t", - "endpoint_addrs": ["http://127.0.0.1:10420/clickhouse-logger/test", + "endpoint_addrs": ["http://127.0.0.1:1980/clickhouse_logger_server", "http://127.0.0.1:10420/clickhouse-logger/test1"], "batch_max_size":1, "inactive_timeout":1 diff --git a/t/plugin/error-log-logger-clickhouse.t b/t/plugin/error-log-logger-clickhouse.t index 25554fe6aeb7..f6a328d7f208 100644 --- a/t/plugin/error-log-logger-clickhouse.t +++ b/t/plugin/error-log-logger-clickhouse.t @@ -35,26 +35,6 @@ plugins: _EOC_ $block->set_value("extra_yaml_config", $extra_yaml_config); } - - my $http_config = $block->http_config // <<_EOC_; - server { - listen 10420; - location /error-logger-clickhouse/test { - content_by_lua_block { - ngx.req.read_body() - local data = ngx.req.get_body_data() - local headers = ngx.req.get_headers() - ngx.log(ngx.WARN, "clickhouse error log body: ", data) - for k, v in pairs(headers) do - ngx.log(ngx.WARN, "clickhouse headers: " .. k .. ":" .. v) - end - ngx.say("ok") - } - } - } -_EOC_ - - $block->set_value("http_config", $http_config); }); run_tests(); @@ -74,7 +54,7 @@ __DATA__ password = "a", database = "default", logtable = "t", - endpoint_addr = "http://127.0.0.1:10420/error-logger-clickhouse/test" + endpoint_addr = "http://127.0.0.1:1980/clickhouse_logger_server" } }, core.schema.TYPE_METADATA @@ -105,7 +85,7 @@ done "password": "a", "database": "default", "logtable": "t", - "endpoint_addr": "http://127.0.0.1:10420/error-logger-clickhouse/test" + "endpoint_addr": "http://127.0.0.1:1980/clickhouse_logger_server" }, "inactive_timeout": 1 }]] @@ -117,7 +97,7 @@ done --- response_body --- error_log this is a warning message for test2 -clickhouse error log body: INSERT INTO t FORMAT JSONEachRow +clickhouse body: INSERT INTO t FORMAT JSONEachRow clickhouse headers: x-clickhouse-key:a clickhouse headers: x-clickhouse-user:default clickhouse headers: x-clickhouse-database:default @@ -139,7 +119,7 @@ clickhouse headers: x-clickhouse-database:default "password": "a", "database": "default", "logtable": "t", - "endpoint_addr": "http://127.0.0.1:10420/error-logger-clickhouse/test" + "endpoint_addr": "http://127.0.0.1:1980/clickhouse_logger_server" }, "batch_max_size": 15, "inactive_timeout": 1 @@ -152,7 +132,7 @@ clickhouse headers: x-clickhouse-database:default --- response_body --- error_log this is a warning message for test3 -clickhouse error log body: INSERT INTO t FORMAT JSONEachRow +clickhouse body: INSERT INTO t FORMAT JSONEachRow clickhouse headers: x-clickhouse-key:a clickhouse headers: x-clickhouse-user:default clickhouse headers: x-clickhouse-database:default @@ -171,7 +151,7 @@ clickhouse headers: x-clickhouse-database:default --- response_body --- error_log this is a warning message for test4 -clickhouse error log body: INSERT INTO t FORMAT JSONEachRow +clickhouse body: INSERT INTO t FORMAT JSONEachRow clickhouse headers: x-clickhouse-key:a clickhouse headers: x-clickhouse-user:default clickhouse headers: x-clickhouse-database:default @@ -190,7 +170,7 @@ clickhouse headers: x-clickhouse-database:default --- response_body --- error_log this is a warning message for test5 -clickhouse error log body: INSERT INTO t FORMAT JSONEachRow +clickhouse body: INSERT INTO t FORMAT JSONEachRow clickhouse headers: x-clickhouse-key:a clickhouse headers: x-clickhouse-user:default clickhouse headers: x-clickhouse-database:default