diff --git a/apisix/consumer.lua b/apisix/consumer.lua index 27ea42768720..32ff69275809 100644 --- a/apisix/consumer.lua +++ b/apisix/consumer.lua @@ -102,7 +102,9 @@ local function create_consume_cache(consumers_conf, key_attr) for _, consumer in ipairs(consumers_conf.nodes) do core.log.info("consumer node: ", core.json.delay_encode(consumer)) - consumer_names[consumer.auth_conf[key_attr]] = consumer + local new_consumer = core.table.clone(consumer) + new_consumer.auth_conf = core.utils.retrieve_secrets_ref(new_consumer.auth_conf) + consumer_names[new_consumer.auth_conf[key_attr]] = new_consumer end return consumer_names @@ -110,7 +112,7 @@ end function _M.consumers_kv(plugin_name, consumer_conf, key_attr) - local consumers = lrucache("consumers_key#".. plugin_name, consumer_conf.conf_version, + local consumers = lrucache("consumers_key#" .. plugin_name, consumer_conf.conf_version, create_consume_cache, consumer_conf, key_attr) return consumers diff --git a/apisix/core.lua b/apisix/core.lua index 7e317f7e553b..8e23b8ed06cd 100644 --- a/apisix/core.lua +++ b/apisix/core.lua @@ -55,4 +55,5 @@ return { pubsub = require("apisix.core.pubsub"), math = require("apisix.core.math"), event = require("apisix.core.event"), + env = require("apisix.core.env"), } diff --git a/apisix/core/env.lua b/apisix/core/env.lua new file mode 100644 index 000000000000..c32f9642a512 --- /dev/null +++ b/apisix/core/env.lua @@ -0,0 +1,103 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +local ffi = require "ffi" + +local json = require("apisix.core.json") +local log = require("apisix.core.log") +local string = require("apisix.core.string") + +local os = os +local type = type +local upper = string.upper +local find = string.find +local sub = string.sub +local str = ffi.string + +local _M = {} + +local ENV_PREFIX = "$ENV://" + +local apisix_env_vars = {} + +ffi.cdef [[ + extern char **environ; +]] + + +function _M.init() + local e = ffi.C.environ + if not e then + log.warn("could not access environment variables") + return + end + + local i = 0 + while e[i] ~= nil do + local var = str(e[i]) + local p = find(var, "=") + if p then + apisix_env_vars[sub(var, 1, p - 1)] = sub(var, p + 1) + end + + i = i + 1 + end +end + + +local function is_env_ref(ref) + -- Avoid the error caused by has_prefix to cause a crash. + return type(ref) == "string" and string.has_prefix(upper(ref), ENV_PREFIX) +end + + +local function parse_ref(ref) + local path = sub(ref, #ENV_PREFIX + 1) + local idx = find(path, "/") + if not idx then + return {key = path, sub_key = ""} + end + local key = sub(path, 1, idx - 1) + local sub_key = sub(path, idx + 1) + + return { + key = key, + sub_key = sub_key + } +end + + +function _M.get(ref) + if not is_env_ref(ref) then + return nil + end + + local opts = parse_ref(ref) + local main_value = apisix_env_vars[opts.key] or os.getenv(opts.key) + if main_value and opts.sub_key ~= "" then + local vt, err = json.decode(main_value) + if not vt then + log.warn("decode failed, err: ", err, " value: ", main_value) + return nil + end + return vt[opts.sub_key] + end + + return main_value +end + + +return _M diff --git a/apisix/core/utils.lua b/apisix/core/utils.lua index f72996b78d99..6b61cf02080e 100644 --- a/apisix/core/utils.lua +++ b/apisix/core/utils.lua @@ -25,6 +25,8 @@ local rfind_char = core_str.rfind_char local table = require("apisix.core.table") local log = require("apisix.core.log") local string = require("apisix.core.string") +local env = require("apisix.core.env") +local lrucache = require("apisix.core.lrucache") local dns_client = require("apisix.core.dns.client") local ngx_re = require("ngx.re") local ipmatcher = require("resty.ipmatcher") @@ -35,6 +37,7 @@ local sub_str = string.sub local str_byte = string.byte local tonumber = tonumber local tostring = tostring +local pairs = pairs local re_gsub = ngx.re.gsub local type = type local io_popen = io.popen @@ -329,4 +332,57 @@ end _M.resolve_var = resolve_var +local secrets_lrucache = lrucache.new({ + ttl = 300, count = 512 +}) + +local retrieve_secrets_ref +do + local retrieve_ref + function retrieve_ref(refs) + for k, v in pairs(refs) do + local typ = type(v) + if typ == "string" then + refs[k] = env.get(v) or v + elseif typ == "table" then + retrieve_ref(v) + end + end + return refs + end + + local function retrieve(refs) + log.info("retrieve secrets refs") + + local new_refs = table.deepcopy(refs) + return retrieve_ref(new_refs) + end + + function retrieve_secrets_ref(refs, cache, key, version) + if not refs or type(refs) ~= "table" then + return nil + end + if not cache then + return retrieve(refs) + end + return secrets_lrucache(key, version, retrieve, refs) + end +end +-- Retrieve all secrets ref in the given table +--- +-- Retrieve all secrets ref in the given table, +-- and then replace them with the values from the environment variables. +-- +-- @function core.utils.retrieve_secrets_ref +-- @tparam table refs The table to be retrieved. +-- @tparam boolean cache Whether to use lrucache to cache results. +-- @tparam string key The cache key for lrucache. +-- @tparam string version The cache version for lrucache. +-- @treturn table The table after the reference is replaced. +-- @usage +-- local new_refs = core.utils.retrieve_secrets_ref(refs) -- "no cache" +-- local new_refs = core.utils.retrieve_secrets_ref(refs, true, key, ver) -- "cache" +_M.retrieve_secrets_ref = retrieve_secrets_ref + + return _M diff --git a/apisix/init.lua b/apisix/init.lua index 6b51ab410b6c..fbb090bee3f6 100644 --- a/apisix/init.lua +++ b/apisix/init.lua @@ -82,6 +82,7 @@ local _M = {version = 0.4} function _M.http_init(args) core.resolver.init_resolver(args) core.id.init() + core.env.init() local process = require("ngx.process") local ok, err = process.enable_privileged_agent() diff --git a/apisix/plugin.lua b/apisix/plugin.lua index 3207683cfc21..4221b7387348 100644 --- a/apisix/plugin.lua +++ b/apisix/plugin.lua @@ -409,6 +409,7 @@ local function trace_plugins_info_for_debug(ctx, plugins) end end + local function meta_filter(ctx, plugin_name, plugin_conf) local filter = plugin_conf._meta and plugin_conf._meta.filter if not filter then @@ -445,6 +446,7 @@ local function meta_filter(ctx, plugin_name, plugin_conf) return ok end + function _M.filter(ctx, conf, plugins, route_conf, phase) local user_plugin_conf = conf.value.plugins if user_plugin_conf == nil or diff --git a/t/core/env.t b/t/core/env.t new file mode 100644 index 000000000000..745fcef1aea5 --- /dev/null +++ b/t/core/env.t @@ -0,0 +1,181 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +BEGIN { + $ENV{TEST_ENV_VAR} = "test-value"; + $ENV{TEST_ENV_SUB_VAR} = '{"main":"main_value","sub":"sub_value"}'; +} + +use t::APISIX 'no_plan'; + +repeat_each(1); +no_long_string(); +no_root_location(); + +run_tests; + +__DATA__ + +=== TEST 1: sanity: start with $env:// +--- config + location /t { + content_by_lua_block { + local env = require("apisix.core.env") + local value = env.get("$env://TEST_ENV_VAR") + ngx.say(value) + } + } +--- request +GET /t +--- response_body +test-value + + + +=== TEST 2: sanity: start with $ENV:// +--- config + location /t { + content_by_lua_block { + local env = require("apisix.core.env") + local value = env.get("$ENV://TEST_ENV_VAR") + ngx.say(value) + } + } +--- request +GET /t +--- response_body +test-value + + + +=== TEST 3: env var case sensitive +--- config + location /t { + content_by_lua_block { + local env = require("apisix.core.env") + local value = env.get("$ENV://test_env_var") + ngx.say(value) + } + } +--- request +GET /t +--- response_body +nil + + + +=== TEST 4: wrong format: wrong type +--- config + location /t { + content_by_lua_block { + local env = require("apisix.core.env") + local value = env.get(1) + ngx.say(value) + + local value = env.get(true) + ngx.say(value) + } + } +--- request +GET /t +--- response_body +nil +nil + + + +=== TEST 5: wrong format: wrong prefix +--- config + location /t { + content_by_lua_block { + local env = require("apisix.core.env") + local value = env.get("env://") + ngx.say(value) + } + } +--- request +GET /t +--- response_body +nil + + + +=== TEST 6: sub value +--- config + location /t { + content_by_lua_block { + local env = require("apisix.core.env") + local value = env.get("$ENV://TEST_ENV_SUB_VAR/main") + ngx.say(value) + local value = env.get("$ENV://TEST_ENV_SUB_VAR/sub") + ngx.say(value) + } + } +--- request +GET /t +--- response_body +main_value +sub_value + + + +=== TEST 7: wrong sub value: error json +--- config + location /t { + content_by_lua_block { + local env = require("apisix.core.env") + local value = env.get("$ENV://TEST_ENV_VAR/main") + ngx.say(value) + } + } +--- request +GET /t +--- response_body +nil + + + +=== TEST 8: wrong sub value: not exits +--- config + location /t { + content_by_lua_block { + local env = require("apisix.core.env") + local value = env.get("$ENV://TEST_ENV_VAR/no") + ngx.say(value) + } + } +--- request +GET /t +--- response_body +nil + + + +=== TEST 9: use nginx env +--- main_config +env ngx_env=apisix-nice; +--- config + location /t { + content_by_lua_block { + local env = require("apisix.core.env") + local value = env.get("$ENV://ngx_env") + ngx.say(value) + } + } +--- request +GET /t +--- response_body +apisix-nice diff --git a/t/core/utils.t b/t/core/utils.t index 4e6b0d76619d..9094ddc1475a 100644 --- a/t/core/utils.t +++ b/t/core/utils.t @@ -361,3 +361,114 @@ apisix: GET /t --- error_log failed to parse domain: ipv6.local + + + +=== TEST 12: retrieve_secrets_ref: no cache +--- main_config +env secret=apisix; +--- extra_init_by_lua +require("apisix.core.env").init() +--- config + location /t { + content_by_lua_block { + local core = require("apisix.core") + local refs = { + key = "jack", + secret = "$env://secret" + } + local new_refs = core.utils.retrieve_secrets_ref(refs) + assert(new_refs ~= refs) + ngx.say(refs.secret) + ngx.say(new_refs.secret) + ngx.say(new_refs.key) + } + } +--- request +GET /t +--- response_body +$env://secret +apisix +jack +--- error_log_like +qr/retrieve secrets refs/ + + + +=== TEST 13: retrieve_secrets_ref: cache +--- main_config +env secret=apisix; +--- extra_init_by_lua +require("apisix.core.env").init() +--- config + location /t { + content_by_lua_block { + local core = require("apisix.core") + local refs = { + key = "jack", + secret = "$env://secret" + } + local refs_1 = core.utils.retrieve_secrets_ref(refs, true, "key", 1) + local refs_2 = core.utils.retrieve_secrets_ref(refs, true, "key", 1) + assert(refs_1 == refs_2) + ngx.say(refs_1.secret) + ngx.say(refs_2.secret) + } + } +--- request +GET /t +--- response_body +apisix +apisix +--- grep_error_log eval +qr/retrieve secrets refs/ +--- grep_error_log_out +retrieve secrets refs + + + +=== TEST 14: retrieve_secrets_ref: table nesting +--- main_config +env secret=apisix; +--- extra_init_by_lua +require("apisix.core.env").init() +--- config + location /t { + content_by_lua_block { + local core = require("apisix.core") + local refs = { + key = "jack", + user = { + username = "apisix", + passsword = "$env://secret" + } + } + local new_refs = core.utils.retrieve_secrets_ref(refs) + ngx.say(new_refs.user.passsword) + } + } +--- request +GET /t +--- response_body +apisix + + + +=== TEST 15: retrieve_secrets_ref: wrong refs type +--- main_config +env secret=apisix; +--- extra_init_by_lua +require("apisix.core.env").init() +--- config + location /t { + content_by_lua_block { + local core = require("apisix.core") + local refs = "wrong" + local new_refs = core.utils.retrieve_secrets_ref(refs) + ngx.say(new_refs) + } + } +--- request +GET /t +--- response_body +nil diff --git a/t/plugin/key-auth.t b/t/plugin/key-auth.t index c08fcc4df22e..4f139bbfeff6 100644 --- a/t/plugin/key-auth.t +++ b/t/plugin/key-auth.t @@ -535,3 +535,43 @@ passed GET /hello?auth=auth-one --- response_args auth: auth-one + + + +=== TEST 26: change consumer with secrets ref +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/consumers', + ngx.HTTP_PUT, + [[{ + "username": "jack", + "plugins": { + "key-auth": { + "key": "$env://test_auth" + } + } + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- request +GET /t +--- response_body +passed + + + +=== TEST 27: verify auth request args should not hidden +--- main_config +env test_auth=authone; +--- request +GET /hello?auth=authone +--- response_args +auth: authone