feat: support store secret in env for auth plugin (#8390)

Co-authored-by: soulbird <zhaothree@gmail.com>

apisix/consumer.lua

@@ -102,15 +102,17 @@ local function create_consume_cache(consumers_conf, key_attr)
 
     for _, consumer in ipairs(consumers_conf.nodes) do
         core.log.info("consumer node: ", core.json.delay_encode(consumer))
-        consumer_names[consumer.auth_conf[key_attr]] = consumer
+        local new_consumer = core.table.clone(consumer)
+        new_consumer.auth_conf = core.utils.retrieve_secrets_ref(new_consumer.auth_conf)
+        consumer_names[new_consumer.auth_conf[key_attr]] = new_consumer
     end
 
     return consumer_names
 end
 
 
 function _M.consumers_kv(plugin_name, consumer_conf, key_attr)
-    local consumers = lrucache("consumers_key#".. plugin_name, consumer_conf.conf_version,
+    local consumers = lrucache("consumers_key#" .. plugin_name, consumer_conf.conf_version,
         create_consume_cache, consumer_conf, key_attr)
 
     return consumers

apisix/core.lua

@@ -55,4 +55,5 @@ return {
     pubsub      = require("apisix.core.pubsub"),
     math        = require("apisix.core.math"),
     event       = require("apisix.core.event"),
+    env         = require("apisix.core.env"),
 }

apisix/core/env.lua

@@ -0,0 +1,103 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements.  See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License.  You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+local ffi         = require "ffi"
+
+local json        = require("apisix.core.json")
+local log         = require("apisix.core.log")
+local string      = require("apisix.core.string")
+
+local os          = os
+local type        = type
+local upper       = string.upper
+local find        = string.find
+local sub         = string.sub
+local str         = ffi.string
+
+local _M = {}
+
+local ENV_PREFIX = "$ENV://"
+
+local apisix_env_vars = {}
+
+ffi.cdef [[
+  extern char **environ;
+]]
+
+
+function _M.init()
+  local e = ffi.C.environ
+  if not e then
+    log.warn("could not access environment variables")
+    return
+  end
+
+  local i = 0
+  while e[i] ~= nil do
+    local var = str(e[i])
+    local p = find(var, "=")
+    if p then
+        apisix_env_vars[sub(var, 1, p - 1)] = sub(var, p + 1)
+    end
+
+    i = i + 1
+  end
+end
+
+
+local function is_env_ref(ref)
+    -- Avoid the error caused by has_prefix to cause a crash.
+    return type(ref) == "string" and string.has_prefix(upper(ref), ENV_PREFIX)
+end
+
+
+local function parse_ref(ref)
+    local path = sub(ref, #ENV_PREFIX + 1)
+    local idx = find(path, "/")
+    if not idx then
+        return {key = path, sub_key = ""}
+    end
+    local key = sub(path, 1, idx - 1)
+    local sub_key = sub(path, idx + 1)
+
+    return {
+      key = key,
+      sub_key = sub_key
+    }
+end
+
+
+function _M.get(ref)
+    if not is_env_ref(ref) then
+        return nil
+    end
+
+    local opts = parse_ref(ref)
+    local main_value = apisix_env_vars[opts.key] or os.getenv(opts.key)
+    if main_value and opts.sub_key ~= "" then
+        local vt, err = json.decode(main_value)
+        if not vt then
+          log.warn("decode failed, err: ", err, " value: ", main_value)
+          return nil
+        end
+        return vt[opts.sub_key]
+    end
+
+    return main_value
+end
+
+
+return _M

apisix/core/utils.lua

@@ -25,6 +25,8 @@ local rfind_char     = core_str.rfind_char
 local table          = require("apisix.core.table")
 local log            = require("apisix.core.log")
 local string         = require("apisix.core.string")
+local env            = require("apisix.core.env")
+local lrucache       = require("apisix.core.lrucache")
 local dns_client     = require("apisix.core.dns.client")
 local ngx_re         = require("ngx.re")
 local ipmatcher      = require("resty.ipmatcher")
@@ -35,6 +37,7 @@ local sub_str        = string.sub
 local str_byte       = string.byte
 local tonumber       = tonumber
 local tostring       = tostring
+local pairs          = pairs
 local re_gsub        = ngx.re.gsub
 local type           = type
 local io_popen       = io.popen
@@ -329,4 +332,57 @@ end
 _M.resolve_var = resolve_var
 
 
+local secrets_lrucache = lrucache.new({
+    ttl = 300, count = 512
+})
+
+local retrieve_secrets_ref
+do
+    local retrieve_ref
+    function retrieve_ref(refs)
+        for k, v in pairs(refs) do
+            local typ = type(v)
+            if typ == "string" then
+                refs[k] = env.get(v) or v
+            elseif typ == "table" then
+                retrieve_ref(v)
+            end
+        end
+        return refs
+    end
+
+    local function retrieve(refs)
+        log.info("retrieve secrets refs")
+
+        local new_refs = table.deepcopy(refs)
+        return retrieve_ref(new_refs)
+    end
+
+    function retrieve_secrets_ref(refs, cache, key, version)
+        if not refs or type(refs) ~= "table" then
+            return nil
+        end
+        if not cache then
+            return retrieve(refs)
+        end
+        return secrets_lrucache(key, version, retrieve, refs)
+    end
+end
+-- Retrieve all secrets ref in the given table
+---
+-- Retrieve all secrets ref in the given table,
+-- and then replace them with the values from the environment variables.
+--
+-- @function core.utils.retrieve_secrets_ref
+-- @tparam table refs The table to be retrieved.
+-- @tparam boolean cache Whether to use lrucache to cache results.
+-- @tparam string key The cache key for lrucache.
+-- @tparam string version The cache version for lrucache.
+-- @treturn table The table after the reference is replaced.
+-- @usage
+-- local new_refs = core.utils.retrieve_secrets_ref(refs) -- "no cache"
+-- local new_refs = core.utils.retrieve_secrets_ref(refs, true, key, ver) -- "cache"
+_M.retrieve_secrets_ref = retrieve_secrets_ref
+
+
 return _M

apisix/init.lua

@@ -82,6 +82,7 @@ local _M = {version = 0.4}
 function _M.http_init(args)
     core.resolver.init_resolver(args)
     core.id.init()
+    core.env.init()
 
     local process = require("ngx.process")
     local ok, err = process.enable_privileged_agent()

apisix/plugin.lua

@@ -409,6 +409,7 @@ local function trace_plugins_info_for_debug(ctx, plugins)
     end
 end
 
+
 local function meta_filter(ctx, plugin_name, plugin_conf)
     local filter = plugin_conf._meta and plugin_conf._meta.filter
     if not filter then
@@ -445,6 +446,7 @@ local function meta_filter(ctx, plugin_name, plugin_conf)
     return ok
 end
 
+
 function _M.filter(ctx, conf, plugins, route_conf, phase)
     local user_plugin_conf = conf.value.plugins
     if user_plugin_conf == nil or