From bf44ed124d777ca64c6a5b18459f798e7b71a8a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=BD=97=E6=B3=BD=E8=BD=A9?= Date: Mon, 2 Nov 2020 16:47:32 +0800 Subject: [PATCH] change: make lrucache lock optional (#2575) --- apisix/core/lrucache.lua | 67 ++++++++++++++++------------ apisix/plugins/basic-auth.lua | 9 ++-- apisix/plugins/cors.lua | 7 ++- apisix/plugins/hmac-auth.lua | 9 ++-- apisix/plugins/jwt-auth.lua | 14 +++--- apisix/plugins/key-auth.lua | 9 ++-- apisix/plugins/limit-conn.lua | 6 ++- apisix/plugins/limit-count.lua | 6 ++- apisix/plugins/limit-req.lua | 8 +++- apisix/plugins/serverless.lua | 7 ++- apisix/plugins/syslog.lua | 8 ++-- apisix/plugins/wolf-rbac.lua | 14 +++--- apisix/plugins/zipkin.lua | 8 +++- t/core/lrucache.t | 51 +++------------------ t/plugin/limit-count-redis-cluster.t | 23 +++++++--- t/plugin/limit-count-redis.t | 31 ++++++++----- t/plugin/syslog.t | 3 ++ 17 files changed, 156 insertions(+), 124 deletions(-) diff --git a/apisix/core/lrucache.lua b/apisix/core/lrucache.lua index c5765546e16b..c2eb8fcadf7e 100644 --- a/apisix/core/lrucache.lua +++ b/apisix/core/lrucache.lua @@ -17,7 +17,9 @@ local lru_new = require("resty.lrucache").new local resty_lock = require("resty.lock") +local log = require("apisix.core.log") local tostring = tostring +local concat = table.concat local ngx = ngx local get_phase = ngx.get_phase @@ -65,14 +67,22 @@ end local function new_lru_fun(opts) - local item_count = opts and opts.count or GLOBAL_ITEMS_COUNT - local item_ttl = opts and opts.ttl or GLOBAL_TTL + local item_count, item_ttl + if opts and opts.type == 'plugin' then + item_count = opts.count or PLUGIN_ITEMS_COUNT + item_ttl = opts.ttl or PLUGIN_TTL + else + item_count = opts and opts.count or GLOBAL_ITEMS_COUNT + item_ttl = opts and opts.ttl or GLOBAL_TTL + end + local item_release = opts and opts.release local invalid_stale = opts and opts.invalid_stale + local serial_creating = opts and opts.serial_creating local lru_obj = lru_new(item_count) return function (key, version, create_obj_fun, ...) - if not can_yield_phases[get_phase()] then + if not serial_creating or not can_yield_phases[get_phase()] then local cache_obj = fetch_valid_cache(lru_obj, invalid_stale, item_ttl, item_release, key, version) if cache_obj then @@ -99,6 +109,8 @@ local function new_lru_fun(opts) end local key_s = tostring(key) + log.info("try to lock with key ", key_s) + local elapsed, err = lock:lock(key_s) if not elapsed then return nil, "failed to acquire the lock: " .. err @@ -108,6 +120,7 @@ local function new_lru_fun(opts) nil, key, version) if cache_obj then lock:unlock() + log.info("unlock with key ", key_s) return cache_obj.val end @@ -116,6 +129,7 @@ local function new_lru_fun(opts) lru_obj:set(key, {val = obj, ver = version}, item_ttl) end lock:unlock() + log.info("unlock with key ", key_s) return obj, err end @@ -125,27 +139,28 @@ end global_lru_fun = new_lru_fun() -local function _plugin(plugin_name, key, version, create_obj_fun, ...) - local lru_global = global_lru_fun("/plugin/" .. plugin_name, nil, - lru_new, PLUGIN_ITEMS_COUNT) - - local obj, stale_obj = lru_global:get(key) - if obj and obj.ver == version then - return obj.val - end - - if stale_obj and stale_obj.ver == version then - lru_global:set(key, stale_obj, PLUGIN_TTL) - return stale_obj - end +local plugin_ctx +do + local key_buf = { + nil, + nil, + nil, + } + + function plugin_ctx(lrucache, api_ctx, extra_key, create_obj_func, ...) + key_buf[1] = api_ctx.conf_type + key_buf[2] = api_ctx.conf_id + + local key + if extra_key then + key_buf[3] = extra_key + key = concat(key_buf, "#", 1, 3) + else + key = concat(key_buf, "#", 1, 2) + end - local err - obj, err = create_obj_fun(...) - if obj ~= nil then - lru_global:set(key, {val = obj, ver = version}, PLUGIN_TTL) + return lrucache(key, api_ctx.conf_version, create_obj_func, ...) end - - return obj, err end @@ -153,14 +168,8 @@ local _M = { version = 0.1, new = new_lru_fun, global = global_lru_fun, - plugin = _plugin, + plugin_ctx = plugin_ctx, } -function _M.plugin_ctx(plugin_name, api_ctx, create_obj_fun, ...) - local key = api_ctx.conf_type .. "#" .. api_ctx.conf_id - return _plugin(plugin_name, key, api_ctx.conf_version, create_obj_fun, ...) -end - - return _M diff --git a/apisix/plugins/basic-auth.lua b/apisix/plugins/basic-auth.lua index 63ec8d19a06f..9a55592d98d5 100644 --- a/apisix/plugins/basic-auth.lua +++ b/apisix/plugins/basic-auth.lua @@ -23,6 +23,9 @@ local consumer = require("apisix.consumer") local lrucache = core.lrucache.new({ ttl = 300, count = 512 }) +local consumers_lrucache = core.lrucache.new({ + type = "plugin", +}) local schema = { type = "object", @@ -143,9 +146,9 @@ function _M.access(conf, ctx) return 401, { message = "Missing related consumer" } end - local consumers = core.lrucache.plugin(plugin_name, "consumers_key", - consumer_conf.conf_version, - create_consume_cache, consumer_conf) + local consumers = consumers_lrucache("consumers_key", + consumer_conf.conf_version, + create_consume_cache, consumer_conf) -- 3. check user exists local cur_consumer = consumers[username] diff --git a/apisix/plugins/cors.lua b/apisix/plugins/cors.lua index 19c01d8b5530..3251729ff6db 100644 --- a/apisix/plugins/cors.lua +++ b/apisix/plugins/cors.lua @@ -20,6 +20,11 @@ local plugin_name = "cors" local str_find = string.find local re_gmatch = ngx.re.gmatch + +local lrucache = core.lrucache.new({ + type = "plugin", +}) + local schema = { type = "object", properties = { @@ -160,7 +165,7 @@ function _M.header_filter(conf, ctx) if allow_origins == "**" then allow_origins = req_origin or '*' end - local multiple_origin, err = core.lrucache.plugin_ctx(plugin_name, ctx, + local multiple_origin, err = core.lrucache.plugin_ctx(lrucache, ctx, nil, create_mutiple_origin_cache, conf) if err then return 500, {message = "get mutiple origin cache failed: " .. err} diff --git a/apisix/plugins/hmac-auth.lua b/apisix/plugins/hmac-auth.lua index b85390ea7996..b5811fa57b37 100644 --- a/apisix/plugins/hmac-auth.lua +++ b/apisix/plugins/hmac-auth.lua @@ -36,6 +36,10 @@ local ACCESS_KEY = "X-HMAC-ACCESS-KEY" local SIGNED_HEADERS_KEY = "X-HMAC-SIGNED-HEADERS" local plugin_name = "hmac-auth" +local lrucache = core.lrucache.new({ + type = "plugin", +}) + local schema = { type = "object", title = "work with route or service object", @@ -158,9 +162,8 @@ local function get_consumer(access_key) return nil, {message = "Missing related consumer"} end - local consumers = core.lrucache.plugin(plugin_name, "consumers_key", - consumer_conf.conf_version, - create_consumer_cache, consumer_conf) + local consumers = lrucache("consumers_key", consumer_conf.conf_version, + create_consumer_cache, consumer_conf) local consumer = consumers[access_key] if not consumer then diff --git a/apisix/plugins/jwt-auth.lua b/apisix/plugins/jwt-auth.lua index 40959f00bf60..92856ee01960 100644 --- a/apisix/plugins/jwt-auth.lua +++ b/apisix/plugins/jwt-auth.lua @@ -29,6 +29,10 @@ local sub_str = string.sub local plugin_name = "jwt-auth" +local lrucache = core.lrucache.new({ + type = "plugin", +}) + local schema = { type = "object", additionalProperties = false, @@ -173,9 +177,8 @@ function _M.rewrite(conf, ctx) return 401, {message = "Missing related consumer"} end - local consumers = core.lrucache.plugin(plugin_name, "consumers_key", - consumer_conf.conf_version, - create_consume_cache, consumer_conf) + local consumers = lrucache("consumers_key", consumer_conf.conf_version, + create_consume_cache, consumer_conf) local consumer = consumers[user_key] if not consumer then @@ -210,9 +213,8 @@ local function gen_token() return core.response.exit(404) end - local consumers = core.lrucache.plugin(plugin_name, "consumers_key", - consumer_conf.conf_version, - create_consume_cache, consumer_conf) + local consumers = lrucache("consumers_key", consumer_conf.conf_version, + create_consume_cache, consumer_conf) core.log.info("consumers: ", core.json.delay_encode(consumers)) local consumer = consumers[key] diff --git a/apisix/plugins/key-auth.lua b/apisix/plugins/key-auth.lua index 50fbdbf5afbb..e72e7294b64f 100644 --- a/apisix/plugins/key-auth.lua +++ b/apisix/plugins/key-auth.lua @@ -20,6 +20,10 @@ local plugin_name = "key-auth" local ipairs = ipairs +local lrucache = core.lrucache.new({ + type = "plugin", +}) + local schema = { type = "object", properties = { @@ -73,9 +77,8 @@ function _M.rewrite(conf, ctx) return 401, {message = "Missing related consumer"} end - local consumers = core.lrucache.plugin(plugin_name, "consumers_key", - consumer_conf.conf_version, - create_consume_cache, consumer_conf) + local consumers = lrucache("consumers_key", consumer_conf.conf_version, + create_consume_cache, consumer_conf) local consumer = consumers[key] if not consumer then diff --git a/apisix/plugins/limit-conn.lua b/apisix/plugins/limit-conn.lua index 2b80a3b3d47d..b255f070a33a 100644 --- a/apisix/plugins/limit-conn.lua +++ b/apisix/plugins/limit-conn.lua @@ -20,6 +20,10 @@ local sleep = core.sleep local plugin_name = "limit-conn" +local lrucache = core.lrucache.new({ + type = "plugin", +}) + local schema = { type = "object", properties = { @@ -61,7 +65,7 @@ end function _M.access(conf, ctx) core.log.info("ver: ", ctx.conf_version) - local lim, err = core.lrucache.plugin_ctx(plugin_name, ctx, + local lim, err = core.lrucache.plugin_ctx(lrucache, ctx, nil, create_limit_obj, conf) if not lim then core.log.error("failed to instantiate a resty.limit.conn object: ", err) diff --git a/apisix/plugins/limit-count.lua b/apisix/plugins/limit-count.lua index ec2fbc8a8938..425dc0dda187 100644 --- a/apisix/plugins/limit-count.lua +++ b/apisix/plugins/limit-count.lua @@ -26,6 +26,9 @@ do local cluster_src = "apisix.plugins.limit-count.limit-count-redis-cluster" limit_redis_cluster_new = require(cluster_src).new end +local lrucache = core.lrucache.new({ + type = 'plugin', serial_creating = true, +}) local schema = { @@ -154,8 +157,7 @@ end function _M.access(conf, ctx) core.log.info("ver: ", ctx.conf_version) - local lim, err = core.lrucache.plugin_ctx(plugin_name, ctx, - create_limit_obj, conf) + local lim, err = core.lrucache.plugin_ctx(lrucache, ctx, conf.policy, create_limit_obj, conf) if not lim then core.log.error("failed to fetch limit.count object: ", err) return 500 diff --git a/apisix/plugins/limit-req.lua b/apisix/plugins/limit-req.lua index 3274f7aad09c..dbbd5d219d90 100644 --- a/apisix/plugins/limit-req.lua +++ b/apisix/plugins/limit-req.lua @@ -20,6 +20,10 @@ local plugin_name = "limit-req" local sleep = core.sleep +local lrucache = core.lrucache.new({ + type = "plugin", +}) + local schema = { type = "object", properties = { @@ -60,8 +64,8 @@ end function _M.access(conf, ctx) - local lim, err = core.lrucache.plugin_ctx(plugin_name, ctx, - create_limit_obj, conf) + local lim, err = core.lrucache.plugin_ctx(lrucache, ctx, nil, + create_limit_obj, conf) if not lim then core.log.error("failed to instantiate a resty.limit.req object: ", err) return 500 diff --git a/apisix/plugins/serverless.lua b/apisix/plugins/serverless.lua index 8ac5abcc4f7f..06d05c91bd4a 100644 --- a/apisix/plugins/serverless.lua +++ b/apisix/plugins/serverless.lua @@ -23,6 +23,11 @@ local type = type return function(plugin_name, priority) local core = require("apisix.core") + + local lrucache = core.lrucache.new({ + type = "plugin", + }) + local schema = { type = "object", properties = { @@ -66,7 +71,7 @@ return function(plugin_name, priority) return end - local functions = core.lrucache.plugin_ctx(plugin_name, ctx, + local functions = core.lrucache.plugin_ctx(lrucache, ctx, nil, load_funcs, conf.functions) for _, func in ipairs(functions) do diff --git a/apisix/plugins/syslog.lua b/apisix/plugins/syslog.lua index cafb442e6135..30262d065902 100644 --- a/apisix/plugins/syslog.lua +++ b/apisix/plugins/syslog.lua @@ -51,7 +51,7 @@ local schema = { local lrucache = core.lrucache.new({ - ttl = 300, count = 512 + ttl = 300, count = 512, serial_creating = true, }) @@ -83,10 +83,8 @@ local function send_syslog_data(conf, log_message, api_ctx) local res = true -- fetch it from lrucache - local logger, err = lrucache( - api_ctx.conf_type .. "#" .. api_ctx.conf_id, - api_ctx.conf_version, - logger_socket.new, logger_socket, { + local logger, err = core.lrucache.plugin_ctx( + lrucache, api_ctx, nil, logger_socket.new, logger_socket, { host = conf.host, port = conf.port, flush_limit = conf.flush_limit, diff --git a/apisix/plugins/wolf-rbac.lua b/apisix/plugins/wolf-rbac.lua index 4a4535ee154b..993e7e45d9b2 100644 --- a/apisix/plugins/wolf-rbac.lua +++ b/apisix/plugins/wolf-rbac.lua @@ -33,6 +33,10 @@ local string = string local plugin_name = "wolf-rbac" +local lrucache = core.lrucache.new({ + type = "plugin", +}) + local schema = { type = "object", properties = { @@ -274,9 +278,8 @@ function _M.rewrite(conf, ctx) return 401, fail_response("Missing related consumer") end - local consumers = core.lrucache.plugin(plugin_name, "consumers_key", - consumer_conf.conf_version, - create_consume_cache, consumer_conf) + local consumers = lrucache("consumers_key", consumer_conf.conf_version, + create_consume_cache, consumer_conf) core.log.info("------ consumers: ", core.json.delay_encode(consumers)) local consumer = consumers[appid] @@ -344,9 +347,8 @@ local function get_consumer(appid) core.response.exit(500) end - local consumers = core.lrucache.plugin(plugin_name, "consumers_key", - consumer_conf.conf_version, - create_consume_cache, consumer_conf) + local consumers = lrucache("consumers_key", consumer_conf.conf_version, + create_consume_cache, consumer_conf) core.log.info("------ consumers: ", core.json.delay_encode(consumers)) local consumer = consumers[appid] diff --git a/apisix/plugins/zipkin.lua b/apisix/plugins/zipkin.lua index ea408ce572f2..ac40386b816e 100644 --- a/apisix/plugins/zipkin.lua +++ b/apisix/plugins/zipkin.lua @@ -26,6 +26,10 @@ local tonumber = tonumber local plugin_name = "zipkin" +local lrucache = core.lrucache.new({ + type = "plugin", +}) + local schema = { type = "object", properties = { @@ -108,8 +112,8 @@ function _M.rewrite(plugin_conf, ctx) conf.server_addr = ctx.var["server_addr"] end - local tracer = core.lrucache.plugin_ctx(plugin_name .. '#' .. conf.server_addr, ctx, - create_tracer, conf, ctx) + local tracer = core.lrucache.plugin_ctx(lrucache, ctx, conf.server_addr, + create_tracer, conf, ctx) ctx.opentracing_sample = tracer.sampler:sample() if not ctx.opentracing_sample then diff --git a/t/core/lrucache.t b/t/core/lrucache.t index 751ebd2d3551..fbea92a2524f 100644 --- a/t/core/lrucache.t +++ b/t/core/lrucache.t @@ -58,44 +58,7 @@ obj: {"idx":2} -=== TEST 2: plugin ---- config - location /t { - content_by_lua_block { - local core = require("apisix.core") - - local idx = 0 - local function create_obj() - idx = idx + 1 - return {idx = idx} - end - - local obj = core.lrucache.plugin("plugin-a", "key", nil, create_obj) - ngx.say("obj: ", core.json.encode(obj)) - - obj = core.lrucache.plugin("plugin-a", "key", nil, create_obj) - ngx.say("obj: ", core.json.encode(obj)) - - obj = core.lrucache.plugin("plugin-a", "key", "1", create_obj) - ngx.say("obj: ", core.json.encode(obj)) - - obj = core.lrucache.plugin("plugin-b", "key", "1", create_obj) - ngx.say("obj: ", core.json.encode(obj)) - } - } ---- request -GET /t ---- response_body -obj: {"idx":1} -obj: {"idx":1} -obj: {"idx":2} -obj: {"idx":3} ---- no_error_log -[error] - - - -=== TEST 3: new +=== TEST 2: new --- config location /t { content_by_lua_block { @@ -138,7 +101,7 @@ obj: {"idx":3} -=== TEST 4: cache the non-table object, eg: number or string +=== TEST 3: cache the non-table object, eg: number or string --- config location /t { content_by_lua_block { @@ -171,7 +134,7 @@ obj: 2 -=== TEST 5: sanity +=== TEST 4: sanity --- config location /t { content_by_lua_block { @@ -207,7 +170,7 @@ obj: {"name":"bbb"} -=== TEST 6: invalid_stale = true +=== TEST 5: invalid_stale = true --- config location /t { content_by_lua_block { @@ -244,7 +207,7 @@ obj: {"idx":2} -=== TEST 7: when creating cached objects, use resty-lock to avoid repeated creation. +=== TEST 6: when creating cached objects, use resty-lock to avoid repeated creation. --- config location /t { content_by_lua_block { @@ -258,7 +221,7 @@ obj: {"idx":2} end local lru_get = core.lrucache.new({ - ttl = 1, count = 256, invalid_stale = true, + ttl = 1, count = 256, invalid_stale = true, serial_creating = true, }) local function f() @@ -282,7 +245,7 @@ obj: {"idx":1} -=== TEST 8: different `key` and `ver`, cached same one table +=== TEST 7: different `key` and `ver`, cached same one table --- config location /t { content_by_lua_block { diff --git a/t/plugin/limit-count-redis-cluster.t b/t/plugin/limit-count-redis-cluster.t index 71e8e07a0c75..ff2989718eb6 100644 --- a/t/plugin/limit-count-redis-cluster.t +++ b/t/plugin/limit-count-redis-cluster.t @@ -192,16 +192,27 @@ passed === TEST 4: up the limit +--- request +GET /hello +--- no_error_log +[error] +--- error_log +try to lock with key route#1#redis-cluster +unlock with key route#1#redis-cluster + + + +=== TEST 5: up the limit --- pipelined_requests eval -["GET /hello", "GET /hello", "GET /hello", "GET /hello"] +["GET /hello", "GET /hello", "GET /hello"] --- error_code eval -[200, 200, 503, 503] +[200, 503, 503] --- no_error_log [error] -=== TEST 5: up the limit again +=== TEST 6: up the limit again --- pipelined_requests eval ["GET /hello1", "GET /hello", "GET /hello2", "GET /hello", "GET /hello"] --- error_code eval @@ -211,7 +222,7 @@ passed -=== TEST 6: set route, four redis nodes, only one is valid +=== TEST 7: set route, four redis nodes, only one is valid --- config location /t { content_by_lua_block { @@ -258,7 +269,7 @@ passed -=== TEST 7: hit route +=== TEST 8: hit route --- config location /t { content_by_lua_block { @@ -299,7 +310,7 @@ code: 200 -=== TEST 8: update route, use new limit configuration +=== TEST 9: update route, use new limit configuration --- config location /t { content_by_lua_block { diff --git a/t/plugin/limit-count-redis.t b/t/plugin/limit-count-redis.t index f87478fa5542..7e2ee052ba86 100644 --- a/t/plugin/limit-count-redis.t +++ b/t/plugin/limit-count-redis.t @@ -193,16 +193,27 @@ passed === TEST 4: up the limit +--- request +GET /hello +--- no_error_log +[error] +--- error_log +try to lock with key route#1#redis +unlock with key route#1#redis + + + +=== TEST 5: up the limit --- pipelined_requests eval -["GET /hello", "GET /hello", "GET /hello", "GET /hello"] +["GET /hello", "GET /hello", "GET /hello"] --- error_code eval -[200, 200, 503, 503] +[200, 503, 503] --- no_error_log [error] -=== TEST 5: up the limit +=== TEST 6: up the limit --- pipelined_requests eval ["GET /hello1", "GET /hello", "GET /hello2", "GET /hello", "GET /hello"] --- error_code eval @@ -212,7 +223,7 @@ passed -=== TEST 6: set route, with redis host, port and right password +=== TEST 7: set route, with redis host, port and right password --- config location /t { content_by_lua_block { @@ -298,7 +309,7 @@ passed -=== TEST 7: up the limit +=== TEST 8: up the limit --- pipelined_requests eval ["GET /hello", "GET /hello", "GET /hello", "GET /hello"] --- error_code eval @@ -308,7 +319,7 @@ passed -=== TEST 8: up the limit +=== TEST 9: up the limit --- pipelined_requests eval ["GET /hello1", "GET /hello", "GET /hello2", "GET /hello", "GET /hello"] --- error_code eval @@ -318,7 +329,7 @@ passed -=== TEST 9: set route, with redis host, port and wrong password +=== TEST 10: set route, with redis host, port and wrong password --- config location /t { content_by_lua_block { @@ -364,7 +375,7 @@ GET /t -=== TEST 10: request for TEST 9 +=== TEST 11: request for TEST 10 --- request GET /hello_new --- error_code eval @@ -376,7 +387,7 @@ failed to limit req: ERR invalid password -=== TEST 11: multi request for TEST 9 +=== TEST 12: multi request for TEST 10 --- pipelined_requests eval ["GET /hello_new", "GET /hello1", "GET /hello1", "GET /hello_new"] --- error_code eval @@ -384,7 +395,7 @@ failed to limit req: ERR invalid password -=== TEST 12: restore redis password to '' +=== TEST 13: restore redis password to '' --- config location /t { content_by_lua_block { diff --git a/t/plugin/syslog.t b/t/plugin/syslog.t index c82dcd2837d7..ace3190808c3 100644 --- a/t/plugin/syslog.t +++ b/t/plugin/syslog.t @@ -263,4 +263,7 @@ passed hello world --- no_error_log [error] +--- error_log +try to lock with key route#1 +unlock with key route#1 --- timeout: 5