Skip to content

Commit

Permalink
feature: plugin limit-count support to use redis cluster (#2406)
Browse files Browse the repository at this point in the history
fix #638

Co-authored-by: liuheng <liuhengloveyou@gmail.com>
  • Loading branch information
membphis and liuhengloveyou authored Oct 17, 2020
1 parent 7cb9563 commit c65f5c9
Show file tree
Hide file tree
Showing 10 changed files with 635 additions and 25 deletions.
19 changes: 19 additions & 0 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,25 @@ jobs:
if: matrix.platform == 'ubuntu-18.04'
run: sudo ./.travis/${{ matrix.os_name }}_runner.sh before_install

- name: Install Redis Cluster
if: matrix.os_name == 'linux_openresty'
uses: vishnudxb/redis-cluster@1.0.5
with:
master1-port: 5000
master2-port: 5001
master3-port: 5002
slave1-port: 5003
slave2-port: 5004
slave3-port: 5005

- name: Running Redis Cluster Test
if: matrix.os_name == 'linux_openresty'
run: |
sudo apt-get install -y redis-tools
docker ps -a
redis-cli -h 127.0.0.1 -p 5000 ping
redis-cli -h 127.0.0.1 -p 5000 cluster nodes
- name: Linux Install
if: matrix.platform == 'ubuntu-18.04'
run: sudo ./.travis/${{ matrix.os_name }}_runner.sh do_install
Expand Down
37 changes: 33 additions & 4 deletions apisix/plugins/limit-count.lua
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,14 @@
local limit_local_new = require("resty.limit.count").new
local core = require("apisix.core")
local plugin_name = "limit-count"
local limit_redis_cluster_new
local limit_redis_new
do
local redis_src = "apisix.plugins.limit-count.limit-count-redis"
limit_redis_new = require(redis_src).new

local cluster_src = "apisix.plugins.limit-count.limit-count-redis-cluster"
limit_redis_cluster_new = require(cluster_src).new
end


Expand All @@ -40,7 +44,7 @@ local schema = {
},
policy = {
type = "string",
enum = {"local", "redis"},
enum = {"local", "redis", "redis-cluster"},
default = "local",
}
},
Expand Down Expand Up @@ -70,11 +74,31 @@ local schema = {
type = "string", minLength = 0,
},
redis_timeout = {
type = "integer", minimum = 1,
default = 1000,
type = "integer", minimum = 1, default = 1000,
},
},
required = {"redis_host"},
},
{
properties = {
policy = {
enum = {"redis-cluster"},
},
redis_cluster_nodes = {
type = "array",
minItems = 2,
items = {
type = "string", minLength = 2, maxLength = 100
},
},
redis_password = {
type = "string", minLength = 0,
},
redis_timeout = {
type = "integer", minimum = 1, default = 1000,
},
},
required = {"redis_cluster_nodes"},
}
}
}
Expand All @@ -83,7 +107,7 @@ local schema = {


local _M = {
version = 0.3,
version = 0.4,
priority = 1002,
name = plugin_name,
schema = schema,
Expand Down Expand Up @@ -119,6 +143,11 @@ local function create_limit_obj(conf)
conf.count, conf.time_window, conf)
end

if conf.policy == "redis-cluster" then
return limit_redis_cluster_new("plugin-" .. plugin_name, conf.count,
conf.time_window, conf)
end

return nil
end

Expand Down
134 changes: 134 additions & 0 deletions apisix/plugins/limit-count/limit-count-redis-cluster.lua
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
--
-- Licensed to the Apache Software Foundation (ASF) under one or more
-- contributor license agreements. See the NOTICE file distributed with
-- this work for additional information regarding copyright ownership.
-- The ASF licenses this file to You under the Apache License, Version 2.0
-- (the "License"); you may not use this file except in compliance with
-- the License. You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
--

local rediscluster = require("resty.rediscluster")
local core = require("apisix.core")
local resty_lock = require("resty.lock")
local setmetatable = setmetatable
local tostring = tostring
local ipairs = ipairs

local _M = {}


local mt = {
__index = _M
}


local function new_redis_cluster(conf)
local config = {
name = "apisix-redis-cluster",
serv_list = {},
read_timeout = conf.redis_timeout,
auth = conf.redis_password,
dict_name = "plugin-limit-count-redis-cluster-slot-lock",
}

for i, conf_item in ipairs(conf.redis_cluster_nodes) do
local host, port, err = core.utils.parse_addr(conf_item)
if err then
return nil, "failed to parse address: " .. conf_item
.. " err: " .. err
end

config.serv_list[i] = {ip = host, port = port}
end

local red_cli, err = rediscluster:new(config)
if not red_cli then
return nil, "failed to new redis cluster: " .. err
end

return red_cli
end


function _M.new(plugin_name, limit, window, conf)
local red_cli, err = new_redis_cluster(conf)
if not red_cli then
return nil, err
end

local self = {
limit = limit, window = window, conf = conf,
plugin_name = plugin_name, red_cli =red_cli
}

return setmetatable(self, mt)
end


function _M.incoming(self, key)
local red = self.red_cli
local limit = self.limit
local window = self.window
local remaining
key = self.plugin_name .. tostring(key)

local ret, err = red:ttl(key)
if not ret then
return false, "failed to get redis `" .. key .."` ttl: " .. err
end

core.log.info("ttl key: ", key, " ret: ", ret, " err: ", err)
if ret < 0 then
local lock, err = resty_lock:new("plugin-limit-count")
if not lock then
return false, "failed to create lock: " .. err
end

local elapsed, err = lock:lock(key)
if not elapsed then
return false, "failed to acquire the lock: " .. err
end

ret = red:ttl(key)
if ret < 0 then
local ok, err = lock:unlock()
if not ok then
return false, "failed to unlock: " .. err
end

ret, err = red:set(key, limit -1, "EX", window)
if not ret then
return nil, err
end

return 0, limit -1
end

local ok, err = lock:unlock()
if not ok then
return false, "failed to unlock: " .. err
end
end

remaining, err = red:incrby(key, -1)
if not remaining then
return nil, err
end

if remaining < 0 then
return nil, "rejected"
end

return 0, remaining
end


return _M
5 changes: 3 additions & 2 deletions apisix/plugins/limit-count/limit-count-redis.lua
Original file line number Diff line number Diff line change
Expand Up @@ -97,12 +97,13 @@ function _M.incoming(self, key)
return false, "failed to unlock: " .. err
end

ret, err = red:set(key, limit -1, "EX", window)
limit = limit -1
ret, err = red:set(key, limit, "EX", window)
if not ret then
return nil, err
end

return 0, limit -1
return 0, limit
end

ok, err = lock:unlock()
Expand Down
1 change: 1 addition & 0 deletions bin/apisix
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,7 @@ http {
lua_shared_dict balancer_ewma 10m;
lua_shared_dict balancer_ewma_locks 10m;
lua_shared_dict balancer_ewma_last_touched_at 10m;
lua_shared_dict plugin-limit-count-redis-cluster-slot-lock 1m;
# for openid-connect plugin
lua_shared_dict discovery 1m; # cache for discovery metadata documents
Expand Down
37 changes: 33 additions & 4 deletions doc/plugins/limit-count.md
Original file line number Diff line number Diff line change
Expand Up @@ -39,11 +39,12 @@ Limit request rate by a fixed number of requests in a given time window.
| time_window | integer | required | | [0,...] | the time window in seconds before the request count is reset. |
| key | string | required | | ["remote_addr", "server_addr", "http_x_real_ip", "http_x_forwarded_for"] | the user specified key to limit the rate. |
| rejected_code | integer | optional | 503 | [200,600] | The HTTP status code returned when the request exceeds the threshold is rejected, default 503. |
| policy | string | optional | "local" | ["local", "redis"] | The rate-limiting policies to use for retrieving and incrementing the limits. Available values are `local`(the counters will be stored locally in-memory on the node) and `redis`(counters are stored on a Redis server and will be shared across the nodes, usually used it to do the global speed limit). |
| policy | string | optional | "local" | ["local", "redis", "redis-cluster"] | The rate-limiting policies to use for retrieving and incrementing the limits. Available values are `local`(the counters will be stored locally in-memory on the node) and `redis`(counters are stored on a Redis server and will be shared across the nodes, usually use it to do the global speed limit). |
| redis_host | string | required for `redis` | | | When using the `redis` policy, this property specifies the address of the Redis server. |
| redis_port | integer | optional | 6379 | [1,...] | When using the `redis` policy, this property specifies the port of the Redis server. |
| redis_password | string | optional | | | When using the `redis` policy, this property specifies the password of the Redis server. |
| redis_timeout | integer | optional | 1000 | [1,...] | When using the `redis` policy, this property specifies the timeout in milliseconds of any command submitted to the Redis server. |
| redis_cluster_nodes | array | optional | | | When using `redis-cluster` policy,This property is a list of addresses of Redis cluster service nodes. |


**Key can be customized by the user, only need to modify a line of code of the plug-in to complete. It is a security consideration that is not open in the plugin.**
Expand Down Expand Up @@ -81,7 +82,7 @@ Then add limit-count plugin:

If you need a cluster-level precision traffic limit, then we can do it with the redis server. The rate limit of the traffic will be shared between different APISIX nodes to limit the rate of cluster traffic.

Here is the example:
Here is the example if we use single `redis` policy:

```shell
curl -i http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
Expand Down Expand Up @@ -109,6 +110,34 @@ curl -i http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335
}'
```

If using `redis-cluster` policy:

```shell
curl -i http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
{
"uri": "/index.html",
"plugins": {
"limit-count": {
"count": 2,
"time_window": 60,
"rejected_code": 503,
"key": "remote_addr",
"policy": "redis-cluster",
"redis_cluster_nodes": [
"127.0.0.1:5000",
"127.0.0.1:5001"
]
}
},
"upstream": {
"type": "roundrobin",
"nodes": {
"39.97.63.215:80": 1
}
}
}'
```

## Test Plugin

The above configuration limits access to only 2 times in 60 seconds. The first two visits will be normally:
Expand All @@ -120,7 +149,7 @@ curl -i http://127.0.0.1:9080/index.html
The response header contains `X-RateLimit-Limit` and `X-RateLimit-Remaining`,
which mean the total number of requests and the remaining number of requests that can be sent:

```
```shell
HTTP/1.1 200 OK
Content-Type: text/html
Content-Length: 13175
Expand All @@ -132,7 +161,7 @@ Server: APISIX web server

When you visit for the third time, you will receive a response with the 503 HTTP code:

```
```shell
HTTP/1.1 503 Service Temporarily Unavailable
Content-Type: text/html
Content-Length: 194
Expand Down
Loading

0 comments on commit c65f5c9

Please sign in to comment.