feature: plugin limit-count support to use redis cluster

.github/workflows/build.yml

@@ -35,6 +35,25 @@ jobs:
         if: matrix.platform == 'ubuntu-18.04'
         run: sudo ./.travis/${{ matrix.os_name }}_runner.sh before_install
 
+      - name: Install Redis Cluster
+        if: matrix.os_name == 'linux_openresty'
+        uses: vishnudxb/redis-cluster@1.0.5
+        with:
+          master1-port: 5000
+          master2-port: 5001
+          master3-port: 5002
+          slave1-port: 5003
+          slave2-port: 5004
+          slave3-port: 5005
+
+      - name: Running Redis Cluster Test
+        if: matrix.os_name == 'linux_openresty'
+        run: |
+          sudo apt-get install -y redis-tools
+          docker ps -a
+          redis-cli -h 127.0.0.1 -p 5000 ping
+          redis-cli -h 127.0.0.1 -p 5000 cluster nodes
+
       - name: Linux Install
         if: matrix.platform == 'ubuntu-18.04'
         run: sudo ./.travis/${{ matrix.os_name }}_runner.sh do_install

apisix/plugins/limit-count.lua

@@ -17,10 +17,14 @@
 local limit_local_new = require("resty.limit.count").new
 local core = require("apisix.core")
 local plugin_name = "limit-count"
+local limit_redis_cluster_new
 local limit_redis_new
 do
     local redis_src = "apisix.plugins.limit-count.limit-count-redis"
     limit_redis_new = require(redis_src).new
+
+    local cluster_src = "apisix.plugins.limit-count.limit-count-redis-cluster"
+    limit_redis_cluster_new = require(cluster_src).new
 end
 
 
@@ -40,7 +44,7 @@ local schema = {
         },
         policy = {
             type = "string",
-            enum = {"local", "redis"},
+            enum = {"local", "redis", "redis-cluster"},
             default = "local",
         }
     },
@@ -70,11 +74,31 @@ local schema = {
                             type = "string", minLength = 0,
                         },
                         redis_timeout = {
-                            type = "integer", minimum = 1,
-                            default = 1000,
+                            type = "integer", minimum = 1, default = 1000,
                         },
                     },
                     required = {"redis_host"},
+                },
+                {
+                    properties = {
+                        policy = {
+                            enum = {"redis-cluster"},
+                        },
+                        redis_cluster_nodes = {
+                            type = "array",
+                            minItems = 2,
+                            items = {
+                                type = "string", minLength = 2, maxLength = 100
+                            },
+                        },
+                        redis_password = {
+                            type = "string", minLength = 0,
+                        },
+                        redis_timeout = {
+                            type = "integer", minimum = 1, default = 1000,
+                        },
+                    },
+                    required = {"redis_cluster_nodes"},
                 }
             }
         }
@@ -83,7 +107,7 @@ local schema = {
 
 
 local _M = {
-    version = 0.3,
+    version = 0.4,
     priority = 1002,
     name = plugin_name,
     schema = schema,
@@ -119,6 +143,11 @@ local function create_limit_obj(conf)
                                conf.count, conf.time_window, conf)
     end
 
+    if conf.policy == "redis-cluster" then
+        return limit_redis_cluster_new("plugin-" .. plugin_name, conf.count,
+                                       conf.time_window, conf)
+    end
+
     return nil
 end
 

apisix/plugins/limit-count/limit-count-redis-cluster.lua

@@ -0,0 +1,134 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements.  See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License.  You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+local rediscluster = require("resty.rediscluster")
+local core = require("apisix.core")
+local resty_lock = require("resty.lock")
+local setmetatable = setmetatable
+local tostring = tostring
+local ipairs = ipairs
+
+local _M = {}
+
+
+local mt = {
+    __index = _M
+}
+
+
+local function new_redis_cluster(conf)
+    local config = {
+        name = "apisix-redis-cluster",
+        serv_list = {},
+        read_timeout = conf.redis_timeout,
+        auth = conf.redis_password,
+        dict_name = "plugin-limit-count-redis-cluster-slot-lock",
+    }
+
+    for i, conf_item in ipairs(conf.redis_cluster_nodes) do
+        local host, port, err = core.utils.parse_addr(conf_item)
+        if err then
+            return nil, "failed to parse address: " .. conf_item
+                        .. " err: " .. err
+        end
+
+        config.serv_list[i] = {ip = host, port = port}
+    end
+
+    local red_cli, err = rediscluster:new(config)
+    if not red_cli then
+        return nil, "failed to new redis cluster: " .. err
+    end
+
+    return red_cli
+end
+
+
+function _M.new(plugin_name, limit, window, conf)
+    local red_cli, err = new_redis_cluster(conf)
+    if not red_cli then
+        return nil, err
+    end
+
+    local self = {
+        limit = limit, window = window, conf = conf,
+        plugin_name = plugin_name, red_cli =red_cli
+    }
+
+    return setmetatable(self, mt)
+end
+
+
+function _M.incoming(self, key)
+    local red = self.red_cli
+    local limit = self.limit
+    local window = self.window
+    local remaining
+    key = self.plugin_name .. tostring(key)
+
+    local ret, err = red:ttl(key)
+    if not ret then
+        return false, "failed to get redis `" .. key .."` ttl: " .. err
+    end
+
+    core.log.info("ttl key: ", key, " ret: ", ret, " err: ", err)
+    if ret < 0 then
+        local lock, err = resty_lock:new("plugin-limit-count")
+        if not lock then
+            return false, "failed to create lock: " .. err
+        end
+
+        local elapsed, err = lock:lock(key)
+        if not elapsed then
+            return false, "failed to acquire the lock: " .. err
+        end
+
+        ret = red:ttl(key)
+        if ret < 0 then
+            local ok, err = lock:unlock()
+            if not ok then
+                return false, "failed to unlock: " .. err
+            end
+
+            ret, err = red:set(key, limit -1, "EX", window)
+            if not ret then
+                return nil, err
+            end
+
+            return 0, limit -1
+        end
+
+        local ok, err = lock:unlock()
+        if not ok then
+            return false, "failed to unlock: " .. err
+        end
+    end
+
+    remaining, err = red:incrby(key, -1)
+    if not remaining then
+        return nil, err
+    end
+
+    if remaining < 0 then
+        return nil, "rejected"
+    end
+
+    return 0, remaining
+end
+
+
+return _M

apisix/plugins/limit-count/limit-count-redis.lua

@@ -97,12 +97,13 @@ function _M.incoming(self, key)
                 return false, "failed to unlock: " .. err
             end
 
-            ret, err = red:set(key, limit -1, "EX", window)
+            limit = limit -1
+            ret, err = red:set(key, limit, "EX", window)
             if not ret then
                 return nil, err
             end
 
-            return 0, limit -1
+            return 0, limit
         end
 
         ok, err = lock:unlock()

bin/apisix

@@ -190,6 +190,7 @@ http {
     lua_shared_dict balancer_ewma        10m;
     lua_shared_dict balancer_ewma_locks  10m;
     lua_shared_dict balancer_ewma_last_touched_at 10m;
+    lua_shared_dict plugin-limit-count-redis-cluster-slot-lock 1m;
 
     # for openid-connect plugin
     lua_shared_dict discovery             1m; # cache for discovery metadata documents

doc/plugins/limit-count.md

@@ -39,11 +39,12 @@ Limit request rate by a fixed number of requests in a given time window.
 | time_window    | integer | required             |         | [0,...]                                                                  | the time window in seconds before the request count is reset.                                                                                                                                                                                                                                               |
 | key            | string  | required             |         | ["remote_addr", "server_addr", "http_x_real_ip", "http_x_forwarded_for"] | the user specified key to limit the rate.                                                                                                                                                                                                                                                                   |
 | rejected_code  | integer | optional             | 503     | [200,600]                                                                | The HTTP status code returned when the request exceeds the threshold is rejected, default 503.                                                                                                                                                                                                              |
-| policy         | string  | optional             | "local" | ["local", "redis"]                                                       | The rate-limiting policies to use for retrieving and incrementing the limits. Available values are `local`(the counters will be stored locally in-memory on the node) and `redis`(counters are stored on a Redis server and will be shared across the nodes, usually used it to do the global speed limit). |
+| policy         | string  | optional             | "local" | ["local", "redis", "redis-cluster"]                                                       | The rate-limiting policies to use for retrieving and incrementing the limits. Available values are `local`(the counters will be stored locally in-memory on the node) and `redis`(counters are stored on a Redis server and will be shared across the nodes, usually use it to do the global speed limit). |
 | redis_host     | string  | required for `redis` |         |                                                                          | When using the `redis` policy, this property specifies the address of the Redis server.                                                                                                                                                                                                                     |
 | redis_port     | integer | optional             | 6379    | [1,...]                                                                  | When using the `redis` policy, this property specifies the port of the Redis server.                                                                                                                                                                                                                        |
 | redis_password | string  | optional             |         |                                                                          | When using the `redis` policy, this property specifies the password of the Redis server.                                                                                                                                                                                                                    |
 | redis_timeout  | integer | optional             | 1000    | [1,...]                                                                  | When using the `redis` policy, this property specifies the timeout in milliseconds of any command submitted to the Redis server.                                                                                                                                                                            |
+| redis_cluster_nodes | array | optional |         |                                                              | When using `redis-cluster` policy，This property is a list of addresses of Redis cluster service nodes. |
 
 
 **Key can be customized by the user, only need to modify a line of code of the plug-in to complete.  It is a security consideration that is not open in the plugin.**
@@ -81,7 +82,7 @@ Then add limit-count plugin:
 
 If you need a cluster-level precision traffic limit, then we can do it with the redis server. The rate limit of the traffic will be shared between different APISIX nodes to limit the rate of cluster traffic.
 
-Here is the example:
+Here is the example if we use single `redis` policy:
 
 ```shell
 curl -i http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
@@ -109,6 +110,34 @@ curl -i http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335
 }'
 ```
 
+If using `redis-cluster` policy:
+
+```shell
+curl -i http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
+{
+    "uri": "/index.html",
+    "plugins": {
+        "limit-count": {
+            "count": 2,
+            "time_window": 60,
+            "rejected_code": 503,
+            "key": "remote_addr",
+            "policy": "redis-cluster",
+            "redis_cluster_nodes": [
+              "127.0.0.1:5000",
+              "127.0.0.1:5001"
+            ]
+        }
+    },
+    "upstream": {
+        "type": "roundrobin",
+        "nodes": {
+            "39.97.63.215:80": 1
+        }
+    }
+}'
+```
+
 ## Test Plugin
 
 The above configuration limits access to only 2 times in 60 seconds. The first two visits will be normally:
@@ -120,7 +149,7 @@ curl -i http://127.0.0.1:9080/index.html
 The response header contains `X-RateLimit-Limit` and `X-RateLimit-Remaining`,
  which mean the total number of requests and the remaining number of requests that can be sent:
 
-```
+```shell
 HTTP/1.1 200 OK
 Content-Type: text/html
 Content-Length: 13175
@@ -132,7 +161,7 @@ Server: APISIX web server
 
 When you visit for the third time, you will receive a response with the 503 HTTP code:
 
-```
+```shell
 HTTP/1.1 503 Service Temporarily Unavailable
 Content-Type: text/html
 Content-Length: 194