feat(plugin): Add new plugin ua-restriction for bot spider restriction (

#4587)

apisix/plugins/ua-restriction.lua

@@ -0,0 +1,124 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements.  See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License.  You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+local ipairs = ipairs
+local core = require("apisix.core")
+local stringx = require('pl.stringx')
+local type = type
+local str_strip = stringx.strip
+local re_find = ngx.re.find
+
+local MATCH_NONE = 0
+local MATCH_ALLOW = 1
+local MATCH_DENY = 2
+
+local lrucache_useragent = core.lrucache.new({ ttl = 300, count = 4096 })
+
+local schema = {
+    type = "object",
+    properties = {
+        bypass_missing = {
+            type = "boolean",
+            default = false,
+        },
+        allowlist = {
+            type = "array",
+            minItems = 1
+        },
+        denylist = {
+            type = "array",
+            minItems = 1
+        },
+        message = {
+            type = "string",
+            minLength = 1,
+            maxLength = 1024,
+            default = "Not allowed"
+        },
+    },
+    additionalProperties = false,
+}
+
+local plugin_name = "ua-restriction"
+
+local _M = {
+    version = 0.1,
+    priority = 2999,
+    name = plugin_name,
+    schema = schema,
+}
+
+local function match_user_agent(user_agent, conf)
+    user_agent = str_strip(user_agent)
+    if conf.allowlist then
+        for _, rule in ipairs(conf.allowlist) do
+            if re_find(user_agent, rule, "jo") then
+                return MATCH_ALLOW
+            end
+        end
+    end
+
+    if conf.denylist then
+        for _, rule in ipairs(conf.denylist) do
+            if re_find(user_agent, rule, "jo") then
+                return MATCH_DENY
+            end
+        end
+    end
+
+    return MATCH_NONE
+end
+
+function _M.check_schema(conf)
+    local ok, err = core.schema.check(schema, conf)
+
+    if not ok then
+        return false, err
+    end
+
+    return true
+end
+
+function _M.access(conf, ctx)
+    local user_agent = core.request.header(ctx, "User-Agent")
+
+    if not user_agent then
+        if conf.bypass_missing then
+            return
+        else
+            return 403, { message = conf.message }
+        end
+    end
+    local match = MATCH_NONE
+    if type(user_agent) == "table" then
+        for _, v in ipairs(user_agent) do
+            if type(v) == "string" then
+                match = lrucache_useragent(v, conf, match_user_agent, v, conf)
+                if match > MATCH_ALLOW then
+                    break
+                end
+            end
+        end
+    else
+        match = lrucache_useragent(user_agent, conf, match_user_agent, user_agent, conf)
+    end
+
+    if match > MATCH_ALLOW then
+        return 403, { message = conf.message }
+    end
+end
+
+return _M

conf/config-default.yaml

@@ -280,6 +280,7 @@ plugins:                          # plugin list (sorted by priority)
   - batch-requests                 # priority: 4010
   - cors                           # priority: 4000
   - ip-restriction                 # priority: 3000
+  - ua-restriction                 # priority: 2999
   - referer-restriction            # priority: 2990
   - uri-blocker                    # priority: 2900
   - request-validation             # priority: 2800

docs/en/latest/config.json

@@ -73,6 +73,7 @@
             "plugins/cors",
             "plugins/uri-blocker",
             "plugins/ip-restriction",
+            "plugins/ua-restriction",
             "plugins/referer-restriction",
             "plugins/consumer-restriction"
           ]

docs/en/latest/plugins/ua-restriction.md

@@ -0,0 +1,130 @@
+---
+title: ua-restriction
+---
+
+<!--
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+-->
+
+## Summary
+
+- [**Name**](#name)
+- [**Attributes**](#attributes)
+- [**How To Enable**](#how-to-enable)
+- [**Test Plugin**](#test-plugin)
+- [**Disable Plugin**](#disable-plugin)
+
+## Name
+
+The `ua-restriction` can restrict access to a Service or a Route by `allowlist` and `denylist` `User-Agent` header.
+
+## Attributes
+
+| Name      | Type          | Requirement | Default | Valid | Description                              |
+| --------- | ------------- | ----------- | ------- | ----- | ---------------------------------------- |
+| bypass_missing  | boolean       | optional    | false   |       | Whether to bypass the check when the User-Agent header is missing |
+| allowlist | array[string] | optional    |         |       | A list of allowed User-Agent headers. |
+| denylist | array[string] | optional    |         |       | A list of denied User-Agent headers. |
+| message | string | optional             | Not allowed. | length range: [1, 1024] | Message of deny reason. |
+
+Any of `allowlist` or `denylist` can be optional, and can work together in this order: allowlist->denylist
+
+The message can be user-defined.
+
+## How To Enable
+
+Creates a route or service object, and enable plugin `ua-restriction`.
+
+```shell
+curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
+{
+    "uri": "/index.html",
+    "upstream": {
+        "type": "roundrobin",
+        "nodes": {
+            "127.0.0.1:1980": 1
+        }
+    },
+    "plugins": {
+        "ua-restriction": {
+             "bypass_missing": true,
+             "allowlist": [
+                 "my-bot1",
+                 "(Baiduspider)/(\\d+)\\.(\\d+)"
+             ],
+             "denylist": [
+                 "my-bot2",
+                 "(Twitterspider)/(\\d+)\\.(\\d+)"
+             ]
+        }
+    }
+}'
+```
+
+Default returns `{"message":"Not allowed"}` when rejected. If you want to use a custom message, you can configure it in the plugin section.
+
+```json
+"plugins": {
+    "ua-restriction": {
+        "denylist": [
+            "my-bot2",
+            "(Twitterspider)/(\\d+)\\.(\\d+)"
+        ],
+        "message": "Do you want to do something bad?"
+    }
+}
+```
+
+## Test Plugin
+
+Requests from normal User-Agent:
+
+```shell
+$ curl http://127.0.0.1:9080/index.html -i
+HTTP/1.1 200 OK
+...
+```
+
+Requests with the bot User-Agent:
+
+```shell
+$ curl http://127.0.0.1:9080/index.html --header 'User-Agent: Twitterspider/2.0'
+HTTP/1.1 403 Forbidden
+```
+
+## Disable Plugin
+
+When you want to disable the `ua-restriction` plugin, it is very simple,
+you can delete the corresponding json configuration in the plugin configuration,
+no need to restart the service, it will take effect immediately:
+
+```shell
+$ curl http://127.0.0.1:2379/v2/keys/apisix/routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d value='
+{
+    "uri": "/index.html",
+    "plugins": {},
+    "upstream": {
+        "type": "roundrobin",
+        "nodes": {
+            "39.97.63.215:80": 1
+        }
+    }
+}'
+```
+
+The `ua-restriction` plugin has been disabled now. It works for other plugins.

docs/zh/latest/config.json

@@ -71,6 +71,7 @@
             "plugins/cors",
             "plugins/uri-blocker",
             "plugins/ip-restriction",
+            "plugins/ua-restriction",
             "plugins/referer-restriction",
             "plugins/consumer-restriction"
           ]