From 6b4313b85e415ca9fac42f3541e2a62ac31677e0 Mon Sep 17 00:00:00 2001 From: Ayeshmantha Perera Date: Wed, 6 May 2020 04:49:18 +0200 Subject: [PATCH] feature: implemented plugin `sys logger`. (#1414) --- apisix/plugins/syslog.lua | 104 ++++++++++++ conf/config.yaml | 1 + doc/plugins/syslog-cn.md | 105 ++++++++++++ doc/plugins/syslog.md | 105 ++++++++++++ rockspec/apisix-master-0.rockspec | 1 + t/admin/plugins.t | 2 +- t/debug/debug-mode.t | 1 + t/plugin/syslog.t | 267 ++++++++++++++++++++++++++++++ 8 files changed, 585 insertions(+), 1 deletion(-) create mode 100644 apisix/plugins/syslog.lua create mode 100644 doc/plugins/syslog-cn.md create mode 100644 doc/plugins/syslog.md create mode 100644 t/plugin/syslog.t diff --git a/apisix/plugins/syslog.lua b/apisix/plugins/syslog.lua new file mode 100644 index 000000000000..bbd53158b34b --- /dev/null +++ b/apisix/plugins/syslog.lua @@ -0,0 +1,104 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +local core = require("apisix.core") +local log_util = require("apisix.utils.log-util") +local logger_socket = require("resty.logger.socket") +local plugin_name = "syslog" +local ngx = ngx + +local schema = { + type = "object", + properties = { + host = {type = "string"}, + port = {type = "integer"}, + flush_limit = {type = "integer", minimum = 1, default = 4096}, + drop_limit = {type = "integer", default = 1048576}, + timeout = {type = "integer", minimum = 1, default = 3}, + sock_type = {type = "string", default = "tcp"}, + max_retry_times = {type = "integer", minimum = 1, default = 3}, + retry_interval = {type = "integer", minimum = 10, default = 100}, + pool_size = {type = "integer", minimum = 5, default = 5}, + tls = {type = "boolean", default = false}, + }, + required = {"host", "port"} +} + +local lrucache = core.lrucache.new({ + ttl = 300, count = 512 +}) + +local _M = { + version = 0.1, + priority = 401, + name = plugin_name, + schema = schema, +} + +function _M.check_schema(conf) + return core.schema.check(schema, conf) +end + +function _M.flush_syslog(logger) + local ok, err = logger:flush(logger) + if not ok then + core.log.error("failed to flush message:", err) + end +end + +-- log phase in APISIX +function _M.log(conf) + local entry = log_util.get_full_log(ngx) + + if not entry.route_id then + core.log.error("failed to obtain the route id for sys logger") + return + end + + -- fetch api_ctx + local api_ctx = ngx.ctx.api_ctx + if not api_ctx then + core.log.error("invalid api_ctx cannot proceed with sys logger plugin") + return core.response.exit(500) + end + + -- fetch it from lrucache + local logger, err = lrucache(api_ctx.conf_type .. "#" .. api_ctx.conf_id, api_ctx.conf_version, + logger_socket.new, logger_socket, { + host = conf.host, + port = conf.port, + flush_limit = conf.flush_limit, + drop_limit = conf.drop_limit, + timeout = conf.timeout, + sock_type = conf.sock_type, + max_retry_times = conf.max_retry_times, + retry_interval = conf.retry_interval, + pool_size = conf.pool_size, + tls = conf.tls, + }) + + if not logger then + core.log.error("failed when initiating the sys logger processor", err) + end + + -- reuse the logger object + local ok, err = logger:log(core.json.encode(entry)) + if not ok then + core.log.error("failed to log message", err) + end +end + +return _M diff --git a/conf/config.yaml b/conf/config.yaml index 3b16de12fded..8de3fe1945d0 100644 --- a/conf/config.yaml +++ b/conf/config.yaml @@ -145,6 +145,7 @@ plugins: # plugin list - proxy-mirror - kafka-logger - cors + - syslog - batch-requests stream_plugins: - mqtt-proxy diff --git a/doc/plugins/syslog-cn.md b/doc/plugins/syslog-cn.md new file mode 100644 index 000000000000..0dfbf7282c3a --- /dev/null +++ b/doc/plugins/syslog-cn.md @@ -0,0 +1,105 @@ + + +# 摘要 +- [**定义**](#name) +- [**属性列表**](#attributes) +- [**如何开启**](#how-to-enable) +- [**测试插件**](#test-plugin) +- [**禁用插件**](#disable-plugin) + + +## 定义 + +`sys` 是一个将Log data请求推送到Syslog的插件。 + +这将提供将Log数据请求作为JSON对象发送的功能。 + +## 属性列表 + +|属性名称 |必选项 |描述| +|--------- |-------- |-----------| +|host |必要的 |IP地址或主机名。| +|port |必要的 |目标上游端口。| +|timeout |可选的 |上游发送数据超时。| +|tls |可选的 |布尔值，用于控制是否执行SSL验证。| +|flush_limit |可选的 |如果缓冲的消息的大小加上当前消息的大小达到（> =）此限制（以字节为单位），则缓冲的日志消息将被写入日志服务器。默认为4096（4KB）。| +|drop_limit |可选的 |如果缓冲的消息的大小加上当前消息的大小大于此限制（以字节为单位），则由于缓冲区大小有限，当前的日志消息将被丢弃。默认drop_limit为1048576（1MB）。| +|sock_type|可选的 |用于传输层的IP协议类型。可以是“ tcp”或“ udp”。默认值为“ tcp”。| +|max_retry_times|可选的 |连接到日志服务器失败或将日志消息发送到日志服务器失败后的最大重试次数。| +|retry_interval|可选的 |重试连接到日志服务器或重试向日志服务器发送日志消息之前的时间延迟（以毫秒为单位），默认为100（0.1s）。| +|pool_size |可选的 |sock：keepalive使用的Keepalive池大小。默认为10。| + +## 如何开启 + +1. 下面例子展示了如何为指定路由开启 `sys-logger` 插件的。 + +```shell +curl http://127.0.0.1:9080/apisix/admin/consumers -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d ' +{ + "username": "foo", + "plugins": { + "plugins": { + "syslog": { + "host" : "127.0.0.1", + "port" : 5044, + "flush_limit" : 1 + } + }, + "upstream": { + "type": "roundrobin", + "nodes": { + "127.0.0.1:1980": 1 + } + }, + "uri": "/hello" + } +}' +``` + +## 测试插件 + +* 成功的情况: + +```shell +$ curl -i http://127.0.0.1:9080/hello +HTTP/1.1 200 OK +... +hello, world +``` + +## 禁用插件 + + +想要禁用“sys-logger”插件，是非常简单的，将对应的插件配置从json配置删除，就会立即生效，不需要重新启动服务： + +```shell +$ curl http://127.0.0.1:2379/apisix/admin/routes/1 -X PUT -d value=' +{ + "methods": ["GET"], + "uri": "/hello", + "plugins": {}, + "upstream": { + "type": "roundrobin", + "nodes": { + "127.0.0.1:1980": 1 + } + } +}' +``` diff --git a/doc/plugins/syslog.md b/doc/plugins/syslog.md new file mode 100644 index 000000000000..2d4307e758ac --- /dev/null +++ b/doc/plugins/syslog.md @@ -0,0 +1,105 @@ + + +# Summary +- [**Name**](#name) +- [**Attributes**](#attributes) +- [**How To Enable**](#how-to-enable) +- [**Test Plugin**](#test-plugin) +- [**Disable Plugin**](#disable-plugin) + + +## Name + +`sys` is a plugin which push Log data requests to Syslog. + +This will provide the ability to send Log data requests as JSON objects. + +## Attributes + +|Name |Requirement |Description| +|--------- |-------- |-----------| +|host |required | IP address or the Hostname.| +|port |required | Target upstream port.| +|timeout |optional |Timeout for the upstream to send data.| +|tls |optional |Boolean value to control whether to perform SSL verification| +|flush_limit |optional |If the buffered messages' size plus the current message size reaches (>=) this limit (in bytes), the buffered log messages will be written to log server. Default to 4096 (4KB).| +|drop_limit |optional |If the buffered messages' size plus the current message size is larger than this limit (in bytes), the current log message will be dropped because of limited buffer size. Default drop_limit is 1048576 (1MB).| +|sock_type|optional |IP protocol type to use for transport layer. Can be either "tcp" or "udp". Default is "tcp".| +|max_retry_times|optional |Max number of retry times after a connect to a log server failed or send log messages to a log server failed.| +|retry_interval|optional |The time delay (in ms) before retry to connect to a log server or retry to send log messages to a log server, default to 100 (0.1s).| +|pool_size |optional |Keepalive pool size used by sock:keepalive. Default to 10.| + +## How To Enable + +The following is an example on how to enable the sys-logger for a specific route. + +```shell +curl http://127.0.0.1:9080/apisix/admin/consumers -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d ' +{ + "username": "foo", + "plugins": { + "plugins": { + "syslog": { + "host" : "127.0.0.1", + "port" : 5044, + "flush_limit" : 1 + } + }, + "upstream": { + "type": "roundrobin", + "nodes": { + "127.0.0.1:1980": 1 + } + }, + "uri": "/hello" + } +}' +``` + +## Test Plugin + +* success: + +```shell +$ curl -i http://127.0.0.1:9080/hello +HTTP/1.1 200 OK +... +hello, world +``` + +## Disable Plugin + +Remove the corresponding json configuration in the plugin configuration to disable the `sys-logger`. +APISIX plugins are hot-reloaded, therefore no need to restart APISIX. + +```shell +$ curl http://127.0.0.1:2379/apisix/admin/routes/1 -X PUT -d value=' +{ + "methods": ["GET"], + "uri": "/hello", + "plugins": {}, + "upstream": { + "type": "roundrobin", + "nodes": { + "127.0.0.1:1980": 1 + } + } +}' +``` diff --git a/rockspec/apisix-master-0.rockspec b/rockspec/apisix-master-0.rockspec index 3c4f344e0273..3a02e1a648b3 100644 --- a/rockspec/apisix-master-0.rockspec +++ b/rockspec/apisix-master-0.rockspec @@ -50,6 +50,7 @@ dependencies = { "jsonschema = 0.8", "lua-resty-ipmatcher = 0.6", "lua-resty-kafka = 0.07", + "lua-resty-logger-socket = 2.0-0", } build = { diff --git a/t/admin/plugins.t b/t/admin/plugins.t index 11939872ff4c..344ffe80a212 100644 --- a/t/admin/plugins.t +++ b/t/admin/plugins.t @@ -30,7 +30,7 @@ __DATA__ --- request GET /apisix/admin/plugins/list --- response_body_like eval -qr/\["limit-req","limit-count","limit-conn","key-auth","basic-auth","prometheus","node-status","jwt-auth","zipkin","ip-restriction","grpc-transcode","serverless-pre-function","serverless-post-function","openid-connect","proxy-rewrite","redirect","response-rewrite","fault-injection","udp-logger","wolf-rbac","proxy-cache","tcp-logger","proxy-mirror","kafka-logger","cors","batch-requests"\]/ +qr/\["limit-req","limit-count","limit-conn","key-auth","basic-auth","prometheus","node-status","jwt-auth","zipkin","ip-restriction","grpc-transcode","serverless-pre-function","serverless-post-function","openid-connect","proxy-rewrite","redirect","response-rewrite","fault-injection","udp-logger","wolf-rbac","proxy-cache","tcp-logger","proxy-mirror","kafka-logger","cors","syslog","batch-requests"\]/ --- no_error_log [error] diff --git a/t/debug/debug-mode.t b/t/debug/debug-mode.t index dcd66e50d4ad..de97e00f8e83 100644 --- a/t/debug/debug-mode.t +++ b/t/debug/debug-mode.t @@ -78,6 +78,7 @@ loaded plugin and sort by priority: 506 name: grpc-transcode loaded plugin and sort by priority: 500 name: prometheus loaded plugin and sort by priority: 405 name: tcp-logger loaded plugin and sort by priority: 403 name: kafka-logger +loaded plugin and sort by priority: 401 name: syslog loaded plugin and sort by priority: 400 name: udp-logger loaded plugin and sort by priority: 0 name: example-plugin loaded plugin and sort by priority: -1000 name: zipkin diff --git a/t/plugin/syslog.t b/t/plugin/syslog.t new file mode 100644 index 000000000000..05a5cd1dd80c --- /dev/null +++ b/t/plugin/syslog.t @@ -0,0 +1,267 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +use t::APISIX 'no_plan'; + +repeat_each(1); +no_long_string(); +no_root_location(); +run_tests; + +__DATA__ + +=== TEST 1: sanity +--- config + location /t { + content_by_lua_block { + local plugin = require("apisix.plugins.syslog") + local ok, err = plugin.check_schema({ + host = "127.0.0.1", + port = 3000, + }) + if not ok then + ngx.say(err) + end + ngx.say("done") + } + } +--- request +GET /t +--- response_body +done +--- no_error_log +[error] + + + +=== TEST 2: missing port +--- config + location /t { + content_by_lua_block { + local plugin = require("apisix.plugins.syslog") + local ok, err = plugin.check_schema({host = "127.0.0.1"}) + if not ok then + ngx.say(err) + end + ngx.say("done") + } + } +--- request +GET /t +--- response_body +property "port" is required +done +--- no_error_log +[error] + + + +=== TEST 3: wrong type of string +--- config + location /t { + content_by_lua_block { + local plugin = require("apisix.plugins.syslog") + local ok, err = plugin.check_schema({ + host = "127.0.0.1", + port = "3000", + }) + if not ok then + ngx.say(err) + end + ngx.say("done") + } + } +--- request +GET /t +--- response_body +property "port" validation failed: wrong type: expected integer, got string +done +--- no_error_log +[error] + + + +=== TEST 4: add plugin +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "syslog": { + "host" : "127.0.0.1", + "port" : 5044 + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]], + [[{ + "node": { + "value": { + "plugins": { + "syslog": { + "host" : "127.0.0.1", + "port" : 5044 + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }, + "key": "/apisix/routes/1" + }, + "action": "set" + }]] + ) + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- request +GET /t +--- response_body +passed +--- no_error_log +[error] + + + +=== TEST 5: access +--- request +GET /hello +--- response_body +hello world +--- no_error_log +[error] +--- wait: 0.2 + + + +=== TEST 6: flush manually +--- config + location /t { + content_by_lua_block { + local plugin = require("apisix.plugins.syslog") + local logger_socket = require "resty.logger.socket" + local logger, err = logger_socket:new({ + host = "127.0.0.1", + port = 5044, + flush_limit = 100, + }) + + local bytes, err = logger:log("abc") + if err then + ngx.log(ngx.ERR, err) + end + + local bytes, err = logger:log("efg") + if err then + ngx.log(ngx.ERR, err) + end + + local ok, err = plugin.flush_syslog(logger) + if not ok then + ngx.say(err) + end + ngx.say("done") + } + } +--- request +GET /t +--- no_error_log +[error] + + + +=== TEST 7: small flush_limit, instant flush +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "syslog": { + "host" : "127.0.0.1", + "port" : 5044, + "flush_limit" : 1 + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]], + [[{ + "node": { + "value": { + "plugins": { + "syslog": { + "host" : "127.0.0.1", + "port" : 5044, + "flush_limit" : 1 + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }, + "key": "/apisix/routes/1" + }, + "action": "set" + }]] + ) + + if code >= 300 then + ngx.status = code + end + + ngx.say(body) + + local http = require "resty.http" + local httpc = http.new() + local uri = "http://127.0.0.1:" .. ngx.var.server_port .. "/hello" + local res, err = httpc:request_uri(uri, {method = "GET"}) + } + } +--- request +GET /hello +hello world +--- no_error_log +[error] +--- wait: 0.2