From 004e616c32cbd7b637212e835347552dddda9b58 Mon Sep 17 00:00:00 2001 From: Fatih USTA Date: Tue, 8 Nov 2022 15:19:18 +0300 Subject: [PATCH 01/10] feat: Added log format support in syslog plugin. --- apisix/plugins/syslog.lua | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/apisix/plugins/syslog.lua b/apisix/plugins/syslog.lua index b57f8a1235eb..5dd591f8aac0 100644 --- a/apisix/plugins/syslog.lua +++ b/apisix/plugins/syslog.lua @@ -19,10 +19,11 @@ local core = require("apisix.core") local log_util = require("apisix.utils.log-util") local bp_manager_mod = require("apisix.utils.batch-processor-manager") local syslog = require("apisix.plugins.syslog.init") +local plugin = require("apisix.plugin") local plugin_name = "syslog" local ngx = ngx -local batch_processor_manager = bp_manager_mod.new("http sys logger") +local batch_processor_manager = bp_manager_mod.new("sys logger") local schema = { type = "object", properties = { @@ -42,27 +43,44 @@ local schema = { local schema = batch_processor_manager:wrap_schema(schema) +local metadata_schema = { + type = "object", + properties = { + log_format = log_util.metadata_schema_log_format, + }, +} + local _M = { version = 0.1, priority = 401, name = plugin_name, schema = schema, + metadata_schema = metadata_schema, flush_syslog = syslog.flush_syslog, } -function _M.check_schema(conf) - local ok, err = core.schema.check(schema, conf) - if not ok then - return false, err +function _M.check_schema(conf, schema_type) + if schema_type == core.schema.TYPE_METADATA then + return core.schema.check(metadata_schema, conf) end - - return true + return core.schema.check(schema, conf) end function _M.log(conf, ctx) - local entry = log_util.get_full_log(ngx, conf) + local metadata = plugin.plugin_metadata(plugin_name) + core.log.info("metadata: ", core.json.delay_encode(metadata)) + + local entry + + if metadata and metadata.value.log_format + and core.table.nkeys(metadata.value.log_format) > 0 + then + entry = log_util.get_custom_format_log(ctx, metadata.value.log_format) + else + entry = log_util.get_full_log(ngx, conf) + end syslog.push_entry(conf, ctx, entry) end From 642c15c4962883e38e76222a9bc584c8dfbc7cf5 Mon Sep 17 00:00:00 2001 From: Fatih USTA Date: Tue, 8 Nov 2022 16:12:13 +0300 Subject: [PATCH 02/10] added custom log format test --- t/plugin/syslog.t | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/t/plugin/syslog.t b/t/plugin/syslog.t index 31d3cc704a7a..986935d0c17a 100644 --- a/t/plugin/syslog.t +++ b/t/plugin/syslog.t @@ -338,3 +338,28 @@ qr/sending a batch logs to 127.0.0.1:(\d+)/ --- grep_error_log_out sending a batch logs to 127.0.0.1:5044 sending a batch logs to 127.0.0.1:5045 + + +=== TEST 9: add log format +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/plugin_metadata/syslog', + ngx.HTTP_PUT, + [[{ + "log_format": { + "host": "$host", + "client_ip": "$remote_addr" + } + }]] + ) + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + From c4a99388d0b43b6147e0cd58c2f4d5420bef2e4e Mon Sep 17 00:00:00 2001 From: Fatih USTA Date: Wed, 9 Nov 2022 12:04:43 +0300 Subject: [PATCH 03/10] Added syslog custom log format doc --- docs/en/latest/plugins/syslog.md | 34 ++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/docs/en/latest/plugins/syslog.md b/docs/en/latest/plugins/syslog.md index f75dccc5b6e6..c10c9eeab6b3 100644 --- a/docs/en/latest/plugins/syslog.md +++ b/docs/en/latest/plugins/syslog.md @@ -49,6 +49,40 @@ Logs can be set as JSON objects. This Plugin supports using batch processors to aggregate and process entries (logs/data) in a batch. This avoids the need for frequently submitting the data. The batch processor submits data every `5` seconds or when the data in the queue reaches `1000`. See [Batch Processor](../batch-processor.md#configuration) for more information or setting your custom configuration. +## Metadata + +You can also set the format of the logs by configuring the Plugin metadata. The following configurations are available: + +| Name | Type | Required | Default | Description | +| ---------- | ------ | -------- | ----------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| log_format | object | False | {"host": "$host", "@timestamp": "$time_iso8601", "client_ip": "$remote_addr"} | Log format declared as key value pairs in JSON format. Values only support strings. [APISIX](../apisix-variable.md) or [Nginx](http://nginx.org/en/docs/varindex.html) variables can be used by prefixing the string with `$`. | + +:::info IMPORTANT + +Configuring the Plugin metadata is global in scope. This means that it will take effect on all Routes and Services which use the `syslog` Plugin. + +::: + +The example below shows how you can configure through the Admin API: + +```shell +curl http://127.0.0.1:9080/apisix/admin/plugin_metadata/syslog -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d ' +{ + "log_format": { + "host": "$host", + "@timestamp": "$time_iso8601", + "client_ip": "$remote_addr" + } +}' +``` + +With this configuration, your logs would be formatted as shown below: + +```shell +{"host":"localhost","@timestamp":"2020-09-23T19:05:05-04:00","client_ip":"127.0.0.1","route_id":"1"} +{"host":"localhost","@timestamp":"2020-09-23T19:05:05-04:00","client_ip":"127.0.0.1","route_id":"1"} +``` + ## Enabling the Plugin The example below shows how you can enable the Plugin for a specific Route: From c218a142aec5e5bd541ae963dbdb0ad64c8ee766 Mon Sep 17 00:00:00 2001 From: Fatih USTA Date: Thu, 10 Nov 2022 15:44:11 +0300 Subject: [PATCH 04/10] Improved syslog plugin "log-format" test cases. --- t/plugin/syslog.t | 55 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/t/plugin/syslog.t b/t/plugin/syslog.t index 986935d0c17a..94248bba9ecf 100644 --- a/t/plugin/syslog.t +++ b/t/plugin/syslog.t @@ -360,6 +360,61 @@ sending a batch logs to 127.0.0.1:5045 ngx.say(body) } } +--- request +GET /t +--- response_body +passed + + + +=== TEST 10: Add route and Enable Syslog Plugin, batch_max_size=1 +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "syslog": { + "batch_max_size": 1, + "disable": false, + "flush_limit": 1, + "host" : "127.0.0.1", + "port" : 5044 + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- request +GET /t --- response_body passed + + +=== TEST 11: hit route and report sys logger +--- request +GET /hello +--- response_body +hello world +--- wait: 0.5 +--- no_error_log +[error] +--- grep_error_log eval +qr/sending a batch logs to 127.0.0.1:(\d+)/ +--- grep_error_log_out +sending a batch logs to 127.0.0.1:5044 From 1f7b6549cc2234848a32d1567304b7e9b5486a4c Mon Sep 17 00:00:00 2001 From: Fatih USTA Date: Mon, 14 Nov 2022 11:45:38 +0300 Subject: [PATCH 05/10] Fixed wrong test check. --- t/plugin/syslog.t | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/t/plugin/syslog.t b/t/plugin/syslog.t index 94248bba9ecf..fdd928229593 100644 --- a/t/plugin/syslog.t +++ b/t/plugin/syslog.t @@ -381,7 +381,7 @@ passed "disable": false, "flush_limit": 1, "host" : "127.0.0.1", - "port" : 5044 + "port" : 5050 } }, "upstream": { @@ -417,4 +417,4 @@ hello world --- grep_error_log eval qr/sending a batch logs to 127.0.0.1:(\d+)/ --- grep_error_log_out -sending a batch logs to 127.0.0.1:5044 +sending a batch logs to 127.0.0.1:5050 From d4ad1fe9cd28c291401b530f1ae1a40e3dc862c0 Mon Sep 17 00:00:00 2001 From: Fatih USTA Date: Mon, 21 Nov 2022 12:09:52 +0300 Subject: [PATCH 06/10] syslog-log-format check content --- t/plugin/syslog.t | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/t/plugin/syslog.t b/t/plugin/syslog.t index fdd928229593..8c279cb9b16a 100644 --- a/t/plugin/syslog.t +++ b/t/plugin/syslog.t @@ -407,6 +407,14 @@ passed === TEST 11: hit route and report sys logger +--- extra_init_by_lua + local syslog = require("apisix.plugins.syslog.init") + local core = require("apisix.core") + local old_f = syslog.push_entry + syslog.push_entry = function(conf, ctx, entry) + core.log.info("syslog-log-format => " .. core.json.encode(entry)) + return old_f(conf, ctx, entry) + end --- request GET /hello --- response_body @@ -414,7 +422,5 @@ hello world --- wait: 0.5 --- no_error_log [error] ---- grep_error_log eval -qr/sending a batch logs to 127.0.0.1:(\d+)/ ---- grep_error_log_out -sending a batch logs to 127.0.0.1:5050 +--- error_log eval +qr/syslog-log-format.*\{.*"host":"localhost"/ From c1dc6c1ee934585ffa3da4ea9563b0fc12db380e Mon Sep 17 00:00:00 2001 From: Fatih USTA Date: Tue, 22 Nov 2022 09:09:39 +0300 Subject: [PATCH 07/10] Just import required libraries for testing --- t/plugin/syslog.t | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/t/plugin/syslog.t b/t/plugin/syslog.t index 655a6da36a30..2857fb3a104d 100644 --- a/t/plugin/syslog.t +++ b/t/plugin/syslog.t @@ -395,10 +395,11 @@ passed === TEST 11: hit route and report sys logger --- extra_init_by_lua local syslog = require("apisix.plugins.syslog.init") - local core = require("apisix.core") + local json = require("apisix.core.json") + local log = require("apisix.core.log") local old_f = syslog.push_entry syslog.push_entry = function(conf, ctx, entry) - core.log.info("syslog-log-format => " .. core.json.encode(entry)) + log.info("syslog-log-format => " .. json.encode(entry)) return old_f(conf, ctx, entry) end --- request From d195b94d88bb8452d5eb4272fc574ce6f6149f2e Mon Sep 17 00:00:00 2001 From: Fatih USTA Date: Thu, 24 Nov 2022 15:36:10 +0300 Subject: [PATCH 08/10] Update docs/en/latest/plugins/syslog.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: 罗泽轩 --- docs/en/latest/plugins/syslog.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/latest/plugins/syslog.md b/docs/en/latest/plugins/syslog.md index c10c9eeab6b3..12fce0b7f075 100644 --- a/docs/en/latest/plugins/syslog.md +++ b/docs/en/latest/plugins/syslog.md @@ -66,7 +66,7 @@ Configuring the Plugin metadata is global in scope. This means that it will take The example below shows how you can configure through the Admin API: ```shell -curl http://127.0.0.1:9080/apisix/admin/plugin_metadata/syslog -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d ' +curl http://127.0.0.1:9180/apisix/admin/plugin_metadata/syslog -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d ' { "log_format": { "host": "$host", From 60315537bab9bd004132357cd33ce5913e66bb1c Mon Sep 17 00:00:00 2001 From: Fatih USTA Date: Thu, 24 Nov 2022 17:11:02 +0300 Subject: [PATCH 09/10] Added upstream parameter into logformat --- t/plugin/syslog.t | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/t/plugin/syslog.t b/t/plugin/syslog.t index 2857fb3a104d..4f6da2864d19 100644 --- a/t/plugin/syslog.t +++ b/t/plugin/syslog.t @@ -336,7 +336,8 @@ sending a batch logs to 127.0.0.1:5045 [[{ "log_format": { "host": "$host", - "client_ip": "$remote_addr" + "client_ip": "$remote_addr", + "upstream": "$upstream_addr" } }]] ) @@ -410,4 +411,4 @@ hello world --- no_error_log [error] --- error_log eval -qr/syslog-log-format.*\{.*"host":"localhost"/ +qr/syslog-log-format.*\{.*"upstream":"127.0.0.1:\d+"/ From 812f1377a641b7a440a054f76c2b8d660dba6fb2 Mon Sep 17 00:00:00 2001 From: Fatih USTA Date: Fri, 25 Nov 2022 10:15:29 +0300 Subject: [PATCH 10/10] Try fix CI lint issue --- t/plugin/syslog.t | 1 + 1 file changed, 1 insertion(+) diff --git a/t/plugin/syslog.t b/t/plugin/syslog.t index 4f6da2864d19..07ed7f115eeb 100644 --- a/t/plugin/syslog.t +++ b/t/plugin/syslog.t @@ -326,6 +326,7 @@ sending a batch logs to 127.0.0.1:5044 sending a batch logs to 127.0.0.1:5045 + === TEST 9: add log format --- config location /t {