Skip to content

Commit

Permalink
feat: Added log format support in syslog plugin. (#8279)
Browse files Browse the repository at this point in the history 
Co-authored-by: 罗泽轩 <spacewanderlzx@gmail.com>
Co-authored-by: Fatih USTA <fatih.usta@trendyol.com>
Fixes #8278
  • Loading branch information
@fatihusta
fatihusta authored Nov 29, 2022
1 parent d6b0487 commit d18db2c
Show file tree
Hide file tree
Showing 3 changed files with 149 additions and 8 deletions.
34 changes: 26 additions & 8 deletions apisix/plugins/syslog.lua
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,11 @@ local core = require("apisix.core")
local log_util = require("apisix.utils.log-util")
local bp_manager_mod = require("apisix.utils.batch-processor-manager")
local syslog = require("apisix.plugins.syslog.init")
local plugin = require("apisix.plugin")
local plugin_name = "syslog"
local ngx = ngx

local batch_processor_manager = bp_manager_mod.new("http sys logger")
local batch_processor_manager = bp_manager_mod.new("sys logger")
local schema = {
type = "object",
properties = {
Expand All @@ -42,27 +43,44 @@ local schema = {

local schema = batch_processor_manager:wrap_schema(schema)

local metadata_schema = {
type = "object",
properties = {
log_format = log_util.metadata_schema_log_format,
},
}

local _M = {
version = 0.1,
priority = 401,
name = plugin_name,
schema = schema,
metadata_schema = metadata_schema,
flush_syslog = syslog.flush_syslog,
}


function _M.check_schema(conf)
local ok, err = core.schema.check(schema, conf)
if not ok then
return false, err
function _M.check_schema(conf, schema_type)
if schema_type == core.schema.TYPE_METADATA then
return core.schema.check(metadata_schema, conf)
end

return true
return core.schema.check(schema, conf)
end


function _M.log(conf, ctx)
local entry = log_util.get_full_log(ngx, conf)
local metadata = plugin.plugin_metadata(plugin_name)
core.log.info("metadata: ", core.json.delay_encode(metadata))

local entry

if metadata and metadata.value.log_format
and core.table.nkeys(metadata.value.log_format) > 0
then
entry = log_util.get_custom_format_log(ctx, metadata.value.log_format)
else
entry = log_util.get_full_log(ngx, conf)
end
syslog.push_entry(conf, ctx, entry)
end

Expand Down
34 changes: 34 additions & 0 deletions docs/en/latest/plugins/syslog.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,40 @@ Logs can be set as JSON objects.

This Plugin supports using batch processors to aggregate and process entries (logs/data) in a batch. This avoids the need for frequently submitting the data. The batch processor submits data every `5` seconds or when the data in the queue reaches `1000`. See [Batch Processor](../batch-processor.md#configuration) for more information or setting your custom configuration.

## Metadata

You can also set the format of the logs by configuring the Plugin metadata. The following configurations are available:

| Name | Type | Required | Default | Description |
| ---------- | ------ | -------- | ----------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| log_format | object | False | {"host": "$host", "@timestamp": "$time_iso8601", "client_ip": "$remote_addr"} | Log format declared as key value pairs in JSON format. Values only support strings. [APISIX](../apisix-variable.md) or [Nginx](http://nginx.org/en/docs/varindex.html) variables can be used by prefixing the string with `$`. |

:::info IMPORTANT

Configuring the Plugin metadata is global in scope. This means that it will take effect on all Routes and Services which use the `syslog` Plugin.

:::

The example below shows how you can configure through the Admin API:

```shell
curl http://127.0.0.1:9180/apisix/admin/plugin_metadata/syslog -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
{
"log_format": {
"host": "$host",
"@timestamp": "$time_iso8601",
"client_ip": "$remote_addr"
}
}'
```

With this configuration, your logs would be formatted as shown below:

```shell
{"host":"localhost","@timestamp":"2020-09-23T19:05:05-04:00","client_ip":"127.0.0.1","route_id":"1"}
{"host":"localhost","@timestamp":"2020-09-23T19:05:05-04:00","client_ip":"127.0.0.1","route_id":"1"}
```

## Enabling the Plugin

The example below shows how you can enable the Plugin for a specific Route:
Expand Down
89 changes: 89 additions & 0 deletions t/plugin/syslog.t
View file
Original file line number Diff line number Diff line change
Expand Up @@ -324,3 +324,92 @@ qr/sending a batch logs to 127.0.0.1:(\d+)/
--- grep_error_log_out
sending a batch logs to 127.0.0.1:5044
sending a batch logs to 127.0.0.1:5045



=== TEST 9: add log format
--- config
location /t {
content_by_lua_block {
local t = require("lib.test_admin").test
local code, body = t('/apisix/admin/plugin_metadata/syslog',
ngx.HTTP_PUT,
[[{
"log_format": {
"host": "$host",
"client_ip": "$remote_addr",
"upstream": "$upstream_addr"
}
}]]
)
if code >= 300 then
ngx.status = code
end
ngx.say(body)
}
}
--- request
GET /t
--- response_body
passed



=== TEST 10: Add route and Enable Syslog Plugin, batch_max_size=1
--- config
location /t {
content_by_lua_block {
local t = require("lib.test_admin").test
local code, body = t('/apisix/admin/routes/1',
ngx.HTTP_PUT,
[[{
"plugins": {
"syslog": {
"batch_max_size": 1,
"disable": false,
"flush_limit": 1,
"host" : "127.0.0.1",
"port" : 5050
}
},
"upstream": {
"nodes": {
"127.0.0.1:1980": 1
},
"type": "roundrobin"
},
"uri": "/hello"
}]]
)
if code >= 300 then
ngx.status = code
end
ngx.say(body)
}
}
--- request
GET /t
--- response_body
passed



=== TEST 11: hit route and report sys logger
--- extra_init_by_lua
local syslog = require("apisix.plugins.syslog.init")
local json = require("apisix.core.json")
local log = require("apisix.core.log")
local old_f = syslog.push_entry
syslog.push_entry = function(conf, ctx, entry)
log.info("syslog-log-format => " .. json.encode(entry))
return old_f(conf, ctx, entry)
end
--- request
GET /hello
--- response_body
hello world
--- wait: 0.5
--- no_error_log
[error]
--- error_log eval
qr/syslog-log-format.*\{.*"upstream":"127.0.0.1:\d+"/

0 comments on commit d18db2c

Please sign in to comment.