From 2f6fd50121c545742a902cf838287406d103566a Mon Sep 17 00:00:00 2001 From: baiyun <337531158@qq.com> Date: Wed, 3 Jan 2024 09:51:46 +0800 Subject: [PATCH] feat: Add schema attribute definition for some log plug-ins (#10738) --- apisix/plugins/file-logger.lua | 8 ++ apisix/plugins/http-logger.lua | 7 + apisix/plugins/loggly.lua | 7 + apisix/plugins/tencent-cloud-cls.lua | 14 ++ docs/en/latest/plugins/file-logger.md | 2 + docs/en/latest/plugins/http-logger.md | 1 + docs/en/latest/plugins/loggly.md | 21 +-- docs/en/latest/plugins/tencent-cloud-cls.md | 6 +- docs/zh/latest/plugins/file-logger.md | 12 +- docs/zh/latest/plugins/http-logger.md | 23 +-- t/plugin/file-logger2.t | 148 ++++++++++++++++++++ t/plugin/http-logger-json.t | 54 +++++++ t/plugin/loggly.t | 118 +++++++++++++++- t/plugin/tencent-cloud-cls.t | 133 ++++++++++++++++++ 14 files changed, 519 insertions(+), 35 deletions(-) diff --git a/apisix/plugins/file-logger.lua b/apisix/plugins/file-logger.lua index 298fae2e4c5b..e0970d821c40 100644 --- a/apisix/plugins/file-logger.lua +++ b/apisix/plugins/file-logger.lua @@ -32,6 +32,14 @@ local schema = { type = "string" }, log_format = {type = "object"}, + include_req_body = {type = "boolean", default = false}, + include_req_body_expr = { + type = "array", + minItems = 1, + items = { + type = "array" + } + }, include_resp_body = {type = "boolean", default = false}, include_resp_body_expr = { type = "array", diff --git a/apisix/plugins/http-logger.lua b/apisix/plugins/http-logger.lua index c417e29f87d4..9da5373d371e 100644 --- a/apisix/plugins/http-logger.lua +++ b/apisix/plugins/http-logger.lua @@ -35,6 +35,13 @@ local schema = { timeout = {type = "integer", minimum = 1, default = 3}, log_format = {type = "object"}, include_req_body = {type = "boolean", default = false}, + include_req_body_expr = { + type = "array", + minItems = 1, + items = { + type = "array" + } + }, include_resp_body = {type = "boolean", default = false}, include_resp_body_expr = { type = "array", diff --git a/apisix/plugins/loggly.lua b/apisix/plugins/loggly.lua index 92fafb78287d..16dc9b4a6879 100644 --- a/apisix/plugins/loggly.lua +++ b/apisix/plugins/loggly.lua @@ -62,6 +62,13 @@ local schema = { description = "base severity log level", }, include_req_body = {type = "boolean", default = false}, + include_req_body_expr = { + type = "array", + minItems = 1, + items = { + type = "array" + } + }, include_resp_body = {type = "boolean", default = false}, include_resp_body_expr = { type = "array", diff --git a/apisix/plugins/tencent-cloud-cls.lua b/apisix/plugins/tencent-cloud-cls.lua index 7a4b7783f35b..38fe56503202 100644 --- a/apisix/plugins/tencent-cloud-cls.lua +++ b/apisix/plugins/tencent-cloud-cls.lua @@ -39,7 +39,21 @@ local schema = { default = 1 }, include_req_body = { type = "boolean", default = false }, + include_req_body_expr = { + type = "array", + minItems = 1, + items = { + type = "array" + } + }, include_resp_body = { type = "boolean", default = false }, + include_resp_body_expr = { + type = "array", + minItems = 1, + items = { + type = "array" + } + }, global_tag = { type = "object" }, log_format = {type = "object"}, }, diff --git a/docs/en/latest/plugins/file-logger.md b/docs/en/latest/plugins/file-logger.md index 5b2974e13c12..f46b5a68c6cd 100644 --- a/docs/en/latest/plugins/file-logger.md +++ b/docs/en/latest/plugins/file-logger.md @@ -47,6 +47,8 @@ The `file-logger` Plugin is used to push log streams to a specific location. | ---- | ------ | -------- | ------------- | | path | string | True | Log file path. | | log_format | object | False | Log format declared as key value pairs in JSON format. Values only support strings. [APISIX](../apisix-variable.md) or [Nginx](http://nginx.org/en/docs/varindex.html) variables can be used by prefixing the string with `$`. | +| include_req_body | boolean | False | When set to `true` includes the request body in the log. If the request body is too big to be kept in the memory, it can't be logged due to Nginx's limitations. | +| include_req_body_expr | array | False | Filter for when the `include_req_body` attribute is set to `true`. Request body is only logged when the expression set here evaluates to `true`. See [lua-resty-expr](https://github.com/api7/lua-resty-expr) for more. | | include_resp_body | boolean | False | When set to `true` includes the response body in the log file. | | include_resp_body_expr | array | False | When the `include_resp_body` attribute is set to `true`, use this to filter based on [lua-resty-expr](https://github.com/api7/lua-resty-expr). If present, only logs the response into file if the expression evaluates to `true`. | | match | array[] | False | Logs will be recorded when the rule matching is successful if the option is set. See [lua-resty-expr](https://github.com/api7/lua-resty-expr#operator-list) for a list of available expressions. | diff --git a/docs/en/latest/plugins/http-logger.md b/docs/en/latest/plugins/http-logger.md index f75869f8bdb4..aef965f0a954 100644 --- a/docs/en/latest/plugins/http-logger.md +++ b/docs/en/latest/plugins/http-logger.md @@ -42,6 +42,7 @@ This will allow the ability to send log data requests as JSON objects to monitor | timeout | integer | False | 3 | [1,...] | Time to keep the connection alive for after sending a request. | | log_format | object | False | | | Log format declared as key value pairs in JSON format. Values only support strings. [APISIX](../apisix-variable.md) or [Nginx](http://nginx.org/en/docs/varindex.html) variables can be used by prefixing the string with `$`. | | include_req_body | boolean | False | false | [false, true] | When set to `true` includes the request body in the log. If the request body is too big to be kept in the memory, it can't be logged due to Nginx's limitations. | +| include_req_body_expr | array | False | | | Filter for when the `include_req_body` attribute is set to `true`. Request body is only logged when the expression set here evaluates to `true`. See [lua-resty-expr](https://github.com/api7/lua-resty-expr) for more. | | include_resp_body | boolean | False | false | [false, true] | When set to `true` includes the response body in the log. | | include_resp_body_expr | array | False | | | When the `include_resp_body` attribute is set to `true`, use this to filter based on [lua-resty-expr](https://github.com/api7/lua-resty-expr). If present, only logs the response if the expression evaluates to `true`. | | concat_method | string | False | "json" | ["json", "new_line"] | Sets how to concatenate logs. When set to `json`, uses `json.encode` for all pending logs and when set to `new_line`, also uses `json.encode` but uses the newline (`\n`) to concatenate lines. | diff --git a/docs/en/latest/plugins/loggly.md b/docs/en/latest/plugins/loggly.md index cee32c8fc02d..663ba05742e4 100644 --- a/docs/en/latest/plugins/loggly.md +++ b/docs/en/latest/plugins/loggly.md @@ -37,16 +37,17 @@ When the maximum batch size is exceeded, the data in the queue is pushed to Logg ## Attributes -| Name | Type | Required | Default | Description | -|------------------------|---------------|----------|---------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| customer_token | string | True | | Unique identifier used when sending logs to Loggly to ensure that they are sent to the right organisation account. | -| severity | string (enum) | False | INFO | Syslog log event severity level. Choose between: `DEBUG`, `INFO`, `NOTICE`, `WARNING`, `ERR`, `CRIT`, `ALERT`, and `EMEGR`. | -| severity_map | object | False | nil | A way to map upstream HTTP response codes to Syslog severity. Key-value pairs where keys are the HTTP response codes and the values are the Syslog severity levels. For example `{"410": "CRIT"}`. | -| tags | array | False | | Metadata to be included with any event log to aid in segmentation and filtering. | -| log_format | object | False | {"host": "$host", "@timestamp": "$time_iso8601", "client_ip": "$remote_addr"} | Log format declared as key value pairs in JSON format. Values only support strings. [APISIX](../apisix-variable.md) or [Nginx](http://nginx.org/en/docs/varindex.html) variables can be used by prefixing the string with `$`. | -| include_req_body | boolean | False | false | When set to `true` includes the request body in the log. If the request body is too big to be kept in the memory, it can't be logged due to Nginx's limitations. | -| include_resp_body | boolean | False | false | When set to `true` includes the response body in the log. | -| include_resp_body_expr | array | False | | When the `include_resp_body` attribute is set to `true`, use this to filter based on [lua-resty-expr](https://github.com/api7/lua-resty-expr). If present, only logs the response if the expression evaluates to `true`. | +| Name | Type | Required | Default | Description | +|------------------------|---------------|----------|---------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| customer_token | string | True | | Unique identifier used when sending logs to Loggly to ensure that they are sent to the right organisation account. | +| severity | string (enum) | False | INFO | Syslog log event severity level. Choose between: `DEBUG`, `INFO`, `NOTICE`, `WARNING`, `ERR`, `CRIT`, `ALERT`, and `EMEGR`. | +| severity_map | object | False | nil | A way to map upstream HTTP response codes to Syslog severity. Key-value pairs where keys are the HTTP response codes and the values are the Syslog severity levels. For example `{"410": "CRIT"}`. | +| tags | array | False | | Metadata to be included with any event log to aid in segmentation and filtering. | +| log_format | object | False | {"host": "$host", "@timestamp": "$time_iso8601", "client_ip": "$remote_addr"} | Log format declared as key value pairs in JSON format. Values only support strings. [APISIX](../apisix-variable.md) or [Nginx](http://nginx.org/en/docs/varindex.html) variables can be used by prefixing the string with `$`. | +| include_req_body | boolean | False | false | When set to `true` includes the request body in the log. If the request body is too big to be kept in the memory, it can't be logged due to Nginx's limitations. | +| include_req_body_expr | array | False | | Filter for when the `include_req_body` attribute is set to `true`. Request body is only logged when the expression set here evaluates to `true`. See [lua-resty-expr](https://github.com/api7/lua-resty-expr) for more. | +| include_resp_body | boolean | False | false | When set to `true` includes the response body in the log. | +| include_resp_body_expr | array | False | | When the `include_resp_body` attribute is set to `true`, use this to filter based on [lua-resty-expr](https://github.com/api7/lua-resty-expr). If present, only logs the response if the expression evaluates to `true`. | This Plugin supports using batch processors to aggregate and process entries (logs/data) in a batch. This avoids the need for frequently submitting the data. The batch processor submits data every `5` seconds or when the data in the queue reaches `1000`. See [Batch Processor](../batch-processor.md#configuration) for more information or setting your custom configuration. diff --git a/docs/en/latest/plugins/tencent-cloud-cls.md b/docs/en/latest/plugins/tencent-cloud-cls.md index a061251f9403..559f13e2d4f0 100644 --- a/docs/en/latest/plugins/tencent-cloud-cls.md +++ b/docs/en/latest/plugins/tencent-cloud-cls.md @@ -35,16 +35,18 @@ The `tencent-cloud-cls` Plugin uses [TencentCloud CLS](https://cloud.tencent.com ## Attributes | Name | Type | Required | Default | Valid values | Description | -| ----------------- | ------- | -------- |---------| ------------- |------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| ----------------- | ------- |----------|---------|---------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------| | cls_host | string | Yes | | | CLS API host，please refer [Uploading Structured Logs](https://www.tencentcloud.com/document/api/614/16873). | | cls_topic | string | Yes | | | topic id of CLS. | | secret_id | string | Yes | | | SecretId of your API key. | | secret_key | string | Yes | | | SecretKey of your API key. | | sample_ratio | number | No | 1 | [0.00001, 1] | How often to sample the requests. Setting to `1` will sample all requests. | | include_req_body | boolean | No | false | [false, true] | When set to `true` includes the request body in the log. If the request body is too big to be kept in the memory, it can't be logged due to NGINX's limitations. | +| include_req_body_expr | array | No | | | Filter for when the `include_req_body` attribute is set to `true`. Request body is only logged when the expression set here evaluates to `true`. See [lua-resty-expr](https://github.com/api7/lua-resty-expr) for more. | | include_resp_body | boolean | No | false | [false, true] | When set to `true` includes the response body in the log. | +| include_resp_body_expr | array | No | | | Filter for when the `include_resp_body` attribute is set to `true`. Response body is only logged when the expression set here evaluates to `true`. See [lua-resty-expr](https://github.com/api7/lua-resty-expr) for more. | | global_tag | object | No | | | kv pairs in JSON，send with each log. | -| log_format | object | No | | | Log format declared as key value pairs in JSON format. Values only support strings. [APISIX](../apisix-variable.md) or [Nginx](http://nginx.org/en/docs/varindex.html) variables can be used by prefixing the string with `$`. | +| log_format | object | No | | | Log format declared as key value pairs in JSON format. Values only support strings. [APISIX](../apisix-variable.md) or [Nginx](http://nginx.org/en/docs/varindex.html) variables can be used by prefixing the string with `$`. | NOTE: `encrypt_fields = {"secret_key"}` is also defined in the schema, which means that the field will be stored encrypted in etcd. See [encrypted storage fields](../plugin-develop.md#encrypted-storage-fields). diff --git a/docs/zh/latest/plugins/file-logger.md b/docs/zh/latest/plugins/file-logger.md index 81b7e6223e2c..ddb9b646b739 100644 --- a/docs/zh/latest/plugins/file-logger.md +++ b/docs/zh/latest/plugins/file-logger.md @@ -46,11 +46,13 @@ description: API 网关 Apache APISIX file-logger 插件可用于将日志数据 ## 属性 | 名称 | 类型 | 必选项 | 描述 | -| ---------------- | ------- | ------ | ------------------------------------------------ | -| path | string | 是 | 自定义输出文件路径。例如：`logs/file.log`。 | -| log_format | object | 否 | 以 JSON 格式的键值对来声明日志格式。对于值部分，仅支持字符串。如果是以 `$` 开头，则表明是要获取 [APISIX 变量](../apisix-variable.md) 或 [NGINX 内置变量](http://nginx.org/en/docs/varindex.html)。 | -| include_resp_body | boolean | 否 | 当设置为 `true` 时，生成的文件包含响应体。 | -| include_resp_body_expr | array | 否 | 当 `include_resp_body` 属性设置为 `true` 时，使用该属性并基于 [lua-resty-expr](https://github.com/api7/lua-resty-expr) 进行过滤。如果存在，则仅在表达式计算结果为 `true` 时记录响应。 | +| ---------------- | ------- |-----| ------------------------------------------------ | +| path | string | 是 | 自定义输出文件路径。例如：`logs/file.log`。 | +| log_format | object | 否 | 以 JSON 格式的键值对来声明日志格式。对于值部分，仅支持字符串。如果是以 `$` 开头，则表明是要获取 [APISIX 变量](../apisix-variable.md) 或 [NGINX 内置变量](http://nginx.org/en/docs/varindex.html)。 | +| include_req_body | boolean | 否 | 当设置为 `true` 时，日志中将包含请求体。如果请求体太大而无法在内存中保存，则由于 Nginx 的限制，无法记录请求体。| +| include_req_body_expr | array | 否 | 当 `include_req_body` 属性设置为 `true` 时的过滤器。只有当此处设置的表达式求值为 `true` 时，才会记录请求体。有关更多信息，请参阅 [lua-resty-expr](https://github.com/api7/lua-resty-expr) 。 | +| include_resp_body | boolean | 否 | 当设置为 `true` 时，生成的文件包含响应体。 | +| include_resp_body_expr | array | 否 | 当 `include_resp_body` 属性设置为 `true` 时，使用该属性并基于 [lua-resty-expr](https://github.com/api7/lua-resty-expr) 进行过滤。如果存在，则仅在表达式计算结果为 `true` 时记录响应。 | | match | array[] | 否 | 当设置了这个选项后，只有匹配规则的日志才会被记录。`match` 是一个表达式列表，具体请参考 [lua-resty-expr](https://github.com/api7/lua-resty-expr#operator-list)。 | ## 插件元数据设置 diff --git a/docs/zh/latest/plugins/http-logger.md b/docs/zh/latest/plugins/http-logger.md index 208ca3c38999..06fc120308cd 100644 --- a/docs/zh/latest/plugins/http-logger.md +++ b/docs/zh/latest/plugins/http-logger.md @@ -35,17 +35,18 @@ description: 本文介绍了 API 网关 Apache APISIX 的 http-logger 插件。 ## 属性 -| 名称 | 类型 | 必选项 | 默认值 | 有效值 | 描述 | -| ---------------------- | ------- | ------ | ------------- | -------------------- | ------------------------------------------------ | -| uri | string | 是 | | | HTTP 或 HTTPS 服务器的 URI。 | -| auth_header | string | 否 | | | 授权 header（如果需要）。 | -| timeout | integer | 否 | 3 | [1,...] | 发送请求后保持连接处于活动状态的时间。 | -| log_format | object | 否 | | | 以 JSON 格式的键值对来声明日志格式。对于值部分，仅支持字符串。如果是以 `$` 开头，则表明是要获取 [APISIX 变量](../apisix-variable.md) 或 [NGINX 内置变量](http://nginx.org/en/docs/varindex.html)。 | -| include_req_body | boolean | 否 | false | [false, true] | 当设置为 `true` 时，将请求体包含在日志中。如果请求体太大而无法保存在内存中，由于 NGINX 的限制，无法记录。 | -| include_resp_body | boolean | 否 | false | [false, true] | 当设置为 `true` 时，包含响应体。 | -| include_resp_body_expr | array | 否 | | | 当 `include_resp_body` 属性设置为 `true` 时，使用该属性并基于 [lua-resty-expr](https://github.com/api7/lua-resty-expr) 进行过滤。如果存在，则仅在表达式计算结果为 `true` 时记录响应。 | -| concat_method | string | 否 | "json" | ["json", "new_line"] | 枚举类型： **json**：对所有待发日志使用 `json.encode` 编码。**new_line**：对每一条待发日志单独使用 `json.encode` 编码并使用 `\n` 连接起来。 | -| ssl_verify | boolean | 否 | false | [false, true] | 当设置为 `true` 时验证证书。 | +| 名称 | 类型 | 必选项 | 默认值 | 有效值 | 描述 | +|-------------------------| ------- |-----| ------------- | -------------------- | ------------------------------------------------ | +| uri | string | 是 | | | HTTP 或 HTTPS 服务器的 URI。 | +| auth_header | string | 否 | | | 授权 header（如果需要）。 | +| timeout | integer | 否 | 3 | [1,...] | 发送请求后保持连接处于活动状态的时间。 | +| log_format | object | 否 | | | 以 JSON 格式的键值对来声明日志格式。对于值部分，仅支持字符串。如果是以 `$` 开头，则表明是要获取 [APISIX 变量](../apisix-variable.md) 或 [NGINX 内置变量](http://nginx.org/en/docs/varindex.html)。 | +| include_req_body | boolean | 否 | false | [false, true] | 当设置为 `true` 时，将请求体包含在日志中。如果请求体太大而无法保存在内存中，由于 NGINX 的限制，无法记录。 | +| include_req_body_expr | array | 否 | | | 当 `include_req_body` 属性设置为 `true` 时的过滤器。只有当此处设置的表达式求值为 `true` 时，才会记录请求体。有关更多信息，请参阅 [lua-resty-expr](https://github.com/api7/lua-resty-expr) 。 | +| include_resp_body | boolean | 否 | false | [false, true] | 当设置为 `true` 时，包含响应体。 | +| include_resp_body_expr | array | 否 | | | 当 `include_resp_body` 属性设置为 `true` 时，使用该属性并基于 [lua-resty-expr](https://github.com/api7/lua-resty-expr) 进行过滤。如果存在，则仅在表达式计算结果为 `true` 时记录响应。 | +| concat_method | string | 否 | "json" | ["json", "new_line"] | 枚举类型： **json**：对所有待发日志使用 `json.encode` 编码。**new_line**：对每一条待发日志单独使用 `json.encode` 编码并使用 `\n` 连接起来。 | +| ssl_verify | boolean | 否 | false | [false, true] | 当设置为 `true` 时验证证书。 | 该插件支持使用批处理器来聚合并批量处理条目（日志和数据）。这样可以避免该插件频繁地提交数据。默认情况下每 `5` 秒钟或队列中的数据达到 `1000` 条时，批处理器会自动提交数据，如需了解更多信息或自定义配置，请参考 [Batch Processor](../batch-processor.md#配置)。 diff --git a/t/plugin/file-logger2.t b/t/plugin/file-logger2.t index 5cf3a76ccf4d..706f401fed58 100644 --- a/t/plugin/file-logger2.t +++ b/t/plugin/file-logger2.t @@ -366,3 +366,151 @@ write file log success } --- response_body not write file log + + + +=== TEST 11: add plugin with 'include_req_body' setting +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + t('/apisix/admin/plugin_metadata/file-logger', ngx.HTTP_DELETE) + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "file-logger": { + "path": "file-with-req-body.log", + "include_req_body": true + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } + + + +=== TEST 12: verify plugin for file-logger with request +--- config + location /t { + content_by_lua_block { + local core = require("apisix.core") + local t = require("lib.test_admin").test + local code = t("/hello", ngx.HTTP_POST, "body-data") + local fd, err = io.open("file-with-req-body.log", 'r') + local msg + + if not fd then + core.log.error("failed to open file: file-with-req-body.log, error info: ", err) + return + end + + -- note only for first line + msg = fd:read() + + local new_msg = core.json.decode(msg) + ngx.status = code + if new_msg.request ~= nil and new_msg.request.body == "body-data" then + ngx.status = code + ngx.say('contain with target') + end + } + } +--- response_body +contain with target + + + +=== TEST 13: check file-logger 'include_req_body' with 'expr' +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "file-logger": { + "path": "file-with-req-expr-body.log", + "include_req_body": true, + "include_req_body_expr": [ + [ + "arg_log_body", + "==", + "yes" + ] + ] + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } + + + +=== TEST 14: verify file-logger req with expression of concern +--- config + location /t { + content_by_lua_block { + local core = require("apisix.core") + local t = require("lib.test_admin").test + local code = t("/hello?log_body=yes", + ngx.HTTP_POST, + [[{"foo": "bar"}]] + ) + local fd, err = io.open("file-with-req-expr-body.log", 'r') + local msg + + if not fd then + core.log.error("failed to open file: file-with-req-expr-body.log, error info: ", err) + return + end + + -- note only for first line + msg = fd:read() + + local new_msg = core.json.decode(msg) + ngx.status = code + if new_msg.request ~= nil and new_msg.request.body ~= nil then + ngx.status = code + ngx.say('contain target body hits with expr') + end + + --- a new request is logged + t("/hello?log_body=no", ngx.HTTP_POST, [[{"foo": "b"}]]) + msg = fd:read("*l") + local new_msg = core.json.decode(msg) + if new_msg.request.body == nil then + ngx.say('skip unconcern body') + end + } + } +--- response_body +contain target body hits with expr +skip unconcern body diff --git a/t/plugin/http-logger-json.t b/t/plugin/http-logger-json.t index 1642ce6d1372..b4a46af55824 100644 --- a/t/plugin/http-logger-json.t +++ b/t/plugin/http-logger-json.t @@ -183,3 +183,57 @@ POST /hello {"sample_payload":"hello"} --- error_log eval qr/(.*"response":\{.*"body":"hello world\\n".*|.*\{\\\"sample_payload\\\":\\\"hello\\\"\}.*){0}/ + + + +=== TEST 7: json body with request_body and request_body expression +--- apisix_yaml +routes: + - + uri: /hello + upstream: + nodes: + "127.0.0.1:1980": 1 + type: roundrobin + plugins: + http-logger: + batch_max_size: 1 + uri: http://127.0.0.1:1980/log + include_req_body: true + include_req_body_expr: + - - arg_bar + - == + - foo +#END +--- request +POST /hello?bar=foo +{"test":"hello"} +--- error_log +"request":{"body":"{\"test\":\"hello\"}" + + + +=== TEST 8: json body with request_body, expr not hit +--- apisix_yaml +routes: + - + uri: /hello + upstream: + nodes: + "127.0.0.1:1980": 1 + type: roundrobin + plugins: + http-logger: + batch_max_size: 1 + uri: http://127.0.0.1:1980/log + include_resp_body: true + include_resp_body_expr: + - - arg_bar + - == + - foo +#END +--- request +POST /hello?bar=bar +{"sample_payload":"hello"} +--- no_error_log +"request":{"body":"{\"test\":\"hello\"}" diff --git a/t/plugin/loggly.t b/t/plugin/loggly.t index f8d2f021d29b..babfa0adfac3 100644 --- a/t/plugin/loggly.t +++ b/t/plugin/loggly.t @@ -491,7 +491,111 @@ qr/message received: <14>1 [\d\-T:.]+Z [\d.]+ apisix [\d]+ - \[tok\@41058 tag="a -=== TEST 11: collect log with log_format +=== TEST 11: collect request log with include_req_body +--- log_level: info +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "loggly": { + "customer_token" : "tok", + "batch_max_size": 1, + "include_req_body": true + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/opentracing" + }]] + ) + + local code, _, body = t("/opentracing", "POST", "body-data") + } + } +--- error_log +"request":{"body":"body-data" + + + +=== TEST 12: collect log with include_req_body_expr +--- log_level: debug +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "loggly": { + "customer_token" : "tok", + "batch_max_size": 1, + "include_req_body": true, + "include_req_body_expr": [ + ["arg_bar", "==", "bar"] + ] + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/opentracing" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + -- this will include resp body + local code, _, body = t("/opentracing?bar=bar", "POST", "body-data") + if code >= 300 then + ngx.status = code + ngx.say("fail") + return + end + ngx.print(body) + + } + } +--- error_log +"request":{"body":"body-data" + + + +=== TEST 13: collect log with include_req_body_expr mismatch +--- log_level: debug +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, _, body = t("/opentracing?foo=bar", "POST", "body-data") + if code >= 300 then + ngx.status = code + ngx.say("fail") + return + end + ngx.print(body) + + } + } +--- no_error_log +"request":{"body":"body-data" + + + +=== TEST 14: collect log with log_format --- config location /t { content_by_lua_block { @@ -534,7 +638,7 @@ qr/message received: <14>1 [\d\-T:.]+Z [\d.]+ apisix [\d]+ - \[tok\@41058 tag="a -=== TEST 12: loggly http protocol +=== TEST 15: loggly http protocol --- config location /t { content_by_lua_block { @@ -576,7 +680,7 @@ loggly tags: "apisix" -=== TEST 13: test setup for collecting syslog with severity based on http response code +=== TEST 16: test setup for collecting syslog with severity based on http response code --- config location /t { content_by_lua_block { @@ -636,7 +740,7 @@ passed -=== TEST 14: syslog PRIVAL 9 for type severity level ALERT +=== TEST 17: syslog PRIVAL 9 for type severity level ALERT --- config location /t { content_by_lua_block { @@ -654,7 +758,7 @@ qr/message received: <9>1 [\d\-T:.]+Z [\d.]+ apisix [\d]+ - \[tok\@41058 tag="ap -=== TEST 15: syslog PRIVAL 11 for type severity level ERR +=== TEST 18: syslog PRIVAL 11 for type severity level ERR --- config location /t { content_by_lua_block { @@ -672,7 +776,7 @@ qr/message received: <11>1 [\d\-T:.]+Z [\d.]+ apisix [\d]+ - \[tok\@41058 tag="a -=== TEST 16: log format in plugin +=== TEST 19: log format in plugin --- config location /t { content_by_lua_block { @@ -729,7 +833,7 @@ passed -=== TEST 17: hit +=== TEST 20: hit --- request GET /opentracing?foo=bar --- response_body diff --git a/t/plugin/tencent-cloud-cls.t b/t/plugin/tencent-cloud-cls.t index 6ad4d585b3b1..6005504da12e 100644 --- a/t/plugin/tencent-cloud-cls.t +++ b/t/plugin/tencent-cloud-cls.t @@ -558,3 +558,136 @@ opentracing --- error_log eval qr/resolve ip failed, hostname: .*, error: address can't be resolved/ --- wait: 0.5 + + + +=== TEST 17: collect log with include_req_body_expr +--- log_level: debug +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + t('/apisix/admin/plugin_metadata/tencent-cloud-cls', ngx.HTTP_DELETE) + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "tencent-cloud-cls": { + "cls_host": "127.0.0.1:10420", + "cls_topic": "143b5d70-139b-4aec-b54e-bb97756916de", + "secret_id": "secret_id", + "secret_key": "secret_key", + "batch_max_size": 1, + "max_retry_count": 1, + "retry_delay": 2, + "buffer_duration": 2, + "inactive_timeout": 2, + "include_req_body": true, + "include_req_body_expr": [ + ["arg_bar", "==", "bar"] + ] + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/opentracing" + }]] + ) + + -- this will include resp body + local code, _, body = t("/opentracing?bar=bar", "POST", "body-data") + } + } +--- error_log +"body":"body-data" + + + +=== TEST 18: collect log with include_req_body_expr mismatch +--- log_level: debug +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, _, body = t("/opentracing?foo=bar", "POST", "body-data") + if code >= 300 then + ngx.status = code + ngx.say("fail") + return + end + ngx.print(body) + + } + } +--- no_error_log +"body":"body-data" + + + +=== TEST 19: collect log with include_resp_body_expr +--- log_level: debug +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "tencent-cloud-cls": { + "cls_host": "127.0.0.1:10420", + "cls_topic": "143b5d70-139b-4aec-b54e-bb97756916de", + "secret_id": "secret_id", + "secret_key": "secret_key", + "batch_max_size": 1, + "max_retry_count": 1, + "retry_delay": 2, + "buffer_duration": 2, + "inactive_timeout": 2, + "include_resp_body": true, + "include_resp_body_expr": [ + ["arg_bar", "==", "bar"] + ] + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/opentracing" + }]] + ) + + -- this will include resp body + local code, _, body = t("/opentracing?bar=bar", "GET") + } + } +--- error_log +"body":"opentracing\n" + + + +=== TEST 20: collect log with include_resp_body_expr mismatch +--- log_level: debug +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, _, body = t("/opentracing?foo=bar", "GET") + if code >= 300 then + ngx.status = code + ngx.say("fail") + return + end + ngx.print(body) + + } + } +--- no_error_log +"body":"opentracing\n"