feat: allow each logger to define custom log format in its conf

Signed-off-by: spacewander <spacewanderlzx@gmail.com>

apisix/plugins/clickhouse-logger.lua

@@ -40,6 +40,7 @@ local schema = {
         timeout = {type = "integer", minimum = 1, default = 3},
         name = {type = "string", default = "clickhouse logger"},
         ssl_verify = {type = "boolean", default = true},
+        log_format = log_util.metadata_schema_log_format,
     },
     oneOf = {
         {required = {"endpoint_addr", "user", "password", "database", "logtable"}},

apisix/plugins/elasticsearch-logger.lua

@@ -52,6 +52,7 @@ local schema = {
             },
             required = {"index"}
         },
+        log_format = log_util.metadata_schema_log_format,
         auth = {
             type = "object",
             properties = {

apisix/plugins/file-logger.lua

@@ -30,6 +30,7 @@ local schema = {
         path = {
             type = "string"
         },
+        log_format = log_util.metadata_schema_log_format,
         include_resp_body = {type = "boolean", default = false},
         include_resp_body_expr = {
             type = "array",

apisix/plugins/google-cloud-logging.lua

@@ -86,6 +86,7 @@ local schema = {
             type = "string",
             default = "apisix.apache.org%2Flogs"
         },
+        log_format = log_util.metadata_schema_log_format,
     },
     oneOf = {
         { required = { "auth_config" } },

apisix/plugins/http-logger.lua

@@ -33,6 +33,7 @@ local schema = {
         uri = core.schema.uri_def,
         auth_header = {type = "string"},
         timeout = {type = "integer", minimum = 1, default = 3},
+        log_format = log_util.metadata_schema_log_format,
         include_req_body = {type = "boolean", default = false},
         include_resp_body = {type = "boolean", default = false},
         include_resp_body_expr = {

apisix/plugins/kafka-logger.lua

@@ -37,6 +37,7 @@ local schema = {
             default = "default",
             enum = {"default", "origin"},
         },
+        log_format = log_util.metadata_schema_log_format,
         -- deprecated, use "brokers" instead
         broker_list = {
             type = "object",

apisix/plugins/loggly.lua

@@ -85,6 +85,7 @@ local schema = {
             type = "boolean",
             default = true
         },
+        log_format = log_util.metadata_schema_log_format,
         severity_map = {
             type = "object",
             description = "upstream response code vs syslog severity mapping",

apisix/plugins/rocketmq-logger.lua

@@ -46,6 +46,7 @@ local schema = {
         topic = {type = "string"},
         key = {type = "string"},
         tag = {type = "string"},
+        log_format = log_util.metadata_schema_log_format,
         timeout = {type = "integer", minimum = 1, default = 3},
         use_tls = {type = "boolean", default = false},
         access_key = {type = "string", default = ""},

apisix/plugins/skywalking-logger.lua

@@ -36,6 +36,7 @@ local schema = {
         endpoint_addr = core.schema.uri_def,
         service_name = {type = "string", default = "APISIX"},
         service_instance_name = {type = "string", default = "APISIX Instance Name"},
+        log_format = log_util.metadata_schema_log_format,
         timeout = {type = "integer", minimum = 1, default = 3},
         include_req_body = {type = "boolean", default = false},
     },

apisix/plugins/sls-logger.lua

@@ -34,6 +34,7 @@ local schema = {
     properties = {
         include_req_body = {type = "boolean", default = false},
         timeout = {type = "integer", minimum = 1, default= 5000},
+        log_format = log_util.metadata_schema_log_format,
         host = {type = "string"},
         port = {type = "integer"},
         project = {type = "string"},

apisix/plugins/splunk-hec-logging.lua

@@ -56,6 +56,7 @@ local schema = {
             type = "boolean",
             default = true
         },
+        log_format = log_util.metadata_schema_log_format,
     },
     required = { "endpoint" },
 }

apisix/plugins/syslog.lua

@@ -33,6 +33,7 @@ local schema = {
         sock_type = {type = "string", default = "tcp", enum = {"tcp", "udp"}},
         pool_size = {type = "integer", minimum = 5, default = 5},
         tls = {type = "boolean", default = false},
+        log_format = log_util.metadata_schema_log_format,
         include_req_body = {type = "boolean", default = false}
     },
     required = {"host", "port"}

apisix/plugins/tcp-logger.lua

@@ -32,6 +32,7 @@ local schema = {
         tls = {type = "boolean", default = false},
         tls_options = {type = "string"},
         timeout = {type = "integer", minimum = 1, default= 1000},
+        log_format = log_util.metadata_schema_log_format,
         include_req_body = {type = "boolean", default = false}
     },
     required = {"host", "port"}

apisix/plugins/tencent-cloud-cls.lua

@@ -41,6 +41,7 @@ local schema = {
         include_req_body = { type = "boolean", default = false },
         include_resp_body = { type = "boolean", default = false },
         global_tag = { type = "object" },
+        log_format = log_util.metadata_schema_log_format,
     },
     encrypt_fields = {"secret_key"},
     required = { "cls_host", "cls_topic", "secret_id", "secret_key" }

apisix/plugins/udp-logger.lua

@@ -30,6 +30,7 @@ local schema = {
         host = {type = "string"},
         port = {type = "integer", minimum = 0},
         timeout = {type = "integer", minimum = 1, default = 3},
+        log_format = log_util.metadata_schema_log_format,
         include_req_body = {type = "boolean", default = false}
     },
     required = {"host", "port"}

apisix/utils/log-util.lua

@@ -217,11 +217,12 @@ function _M.get_log_entry(plugin_name, conf, ctx)
     local entry
     local customized = false
 
-    if metadata and metadata.value.log_format
+    local has_meta_log_format = metadata and metadata.value.log_format
         and core.table.nkeys(metadata.value.log_format) > 0
-    then
+
+    if conf.log_format or has_meta_log_format then
         customized = true
-        entry = get_custom_format_log(ctx, metadata.value.log_format)
+        entry = get_custom_format_log(ctx, conf.log_format or metadata.value.log_format)
     else
         if is_http then
             entry = get_full_log(ngx, conf)

docs/en/latest/plugins/clickhouse-logger.md

@@ -44,6 +44,7 @@ The `clickhouse-logger` Plugin is used to push logs to [ClickHouse](https://clic
 | timeout       | integer | False    | 3                   | [1,...]      | Time to keep the connection alive for after sending a request. |
 | name          | string  | False    | "clickhouse logger" |              | Unique identifier for the logger.                              |
 | ssl_verify    | boolean | False    | true                | [true,false] | When set to `true`, verifies SSL.                              |
+| log_format       | object  | False    |              |              | Log format declared as key value pairs in JSON format. Values only support strings. [APISIX](../apisix-variable.md) or [Nginx](http://nginx.org/en/docs/varindex.html) variables can be used by prefixing the string with `$`. |
 
 NOTE: `encrypt_fields = {"password"}` is also defined in the schema, which means that the field will be stored encrypted in etcd. See [encrypted storage fields](../plugin-develop.md#encrypted-storage-fields).
 

docs/en/latest/plugins/elasticsearch-logger.md

@@ -42,6 +42,7 @@ When the Plugin is enabled, APISIX will serialize the request context informatio
 | field         | array   | True     |                             | Elasticsearch `field` configuration.                          |
 | field.index   | string  | True     |                             | Elasticsearch [_index field](https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-index-field.html#mapping-index-field). |
 | field.type    | string  | False    | Elasticsearch default value | Elasticsearch [_type field](https://www.elastic.co/guide/en/elasticsearch/reference/7.17/mapping-type-field.html#mapping-type-field). |
+| log_format | object | False    |          | Log format declared as key value pairs in JSON format. Values only support strings. [APISIX](../apisix-variable.md) or [Nginx](http://nginx.org/en/docs/varindex.html) variables can be used by prefixing the string with `$`. |
 | auth          | array   | False    |                             | Elasticsearch [authentication](https://www.elastic.co/guide/en/elasticsearch/reference/current/setting-up-authentication.html) configuration. |
 | auth.username | string  | True     |                             | Elasticsearch [authentication](https://www.elastic.co/guide/en/elasticsearch/reference/current/setting-up-authentication.html) username. |
 | auth.password | string  | True     |                             | Elasticsearch [authentication](https://www.elastic.co/guide/en/elasticsearch/reference/current/setting-up-authentication.html) password. |

docs/en/latest/plugins/file-logger.md

@@ -46,6 +46,7 @@ The `file-logger` Plugin is used to push log streams to a specific location.
 | Name | Type   | Required | Description   |
 | ---- | ------ | -------- | ------------- |
 | path | string | True     | Log file path. |
+| log_format | object | False    | Log format declared as key value pairs in JSON format. Values only support strings. [APISIX](../apisix-variable.md) or [Nginx](http://nginx.org/en/docs/varindex.html) variables can be used by prefixing the string with `$`. |
 | include_resp_body      | boolean | False     | When set to `true` includes the response body in the log file.                                                                                                                                                                |
 | include_resp_body_expr | array   | False     | When the `include_resp_body` attribute is set to `true`, use this to filter based on [lua-resty-expr](https://github.com/api7/lua-resty-expr). If present, only logs the response into file if the expression evaluates to `true`. |
 

docs/en/latest/plugins/google-cloud-logging.md

@@ -46,6 +46,7 @@ This plugin also allows to push logs as a batch to your Google Cloud Logging Ser
 | ssl_verify              | False    | true                                                                                                                                                                                                 | When set to `true`, enables SSL verification as mentioned in [OpenResty docs](https://github.com/openresty/lua-nginx-module#tcpsocksslhandshake).                  |
 | resource                | False    | {"type": "global"}                                                                                                                                                                                   | Google monitor resource. See [MonitoredResource](https://cloud.google.com/logging/docs/reference/v2/rest/v2/MonitoredResource) for more details.                   |
 | log_id                  | False    | apisix.apache.org%2Flogs                                                                                                                                                                             | Google Cloud logging ID. See [LogEntry](https://cloud.google.com/logging/docs/reference/v2/rest/v2/LogEntry) for details.                                          |
+| log_format       | False    |                             | Log format declared as key value pairs in JSON format. Values only support strings. [APISIX](../apisix-variable.md) or [Nginx](http://nginx.org/en/docs/varindex.html) variables can be used by prefixing the string with `$`. |
 
 NOTE: `encrypt_fields = {"auth_config.private_key"}` is also defined in the schema, which means that the field will be stored encrypted in etcd. See [encrypted storage fields](../plugin-develop.md#encrypted-storage-fields).
 

docs/en/latest/plugins/http-logger.md

@@ -40,6 +40,7 @@ This will allow the ability to send log data requests as JSON objects to monitor
 | uri                    | string  | True     |               |                      | URI of the HTTP/HTTPS server.                                                                                                                                                                                            |
 | auth_header            | string  | False    |               |                      | Authorization headers if required.                                                                                                                                                                                       |
 | timeout                | integer | False    | 3             | [1,...]              | Time to keep the connection alive for after sending a request.                                                                                                                                                           |
+| log_format | object | False    |      |               | Log format declared as key value pairs in JSON format. Values only support strings. [APISIX](../apisix-variable.md) or [Nginx](http://nginx.org/en/docs/varindex.html) variables can be used by prefixing the string with `$`. |
 | include_req_body       | boolean | False    | false         | [false, true]        | When set to `true` includes the request body in the log. If the request body is too big to be kept in the memory, it can't be logged due to Nginx's limitations.                                                         |
 | include_resp_body      | boolean | False    | false         | [false, true]        | When set to `true` includes the response body in the log.                                                                                                                                                                |
 | include_resp_body_expr | array   | False    |               |                      | When the `include_resp_body` attribute is set to `true`, use this to filter based on [lua-resty-expr](https://github.com/api7/lua-resty-expr). If present, only logs the response if the expression evaluates to `true`. |

docs/en/latest/plugins/kafka-logger.md

@@ -52,6 +52,7 @@ It might take some time to receive the log data. It will be automatically sent a
 | timeout                | integer | False    | 3              | [1,...]               | Timeout for the upstream to send data.                                                                                                                                                                                                                                                                                                           |
 | name                   | string  | False    | "kafka logger" |                       | Unique identifier for the batch processor.                                                                                                                                                                                                                                                                                                       |
 | meta_format            | enum    | False    | "default"      | ["default"，"origin"] | Format to collect the request information. Setting to `default` collects the information in JSON format and `origin` collects the information with the original HTTP request. See [examples](#meta_format-example) below.                                                                                                                        |
+| log_format | object | False    |      |               | Log format declared as key value pairs in JSON format. Values only support strings. [APISIX](../apisix-variable.md) or [Nginx](http://nginx.org/en/docs/varindex.html) variables can be used by prefixing the string with `$`. |
 | include_req_body       | boolean | False    | false          | [false, true]         | When set to `true` includes the request body in the log. If the request body is too big to be kept in the memory, it can't be logged due to Nginx's limitations.                                                                                                                                                                                 |
 | include_req_body_expr  | array   | False    |                |                       | Filter for when the `include_req_body` attribute is set to `true`. Request body is only logged when the expression set here evaluates to `true`. See [lua-resty-expr](https://github.com/api7/lua-resty-expr) for more.                                                                                                                          |
 | include_resp_body      | boolean | False    | false          | [false, true]         | When set to `true` includes the response body in the log.                                                                                                                                                                                                                                                                                        |

docs/en/latest/plugins/loggly.md

@@ -43,6 +43,7 @@ When the maximum batch size is exceeded, the data in the queue is pushed to Logg
 | severity               | string (enum) | False    | INFO    | Syslog log event severity level. Choose between: `DEBUG`, `INFO`, `NOTICE`, `WARNING`, `ERR`, `CRIT`, `ALERT`, and `EMEGR`.                                                                                               |
 | severity_map           | object        | False    | nil     | A way to map upstream HTTP response codes to Syslog severity. Key-value pairs where keys are the HTTP response codes and the values are the Syslog severity levels. For example `{"410": "CRIT"}`.                       |
 | tags                   | array         | False    |         | Metadata to be included with any event log to aid in segmentation and filtering.                                                                                                                                         |
+| log_format       | object  | False    |              | Log format declared as key value pairs in JSON format. Values only support strings. [APISIX](../apisix-variable.md) or [Nginx](http://nginx.org/en/docs/varindex.html) variables can be used by prefixing the string with `$`. |
 | include_req_body       | boolean       | False    | false   | When set to `true` includes the request body in the log. If the request body is too big to be kept in the memory, it can't be logged due to Nginx's limitations.                                                         |
 | include_resp_body      | boolean       | False    | false   | When set to `true` includes the response body in the log.                                                                                                                                                                 |
 | include_resp_body_expr | array         | False    |         | When the `include_resp_body` attribute is set to `true`, use this to filter based on [lua-resty-expr](https://github.com/api7/lua-resty-expr). If present, only logs the response if the expression evaluates to `true`. |