From 721bac884d98908d442781f31e2e204d7cff9134 Mon Sep 17 00:00:00 2001 From: sshniro Date: Sat, 4 Apr 2020 08:37:09 +0200 Subject: [PATCH 1/7] Initial commit for HTTP logger --- apisix/plugins/http-logger.lua | 173 +++++++++++++++++++++++++++++++++ doc/plugins/http-logger.md | 103 ++++++++++++++++++++ t/plugin/http-logger.t | 44 +++++++++ 3 files changed, 320 insertions(+) create mode 100644 apisix/plugins/http-logger.lua create mode 100644 doc/plugins/http-logger.md create mode 100644 t/plugin/http-logger.t diff --git a/apisix/plugins/http-logger.lua b/apisix/plugins/http-logger.lua new file mode 100644 index 000000000000..dbba90cff0a8 --- /dev/null +++ b/apisix/plugins/http-logger.lua @@ -0,0 +1,173 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +local core = require("apisix.core") +local log_util = require("apisix.utils.log-util") +local batch_processor = require("apisix.utils.batch-processor") +local plugin_name = "udp-logger" +local ngx = ngx +local tostring = tostring +local http = require "resty.http" +local url = require "socket.url" +local buffers = {} + +local schema = { + type = "object", + properties = { + uri = {type = "string"}, + auth_header = {type = "string", default = ""}, + timeout = {type = "integer", minimum = 1, default = 3}, + name = {type = "string", default = "http logger"}, + max_retry_count = {type = "integer", minimum = 0, default = 0}, + retry_delay = {type = "integer", minimum = 0, default = 1}, + buffer_duration = {type = "integer", minimum = 1, default = 60}, + inactive_timeout = {type = "integer", minimum = 1, default = 5}, + batch_max_size = {type = "integer", minimum = 1, default = 1000}, + }, + required = {"uri"} +} + + +local _M = { + version = 0.1, + priority = 410, + name = plugin_name, + schema = schema, +} + + +function _M.check_schema(conf) + return core.schema.check(schema, conf) +end + + +local function send_http_data(conf, log_message) + local err_msg + local res = true + local url_decoded = url.parse(conf.uri) + local host = url_decoded.host + local port = url_decoded.port + + if url_decoded.scheme == "https" then + port = 443 + end + + local httpc = http.new() + httpc:set_timeout(conf.timeout) + local ok, err = httpc:connect(host, port) + + if not ok then + return false, "failed to connect to host[" .. host .. "] port[" + .. tostring(port) .. "] " .. err + end + + if url_decoded.scheme == "https" then + ok, err = httpc:ssl_handshake(true, host, false) + if err then + return nil, "failed to perform SSL with host[" .. host .. "] " + .. "port[" .. tostring(port) .. "] " .. err + end + end + + local httpc_res, err = httpc:request({ + method = "POST", + path = url_decoded.path, + query = url_decoded.query, + body = log_message, + headers = { + ["Host"] = url_decoded.host, + ["Content-Type"] = "application/json", + ["Authorization"] = conf.auth_header + } + }) + + if not httpc_res then + return false, "error while sending data to [" .. host .. "] port[" + .. tostring(port) .. "] " .. err + end + + -- some error occurred in the server + if httpc_res.status >= 400 then + res = false + err_msg = "server returned status code[" .. httpc_res.status .. "] host[" + .. host .. "] port[" .. tostring(port) .. "] " + .. "body[" .. httpc_res:read_body() .. "]" + end + + -- keep the connection alive + ok, err = httpc:set_keepalive(conf.keepalive) + + if not ok then + core.log.error("failed to keep the connection alive", err) + end + + return res, err_msg +end + + +function _M.log(conf) + local entry = log_util.get_full_log(ngx) + + if not entry.route_id then + core.log.error("failed to obtain the route id for tcp logger") + return + end + + local log_buffer = buffers[entry.route_id] + + if log_buffer then + log_buffer:push(entry) + return + end + + -- Generate a function to be executed by the batch processor + local func = function(entries, batch_max_size) + local data, err + if batch_max_size == 1 then + data, err = core.json.encode(entries[1]) -- encode as single {} + else + data, err = core.json.encode(entries) -- encode as array [{}] + end + + if not data then + return false, 'error occurred while encoding the data: ' .. err + end + + return send_http_data(conf, data) + end + + local config = { + name = conf.name, + retry_delay = conf.retry_delay, + batch_max_size = conf.batch_max_size, + max_retry_count = conf.max_retry_count, + buffer_duration = conf.buffer_duration, + inactive_timeout = conf.inactive_timeout, + } + + local err + log_buffer, err = batch_processor:new(func, config) + + if not log_buffer then + core.log.error("error when creating the batch processor: ", err) + return + end + + buffers[entry.route_id] = log_buffer + log_buffer:push(entry) +end + +return _M diff --git a/doc/plugins/http-logger.md b/doc/plugins/http-logger.md new file mode 100644 index 000000000000..6bd5217c5e03 --- /dev/null +++ b/doc/plugins/http-logger.md @@ -0,0 +1,103 @@ + + +# Summary +- [**Name**](#name) +- [**Attributes**](#attributes) +- [**How To Enable**](#how-to-enable) +- [**Test Plugin**](#test-plugin) +- [**Disable Plugin**](#disable-plugin) + + +## Name + +`http-logger` is a plugin which push Log data requests to TCP servers. + +This will provide the ability to send Log data requests as JSON objects to Monitoring tools and other TCP servers. + +## Attributes + +|Name |Requirement |Description| +|--------- |-------- |-----------| +|uri |required |URI of the server| +|authorization |optional |Any authorization headers| +|keepalive |optional |Time to keep the connection alive after sending a request| +|name |optional |A unique identifier to identity the logger| +|batch_max_size |optional |Max size of each batch, default is 1000| +|inactive_timeout|optional |maximum age in seconds when the buffer will be flushed if inactive, default is 5s| +|buffer_duration|optional |Maximum age in seconds of the oldest entry in a batch before the batch must be processed, default is 5| +|max_retry_count|optional |Maximum number of retries before removing from the processing pipe line; default is zero| +|retry_delay |optional |Number of seconds the process execution should be delayed if the execution fails; default is 1| + + +## How To Enable + +The following is an example on how to enable the http-logger for a specific route. + +```shell +curl http://127.0.0.1:9080/apisix/admin/consumers -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d ' +{ + "username": "foo", + "plugins": { + "plugins": { + "http-logger": { + "uri": "127.0.0.1:80/postendpoint?param=1", + } + }, + "upstream": { + "type": "roundrobin", + "nodes": { + "127.0.0.1:1980": 1 + } + }, + "uri": "/hello" + } +}' +``` + +## Test Plugin + +* success: + +```shell +$ curl -i http://127.0.0.1:9080/hello +HTTP/1.1 200 OK +... +hello, world +``` + +## Disable Plugin + +Remove the corresponding json configuration in the plugin configuration to disable the `http-logger`. +APISIX plugins are hot-reloaded, therefore no need to restart APISIX. + +```shell +$ curl http://127.0.0.1:2379/apisix/admin/routes/1 -X PUT -d value=' +{ + "methods": ["GET"], + "uri": "/hello", + "plugins": {}, + "upstream": { + "type": "roundrobin", + "nodes": { + "127.0.0.1:1980": 1 + } + } +}' +``` diff --git a/t/plugin/http-logger.t b/t/plugin/http-logger.t new file mode 100644 index 000000000000..4d78ae1bcd36 --- /dev/null +++ b/t/plugin/http-logger.t @@ -0,0 +1,44 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +use t::APISIX 'no_plan'; + +repeat_each(1); +no_long_string(); +no_root_location(); +run_tests; + +__DATA__ + +=== TEST 1: sanity +--- config + location /t { + content_by_lua_block { + local plugin = require("apisix.plugins.http-logger") + local ok, err = plugin.check_schema({uri = "127.0.0.1"}) + if not ok then + ngx.say(err) + end + + ngx.say("done") + } + } +--- request +GET /t +--- response_body +done +--- no_error_log +[error] From 22d7b0ec0006fa7631db61c57baefbd49e6c02d6 Mon Sep 17 00:00:00 2001 From: sshniro Date: Tue, 14 Apr 2020 07:20:26 +0200 Subject: [PATCH 2/7] Adding test cases --- apisix/plugins/http-logger.lua | 8 ++- conf/config.yaml | 1 + t/admin/plugins.t | 2 +- t/debug/debug-mode.t | 1 + t/plugin/http-logger.t | 103 +++++++++++++++++++++++++++++++++ 5 files changed, 111 insertions(+), 4 deletions(-) diff --git a/apisix/plugins/http-logger.lua b/apisix/plugins/http-logger.lua index dbba90cff0a8..7fc18f47a640 100644 --- a/apisix/plugins/http-logger.lua +++ b/apisix/plugins/http-logger.lua @@ -21,7 +21,7 @@ local plugin_name = "udp-logger" local ngx = ngx local tostring = tostring local http = require "resty.http" -local url = require "socket.url" +local url = require "net.url" local buffers = {} local schema = { @@ -63,6 +63,8 @@ local function send_http_data(conf, log_message) if url_decoded.scheme == "https" then port = 443 + elseif not url_decoded.port then + port = 80 end local httpc = http.new() @@ -76,7 +78,7 @@ local function send_http_data(conf, log_message) if url_decoded.scheme == "https" then ok, err = httpc:ssl_handshake(true, host, false) - if err then + if not ok then return nil, "failed to perform SSL with host[" .. host .. "] " .. "port[" .. tostring(port) .. "] " .. err end @@ -122,7 +124,7 @@ function _M.log(conf) local entry = log_util.get_full_log(ngx) if not entry.route_id then - core.log.error("failed to obtain the route id for tcp logger") + core.log.error("failed to obtain the route id for http logger") return end diff --git a/conf/config.yaml b/conf/config.yaml index a3630b6489c4..facbb622b578 100644 --- a/conf/config.yaml +++ b/conf/config.yaml @@ -145,5 +145,6 @@ plugins: # plugin list - proxy-mirror - kafka-logger - cors + - http-logger stream_plugins: - mqtt-proxy diff --git a/t/admin/plugins.t b/t/admin/plugins.t index 20ee9ed88f7c..c1edf77ef822 100644 --- a/t/admin/plugins.t +++ b/t/admin/plugins.t @@ -30,7 +30,7 @@ __DATA__ --- request GET /apisix/admin/plugins/list --- response_body_like eval -qr/\["limit-req","limit-count","limit-conn","key-auth","basic-auth","prometheus","node-status","jwt-auth","zipkin","ip-restriction","grpc-transcode","serverless-pre-function","serverless-post-function","openid-connect","proxy-rewrite","redirect","response-rewrite","fault-injection","udp-logger","wolf-rbac","proxy-cache","tcp-logger","proxy-mirror","kafka-logger","cors"\]/ +qr/\["limit-req","limit-count","limit-conn","key-auth","basic-auth","prometheus","node-status","jwt-auth","zipkin","ip-restriction","grpc-transcode","serverless-pre-function","serverless-post-function","openid-connect","proxy-rewrite","redirect","response-rewrite","fault-injection","udp-logger","wolf-rbac","proxy-cache","tcp-logger","proxy-mirror","kafka-logger","cors","http-logger"\]/ --- no_error_log [error] diff --git a/t/debug/debug-mode.t b/t/debug/debug-mode.t index 0acdb8244347..f445aae28695 100644 --- a/t/debug/debug-mode.t +++ b/t/debug/debug-mode.t @@ -75,6 +75,7 @@ loaded plugin and sort by priority: 900 name: redirect loaded plugin and sort by priority: 899 name: response-rewrite loaded plugin and sort by priority: 506 name: grpc-transcode loaded plugin and sort by priority: 500 name: prometheus +loaded plugin and sort by priority: 410 name: http-logger loaded plugin and sort by priority: 405 name: tcp-logger loaded plugin and sort by priority: 403 name: kafka-logger loaded plugin and sort by priority: 400 name: udp-logger diff --git a/t/plugin/http-logger.t b/t/plugin/http-logger.t index 4d78ae1bcd36..956bda2b8109 100644 --- a/t/plugin/http-logger.t +++ b/t/plugin/http-logger.t @@ -42,3 +42,106 @@ GET /t done --- no_error_log [error] + + + +=== TEST 2: full schema check +--- config + location /t { + content_by_lua_block { + local plugin = require("apisix.plugins.http-logger") + local ok, err = plugin.check_schema({uri = "127.0.0.1", + auth_header = "Basic 123", + timeout = 3, + name = "http-logger", + max_retry_count = 2, + retry_delay = 2, + buffer_duration = 2, + inactive_timeout = 2, + batch_max_size = 500, + }) + if not ok then + ngx.say(err) + end + + ngx.say("done") + } + } +--- request +GET /t +--- response_body +done +--- no_error_log +[error] + + + +=== TEST 4: add plugin +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "http-logger": { + "uri": "http://127.0.0.1", + "batch_max_size": 1 + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/opentracing" + }]], + [[{ + "node": { + "value": { + "plugins": { + "http-logger": { + "uri": "http://127.0.0.1", + "batch_max_size": 1 + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/opentracing" + }, + "key": "/apisix/routes/1" + }, + "action": "set" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- request +GET /t +--- response_body +passed +--- no_error_log +[error] + + + +=== TEST 5: access +--- request +GET /opentracing +--- response_body +opentracing +--- no_error_log +[error] +--- wait: 0.2 + From 358a16eb6aac2dd8ac569f8a5d606b1c5f2b24a8 Mon Sep 17 00:00:00 2001 From: sshniro Date: Tue, 14 Apr 2020 07:31:50 +0200 Subject: [PATCH 3/7] Adding debug loglevel to the http logger --- apisix/plugins/http-logger.lua | 2 +- t/plugin/http-logger.t | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/apisix/plugins/http-logger.lua b/apisix/plugins/http-logger.lua index 7fc18f47a640..7ec3d3dca8ff 100644 --- a/apisix/plugins/http-logger.lua +++ b/apisix/plugins/http-logger.lua @@ -113,7 +113,7 @@ local function send_http_data(conf, log_message) ok, err = httpc:set_keepalive(conf.keepalive) if not ok then - core.log.error("failed to keep the connection alive", err) + core.log.debug("failed to keep the connection alive", err) end return res, err_msg diff --git a/t/plugin/http-logger.t b/t/plugin/http-logger.t index 956bda2b8109..4a7e9843744c 100644 --- a/t/plugin/http-logger.t +++ b/t/plugin/http-logger.t @@ -16,6 +16,7 @@ # use t::APISIX 'no_plan'; +log_level('debug'); repeat_each(1); no_long_string(); no_root_location(); @@ -86,7 +87,7 @@ done [[{ "plugins": { "http-logger": { - "uri": "http://127.0.0.1", + "uri": "http://127.0.0.1:1982/hello", "batch_max_size": 1 } }, @@ -103,7 +104,7 @@ done "value": { "plugins": { "http-logger": { - "uri": "http://127.0.0.1", + "uri": "http://127.0.0.1:1982/hello", "batch_max_size": 1 } }, @@ -141,7 +142,8 @@ passed GET /opentracing --- response_body opentracing ---- no_error_log -[error] ---- wait: 0.2 +--- error_log +failed to keep the connection +Batch Processor[http logger] successfully processed the entries +--- wait: 0.5 From cffe5d815e5f539f754f2fa8691a62675c51eb08 Mon Sep 17 00:00:00 2001 From: sshniro Date: Wed, 15 Apr 2020 11:01:27 +0200 Subject: [PATCH 4/7] Adding more test cases --- .travis/linux_openresty_runner.sh | 1 + .travis/linux_tengine_runner.sh | 1 + apisix/plugins/http-logger.lua | 10 +- t/plugin/http-logger.t | 420 +++++++++++++++++++++++++++++- 4 files changed, 425 insertions(+), 7 deletions(-) diff --git a/.travis/linux_openresty_runner.sh b/.travis/linux_openresty_runner.sh index f941f9bac75d..b315227fb150 100755 --- a/.travis/linux_openresty_runner.sh +++ b/.travis/linux_openresty_runner.sh @@ -34,6 +34,7 @@ before_install() { sudo cpanm --notest Test::Nginx >build.log 2>&1 || (cat build.log && exit 1) docker pull redis:3.0-alpine docker run --rm -itd -p 6379:6379 --name apisix_redis redis:3.0-alpine + docker run --rm -itd -e HTTP_PORT=8888 -e HTTPS_PORT=9999 -p 8888:8888 -p 9999:9999 mendhak/http-https-echo # spin up kafka cluster for tests (1 zookeper and 1 kafka instance) docker pull bitnami/zookeeper:3.6.0 docker pull bitnami/kafka:latest diff --git a/.travis/linux_tengine_runner.sh b/.travis/linux_tengine_runner.sh index 13d28d8a1de4..422e01a4d376 100755 --- a/.travis/linux_tengine_runner.sh +++ b/.travis/linux_tengine_runner.sh @@ -34,6 +34,7 @@ before_install() { sudo cpanm --notest Test::Nginx >build.log 2>&1 || (cat build.log && exit 1) docker pull redis:3.0-alpine docker run --rm -itd -p 6379:6379 --name apisix_redis redis:3.0-alpine + docker run --rm -itd -e HTTP_PORT=8888 -e HTTPS_PORT=9999 -p 8888:8888 -p 9999:9999 mendhak/http-https-echo # spin up kafka cluster for tests (1 zookeper and 1 kafka instance) docker pull bitnami/zookeeper:3.6.0 docker pull bitnami/kafka:latest diff --git a/apisix/plugins/http-logger.lua b/apisix/plugins/http-logger.lua index 7ec3d3dca8ff..90e3fc4643b6 100644 --- a/apisix/plugins/http-logger.lua +++ b/apisix/plugins/http-logger.lua @@ -61,14 +61,14 @@ local function send_http_data(conf, log_message) local host = url_decoded.host local port = url_decoded.port - if url_decoded.scheme == "https" then + if ((not port) and url_decoded.scheme == "https") then port = 443 - elseif not url_decoded.port then + elseif not port then port = 80 end local httpc = http.new() - httpc:set_timeout(conf.timeout) + httpc:set_timeout(conf.timeout * 1000) local ok, err = httpc:connect(host, port) if not ok then @@ -84,7 +84,7 @@ local function send_http_data(conf, log_message) end end - local httpc_res, err = httpc:request({ + local httpc_res, httpc_err = httpc:request({ method = "POST", path = url_decoded.path, query = url_decoded.query, @@ -98,7 +98,7 @@ local function send_http_data(conf, log_message) if not httpc_res then return false, "error while sending data to [" .. host .. "] port[" - .. tostring(port) .. "] " .. err + .. tostring(port) .. "] " .. httpc_err end -- some error occurred in the server diff --git a/t/plugin/http-logger.t b/t/plugin/http-logger.t index 4a7e9843744c..b8e8c639d573 100644 --- a/t/plugin/http-logger.t +++ b/t/plugin/http-logger.t @@ -77,6 +77,37 @@ done +=== TEST 3: uri is missing +--- config + location /t { + content_by_lua_block { + local plugin = require("apisix.plugins.http-logger") + local ok, err = plugin.check_schema({auth_header = "Basic 123", + timeout = 3, + name = "http-logger", + max_retry_count = 2, + retry_delay = 2, + buffer_duration = 2, + inactive_timeout = 2, + batch_max_size = 500, + }) + if not ok then + ngx.say(err) + end + + ngx.say("done") + } + } +--- request +GET /t +--- response_body +property "uri" is required +done +--- no_error_log +[error] + + + === TEST 4: add plugin --- config location /t { @@ -137,13 +168,398 @@ passed -=== TEST 5: access +=== TEST 5: access local server --- request GET /opentracing --- response_body opentracing --- error_log -failed to keep the connection Batch Processor[http logger] successfully processed the entries --- wait: 0.5 + + +=== TEST 6: set to the http external endpoint +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "http-logger": { + "uri": "http://127.0.0.1:8888/hello-world-http", + "batch_max_size": 1 + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]], + [[{ + "node": { + "value": { + "plugins": { + "http-logger": { + "uri": "http://127.0.0.1:8888/hello-world-http", + "batch_max_size": 1 + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }, + "key": "/apisix/routes/1" + }, + "action": "set" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- request +GET /t +--- response_body +passed +--- no_error_log +[error] + + + +=== TEST 7: access external endpoint +--- request +GET /hello +--- response_body +hello world +--- error_log +Batch Processor[http logger] successfully processed the entries +--- wait: 1.5 + + + +=== TEST 8: set wrong https endpoint +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "http-logger": { + "uri": "https://127.0.0.1:8888/hello-world-http", + "batch_max_size": 1 + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello1" + }]], + [[{ + "node": { + "value": { + "plugins": { + "http-logger": { + "uri": "https://127.0.0.1:8888/hello-world-http", + "batch_max_size": 1 + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello1" + }, + "key": "/apisix/routes/1" + }, + "action": "set" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- request +GET /t +--- response_body +passed +--- no_error_log +[error] + + + +=== TEST 9: access wrong https endpoint +--- request +GET /hello1 +--- response_body +hello1 world +--- error_log +failed to perform SSL with host[127.0.0.1] port[8888] handshake failed +--- wait: 1.5 + + + +=== TEST 10: set correct https endpoint +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "http-logger": { + "uri": "https://127.0.0.1:9999/hello-world-http", + "batch_max_size": 1 + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello1" + }]], + [[{ + "node": { + "value": { + "plugins": { + "http-logger": { + "uri": "https://127.0.0.1:9999/hello-world-http", + "batch_max_size": 1 + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello1" + }, + "key": "/apisix/routes/1" + }, + "action": "set" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- request +GET /t +--- response_body +passed +--- no_error_log +[error] + + + +=== TEST 11: access correct https endpoint +--- request +GET /hello1 +--- response_body +hello1 world +--- error_log +Batch Processor[http logger] successfully processed the entries +--- wait: 1.5 + + + +=== TEST 12: set batch max size to two +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "http-logger": { + "uri": "https://127.0.0.1:9999/hello-world-http", + "batch_max_size": 2, + "max_retry_count": 1, + "retry_delay": 2, + "buffer_duration": 2, + "inactive_timeout": 2 + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello1" + }]], + [[{ + "node": { + "value": { + "plugins": { + "http-logger": { + "uri": "https://127.0.0.1:9999/hello-world-http", + "batch_max_size": 2, + "max_retry_count": 1, + "retry_delay": 2, + "buffer_duration": 2, + "inactive_timeout": 2 + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello1" + }, + "key": "/apisix/routes/1" + }, + "action": "set" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- request +GET /t +--- response_body +passed +--- no_error_log +[error] + + + +=== TEST 13: access route with batch max size twice +--- config + location /t { + content_by_lua_block { + local http = require "resty.http" + local httpc = http.new() + local uri = "http://127.0.0.1:" .. ngx.var.server_port .. "/hello1" + local res, err = httpc:request_uri(uri, { method = "GET"}) + res, err = httpc:request_uri(uri, { method = "GET"}) + ngx.status = res.status + if res.status == 200 then + ngx.say("hello1 world") + end + } + } +--- request +GET /t +--- response_body +hello1 world +--- error_log +Batch Processor[http logger] batch max size has exceeded +tranferring buffer entries to processing pipe line, buffercount[2] +Batch Processor[http logger] successfully processed the entries +--- wait: 1.5 + + + +=== TEST 14: set wrong port +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "http-logger": { + "uri": "http://127.0.0.1:9991/hello-world-http", + "batch_max_size": 1, + "max_retry_count": 1, + "retry_delay": 2, + "buffer_duration": 2, + "inactive_timeout": 2 + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello1" + }]], + [[{ + "node": { + "value": { + "plugins": { + "http-logger": { + "uri": "http://127.0.0.1:9991/hello-world-http", + "batch_max_size": 1, + "max_retry_count": 1, + "retry_delay": 2, + "buffer_duration": 2, + "inactive_timeout": 2 + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello1" + }, + "key": "/apisix/routes/1" + }, + "action": "set" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- request +GET /t +--- response_body +passed +--- no_error_log +[error] + + + +=== TEST 15: access wrong port +--- request +GET /hello1 +--- response_body +hello1 world +--- error_log +Batch Processor[http logger] failed to process entries: failed to connect to host[127.0.0.1] port[9991] connection refused +--- wait: 1.5 From 23fe1abc5f13f2caac76032b5b3de61c093ff63c Mon Sep 17 00:00:00 2001 From: sshniro Date: Wed, 15 Apr 2020 11:04:31 +0200 Subject: [PATCH 5/7] Updating the doc --- doc/plugins/http-logger.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/plugins/http-logger.md b/doc/plugins/http-logger.md index 6bd5217c5e03..b4704130f193 100644 --- a/doc/plugins/http-logger.md +++ b/doc/plugins/http-logger.md @@ -27,9 +27,9 @@ ## Name -`http-logger` is a plugin which push Log data requests to TCP servers. +`http-logger` is a plugin which push Log data requests to HTTP/HTTPS servers. -This will provide the ability to send Log data requests as JSON objects to Monitoring tools and other TCP servers. +This will provide the ability to send Log data requests as JSON objects to Monitoring tools and other HTTP servers. ## Attributes From 93629e5c49d63e20a7c73f2e3355aa455ff65ca9 Mon Sep 17 00:00:00 2001 From: sshniro Date: Wed, 15 Apr 2020 12:40:24 +0200 Subject: [PATCH 6/7] Adding additional parameters --- t/plugin/http-logger.t | 48 +++++++++++++++++++++++++++++++++++------- 1 file changed, 40 insertions(+), 8 deletions(-) diff --git a/t/plugin/http-logger.t b/t/plugin/http-logger.t index b8e8c639d573..029a85e17413 100644 --- a/t/plugin/http-logger.t +++ b/t/plugin/http-logger.t @@ -119,7 +119,11 @@ done "plugins": { "http-logger": { "uri": "http://127.0.0.1:1982/hello", - "batch_max_size": 1 + "batch_max_size": 1, + "max_retry_count": 1, + "retry_delay": 2, + "buffer_duration": 2, + "inactive_timeout": 2 } }, "upstream": { @@ -136,7 +140,11 @@ done "plugins": { "http-logger": { "uri": "http://127.0.0.1:1982/hello", - "batch_max_size": 1 + "batch_max_size": 1, + "max_retry_count": 1, + "retry_delay": 2, + "buffer_duration": 2, + "inactive_timeout": 2 } }, "upstream": { @@ -190,7 +198,11 @@ Batch Processor[http logger] successfully processed the entries "plugins": { "http-logger": { "uri": "http://127.0.0.1:8888/hello-world-http", - "batch_max_size": 1 + "batch_max_size": 1, + "max_retry_count": 1, + "retry_delay": 2, + "buffer_duration": 2, + "inactive_timeout": 2 } }, "upstream": { @@ -207,7 +219,11 @@ Batch Processor[http logger] successfully processed the entries "plugins": { "http-logger": { "uri": "http://127.0.0.1:8888/hello-world-http", - "batch_max_size": 1 + "batch_max_size": 1, + "max_retry_count": 1, + "retry_delay": 2, + "buffer_duration": 2, + "inactive_timeout": 2 } }, "upstream": { @@ -261,7 +277,11 @@ Batch Processor[http logger] successfully processed the entries "plugins": { "http-logger": { "uri": "https://127.0.0.1:8888/hello-world-http", - "batch_max_size": 1 + "batch_max_size": 1, + "max_retry_count": 1, + "retry_delay": 2, + "buffer_duration": 2, + "inactive_timeout": 2 } }, "upstream": { @@ -278,7 +298,11 @@ Batch Processor[http logger] successfully processed the entries "plugins": { "http-logger": { "uri": "https://127.0.0.1:8888/hello-world-http", - "batch_max_size": 1 + "batch_max_size": 1, + "max_retry_count": 1, + "retry_delay": 2, + "buffer_duration": 2, + "inactive_timeout": 2 } }, "upstream": { @@ -332,7 +356,11 @@ failed to perform SSL with host[127.0.0.1] port[8888] handshake failed "plugins": { "http-logger": { "uri": "https://127.0.0.1:9999/hello-world-http", - "batch_max_size": 1 + "batch_max_size": 1, + "max_retry_count": 1, + "retry_delay": 2, + "buffer_duration": 2, + "inactive_timeout": 2 } }, "upstream": { @@ -349,7 +377,11 @@ failed to perform SSL with host[127.0.0.1] port[8888] handshake failed "plugins": { "http-logger": { "uri": "https://127.0.0.1:9999/hello-world-http", - "batch_max_size": 1 + "batch_max_size": 1, + "max_retry_count": 1, + "retry_delay": 2, + "buffer_duration": 2, + "inactive_timeout": 2 } }, "upstream": { From 902428b6538cee9a9e711b46db6c1b6b40bb5310 Mon Sep 17 00:00:00 2001 From: sshniro Date: Mon, 4 May 2020 16:09:56 +0200 Subject: [PATCH 7/7] Fixing typo --- apisix/plugins/http-logger.lua | 2 +- doc/plugins/http-logger.md | 29 +++++++++++++---------------- 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/apisix/plugins/http-logger.lua b/apisix/plugins/http-logger.lua index 90e3fc4643b6..523d02d09796 100644 --- a/apisix/plugins/http-logger.lua +++ b/apisix/plugins/http-logger.lua @@ -17,7 +17,7 @@ local core = require("apisix.core") local log_util = require("apisix.utils.log-util") local batch_processor = require("apisix.utils.batch-processor") -local plugin_name = "udp-logger" +local plugin_name = "http-logger" local ngx = ngx local tostring = tostring local http = require "resty.http" diff --git a/doc/plugins/http-logger.md b/doc/plugins/http-logger.md index b4704130f193..55d0bc4eb9e7 100644 --- a/doc/plugins/http-logger.md +++ b/doc/plugins/http-logger.md @@ -51,23 +51,20 @@ This will provide the ability to send Log data requests as JSON objects to Monit The following is an example on how to enable the http-logger for a specific route. ```shell -curl http://127.0.0.1:9080/apisix/admin/consumers -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d ' +curl http://127.0.0.1:9080/apisix/admin/routes/5 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d ' { - "username": "foo", - "plugins": { - "plugins": { - "http-logger": { - "uri": "127.0.0.1:80/postendpoint?param=1", - } - }, - "upstream": { - "type": "roundrobin", - "nodes": { - "127.0.0.1:1980": 1 - } - }, - "uri": "/hello" - } + "plugins": { + "http-logger": { + "uri": "127.0.0.1:80/postendpoint?param=1", + } + }, + "upstream": { + "type": "roundrobin", + "nodes": { + "127.0.0.1:1980": 1 + } + }, + "uri": "/hello" }' ```