From 08aee2a6da49725a1e2943d45218360d1e0a24a3 Mon Sep 17 00:00:00 2001
From: Zeping Bai <bzp2010@apache.org>
Date: Sat, 29 Apr 2023 17:24:32 +0800
Subject: [PATCH 01/23] feat: add loki logger

---
 apisix/plugins/loki-logger.lua | 231 +++++++++++++++++++++++++++++++++
 conf/config-default.yaml       |   1 +
 2 files changed, 232 insertions(+)
 create mode 100644 apisix/plugins/loki-logger.lua

diff --git a/apisix/plugins/loki-logger.lua b/apisix/plugins/loki-logger.lua
new file mode 100644
index 000000000000..86c961b5f5f9
--- /dev/null
+++ b/apisix/plugins/loki-logger.lua
@@ -0,0 +1,231 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements.  See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License.  You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+local bp_manager_mod  = require("apisix.utils.batch-processor-manager")
+local log_util        = require("apisix.utils.log-util")
+local core            = require("apisix.core")
+local http            = require("resty.http")
+local new_tab         = require("table.new")
+
+local ipairs       = ipairs
+local math_random  = math.random
+local table_insert = table.insert
+local str_format   = core.string.format
+
+local plugin_name = "loki-logger"
+local batch_processor_manager = bp_manager_mod.new("loki logger")
+
+local schema = {
+    type = "object",
+    properties = {
+        -- core configurations
+        endpoint_addrs = {
+            type = "array",
+            minItems = 1,
+            items = core.schema.uri_def,
+        },
+        endpoint_uri = {
+            type = "string",
+            minLength = 1,
+            default = "/loki/api/v1/push"
+        },
+        tenant_id = {type = "string", default = nil},
+        log_labels = {
+            type = "object",
+            patternProperties = {
+                [".*"] = {
+                    type = "string",
+                    minLength = 1,
+                },
+            },
+            default = {
+                job = "apisix",
+            },
+        },
+
+        -- connection layer configurations
+        ssl_verify = {type = "boolean", default = false},
+        timeout = {
+            type = "integer",
+            minimum = 1,
+            maximum = 60000,
+            default = 3000,
+            description = "timeout in milliseconds",
+        },
+        keepalive = {type = "boolean", default = true},
+        keepalive_timeout = {
+            type = "integer",
+            minimum = 1000,
+            default = 60000,
+            description = "keepalive timeout in milliseconds",
+        },
+        keepalive_pool = {type = "integer", minimum = 1, default = 5},
+
+        -- logger related configurations
+        log_format = {type = "object"},
+        include_req_body = {type = "boolean", default = false},
+        include_req_body_expr = {
+            type = "array",
+            minItems = 1,
+            items = {
+                type = "array"
+            }
+        },
+        include_resp_body = {type = "boolean", default = false},
+        include_resp_body_expr = {
+            type = "array",
+            minItems = 1,
+            items = {
+                type = "array"
+            }
+        },
+    },
+    required = {"endpoint_addrs"}
+}
+
+
+local metadata_schema = {
+    type = "object",
+    properties = {
+        log_format = log_util.metadata_schema_log_format,
+    },
+}
+
+
+local _M = {
+    version = 0.1,
+    priority = 410,
+    name = plugin_name,
+    schema = batch_processor_manager:wrap_schema(schema),
+    metadata_schema = metadata_schema,
+}
+
+
+function _M.check_schema(conf, schema_type)
+    if schema_type == core.schema.TYPE_METADATA then
+        return core.schema.check(metadata_schema, conf)
+    end
+
+    local ok, err = core.schema.check(schema, conf)
+    if not ok then
+        return nil, err
+    end
+    return log_util.check_log_schema(conf)
+end
+
+
+local function send_http_data(conf, log)
+    local params = {
+        headers = {
+            ["Content-Type"] = "application/json",
+            ["X-Scope-OrgID"] = conf.tenant_id,
+        },
+        keepalive = conf.keepalive,
+        ssl_verify = conf.ssl_verify,
+        method = "POST",
+        body = core.json.encode(log)
+    }
+
+    if conf.keepalive then
+        params.keepalive_timeout = conf.keepalive_timeout
+        params.keepalive_pool = conf.keepalive_pool
+    end
+
+    local httpc, err = http.new()
+    if not httpc then
+        return false, str_format("create http error: %s", err)
+    end
+    httpc:set_timeout(conf.timeout)
+
+    -- select an random endpoint and build URL
+    local endpoint_url = conf.endpoint_addrs[math_random(#conf.endpoint_addrs)] .. conf.endpoint_uri
+    local res, err = httpc:request_uri(endpoint_url, params)
+    if not res then
+        return false, err
+    end
+
+    if res.status >= 300 then
+        return false, str_format("loki server returned status: %d, body: %s",
+            res.status, res.body or "")
+    end
+
+    return true
+end
+
+
+function _M.body_filter(conf, ctx)
+    log_util.collect_body(conf, ctx)
+end
+
+
+function _M.log(conf, ctx)
+    local entry = log_util.get_log_entry(plugin_name, conf, ctx)
+
+    if not entry.route_id then
+        entry.route_id = "no-matched"
+    end
+
+    -- insert start time as log time, multiply to nanoseconds
+    -- use string concat to circumvent 64bit integers that LuaVM cannot handle
+    -- that is, first process the decimal part of the millisecond value
+    -- and then add 6 zeros by string concatenation
+    entry.loki_log_time = tostring(ngx.req.start_time() * 1000) .. "000000"
+
+    if batch_processor_manager:add_entry(conf, entry) then
+        return
+    end
+
+    -- generate a function to be executed by the batch processor
+    local func = function(entries)
+        local labels = conf.log_labels
+
+        -- parsing possible variables in label value
+        for key, value in pairs(labels) do
+            local new_val, err, n_resolved = core.utils.resolve_var(value, ctx.var)
+            if not err and n_resolved > 0 then
+                labels[key] = new_val
+            end
+        end
+
+        -- build loki request data
+        local data = {
+            streams = {
+                {
+                    stream = labels,
+                    values = new_tab(1, 0),
+                }
+            }
+        }
+
+        -- add all entries to the batch
+        for _, entry in ipairs(entries) do
+            local log_time = entry.loki_log_time
+            entry.loki_log_time = nil -- clean logger internal field
+
+            table_insert(data.streams[1].values, {
+                log_time, core.json.encode(entry)
+            })
+        end
+
+        return send_http_data(conf, data)
+    end
+
+    batch_processor_manager:add_entry_to_new_processor(conf, entry, ctx, func)
+end
+
+
+return _M
diff --git a/conf/config-default.yaml b/conf/config-default.yaml
index 4f97adc4e4f5..d41df397b9e5 100755
--- a/conf/config-default.yaml
+++ b/conf/config-default.yaml
@@ -467,6 +467,7 @@ plugins:                          # plugin list (sorted by priority)
   - public-api                     # priority: 501
   - prometheus                     # priority: 500
   - datadog                        # priority: 495
+  - loki-logger                    # priority: 414
   - elasticsearch-logger           # priority: 413
   - echo                           # priority: 412
   - loggly                         # priority: 411

From 974e0fb036e159cbb19008c552cd03a41b2dfd0b Mon Sep 17 00:00:00 2001
From: Zeping Bai <bzp2010@apache.org>
Date: Sat, 29 Apr 2023 17:25:51 +0800
Subject: [PATCH 02/23] chore: change default priority

---
 apisix/plugins/loki-logger.lua | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apisix/plugins/loki-logger.lua b/apisix/plugins/loki-logger.lua
index 86c961b5f5f9..119dd8e27009 100644
--- a/apisix/plugins/loki-logger.lua
+++ b/apisix/plugins/loki-logger.lua
@@ -108,7 +108,7 @@ local metadata_schema = {
 
 local _M = {
     version = 0.1,
-    priority = 410,
+    priority = 414,
     name = plugin_name,
     schema = batch_processor_manager:wrap_schema(schema),
     metadata_schema = metadata_schema,

From 35fa4e87731b2039575e2a2c491c7cf4dbae953a Mon Sep 17 00:00:00 2001
From: Zeping Bai <bzp2010@apache.org>
Date: Sat, 29 Apr 2023 18:13:37 +0800
Subject: [PATCH 03/23] test: fix lint

---
 apisix/plugins/loki-logger.lua | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/apisix/plugins/loki-logger.lua b/apisix/plugins/loki-logger.lua
index 119dd8e27009..91fd5a8db763 100644
--- a/apisix/plugins/loki-logger.lua
+++ b/apisix/plugins/loki-logger.lua
@@ -21,9 +21,12 @@ local core            = require("apisix.core")
 local http            = require("resty.http")
 local new_tab         = require("table.new")
 
+local pairs        = pairs
 local ipairs       = ipairs
+local tostring     = tostring
 local math_random  = math.random
 local table_insert = table.insert
+local ngx          = ngx
 local str_format   = core.string.format
 
 local plugin_name = "loki-logger"

From ea4012ef92aa14b8b476d1576b52abcf7276c94c Mon Sep 17 00:00:00 2001
From: Zeping Bai <bzp2010@apache.org>
Date: Sat, 29 Apr 2023 18:16:51 +0800
Subject: [PATCH 04/23] ci: add loki to ci env

---
 ci/pod/docker-compose.plugin.yml | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/ci/pod/docker-compose.plugin.yml b/ci/pod/docker-compose.plugin.yml
index f4e3916987ed..08f257cac0ef 100644
--- a/ci/pod/docker-compose.plugin.yml
+++ b/ci/pod/docker-compose.plugin.yml
@@ -143,6 +143,15 @@ services:
       - ./t/certs:/certs
 
 
+  ## Grafana Loki
+  loki:
+    image: grafana/loki:2.8.0
+    command: -config.file=/etc/loki/local-config.yaml
+    ports:
+      - "3100:3100"
+    networks:
+      - loki_net
+
   rocketmq_namesrv:
     image: apacherocketmq/rocketmq:4.6.0
     container_name: rmqnamesrv
@@ -331,3 +340,4 @@ networks:
   rocketmq_net:
   opa_net:
   vector_net:
+  loki_net:

From 09f5c44cc0f7c6e52d765d3a36531d1fa27249cf Mon Sep 17 00:00:00 2001
From: Zeping Bai <bzp2010@apache.org>
Date: Sun, 30 Apr 2023 01:20:23 +0800
Subject: [PATCH 05/23] chore: change default tenant id to fake

---
 apisix/plugins/loki-logger.lua | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apisix/plugins/loki-logger.lua b/apisix/plugins/loki-logger.lua
index 91fd5a8db763..f5afbfa591d2 100644
--- a/apisix/plugins/loki-logger.lua
+++ b/apisix/plugins/loki-logger.lua
@@ -46,7 +46,7 @@ local schema = {
             minLength = 1,
             default = "/loki/api/v1/push"
         },
-        tenant_id = {type = "string", default = nil},
+        tenant_id = {type = "string", default = "fake"},
         log_labels = {
             type = "object",
             patternProperties = {

From 8210f18ec83bfd55a8a58b1b47788f864f910d4e Mon Sep 17 00:00:00 2001
From: Zeping Bai <bzp2010@apache.org>
Date: Sun, 30 Apr 2023 01:20:32 +0800
Subject: [PATCH 06/23] ci: update env

---
 ci/pod/docker-compose.plugin.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/pod/docker-compose.plugin.yml b/ci/pod/docker-compose.plugin.yml
index 08f257cac0ef..808a82a4b39a 100644
--- a/ci/pod/docker-compose.plugin.yml
+++ b/ci/pod/docker-compose.plugin.yml
@@ -146,7 +146,7 @@ services:
   ## Grafana Loki
   loki:
     image: grafana/loki:2.8.0
-    command: -config.file=/etc/loki/local-config.yaml
+    command: -config.file=/etc/loki/local-config.yaml -auth.enabled -querier.multi-tenant-queries-enabled
     ports:
       - "3100:3100"
     networks:

From ea51258a543a4e2a50e16d0ed5448bb66acf2cf8 Mon Sep 17 00:00:00 2001
From: Zeping Bai <bzp2010@apache.org>
Date: Sun, 30 Apr 2023 01:21:12 +0800
Subject: [PATCH 07/23] test: add cases

---
 t/plugin/loki-logger.t | 356 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 356 insertions(+)
 create mode 100644 t/plugin/loki-logger.t

diff --git a/t/plugin/loki-logger.t b/t/plugin/loki-logger.t
new file mode 100644
index 000000000000..d3912e753361
--- /dev/null
+++ b/t/plugin/loki-logger.t
@@ -0,0 +1,356 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+use t::APISIX 'no_plan';
+
+repeat_each(1);
+no_long_string();
+no_root_location();
+
+add_block_preprocessor(sub {
+    my ($block) = @_;
+
+    if (!defined $block->request) {
+        $block->set_value("request", "GET /t");
+    }
+});
+
+run_tests();
+
+__DATA__
+
+=== TEST 1: sanity
+--- config
+    location /t {
+        content_by_lua_block {
+            local test_cases = {
+                {endpoint_addrs = {"http://127.0.0.1:8199"}},
+                {endpoint_addrs = "http://127.0.0.1:8199"},
+                {endpoint_addrs = {}},
+                {},
+                {endpoint_addrs = {"http://127.0.0.1:8199"}, endpoint_uri = "/loki/api/v1/push"},
+                {endpoint_addrs = {"http://127.0.0.1:8199"}, endpoint_uri = 1234},
+                {endpoint_addrs = {"http://127.0.0.1:8199"}, tenant_id = 1234},
+                {endpoint_addrs = {"http://127.0.0.1:8199"}, log_labels = "1234"},
+                {endpoint_addrs = {"http://127.0.0.1:8199"}, log_labels = {job = "apisix6"}},
+            }
+            local plugin = require("apisix.plugins.loki-logger")
+
+            for _, case in ipairs(test_cases) do
+                local ok, err = plugin.check_schema(case)
+                ngx.say(ok and "done" or err)
+            end
+        }
+    }
+--- response_body
+done
+property "endpoint_addrs" validation failed: wrong type: expected array, got string
+property "endpoint_addrs" validation failed: expect array to have at least 1 items
+property "endpoint_addrs" is required
+done
+property "endpoint_uri" validation failed: wrong type: expected string, got number
+property "tenant_id" validation failed: wrong type: expected string, got number
+property "log_labels" validation failed: wrong type: expected object, got string
+done
+
+
+
+=== TEST 2: setup route
+--- config
+    location /t {
+        content_by_lua_block {
+            local t = require("lib.test_admin").test
+            local code, body = t('/apisix/admin/routes/1',
+                ngx.HTTP_PUT,
+                [[{
+                    "plugins": {
+                        "loki-logger": {
+                            "endpoint_addrs": ["http://127.0.0.1:3100"],
+                            "tenant_id": "tenant_1",
+                            "batch_max_size": 1
+                        }
+                    },
+                    "upstream": {
+                        "nodes": {
+                            "127.0.0.1:1980": 1
+                        },
+                        "type": "roundrobin"
+                    },
+                    "uri": "/hello"
+                }]]
+            )
+
+            if code >= 300 then
+                ngx.status = code
+            end
+            ngx.say(body)
+        }
+    }
+--- response_body
+passed
+
+
+
+=== TEST 3: hit route
+--- request
+GET /hello
+--- more_headers
+test-header: only-for-test#1
+--- response_body
+hello world
+
+
+
+=== TEST 4: check loki log
+--- config
+    location /t {
+        content_by_lua_block {
+            local cjson = require("cjson")
+            local httpc = require("resty.http").new()            
+            local now = ngx.now() * 1000
+            local res, err = httpc:request_uri("http://127.0.0.1:3100/loki/api/v1/query_range", {
+                query = {
+                    direction = "backward",
+                    start = tostring(now - 1000).."000000",
+                    ["end"] = tostring(now).."000000",
+                    limit = "10",
+                    query = [[{job="apisix"} | json]],
+                },
+                headers = {
+                    ["X-Scope-OrgID"] = "tenant_1"
+                }
+            })
+
+            assert(res ~= nil, "request error: " .. (err or ""))
+            assert(res.status == 200, "loki error: " .. res.status .. " " .. res.body)
+            
+            local data = cjson.decode(res.body)
+            assert(data ~= nil, "loki response error: " .. res.body)
+            assert(data.status == "success", "loki response error: " .. res.body)
+            assert(#data.data.result > 0, "loki log empty: " .. res.body)
+
+            local entry = data.data.result[1]
+            assert(entry.stream.request_headers_test_header == "only-for-test#1",
+                  "expected field request_headers_test_header value: " .. cjson.encode(entry))
+        }
+    }
+--- error_code: 200
+
+
+
+=== TEST 5: setup route (with log_labels)
+--- config
+    location /t {
+        content_by_lua_block {
+            local t = require("lib.test_admin").test
+            local code, body = t('/apisix/admin/routes/1',
+                ngx.HTTP_PUT,
+                [[{
+                    "plugins": {
+                        "loki-logger": {
+                            "endpoint_addrs": ["http://127.0.0.1:3100"],
+                            "tenant_id": "tenant_1",
+                            "log_labels": {
+                                "custom_label": "custom_label_value"
+                            },
+                            "batch_max_size": 1
+                        }
+                    },
+                    "upstream": {
+                        "nodes": {
+                            "127.0.0.1:1980": 1
+                        },
+                        "type": "roundrobin"
+                    },
+                    "uri": "/hello"
+                }]]
+            )
+
+            if code >= 300 then
+                ngx.status = code
+            end
+            ngx.say(body)
+        }
+    }
+--- response_body
+passed
+
+
+
+=== TEST 6: hit route
+--- request
+GET /hello
+--- more_headers
+test-header: only-for-test#2
+--- response_body
+hello world
+
+
+
+=== TEST 7: check loki log (with custom_label)
+--- config
+    location /t {
+        content_by_lua_block {
+            local cjson = require("cjson")
+            local httpc = require("resty.http").new()            
+            local now = ngx.now() * 1000
+            local res, err = httpc:request_uri("http://127.0.0.1:3100/loki/api/v1/query_range", {
+                query = {
+                    direction = "backward",
+                    start = tostring(now - 1000).."000000",
+                    ["end"] = tostring(now).."000000",
+                    limit = "10",
+                    query = [[{custom_label="custom_label_value"} | json]],
+                },
+                headers = {
+                    ["X-Scope-OrgID"] = "tenant_1"
+                }
+            })
+
+            assert(res ~= nil, "request error: " .. (err or ""))
+            assert(res.status == 200, "loki error: " .. res.status .. " " .. res.body)
+            
+            local data = cjson.decode(res.body)
+            assert(data ~= nil, "loki response error: " .. res.body)
+            assert(data.status == "success", "loki response error: " .. res.body)
+            assert(#data.data.result > 0, "loki log empty: " .. res.body)
+
+            local entry = data.data.result[1]
+            assert(entry.stream.request_headers_test_header == "only-for-test#2",
+                  "expected field request_headers_test_header value: " .. cjson.encode(entry))
+        }
+    }
+--- error_code: 200
+
+
+
+=== TEST 8: setup route (with tenant_id)
+--- config
+    location /t {
+        content_by_lua_block {
+            local t = require("lib.test_admin").test
+            local code, body = t('/apisix/admin/routes/1',
+                ngx.HTTP_PUT,
+                [[{
+                    "plugins": {
+                        "loki-logger": {
+                            "endpoint_addrs": ["http://127.0.0.1:3100"],
+                            "tenant_id": "tenant_2",
+                            "batch_max_size": 1
+                        }
+                    },
+                    "upstream": {
+                        "nodes": {
+                            "127.0.0.1:1980": 1
+                        },
+                        "type": "roundrobin"
+                    },
+                    "uri": "/hello"
+                }]]
+            )
+
+            if code >= 300 then
+                ngx.status = code
+            end
+            ngx.say(body)
+        }
+    }
+--- response_body
+passed
+
+
+
+=== TEST 9: hit route
+--- request
+GET /hello
+--- more_headers
+test-header: only-for-test#3
+--- response_body
+hello world
+
+
+
+=== TEST 10: check loki log (with tenant_id tenant_1)
+--- config
+    location /t {
+        content_by_lua_block {
+            local cjson = require("cjson")
+            local httpc = require("resty.http").new()            
+            local now = ngx.now() * 1000
+            local res, err = httpc:request_uri("http://127.0.0.1:3100/loki/api/v1/query_range", {
+                query = {
+                    direction = "backward",
+                    start = tostring(now - 5000).."000000",
+                    ["end"] = tostring(now).."000000",
+                    limit = "10",
+                    query = [[{job="apisix"} | json]],
+                },
+                headers = {
+                    ["X-Scope-OrgID"] = "tenant_1"
+                }
+            })
+
+            assert(res ~= nil, "request error: " .. (err or ""))
+            assert(res.status == 200, "loki error: " .. res.status .. " " .. res.body)
+            
+            local data = cjson.decode(res.body)
+            assert(data ~= nil, "loki response error: " .. res.body)
+            assert(data.status == "success", "loki response error: " .. res.body)
+            assert(#data.data.result > 0, "loki log empty: " .. res.body)
+
+            local entry = data.data.result[1]
+            assert(entry.stream.request_headers_test_header ~= "only-for-test#3",
+                  "expected field request_headers_test_header value: " .. cjson.encode(entry))
+        }
+    }
+--- error_code: 200
+
+
+
+=== TEST 11: check loki log (with tenant_id tenant_2)
+--- config
+    location /t {
+        content_by_lua_block {
+            local cjson = require("cjson")
+            local httpc = require("resty.http").new()            
+            local now = ngx.now() * 1000
+            local res, err = httpc:request_uri("http://127.0.0.1:3100/loki/api/v1/query_range", {
+                query = {
+                    direction = "backward",
+                    start = tostring(now - 1000).."000000",
+                    ["end"] = tostring(now).."000000",
+                    limit = "10",
+                    query = [[{job="apisix"} | json]],
+                },
+                headers = {
+                    ["X-Scope-OrgID"] = "tenant_2"
+                }
+            })
+
+            assert(res ~= nil, "request error: " .. (err or ""))
+            assert(res.status == 200, "loki error: " .. res.status .. " " .. res.body)
+            
+            local data = cjson.decode(res.body)
+            assert(data ~= nil, "loki response error: " .. res.body)
+            assert(data.status == "success", "loki response error: " .. res.body)
+            assert(#data.data.result > 0, "loki log empty: " .. res.body)
+
+            local entry = data.data.result[1]
+            assert(entry.stream.request_headers_test_header == "only-for-test#3",
+                  "expected field request_headers_test_header value: " .. cjson.encode(entry))
+        }
+    }
+--- error_code: 200

From 1730473d66a49a1c81cb3d92ca3daf9d411f69be Mon Sep 17 00:00:00 2001
From: Zeping Bai <bzp2010@apache.org>
Date: Sun, 30 Apr 2023 01:26:09 +0800
Subject: [PATCH 08/23] test: fix lint

---
 t/plugin/loki-logger.t | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/t/plugin/loki-logger.t b/t/plugin/loki-logger.t
index d3912e753361..22fe2ab4f7f3 100644
--- a/t/plugin/loki-logger.t
+++ b/t/plugin/loki-logger.t
@@ -119,7 +119,7 @@ hello world
     location /t {
         content_by_lua_block {
             local cjson = require("cjson")
-            local httpc = require("resty.http").new()            
+            local httpc = require("resty.http").new()
             local now = ngx.now() * 1000
             local res, err = httpc:request_uri("http://127.0.0.1:3100/loki/api/v1/query_range", {
                 query = {
@@ -136,7 +136,7 @@ hello world
 
             assert(res ~= nil, "request error: " .. (err or ""))
             assert(res.status == 200, "loki error: " .. res.status .. " " .. res.body)
-            
+
             local data = cjson.decode(res.body)
             assert(data ~= nil, "loki response error: " .. res.body)
             assert(data.status == "success", "loki response error: " .. res.body)
@@ -205,7 +205,7 @@ hello world
     location /t {
         content_by_lua_block {
             local cjson = require("cjson")
-            local httpc = require("resty.http").new()            
+            local httpc = require("resty.http").new()
             local now = ngx.now() * 1000
             local res, err = httpc:request_uri("http://127.0.0.1:3100/loki/api/v1/query_range", {
                 query = {
@@ -222,7 +222,7 @@ hello world
 
             assert(res ~= nil, "request error: " .. (err or ""))
             assert(res.status == 200, "loki error: " .. res.status .. " " .. res.body)
-            
+
             local data = cjson.decode(res.body)
             assert(data ~= nil, "loki response error: " .. res.body)
             assert(data.status == "success", "loki response error: " .. res.body)
@@ -288,7 +288,7 @@ hello world
     location /t {
         content_by_lua_block {
             local cjson = require("cjson")
-            local httpc = require("resty.http").new()            
+            local httpc = require("resty.http").new()
             local now = ngx.now() * 1000
             local res, err = httpc:request_uri("http://127.0.0.1:3100/loki/api/v1/query_range", {
                 query = {
@@ -305,7 +305,7 @@ hello world
 
             assert(res ~= nil, "request error: " .. (err or ""))
             assert(res.status == 200, "loki error: " .. res.status .. " " .. res.body)
-            
+
             local data = cjson.decode(res.body)
             assert(data ~= nil, "loki response error: " .. res.body)
             assert(data.status == "success", "loki response error: " .. res.body)
@@ -325,7 +325,7 @@ hello world
     location /t {
         content_by_lua_block {
             local cjson = require("cjson")
-            local httpc = require("resty.http").new()            
+            local httpc = require("resty.http").new()
             local now = ngx.now() * 1000
             local res, err = httpc:request_uri("http://127.0.0.1:3100/loki/api/v1/query_range", {
                 query = {
@@ -342,7 +342,7 @@ hello world
 
             assert(res ~= nil, "request error: " .. (err or ""))
             assert(res.status == 200, "loki error: " .. res.status .. " " .. res.body)
-            
+
             local data = cjson.decode(res.body)
             assert(data ~= nil, "loki response error: " .. res.body)
             assert(data.status == "success", "loki response error: " .. res.body)

From 498ac657a0f5ac16f34aedeca68d1d5aba4ff65d Mon Sep 17 00:00:00 2001
From: Zeping Bai <bzp2010@apache.org>
Date: Sun, 30 Apr 2023 12:12:16 +0800
Subject: [PATCH 09/23] test: update plugin list

---
 t/admin/plugins.t | 1 +
 1 file changed, 1 insertion(+)

diff --git a/t/admin/plugins.t b/t/admin/plugins.t
index ceb4df15bd70..ae7617dfd23d 100644
--- a/t/admin/plugins.t
+++ b/t/admin/plugins.t
@@ -109,6 +109,7 @@ grpc-web
 public-api
 prometheus
 datadog
+loki-logger
 elasticsearch-logger
 echo
 loggly

From 35ef5e1834d749b6bdd758939926b7d14f8d19d7 Mon Sep 17 00:00:00 2001
From: Zeping Bai <bzp2010@apache.org>
Date: Sun, 30 Apr 2023 12:48:23 +0800
Subject: [PATCH 10/23] test: debug

---
 t/plugin/loki-logger.t | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/t/plugin/loki-logger.t b/t/plugin/loki-logger.t
index 22fe2ab4f7f3..889099951620 100644
--- a/t/plugin/loki-logger.t
+++ b/t/plugin/loki-logger.t
@@ -124,7 +124,7 @@ hello world
             local res, err = httpc:request_uri("http://127.0.0.1:3100/loki/api/v1/query_range", {
                 query = {
                     direction = "backward",
-                    start = tostring(now - 1000).."000000",
+                    start = tostring(now - 3000).."000000",
                     ["end"] = tostring(now).."000000",
                     limit = "10",
                     query = [[{job="apisix"} | json]],
@@ -210,7 +210,7 @@ hello world
             local res, err = httpc:request_uri("http://127.0.0.1:3100/loki/api/v1/query_range", {
                 query = {
                     direction = "backward",
-                    start = tostring(now - 1000).."000000",
+                    start = tostring(now - 3000).."000000",
                     ["end"] = tostring(now).."000000",
                     limit = "10",
                     query = [[{custom_label="custom_label_value"} | json]],
@@ -293,7 +293,7 @@ hello world
             local res, err = httpc:request_uri("http://127.0.0.1:3100/loki/api/v1/query_range", {
                 query = {
                     direction = "backward",
-                    start = tostring(now - 5000).."000000",
+                    start = tostring(now - 3000).."000000",
                     ["end"] = tostring(now).."000000",
                     limit = "10",
                     query = [[{job="apisix"} | json]],
@@ -330,7 +330,7 @@ hello world
             local res, err = httpc:request_uri("http://127.0.0.1:3100/loki/api/v1/query_range", {
                 query = {
                     direction = "backward",
-                    start = tostring(now - 1000).."000000",
+                    start = tostring(now - 3000).."000000",
                     ["end"] = tostring(now).."000000",
                     limit = "10",
                     query = [[{job="apisix"} | json]],

From 927498cfa2a3b9c7aea2eb3eae9f23a598703181 Mon Sep 17 00:00:00 2001
From: Zeping Bai <bzp2010@apache.org>
Date: Sun, 30 Apr 2023 13:53:15 +0800
Subject: [PATCH 11/23] docs: add

---
 docs/en/latest/plugins/loki-logger.md | 165 ++++++++++++++++++++++++++
 1 file changed, 165 insertions(+)
 create mode 100644 docs/en/latest/plugins/loki-logger.md

diff --git a/docs/en/latest/plugins/loki-logger.md b/docs/en/latest/plugins/loki-logger.md
new file mode 100644
index 000000000000..179f03533093
--- /dev/null
+++ b/docs/en/latest/plugins/loki-logger.md
@@ -0,0 +1,165 @@
+---
+title: loki-logger
+keywords:
+  - Apache APISIX
+  - API Gateway
+  - Plugin
+  - Loki-logger
+  - Grafana Loki
+description: This document contains information about the Apache APISIX loki-logger Plugin.
+---
+
+<!--
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+-->
+
+## Description
+
+The `loki-logger` plugin is used to forward logs to [Grafana Loki](https://grafana.com/oss/loki/) for analysis and storage.
+
+When the Plugin is enabled, APISIX will serialize the request context information to [Log entries in JSON](https://grafana.com/docs/loki/latest/api/#push-log-entries-to-loki) and submit it to the batch queue. When the maximum batch size is exceeded, the data in the queue is pushed to Elasticsearch. See [batch processor](../batch-processor.md) for more details.
+
+## Attributes
+
+| Name | Type | Required | Default | Description |
+|---|---|---|---|---|
+| endpoint_addrs | array | True |  | Loki API base URL, format like http://127.0.0.1:3100, supports HTTPS and domain names. If multiple endpoints are configured, they will be written randomly. |
+| endpoint_uri | string | False | /loki/api/v1/push | If you are using a log collection service that is compatible with the Loki Push API, you can use this configuration item to customize the API path. |
+| tenant_id | string | False | fake | Loki tenant ID. According to Loki's [multi-tenancy documentation](https://grafana.com/docs/loki/latest/operations/multi-tenancy/#multi-tenancy), its default value is set to the default value `fake` under single-tenancy. |
+| log_labels | object | False | {job = "apisix"} | Loki log label. [APISIX variables](../apisix-variable.md) and [Nginx variables](http://nginx.org/en/docs/varindex.html) can be used by prefixing the string with `$`, both individual and combined, such as `$host` or `$remote_addr:$remote_port`. |
+| ssl_verify        | boolean       | False    | true | When set to `true`, verifies the SSL certificate. |
+| timeout           | integer       | False    | 3000ms  | [1, 60000]ms   | Timeout for the authorization service HTTP call. |
+| keepalive         | boolean       | False    | true | When set to `true`, keeps the connection alive for multiple requests. |
+| keepalive_timeout | integer       | False    | 60000ms | [1000, ...]ms  | Idle time after which the connection is closed. |
+| keepalive_pool    | integer       | False    | 5       | [1, ...]ms     | Connection pool limit. |
+| log_format | object | False    |          | Log format declared as key value pairs in JSON format. Values only support strings. [APISIX variables](../apisix-variable.md) and [Nginx variables](http://nginx.org/en/docs/varindex.html) can be used by prefixing the string with `$`. |
+| include_req_body       | boolean | False    | false | When set to `true` includes the request body in the log. If the request body is too big to be kept in the memory, it can't be logged due to Nginx's limitations. |
+| include_req_body_expr  | array   | False    |  | Filter for when the `include_req_body` attribute is set to `true`. Request body is only logged when the expression set here evaluates to `true`. See [lua-resty-expr](https://github.com/api7/lua-resty-expr) for more. |
+| include_resp_body      | boolean | False    | false | When set to `true` includes the response body in the log. |
+| include_resp_body_expr | array   | False    |  | Filter for when the `include_resp_body` attribute is set to `true`. Response body is only logged when the expression set here evaluates to `true`. See [lua-resty-expr](https://github.com/api7/lua-resty-expr) for more. |
+
+This plugin supports using batch processors to aggregate and process entries (logs/data) in a batch. This avoids the need for frequently submitting the data. The batch processor submits data every `5` seconds or when the data in the queue reaches `1000`. See [Batch Processor](../batch-processor.md#configuration) for more information or setting your custom configuration.
+
+## Metadata
+
+You can also set the format of the logs by configuring the Plugin metadata. The following configurations are available:
+
+| Name | Type | Required | Default | Description |
+|------|------|----------|---------|-------------|
+| log_format | object | False | {"host": "$host", "@timestamp": "$time_iso8601", "client_ip": "$remote_addr"} | Log format declared as key value pairs in JSON format. Values only support strings. [APISIX variables](../apisix-variable.md) and [Nginx variables](http://nginx.org/en/docs/varindex.html) can be used by prefixing the string with `$`. |
+
+:::info IMPORTANT
+
+Configuring the plugin metadata is global in scope. This means that it will take effect on all Routes and Services which use the `loki-logger` plugin.
+
+:::
+
+The example below shows how you can configure through the Admin API:
+
+```shell
+curl http://127.0.0.1:9180/apisix/admin/plugin_metadata/loki-logger -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
+{
+    "log_format": {
+        "host": "$host",
+        "@timestamp": "$time_iso8601",
+        "client_ip": "$remote_addr"
+    }
+}'
+```
+
+With this configuration, your logs would be formatted as shown below:
+
+```shell
+{"host":"localhost","@timestamp":"2020-09-23T19:05:05-04:00","client_ip":"127.0.0.1","route_id":"1"}
+{"host":"localhost","@timestamp":"2020-09-23T19:05:05-04:00","client_ip":"127.0.0.1","route_id":"1"}
+```
+
+## Enabling the plugin
+
+The example below shows how you can enable the `loki-logger` plugin on a specific Route:
+
+```shell
+curl http://127.0.0.1:9180/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
+{
+    "plugins": {
+        "loki-logger": {
+            "endpoint_addrs" : ["http://127.0.0.1:3100"]
+        }
+    },
+    "upstream": {
+       "nodes": {
+           "127.0.0.1:1980": 1
+       },
+       "type": "roundrobin"
+    },
+    "uri": "/hello"
+}'
+```
+
+## Example usage
+
+Now, if you make a request to APISIX, it will be logged in your Loki server:
+
+```shell
+curl -i http://127.0.0.1:9080/hello
+```
+
+## Disable plugin
+
+To disable the `loki-logger` plugin, you can delete the corresponding JSON configuration from the plugin configuration. APISIX will automatically reload and you do not have to restart for this to take effect.
+
+```shell
+curl http://127.0.0.1:9180/apisix/admin/routes/1  -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
+{
+    "methods": ["GET"],
+    "uri": "/hello",
+    "plugins": {},
+    "upstream": {
+        "type": "roundrobin",
+        "nodes": {
+            "127.0.0.1:1980": 1
+        }
+    }
+}'
+```
+
+## FAQ
+
+### Logs are not pushed properly
+
+Look at `error.log` for such a log.
+
+```text
+2023/04/30 13:45:46 [error] 19381#19381: *1075673 [lua] batch-processor.lua:95: Batch Processor[loki logger] failed to process entries: loki server returned status: 401, body: no org id, context: ngx.timer, client: 127.0.0.1, server: 0.0.0.0:9081
+```
+
+The error can be diagnosed based on the error code in the `failed to process entries: loki server returned status: 401, body: no org id` and the response body of the loki server.
+
+### Getting errors when QPS is high?
+
+- Make sure to `keepalive` related configuration is set properly. See [Attributes](#attributes) for more information.
+- Check the logs in `error.log`, look for such a log.
+  
+    ```text
+    2023/04/30 13:49:34 [error] 19381#19381: *1082680 [lua] batch-processor.lua:95: Batch Processor[loki logger] failed to process entries: loki server returned status: 429, body: Ingestion rate limit exceeded for user tenant_1 (limit: 4194304 bytes/sec) while attempting to ingest '1000' lines totaling '616307' bytes, reduce log volume or contact your Loki administrator to see if the limit can be increased, context: ngx.timer, client: 127.0.0.1, server: 0.0.0.0:9081
+    ```
+
+  - The logs usually associated with high QPS look like the above. The error is: `Ingestion rate limit exceeded for user tenant_1 (limit: 4194304 bytes/sec) while attempting to ingest '1000' lines totaling '616307' bytes, reduce log volume or contact your Loki administrator to see if the limit can be increased`.
+  - Refer to [Loki documentation](https://grafana.com/docs/loki/latest/configuration/#limits_config) to add limits on the amount of default and burst logs, such as `ingestion_rate_mb` and `ingestion_burst_size_mb`.
+
+    As the test during development, setting the `ingestion_burst_size_mb` to 100 allows APISIX to push the logs correctly at least at 10000 RPS.

From b4816f6c1c6a534f45d91016bf14ad829c7e83cd Mon Sep 17 00:00:00 2001
From: Zeping Bai <bzp2010@apache.org>
Date: Sun, 30 Apr 2023 14:11:00 +0800
Subject: [PATCH 12/23] test: fix lint

---
 docs/en/latest/plugins/loki-logger.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/en/latest/plugins/loki-logger.md b/docs/en/latest/plugins/loki-logger.md
index 179f03533093..a862e37780f9 100644
--- a/docs/en/latest/plugins/loki-logger.md
+++ b/docs/en/latest/plugins/loki-logger.md
@@ -154,7 +154,7 @@ The error can be diagnosed based on the error code in the `failed to process ent
 
 - Make sure to `keepalive` related configuration is set properly. See [Attributes](#attributes) for more information.
 - Check the logs in `error.log`, look for such a log.
-  
+
     ```text
     2023/04/30 13:49:34 [error] 19381#19381: *1082680 [lua] batch-processor.lua:95: Batch Processor[loki logger] failed to process entries: loki server returned status: 429, body: Ingestion rate limit exceeded for user tenant_1 (limit: 4194304 bytes/sec) while attempting to ingest '1000' lines totaling '616307' bytes, reduce log volume or contact your Loki administrator to see if the limit can be increased, context: ngx.timer, client: 127.0.0.1, server: 0.0.0.0:9081
     ```

From 01262ada081c5bab7f72172f41395e0d597a2f81 Mon Sep 17 00:00:00 2001
From: Zeping Bai <bzp2010@apache.org>
Date: Sun, 30 Apr 2023 14:12:28 +0800
Subject: [PATCH 13/23] docs: add to config json

---
 docs/en/latest/config.json | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/en/latest/config.json b/docs/en/latest/config.json
index d752359a6089..496a44ff98f1 100644
--- a/docs/en/latest/config.json
+++ b/docs/en/latest/config.json
@@ -181,7 +181,8 @@
                 "plugins/file-logger",
                 "plugins/loggly",
                 "plugins/elasticsearch-logger",
-                "plugins/tencent-cloud-cls"
+                "plugins/tencent-cloud-cls",
+                "plugins/loki-logger"
               ]
             }
           ]

From f5a13de1a03707da94e0f0c81d9b86b48f5e874f Mon Sep 17 00:00:00 2001
From: Zeping Bai <bzp2010@apache.org>
Date: Sun, 30 Apr 2023 14:29:27 +0800
Subject: [PATCH 14/23] chore: update log

---
 apisix/plugins/loki-logger.lua | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apisix/plugins/loki-logger.lua b/apisix/plugins/loki-logger.lua
index f5afbfa591d2..593fc8b18e6c 100644
--- a/apisix/plugins/loki-logger.lua
+++ b/apisix/plugins/loki-logger.lua
@@ -150,7 +150,7 @@ local function send_http_data(conf, log)
 
     local httpc, err = http.new()
     if not httpc then
-        return false, str_format("create http error: %s", err)
+        return false, str_format("create http client error: %s", err)
     end
     httpc:set_timeout(conf.timeout)
 

From 6fa098188cf3e74bb2b8189ffbae703b83ac9360 Mon Sep 17 00:00:00 2001
From: Zeping Bai <bzp2010@apache.org>
Date: Sun, 30 Apr 2023 14:31:47 +0800
Subject: [PATCH 15/23] test: fix

---
 t/plugin/loki-logger.t | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/t/plugin/loki-logger.t b/t/plugin/loki-logger.t
index 889099951620..68b53743e4f0 100644
--- a/t/plugin/loki-logger.t
+++ b/t/plugin/loki-logger.t
@@ -293,7 +293,7 @@ hello world
             local res, err = httpc:request_uri("http://127.0.0.1:3100/loki/api/v1/query_range", {
                 query = {
                     direction = "backward",
-                    start = tostring(now - 3000).."000000",
+                    start = tostring(now - 10000).."000000",
                     ["end"] = tostring(now).."000000",
                     limit = "10",
                     query = [[{job="apisix"} | json]],

From 72f93df9b395d9668578db71a359f8270e099899 Mon Sep 17 00:00:00 2001
From: Zeping Bai <bzp2010@apache.org>
Date: Sun, 7 May 2023 21:38:34 +0800
Subject: [PATCH 16/23] feat: decouple logs fetcher

---
 t/lib/grafana_loki.lua |  60 +++++++++++++++++++++
 t/plugin/loki-logger.t | 116 ++++++++++++-----------------------------
 2 files changed, 92 insertions(+), 84 deletions(-)
 create mode 100644 t/lib/grafana_loki.lua

diff --git a/t/lib/grafana_loki.lua b/t/lib/grafana_loki.lua
new file mode 100644
index 000000000000..910da1219d8d
--- /dev/null
+++ b/t/lib/grafana_loki.lua
@@ -0,0 +1,60 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements.  See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License.  You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+local cjson = require("cjson")
+local http = require("resty.http")
+
+local _M = {}
+
+
+function _M.fetch_logs_from_loki(from, to, options)
+    options = options or {}
+
+    local now = ngx.now() * 1000
+    local direction = options.direction or "backward"
+    local limit = options.limit or "10"
+    local query = options.query or [[{job="apisix"} | json]]
+    local url = options.url or "http://127.0.0.1:3100/loki/api/v1/query_range"
+    local headers = options.headers or {
+        ["X-Scope-OrgID"] = "tenant_1"
+    }
+
+    local httpc = http.new()
+    local res, err = httpc:request_uri(url, {
+        query = {
+            start = from,
+            ["end"] = to,
+            direction = direction,
+            limit = limit,
+            query = query,
+        },
+        headers = headers
+    })
+
+    if not res or err then
+        return nil, err
+    end
+
+    if res.status > 300 then
+        return nil, "HTTP status code: " .. res.status .. ", body: " .. res.body
+    end
+
+    return cjson.decode(res.body)
+end
+
+
+return _M
diff --git a/t/plugin/loki-logger.t b/t/plugin/loki-logger.t
index 68b53743e4f0..f78307890986 100644
--- a/t/plugin/loki-logger.t
+++ b/t/plugin/loki-logger.t
@@ -118,27 +118,13 @@ hello world
 --- config
     location /t {
         content_by_lua_block {
-            local cjson = require("cjson")
-            local httpc = require("resty.http").new()
-            local now = ngx.now() * 1000
-            local res, err = httpc:request_uri("http://127.0.0.1:3100/loki/api/v1/query_range", {
-                query = {
-                    direction = "backward",
-                    start = tostring(now - 3000).."000000",
-                    ["end"] = tostring(now).."000000",
-                    limit = "10",
-                    query = [[{job="apisix"} | json]],
-                },
-                headers = {
-                    ["X-Scope-OrgID"] = "tenant_1"
-                }
-            })
-
-            assert(res ~= nil, "request error: " .. (err or ""))
-            assert(res.status == 200, "loki error: " .. res.status .. " " .. res.body)
-
-            local data = cjson.decode(res.body)
-            assert(data ~= nil, "loki response error: " .. res.body)
+            local data, err = require("lib.grafana_loki").fetch_logs_from_loki(
+                tostring(now - 3000) .. "000000", -- from
+                tostring(now) .. "000000",        -- to
+            )
+
+            assert(err == nil, "fetch logs error: " .. err)
+            assert(data ~= nil, "loki response decode error: " .. res.body)
             assert(data.status == "success", "loki response error: " .. res.body)
             assert(#data.data.result > 0, "loki log empty: " .. res.body)
 
@@ -204,27 +190,14 @@ hello world
 --- config
     location /t {
         content_by_lua_block {
-            local cjson = require("cjson")
-            local httpc = require("resty.http").new()
-            local now = ngx.now() * 1000
-            local res, err = httpc:request_uri("http://127.0.0.1:3100/loki/api/v1/query_range", {
-                query = {
-                    direction = "backward",
-                    start = tostring(now - 3000).."000000",
-                    ["end"] = tostring(now).."000000",
-                    limit = "10",
-                    query = [[{custom_label="custom_label_value"} | json]],
-                },
-                headers = {
-                    ["X-Scope-OrgID"] = "tenant_1"
-                }
-            })
-
-            assert(res ~= nil, "request error: " .. (err or ""))
-            assert(res.status == 200, "loki error: " .. res.status .. " " .. res.body)
-
-            local data = cjson.decode(res.body)
-            assert(data ~= nil, "loki response error: " .. res.body)
+            local data, err = require("lib.grafana_loki").fetch_logs_from_loki(
+                tostring(now - 3000) .. "000000", -- from
+                tostring(now) .. "000000",        -- to
+                { query = [[{custom_label="custom_label_value"} | json]] }
+            )
+
+            assert(err == nil, "fetch logs error: " .. err)
+            assert(data ~= nil, "loki response decode error: " .. res.body)
             assert(data.status == "success", "loki response error: " .. res.body)
             assert(#data.data.result > 0, "loki log empty: " .. res.body)
 
@@ -287,27 +260,13 @@ hello world
 --- config
     location /t {
         content_by_lua_block {
-            local cjson = require("cjson")
-            local httpc = require("resty.http").new()
-            local now = ngx.now() * 1000
-            local res, err = httpc:request_uri("http://127.0.0.1:3100/loki/api/v1/query_range", {
-                query = {
-                    direction = "backward",
-                    start = tostring(now - 10000).."000000",
-                    ["end"] = tostring(now).."000000",
-                    limit = "10",
-                    query = [[{job="apisix"} | json]],
-                },
-                headers = {
-                    ["X-Scope-OrgID"] = "tenant_1"
-                }
-            })
-
-            assert(res ~= nil, "request error: " .. (err or ""))
-            assert(res.status == 200, "loki error: " .. res.status .. " " .. res.body)
-
-            local data = cjson.decode(res.body)
-            assert(data ~= nil, "loki response error: " .. res.body)
+            local data, err = require("lib.grafana_loki").fetch_logs_from_loki(
+                tostring(now - 10000) .. "000000", -- from
+                tostring(now) .. "000000",        -- to
+            )
+
+            assert(err == nil, "fetch logs error: " .. err)
+            assert(data ~= nil, "loki response decode error: " .. res.body)
             assert(data.status == "success", "loki response error: " .. res.body)
             assert(#data.data.result > 0, "loki log empty: " .. res.body)
 
@@ -324,27 +283,16 @@ hello world
 --- config
     location /t {
         content_by_lua_block {
-            local cjson = require("cjson")
-            local httpc = require("resty.http").new()
-            local now = ngx.now() * 1000
-            local res, err = httpc:request_uri("http://127.0.0.1:3100/loki/api/v1/query_range", {
-                query = {
-                    direction = "backward",
-                    start = tostring(now - 3000).."000000",
-                    ["end"] = tostring(now).."000000",
-                    limit = "10",
-                    query = [[{job="apisix"} | json]],
-                },
-                headers = {
-                    ["X-Scope-OrgID"] = "tenant_2"
-                }
-            })
-
-            assert(res ~= nil, "request error: " .. (err or ""))
-            assert(res.status == 200, "loki error: " .. res.status .. " " .. res.body)
-
-            local data = cjson.decode(res.body)
-            assert(data ~= nil, "loki response error: " .. res.body)
+            local data, err = require("lib.grafana_loki").fetch_logs_from_loki(
+                tostring(now - 3000) .. "000000", -- from
+                tostring(now) .. "000000",        -- to
+                { headers = {
+                        ["X-Scope-OrgID"] = "tenant_2"
+                } }
+            )
+
+            assert(err == nil, "fetch logs error: " .. err)
+            assert(data ~= nil, "loki response decode error: " .. res.body)
             assert(data.status == "success", "loki response error: " .. res.body)
             assert(#data.data.result > 0, "loki log empty: " .. res.body)
 

From de9c5f188e1064b7917fd95efe67b9040a918867 Mon Sep 17 00:00:00 2001
From: Zeping Bai <bzp2010@apache.org>
Date: Sun, 7 May 2023 21:44:19 +0800
Subject: [PATCH 17/23] test: fix lint

---
 t/lib/grafana_loki.lua | 1 -
 1 file changed, 1 deletion(-)

diff --git a/t/lib/grafana_loki.lua b/t/lib/grafana_loki.lua
index 910da1219d8d..3152ccbbeaff 100644
--- a/t/lib/grafana_loki.lua
+++ b/t/lib/grafana_loki.lua
@@ -24,7 +24,6 @@ local _M = {}
 function _M.fetch_logs_from_loki(from, to, options)
     options = options or {}
 
-    local now = ngx.now() * 1000
     local direction = options.direction or "backward"
     local limit = options.limit or "10"
     local query = options.query or [[{job="apisix"} | json]]

From 6dfff667c53e6363b37fc48ebaa5f5b7169673c8 Mon Sep 17 00:00:00 2001
From: Zeping Bai <bzp2010@apache.org>
Date: Tue, 9 May 2023 10:08:28 +0800
Subject: [PATCH 18/23] test: fix

---
 t/plugin/loki-logger.t | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/t/plugin/loki-logger.t b/t/plugin/loki-logger.t
index f78307890986..75e308835a6a 100644
--- a/t/plugin/loki-logger.t
+++ b/t/plugin/loki-logger.t
@@ -120,7 +120,7 @@ hello world
         content_by_lua_block {
             local data, err = require("lib.grafana_loki").fetch_logs_from_loki(
                 tostring(now - 3000) .. "000000", -- from
-                tostring(now) .. "000000",        -- to
+                tostring(now) .. "000000"         -- to
             )
 
             assert(err == nil, "fetch logs error: " .. err)

From 80c0021f8c2efd4701e104cf38710bc8c88ad0ea Mon Sep 17 00:00:00 2001
From: Zeping Bai <bzp2010@apache.org>
Date: Tue, 9 May 2023 11:01:33 +0800
Subject: [PATCH 19/23] test: fix

---
 t/plugin/loki-logger.t | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/t/plugin/loki-logger.t b/t/plugin/loki-logger.t
index 75e308835a6a..36b716658656 100644
--- a/t/plugin/loki-logger.t
+++ b/t/plugin/loki-logger.t
@@ -118,6 +118,7 @@ hello world
 --- config
     location /t {
         content_by_lua_block {
+            local now = ngx.now() * 1000
             local data, err = require("lib.grafana_loki").fetch_logs_from_loki(
                 tostring(now - 3000) .. "000000", -- from
                 tostring(now) .. "000000"         -- to
@@ -190,6 +191,7 @@ hello world
 --- config
     location /t {
         content_by_lua_block {
+            local now = ngx.now() * 1000
             local data, err = require("lib.grafana_loki").fetch_logs_from_loki(
                 tostring(now - 3000) .. "000000", -- from
                 tostring(now) .. "000000",        -- to
@@ -260,6 +262,7 @@ hello world
 --- config
     location /t {
         content_by_lua_block {
+            local now = ngx.now() * 1000
             local data, err = require("lib.grafana_loki").fetch_logs_from_loki(
                 tostring(now - 10000) .. "000000", -- from
                 tostring(now) .. "000000",        -- to
@@ -283,6 +286,7 @@ hello world
 --- config
     location /t {
         content_by_lua_block {
+            local now = ngx.now() * 1000
             local data, err = require("lib.grafana_loki").fetch_logs_from_loki(
                 tostring(now - 3000) .. "000000", -- from
                 tostring(now) .. "000000",        -- to

From f92fa42b0bf699ebe61ab97ddde9c4d48a15356c Mon Sep 17 00:00:00 2001
From: Zeping Bai <bzp2010@apache.org>
Date: Tue, 9 May 2023 11:05:34 +0800
Subject: [PATCH 20/23] docs: fix

---
 docs/en/latest/plugins/loki-logger.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/en/latest/plugins/loki-logger.md b/docs/en/latest/plugins/loki-logger.md
index a862e37780f9..5e7a2c7a4286 100644
--- a/docs/en/latest/plugins/loki-logger.md
+++ b/docs/en/latest/plugins/loki-logger.md
@@ -32,7 +32,7 @@ description: This document contains information about the Apache APISIX loki-log
 
 The `loki-logger` plugin is used to forward logs to [Grafana Loki](https://grafana.com/oss/loki/) for analysis and storage.
 
-When the Plugin is enabled, APISIX will serialize the request context information to [Log entries in JSON](https://grafana.com/docs/loki/latest/api/#push-log-entries-to-loki) and submit it to the batch queue. When the maximum batch size is exceeded, the data in the queue is pushed to Elasticsearch. See [batch processor](../batch-processor.md) for more details.
+When the Plugin is enabled, APISIX will serialize the request context information to [Log entries in JSON](https://grafana.com/docs/loki/latest/api/#push-log-entries-to-loki) and submit it to the batch queue. When the maximum batch size is exceeded, the data in the queue is pushed to Grafana Loki. See [batch processor](../batch-processor.md) for more details.
 
 ## Attributes
 

From fe9e1ebe483ccb114df91cca4b6ea2f53564d668 Mon Sep 17 00:00:00 2001
From: Zeping Bai <bzp2010@apache.org>
Date: Tue, 9 May 2023 15:42:02 +0800
Subject: [PATCH 21/23] test: fix

---
 t/plugin/loki-logger.t | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/t/plugin/loki-logger.t b/t/plugin/loki-logger.t
index 36b716658656..3afb8c0b028d 100644
--- a/t/plugin/loki-logger.t
+++ b/t/plugin/loki-logger.t
@@ -124,7 +124,7 @@ hello world
                 tostring(now) .. "000000"         -- to
             )
 
-            assert(err == nil, "fetch logs error: " .. err)
+            assert(err == nil, "fetch logs error: " .. err or "")
             assert(data ~= nil, "loki response decode error: " .. res.body)
             assert(data.status == "success", "loki response error: " .. res.body)
             assert(#data.data.result > 0, "loki log empty: " .. res.body)
@@ -198,7 +198,7 @@ hello world
                 { query = [[{custom_label="custom_label_value"} | json]] }
             )
 
-            assert(err == nil, "fetch logs error: " .. err)
+            assert(err == nil, "fetch logs error: " .. err or "")
             assert(data ~= nil, "loki response decode error: " .. res.body)
             assert(data.status == "success", "loki response error: " .. res.body)
             assert(#data.data.result > 0, "loki log empty: " .. res.body)
@@ -268,7 +268,7 @@ hello world
                 tostring(now) .. "000000",        -- to
             )
 
-            assert(err == nil, "fetch logs error: " .. err)
+            assert(err == nil, "fetch logs error: " .. err or "")
             assert(data ~= nil, "loki response decode error: " .. res.body)
             assert(data.status == "success", "loki response error: " .. res.body)
             assert(#data.data.result > 0, "loki log empty: " .. res.body)
@@ -295,7 +295,7 @@ hello world
                 } }
             )
 
-            assert(err == nil, "fetch logs error: " .. err)
+            assert(err == nil, "fetch logs error: " .. err or "")
             assert(data ~= nil, "loki response decode error: " .. res.body)
             assert(data.status == "success", "loki response error: " .. res.body)
             assert(#data.data.result > 0, "loki log empty: " .. res.body)

From be8919d685fd763e97d0525c9bc4f832a90de21f Mon Sep 17 00:00:00 2001
From: Zeping Bai <bzp2010@apache.org>
Date: Tue, 9 May 2023 16:24:31 +0800
Subject: [PATCH 22/23] test: fix

---
 t/lib/grafana_loki.lua |  6 +++++-
 t/plugin/loki-logger.t | 34 +++++++++++++++++-----------------
 2 files changed, 22 insertions(+), 18 deletions(-)

diff --git a/t/lib/grafana_loki.lua b/t/lib/grafana_loki.lua
index 3152ccbbeaff..fc1173982130 100644
--- a/t/lib/grafana_loki.lua
+++ b/t/lib/grafana_loki.lua
@@ -52,7 +52,11 @@ function _M.fetch_logs_from_loki(from, to, options)
         return nil, "HTTP status code: " .. res.status .. ", body: " .. res.body
     end
 
-    return cjson.decode(res.body)
+    local data = cjson.decode(res.body)
+    if not data then
+        return nil, "failed to decode response body: " .. res.body
+    end
+    return data, nil
 end
 
 
diff --git a/t/plugin/loki-logger.t b/t/plugin/loki-logger.t
index 3afb8c0b028d..faa8749a917d 100644
--- a/t/plugin/loki-logger.t
+++ b/t/plugin/loki-logger.t
@@ -118,16 +118,16 @@ hello world
 --- config
     location /t {
         content_by_lua_block {
+            local cjson = require("cjson")
             local now = ngx.now() * 1000
             local data, err = require("lib.grafana_loki").fetch_logs_from_loki(
                 tostring(now - 3000) .. "000000", -- from
                 tostring(now) .. "000000"         -- to
             )
 
-            assert(err == nil, "fetch logs error: " .. err or "")
-            assert(data ~= nil, "loki response decode error: " .. res.body)
-            assert(data.status == "success", "loki response error: " .. res.body)
-            assert(#data.data.result > 0, "loki log empty: " .. res.body)
+            assert(err == nil, "fetch logs error: " .. (err or ""))
+            assert(data.status == "success", "loki response error: " .. cjson.encode(data))
+            assert(#data.data.result > 0, "loki log empty: " .. cjson.encode(data))
 
             local entry = data.data.result[1]
             assert(entry.stream.request_headers_test_header == "only-for-test#1",
@@ -191,6 +191,7 @@ hello world
 --- config
     location /t {
         content_by_lua_block {
+            local cjson = require("cjson")
             local now = ngx.now() * 1000
             local data, err = require("lib.grafana_loki").fetch_logs_from_loki(
                 tostring(now - 3000) .. "000000", -- from
@@ -198,10 +199,9 @@ hello world
                 { query = [[{custom_label="custom_label_value"} | json]] }
             )
 
-            assert(err == nil, "fetch logs error: " .. err or "")
-            assert(data ~= nil, "loki response decode error: " .. res.body)
-            assert(data.status == "success", "loki response error: " .. res.body)
-            assert(#data.data.result > 0, "loki log empty: " .. res.body)
+            assert(err == nil, "fetch logs error: " .. (err or ""))
+            assert(data.status == "success", "loki response error: " .. cjson.encode(data))
+            assert(#data.data.result > 0, "loki log empty: " .. cjson.encode(data))
 
             local entry = data.data.result[1]
             assert(entry.stream.request_headers_test_header == "only-for-test#2",
@@ -262,16 +262,16 @@ hello world
 --- config
     location /t {
         content_by_lua_block {
+            local cjson = require("cjson")
             local now = ngx.now() * 1000
             local data, err = require("lib.grafana_loki").fetch_logs_from_loki(
                 tostring(now - 10000) .. "000000", -- from
-                tostring(now) .. "000000",        -- to
+                tostring(now) .. "000000"          -- to
             )
 
-            assert(err == nil, "fetch logs error: " .. err or "")
-            assert(data ~= nil, "loki response decode error: " .. res.body)
-            assert(data.status == "success", "loki response error: " .. res.body)
-            assert(#data.data.result > 0, "loki log empty: " .. res.body)
+            assert(err == nil, "fetch logs error: " .. (err or ""))
+            assert(data.status == "success", "loki response error: " .. cjson.encode(data))
+            assert(#data.data.result > 0, "loki log empty: " .. cjson.encode(data))
 
             local entry = data.data.result[1]
             assert(entry.stream.request_headers_test_header ~= "only-for-test#3",
@@ -286,6 +286,7 @@ hello world
 --- config
     location /t {
         content_by_lua_block {
+            local cjson = require("cjson")
             local now = ngx.now() * 1000
             local data, err = require("lib.grafana_loki").fetch_logs_from_loki(
                 tostring(now - 3000) .. "000000", -- from
@@ -295,10 +296,9 @@ hello world
                 } }
             )
 
-            assert(err == nil, "fetch logs error: " .. err or "")
-            assert(data ~= nil, "loki response decode error: " .. res.body)
-            assert(data.status == "success", "loki response error: " .. res.body)
-            assert(#data.data.result > 0, "loki log empty: " .. res.body)
+            assert(err == nil, "fetch logs error: " .. (err or ""))
+            assert(data.status == "success", "loki response error: " .. cjson.encode(data))
+            assert(#data.data.result > 0, "loki log empty: " .. cjson.encode(data))
 
             local entry = data.data.result[1]
             assert(entry.stream.request_headers_test_header == "only-for-test#3",

From bf40902e29d4561f53cced0bec75256463d80838 Mon Sep 17 00:00:00 2001
From: Zeping Bai <bzp2010@apache.org>
Date: Thu, 25 May 2023 10:38:49 +0800
Subject: [PATCH 23/23] fix: review

---
 ci/pod/docker-compose.plugin.yml      | 1 +
 docs/en/latest/plugins/loki-logger.md | 6 +++---
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/ci/pod/docker-compose.plugin.yml b/ci/pod/docker-compose.plugin.yml
index 808a82a4b39a..b79e492c7c54 100644
--- a/ci/pod/docker-compose.plugin.yml
+++ b/ci/pod/docker-compose.plugin.yml
@@ -340,4 +340,5 @@ networks:
   rocketmq_net:
   opa_net:
   vector_net:
+  clickhouse_net:
   loki_net:
diff --git a/docs/en/latest/plugins/loki-logger.md b/docs/en/latest/plugins/loki-logger.md
index 5e7a2c7a4286..6bd3ae8d68ed 100644
--- a/docs/en/latest/plugins/loki-logger.md
+++ b/docs/en/latest/plugins/loki-logger.md
@@ -38,7 +38,7 @@ When the Plugin is enabled, APISIX will serialize the request context informatio
 
 | Name | Type | Required | Default | Description |
 |---|---|---|---|---|
-| endpoint_addrs | array | True |  | Loki API base URL, format like http://127.0.0.1:3100, supports HTTPS and domain names. If multiple endpoints are configured, they will be written randomly. |
+| endpoint_addrs | array[string] | True |  | Loki API base URL, format like http://127.0.0.1:3100, supports HTTPS and domain names. If multiple endpoints are configured, they will be written randomly. |
 | endpoint_uri | string | False | /loki/api/v1/push | If you are using a log collection service that is compatible with the Loki Push API, you can use this configuration item to customize the API path. |
 | tenant_id | string | False | fake | Loki tenant ID. According to Loki's [multi-tenancy documentation](https://grafana.com/docs/loki/latest/operations/multi-tenancy/#multi-tenancy), its default value is set to the default value `fake` under single-tenancy. |
 | log_labels | object | False | {job = "apisix"} | Loki log label. [APISIX variables](../apisix-variable.md) and [Nginx variables](http://nginx.org/en/docs/varindex.html) can be used by prefixing the string with `$`, both individual and combined, such as `$host` or `$remote_addr:$remote_port`. |
@@ -119,9 +119,9 @@ Now, if you make a request to APISIX, it will be logged in your Loki server:
 curl -i http://127.0.0.1:9080/hello
 ```
 
-## Disable plugin
+## Delete the plugin
 
-To disable the `loki-logger` plugin, you can delete the corresponding JSON configuration from the plugin configuration. APISIX will automatically reload and you do not have to restart for this to take effect.
+When you need to remove the `loki-logger` plugin, you can delete the corresponding JSON configuration with the following command and APISIX will automatically reload the relevant configuration without restarting the service:
 
 ```shell
 curl http://127.0.0.1:9180/apisix/admin/routes/1  -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '