From 66cd80f77cea3631996a13702ed131d4b2a84246 Mon Sep 17 00:00:00 2001 From: Zeping Bai Date: Mon, 29 May 2023 10:01:44 +0800 Subject: [PATCH] feat: add loki-logger plugin (#9399) --- apisix/plugins/loki-logger.lua | 234 +++++++++++++++++++ ci/pod/docker-compose.plugin.yml | 10 + conf/config-default.yaml | 1 + docs/en/latest/config.json | 3 +- docs/en/latest/plugins/loki-logger.md | 165 ++++++++++++++ t/admin/plugins.t | 1 + t/lib/grafana_loki.lua | 63 ++++++ t/plugin/loki-logger.t | 308 ++++++++++++++++++++++++++ 8 files changed, 784 insertions(+), 1 deletion(-) create mode 100644 apisix/plugins/loki-logger.lua create mode 100644 docs/en/latest/plugins/loki-logger.md create mode 100644 t/lib/grafana_loki.lua create mode 100644 t/plugin/loki-logger.t diff --git a/apisix/plugins/loki-logger.lua b/apisix/plugins/loki-logger.lua new file mode 100644 index 000000000000..593fc8b18e6c --- /dev/null +++ b/apisix/plugins/loki-logger.lua @@ -0,0 +1,234 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +local bp_manager_mod = require("apisix.utils.batch-processor-manager") +local log_util = require("apisix.utils.log-util") +local core = require("apisix.core") +local http = require("resty.http") +local new_tab = require("table.new") + +local pairs = pairs +local ipairs = ipairs +local tostring = tostring +local math_random = math.random +local table_insert = table.insert +local ngx = ngx +local str_format = core.string.format + +local plugin_name = "loki-logger" +local batch_processor_manager = bp_manager_mod.new("loki logger") + +local schema = { + type = "object", + properties = { + -- core configurations + endpoint_addrs = { + type = "array", + minItems = 1, + items = core.schema.uri_def, + }, + endpoint_uri = { + type = "string", + minLength = 1, + default = "/loki/api/v1/push" + }, + tenant_id = {type = "string", default = "fake"}, + log_labels = { + type = "object", + patternProperties = { + [".*"] = { + type = "string", + minLength = 1, + }, + }, + default = { + job = "apisix", + }, + }, + + -- connection layer configurations + ssl_verify = {type = "boolean", default = false}, + timeout = { + type = "integer", + minimum = 1, + maximum = 60000, + default = 3000, + description = "timeout in milliseconds", + }, + keepalive = {type = "boolean", default = true}, + keepalive_timeout = { + type = "integer", + minimum = 1000, + default = 60000, + description = "keepalive timeout in milliseconds", + }, + keepalive_pool = {type = "integer", minimum = 1, default = 5}, + + -- logger related configurations + log_format = {type = "object"}, + include_req_body = {type = "boolean", default = false}, + include_req_body_expr = { + type = "array", + minItems = 1, + items = { + type = "array" + } + }, + include_resp_body = {type = "boolean", default = false}, + include_resp_body_expr = { + type = "array", + minItems = 1, + items = { + type = "array" + } + }, + }, + required = {"endpoint_addrs"} +} + + +local metadata_schema = { + type = "object", + properties = { + log_format = log_util.metadata_schema_log_format, + }, +} + + +local _M = { + version = 0.1, + priority = 414, + name = plugin_name, + schema = batch_processor_manager:wrap_schema(schema), + metadata_schema = metadata_schema, +} + + +function _M.check_schema(conf, schema_type) + if schema_type == core.schema.TYPE_METADATA then + return core.schema.check(metadata_schema, conf) + end + + local ok, err = core.schema.check(schema, conf) + if not ok then + return nil, err + end + return log_util.check_log_schema(conf) +end + + +local function send_http_data(conf, log) + local params = { + headers = { + ["Content-Type"] = "application/json", + ["X-Scope-OrgID"] = conf.tenant_id, + }, + keepalive = conf.keepalive, + ssl_verify = conf.ssl_verify, + method = "POST", + body = core.json.encode(log) + } + + if conf.keepalive then + params.keepalive_timeout = conf.keepalive_timeout + params.keepalive_pool = conf.keepalive_pool + end + + local httpc, err = http.new() + if not httpc then + return false, str_format("create http client error: %s", err) + end + httpc:set_timeout(conf.timeout) + + -- select an random endpoint and build URL + local endpoint_url = conf.endpoint_addrs[math_random(#conf.endpoint_addrs)] .. conf.endpoint_uri + local res, err = httpc:request_uri(endpoint_url, params) + if not res then + return false, err + end + + if res.status >= 300 then + return false, str_format("loki server returned status: %d, body: %s", + res.status, res.body or "") + end + + return true +end + + +function _M.body_filter(conf, ctx) + log_util.collect_body(conf, ctx) +end + + +function _M.log(conf, ctx) + local entry = log_util.get_log_entry(plugin_name, conf, ctx) + + if not entry.route_id then + entry.route_id = "no-matched" + end + + -- insert start time as log time, multiply to nanoseconds + -- use string concat to circumvent 64bit integers that LuaVM cannot handle + -- that is, first process the decimal part of the millisecond value + -- and then add 6 zeros by string concatenation + entry.loki_log_time = tostring(ngx.req.start_time() * 1000) .. "000000" + + if batch_processor_manager:add_entry(conf, entry) then + return + end + + -- generate a function to be executed by the batch processor + local func = function(entries) + local labels = conf.log_labels + + -- parsing possible variables in label value + for key, value in pairs(labels) do + local new_val, err, n_resolved = core.utils.resolve_var(value, ctx.var) + if not err and n_resolved > 0 then + labels[key] = new_val + end + end + + -- build loki request data + local data = { + streams = { + { + stream = labels, + values = new_tab(1, 0), + } + } + } + + -- add all entries to the batch + for _, entry in ipairs(entries) do + local log_time = entry.loki_log_time + entry.loki_log_time = nil -- clean logger internal field + + table_insert(data.streams[1].values, { + log_time, core.json.encode(entry) + }) + end + + return send_http_data(conf, data) + end + + batch_processor_manager:add_entry_to_new_processor(conf, entry, ctx, func) +end + + +return _M diff --git a/ci/pod/docker-compose.plugin.yml b/ci/pod/docker-compose.plugin.yml index 606929125393..58e305c8faa7 100644 --- a/ci/pod/docker-compose.plugin.yml +++ b/ci/pod/docker-compose.plugin.yml @@ -143,6 +143,15 @@ services: - ./t/certs:/certs + ## Grafana Loki + loki: + image: grafana/loki:2.8.0 + command: -config.file=/etc/loki/local-config.yaml -auth.enabled -querier.multi-tenant-queries-enabled + ports: + - "3100:3100" + networks: + - loki_net + rocketmq_namesrv: image: apacherocketmq/rocketmq:4.6.0 container_name: rmqnamesrv @@ -351,3 +360,4 @@ networks: opa_net: vector_net: clickhouse_net: + loki_net: diff --git a/conf/config-default.yaml b/conf/config-default.yaml index 4f97adc4e4f5..d41df397b9e5 100755 --- a/conf/config-default.yaml +++ b/conf/config-default.yaml @@ -467,6 +467,7 @@ plugins: # plugin list (sorted by priority) - public-api # priority: 501 - prometheus # priority: 500 - datadog # priority: 495 + - loki-logger # priority: 414 - elasticsearch-logger # priority: 413 - echo # priority: 412 - loggly # priority: 411 diff --git a/docs/en/latest/config.json b/docs/en/latest/config.json index d752359a6089..496a44ff98f1 100644 --- a/docs/en/latest/config.json +++ b/docs/en/latest/config.json @@ -181,7 +181,8 @@ "plugins/file-logger", "plugins/loggly", "plugins/elasticsearch-logger", - "plugins/tencent-cloud-cls" + "plugins/tencent-cloud-cls", + "plugins/loki-logger" ] } ] diff --git a/docs/en/latest/plugins/loki-logger.md b/docs/en/latest/plugins/loki-logger.md new file mode 100644 index 000000000000..6bd3ae8d68ed --- /dev/null +++ b/docs/en/latest/plugins/loki-logger.md @@ -0,0 +1,165 @@ +--- +title: loki-logger +keywords: + - Apache APISIX + - API Gateway + - Plugin + - Loki-logger + - Grafana Loki +description: This document contains information about the Apache APISIX loki-logger Plugin. +--- + + + +## Description + +The `loki-logger` plugin is used to forward logs to [Grafana Loki](https://grafana.com/oss/loki/) for analysis and storage. + +When the Plugin is enabled, APISIX will serialize the request context information to [Log entries in JSON](https://grafana.com/docs/loki/latest/api/#push-log-entries-to-loki) and submit it to the batch queue. When the maximum batch size is exceeded, the data in the queue is pushed to Grafana Loki. See [batch processor](../batch-processor.md) for more details. + +## Attributes + +| Name | Type | Required | Default | Description | +|---|---|---|---|---| +| endpoint_addrs | array[string] | True | | Loki API base URL, format like http://127.0.0.1:3100, supports HTTPS and domain names. If multiple endpoints are configured, they will be written randomly. | +| endpoint_uri | string | False | /loki/api/v1/push | If you are using a log collection service that is compatible with the Loki Push API, you can use this configuration item to customize the API path. | +| tenant_id | string | False | fake | Loki tenant ID. According to Loki's [multi-tenancy documentation](https://grafana.com/docs/loki/latest/operations/multi-tenancy/#multi-tenancy), its default value is set to the default value `fake` under single-tenancy. | +| log_labels | object | False | {job = "apisix"} | Loki log label. [APISIX variables](../apisix-variable.md) and [Nginx variables](http://nginx.org/en/docs/varindex.html) can be used by prefixing the string with `$`, both individual and combined, such as `$host` or `$remote_addr:$remote_port`. | +| ssl_verify | boolean | False | true | When set to `true`, verifies the SSL certificate. | +| timeout | integer | False | 3000ms | [1, 60000]ms | Timeout for the authorization service HTTP call. | +| keepalive | boolean | False | true | When set to `true`, keeps the connection alive for multiple requests. | +| keepalive_timeout | integer | False | 60000ms | [1000, ...]ms | Idle time after which the connection is closed. | +| keepalive_pool | integer | False | 5 | [1, ...]ms | Connection pool limit. | +| log_format | object | False | | Log format declared as key value pairs in JSON format. Values only support strings. [APISIX variables](../apisix-variable.md) and [Nginx variables](http://nginx.org/en/docs/varindex.html) can be used by prefixing the string with `$`. | +| include_req_body | boolean | False | false | When set to `true` includes the request body in the log. If the request body is too big to be kept in the memory, it can't be logged due to Nginx's limitations. | +| include_req_body_expr | array | False | | Filter for when the `include_req_body` attribute is set to `true`. Request body is only logged when the expression set here evaluates to `true`. See [lua-resty-expr](https://github.com/api7/lua-resty-expr) for more. | +| include_resp_body | boolean | False | false | When set to `true` includes the response body in the log. | +| include_resp_body_expr | array | False | | Filter for when the `include_resp_body` attribute is set to `true`. Response body is only logged when the expression set here evaluates to `true`. See [lua-resty-expr](https://github.com/api7/lua-resty-expr) for more. | + +This plugin supports using batch processors to aggregate and process entries (logs/data) in a batch. This avoids the need for frequently submitting the data. The batch processor submits data every `5` seconds or when the data in the queue reaches `1000`. See [Batch Processor](../batch-processor.md#configuration) for more information or setting your custom configuration. + +## Metadata + +You can also set the format of the logs by configuring the Plugin metadata. The following configurations are available: + +| Name | Type | Required | Default | Description | +|------|------|----------|---------|-------------| +| log_format | object | False | {"host": "$host", "@timestamp": "$time_iso8601", "client_ip": "$remote_addr"} | Log format declared as key value pairs in JSON format. Values only support strings. [APISIX variables](../apisix-variable.md) and [Nginx variables](http://nginx.org/en/docs/varindex.html) can be used by prefixing the string with `$`. | + +:::info IMPORTANT + +Configuring the plugin metadata is global in scope. This means that it will take effect on all Routes and Services which use the `loki-logger` plugin. + +::: + +The example below shows how you can configure through the Admin API: + +```shell +curl http://127.0.0.1:9180/apisix/admin/plugin_metadata/loki-logger -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d ' +{ + "log_format": { + "host": "$host", + "@timestamp": "$time_iso8601", + "client_ip": "$remote_addr" + } +}' +``` + +With this configuration, your logs would be formatted as shown below: + +```shell +{"host":"localhost","@timestamp":"2020-09-23T19:05:05-04:00","client_ip":"127.0.0.1","route_id":"1"} +{"host":"localhost","@timestamp":"2020-09-23T19:05:05-04:00","client_ip":"127.0.0.1","route_id":"1"} +``` + +## Enabling the plugin + +The example below shows how you can enable the `loki-logger` plugin on a specific Route: + +```shell +curl http://127.0.0.1:9180/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d ' +{ + "plugins": { + "loki-logger": { + "endpoint_addrs" : ["http://127.0.0.1:3100"] + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" +}' +``` + +## Example usage + +Now, if you make a request to APISIX, it will be logged in your Loki server: + +```shell +curl -i http://127.0.0.1:9080/hello +``` + +## Delete the plugin + +When you need to remove the `loki-logger` plugin, you can delete the corresponding JSON configuration with the following command and APISIX will automatically reload the relevant configuration without restarting the service: + +```shell +curl http://127.0.0.1:9180/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d ' +{ + "methods": ["GET"], + "uri": "/hello", + "plugins": {}, + "upstream": { + "type": "roundrobin", + "nodes": { + "127.0.0.1:1980": 1 + } + } +}' +``` + +## FAQ + +### Logs are not pushed properly + +Look at `error.log` for such a log. + +```text +2023/04/30 13:45:46 [error] 19381#19381: *1075673 [lua] batch-processor.lua:95: Batch Processor[loki logger] failed to process entries: loki server returned status: 401, body: no org id, context: ngx.timer, client: 127.0.0.1, server: 0.0.0.0:9081 +``` + +The error can be diagnosed based on the error code in the `failed to process entries: loki server returned status: 401, body: no org id` and the response body of the loki server. + +### Getting errors when QPS is high? + +- Make sure to `keepalive` related configuration is set properly. See [Attributes](#attributes) for more information. +- Check the logs in `error.log`, look for such a log. + + ```text + 2023/04/30 13:49:34 [error] 19381#19381: *1082680 [lua] batch-processor.lua:95: Batch Processor[loki logger] failed to process entries: loki server returned status: 429, body: Ingestion rate limit exceeded for user tenant_1 (limit: 4194304 bytes/sec) while attempting to ingest '1000' lines totaling '616307' bytes, reduce log volume or contact your Loki administrator to see if the limit can be increased, context: ngx.timer, client: 127.0.0.1, server: 0.0.0.0:9081 + ``` + + - The logs usually associated with high QPS look like the above. The error is: `Ingestion rate limit exceeded for user tenant_1 (limit: 4194304 bytes/sec) while attempting to ingest '1000' lines totaling '616307' bytes, reduce log volume or contact your Loki administrator to see if the limit can be increased`. + - Refer to [Loki documentation](https://grafana.com/docs/loki/latest/configuration/#limits_config) to add limits on the amount of default and burst logs, such as `ingestion_rate_mb` and `ingestion_burst_size_mb`. + + As the test during development, setting the `ingestion_burst_size_mb` to 100 allows APISIX to push the logs correctly at least at 10000 RPS. diff --git a/t/admin/plugins.t b/t/admin/plugins.t index ceb4df15bd70..ae7617dfd23d 100644 --- a/t/admin/plugins.t +++ b/t/admin/plugins.t @@ -109,6 +109,7 @@ grpc-web public-api prometheus datadog +loki-logger elasticsearch-logger echo loggly diff --git a/t/lib/grafana_loki.lua b/t/lib/grafana_loki.lua new file mode 100644 index 000000000000..fc1173982130 --- /dev/null +++ b/t/lib/grafana_loki.lua @@ -0,0 +1,63 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +local cjson = require("cjson") +local http = require("resty.http") + +local _M = {} + + +function _M.fetch_logs_from_loki(from, to, options) + options = options or {} + + local direction = options.direction or "backward" + local limit = options.limit or "10" + local query = options.query or [[{job="apisix"} | json]] + local url = options.url or "http://127.0.0.1:3100/loki/api/v1/query_range" + local headers = options.headers or { + ["X-Scope-OrgID"] = "tenant_1" + } + + local httpc = http.new() + local res, err = httpc:request_uri(url, { + query = { + start = from, + ["end"] = to, + direction = direction, + limit = limit, + query = query, + }, + headers = headers + }) + + if not res or err then + return nil, err + end + + if res.status > 300 then + return nil, "HTTP status code: " .. res.status .. ", body: " .. res.body + end + + local data = cjson.decode(res.body) + if not data then + return nil, "failed to decode response body: " .. res.body + end + return data, nil +end + + +return _M diff --git a/t/plugin/loki-logger.t b/t/plugin/loki-logger.t new file mode 100644 index 000000000000..faa8749a917d --- /dev/null +++ b/t/plugin/loki-logger.t @@ -0,0 +1,308 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +use t::APISIX 'no_plan'; + +repeat_each(1); +no_long_string(); +no_root_location(); + +add_block_preprocessor(sub { + my ($block) = @_; + + if (!defined $block->request) { + $block->set_value("request", "GET /t"); + } +}); + +run_tests(); + +__DATA__ + +=== TEST 1: sanity +--- config + location /t { + content_by_lua_block { + local test_cases = { + {endpoint_addrs = {"http://127.0.0.1:8199"}}, + {endpoint_addrs = "http://127.0.0.1:8199"}, + {endpoint_addrs = {}}, + {}, + {endpoint_addrs = {"http://127.0.0.1:8199"}, endpoint_uri = "/loki/api/v1/push"}, + {endpoint_addrs = {"http://127.0.0.1:8199"}, endpoint_uri = 1234}, + {endpoint_addrs = {"http://127.0.0.1:8199"}, tenant_id = 1234}, + {endpoint_addrs = {"http://127.0.0.1:8199"}, log_labels = "1234"}, + {endpoint_addrs = {"http://127.0.0.1:8199"}, log_labels = {job = "apisix6"}}, + } + local plugin = require("apisix.plugins.loki-logger") + + for _, case in ipairs(test_cases) do + local ok, err = plugin.check_schema(case) + ngx.say(ok and "done" or err) + end + } + } +--- response_body +done +property "endpoint_addrs" validation failed: wrong type: expected array, got string +property "endpoint_addrs" validation failed: expect array to have at least 1 items +property "endpoint_addrs" is required +done +property "endpoint_uri" validation failed: wrong type: expected string, got number +property "tenant_id" validation failed: wrong type: expected string, got number +property "log_labels" validation failed: wrong type: expected object, got string +done + + + +=== TEST 2: setup route +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "loki-logger": { + "endpoint_addrs": ["http://127.0.0.1:3100"], + "tenant_id": "tenant_1", + "batch_max_size": 1 + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 3: hit route +--- request +GET /hello +--- more_headers +test-header: only-for-test#1 +--- response_body +hello world + + + +=== TEST 4: check loki log +--- config + location /t { + content_by_lua_block { + local cjson = require("cjson") + local now = ngx.now() * 1000 + local data, err = require("lib.grafana_loki").fetch_logs_from_loki( + tostring(now - 3000) .. "000000", -- from + tostring(now) .. "000000" -- to + ) + + assert(err == nil, "fetch logs error: " .. (err or "")) + assert(data.status == "success", "loki response error: " .. cjson.encode(data)) + assert(#data.data.result > 0, "loki log empty: " .. cjson.encode(data)) + + local entry = data.data.result[1] + assert(entry.stream.request_headers_test_header == "only-for-test#1", + "expected field request_headers_test_header value: " .. cjson.encode(entry)) + } + } +--- error_code: 200 + + + +=== TEST 5: setup route (with log_labels) +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "loki-logger": { + "endpoint_addrs": ["http://127.0.0.1:3100"], + "tenant_id": "tenant_1", + "log_labels": { + "custom_label": "custom_label_value" + }, + "batch_max_size": 1 + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 6: hit route +--- request +GET /hello +--- more_headers +test-header: only-for-test#2 +--- response_body +hello world + + + +=== TEST 7: check loki log (with custom_label) +--- config + location /t { + content_by_lua_block { + local cjson = require("cjson") + local now = ngx.now() * 1000 + local data, err = require("lib.grafana_loki").fetch_logs_from_loki( + tostring(now - 3000) .. "000000", -- from + tostring(now) .. "000000", -- to + { query = [[{custom_label="custom_label_value"} | json]] } + ) + + assert(err == nil, "fetch logs error: " .. (err or "")) + assert(data.status == "success", "loki response error: " .. cjson.encode(data)) + assert(#data.data.result > 0, "loki log empty: " .. cjson.encode(data)) + + local entry = data.data.result[1] + assert(entry.stream.request_headers_test_header == "only-for-test#2", + "expected field request_headers_test_header value: " .. cjson.encode(entry)) + } + } +--- error_code: 200 + + + +=== TEST 8: setup route (with tenant_id) +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "loki-logger": { + "endpoint_addrs": ["http://127.0.0.1:3100"], + "tenant_id": "tenant_2", + "batch_max_size": 1 + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 9: hit route +--- request +GET /hello +--- more_headers +test-header: only-for-test#3 +--- response_body +hello world + + + +=== TEST 10: check loki log (with tenant_id tenant_1) +--- config + location /t { + content_by_lua_block { + local cjson = require("cjson") + local now = ngx.now() * 1000 + local data, err = require("lib.grafana_loki").fetch_logs_from_loki( + tostring(now - 10000) .. "000000", -- from + tostring(now) .. "000000" -- to + ) + + assert(err == nil, "fetch logs error: " .. (err or "")) + assert(data.status == "success", "loki response error: " .. cjson.encode(data)) + assert(#data.data.result > 0, "loki log empty: " .. cjson.encode(data)) + + local entry = data.data.result[1] + assert(entry.stream.request_headers_test_header ~= "only-for-test#3", + "expected field request_headers_test_header value: " .. cjson.encode(entry)) + } + } +--- error_code: 200 + + + +=== TEST 11: check loki log (with tenant_id tenant_2) +--- config + location /t { + content_by_lua_block { + local cjson = require("cjson") + local now = ngx.now() * 1000 + local data, err = require("lib.grafana_loki").fetch_logs_from_loki( + tostring(now - 3000) .. "000000", -- from + tostring(now) .. "000000", -- to + { headers = { + ["X-Scope-OrgID"] = "tenant_2" + } } + ) + + assert(err == nil, "fetch logs error: " .. (err or "")) + assert(data.status == "success", "loki response error: " .. cjson.encode(data)) + assert(#data.data.result > 0, "loki log empty: " .. cjson.encode(data)) + + local entry = data.data.result[1] + assert(entry.stream.request_headers_test_header == "only-for-test#3", + "expected field request_headers_test_header value: " .. cjson.encode(entry)) + } + } +--- error_code: 200