From bd8c57976a80d9393635065887559844baaeb29c Mon Sep 17 00:00:00 2001
From: xinchenyuan <xinchenyuan@tencent.com>
Date: Sat, 30 Jul 2022 23:58:38 +0800
Subject: [PATCH 01/24] add tencent-cloud-cls

add tencent-cloud-cls

add tencent-cloud-cls

add tencent-cloud-cls

add tencent-cloud-cls

add tencent-cloud-cls
---
 apisix/plugins/tencent-cloud-cls.lua         |  69 +++++++
 apisix/plugins/tencent-cloud-cls/cls-sdk.lua | 193 +++++++++++++++++++
 apisix/plugins/tencent-cloud-cls/cls.pb      |  20 ++
 apisix/plugins/tencent-cloud-cls/cls.proto   |  34 ++++
 4 files changed, 316 insertions(+)
 create mode 100644 apisix/plugins/tencent-cloud-cls.lua
 create mode 100644 apisix/plugins/tencent-cloud-cls/cls-sdk.lua
 create mode 100644 apisix/plugins/tencent-cloud-cls/cls.pb
 create mode 100644 apisix/plugins/tencent-cloud-cls/cls.proto

diff --git a/apisix/plugins/tencent-cloud-cls.lua b/apisix/plugins/tencent-cloud-cls.lua
new file mode 100644
index 000000000000..e63091f7dad3
--- /dev/null
+++ b/apisix/plugins/tencent-cloud-cls.lua
@@ -0,0 +1,69 @@
+local core = require("apisix.core")
+local log_util = require("apisix.utils.log-util")
+local bp_manager_mod = require("apisix.utils.batch-processor-manager")
+local cls_sdk = require("apisix.plugins.tencent-cloud-cls.cls-sdk")
+local random = math.random
+local ngx = ngx
+math.randomseed(ngx.time() + ngx.worker.pid())
+
+local plugin_name = "tencent-cloud-cls"
+local batch_processor_manager = bp_manager_mod.new(plugin_name)
+local schema = {
+    type = "object",
+    properties = {
+        cls_host = { type = "string" },
+        cls_topic = { type = "string" },
+        -- https://console.cloud.tencent.com/capi
+        secret_id = { type = "string" },
+        secret_key = { type = "string" },
+        sample_rate = { type = "integer", minimum = 1, maximum = 100, default = 100 },
+        include_req_body = { type = "boolean", default = false },
+        include_resp_body = { type = "boolean", default = false },
+    },
+    required = { "cls_host", "cls_topic", "secret_id", "secret_key" }
+}
+
+local _M = {
+    version = 0.1,
+    priority = 413,
+    name = plugin_name,
+    schema = batch_processor_manager:wrap_schema(schema),
+}
+
+function _M.check_schema(conf)
+    return core.schema.check(schema, conf)
+end
+
+function _M.body_filter(conf, ctx)
+    -- sample if set
+    if conf.sample_rate < 100 and random(1, 100) > conf.sample_rate then
+        core.log.debug("not sampled")
+        return
+    end
+    log_util.collect_body(conf, ctx)
+    ctx.cls_sample = true
+end
+
+function _M.log(conf, ctx)
+    -- sample if set
+    if ctx.cls_sample == nil then
+        core.log.debug("not sampled")
+        return
+    end
+    local entry = log_util.get_full_log(ngx, conf)
+    if not entry.route_id then
+        entry.route_id = "no-matched"
+    end
+
+    if batch_processor_manager:add_entry(conf, entry) then
+        return
+    end
+
+    local process = function(entries)
+        return cls_sdk.send_to_cls(conf.secret_id, conf.secret_key, conf.cls_host, conf.cls_topic, entries)
+    end
+
+    batch_processor_manager:add_entry_to_new_processor(conf, entry, ctx, process)
+end
+
+return _M
diff --git a/apisix/plugins/tencent-cloud-cls/cls-sdk.lua b/apisix/plugins/tencent-cloud-cls/cls-sdk.lua
new file mode 100644
index 000000000000..7b65fe037e4c
--- /dev/null
+++ b/apisix/plugins/tencent-cloud-cls/cls-sdk.lua
@@ -0,0 +1,193 @@
+-- https://cloud.tencent.com/document/api/614/16873
+local pb = require "pb"
+assert(pb.loadfile("apisix/plugins/tencent-cloud-cls/cls.pb"))
+local http = require("resty.http")
+local socket = require("socket")
+local str_util = require("resty.string")
+local core = require("apisix.core")
+local json = core.json
+local json_encode = json.encode
+
+local ngx = ngx
+local ngx_time = ngx.time
+local ngx_now = ngx.now
+local ngx_sha1_bin = ngx.sha1_bin
+local ngx_hmac_sha1 = ngx.hmac_sha1
+
+local fmt = string.format
+local concat_tab = table.concat
+local clear_tab = table.clear
+local new_tab = table.new
+local insert_tab = table.insert
+
+local MAX_SINGLE_VALUE_SIZE = 1 * 1024 * 1024
+local MAX_LOG_GROUP_VALUE_SIZE = 5 * 1024 * 1024 -- 5MB
+
+local cls_api_path = "/structuredlog"
+local auth_expire_time = 60
+local cls_conn_timeout = 1000
+local cls_read_timeout = 10000
+local cls_send_timeout = 10000
+
+local headers_cache = {}
+local params_cache = {
+    ssl_verify = false,
+    headers = headers_cache,
+}
+
+local function get_ip(hostname)
+    local _, resolved = socket.dns.toip(hostname)
+    local ListTab = {}
+    for _, v in ipairs(resolved.ip) do
+        table.insert(ListTab, v)
+    end
+    return ListTab
+end
+
+local host_ip = tostring(unpack(get_ip(socket.dns.gethostname())))
+local log_group_list = {}
+local log_group_list_pb = {
+    logGroupList = log_group_list,
+}
+
+local function sha1(msg)
+    return str_util.to_hex(ngx_sha1_bin(msg))
+end
+
+local function sha1_hmac(key, msg)
+    return str_util.to_hex(ngx_hmac_sha1(key, msg))
+end
+
+-- sign algorithm https://cloud.tencent.com/document/product/614/12445
+local function sign(secret_id, secret_key)
+    local method = "post"
+    local format_params = ""
+    local format_headers = ""
+    local sign_algorithm = "sha1"
+    local http_request_info = fmt("%s\n%s\n%s\n%s\n", method, cls_api_path, format_params, format_headers)
+    local cur_time = ngx_time()
+    local sign_time = fmt("%d;%d", cur_time, cur_time + auth_expire_time)
+    local string_to_sign = fmt("%s\n%s\n%s\n", sign_algorithm, sign_time, sha1(http_request_info))
+
+    local sign_key = sha1_hmac(secret_key, sign_time)
+    local signature = sha1_hmac(sign_key, string_to_sign)
+
+    local arr = {
+        "q-sign-algorithm=sha1",
+        "q-ak=" .. secret_id,
+        "q-sign-time=" .. sign_time,
+        "q-key-time=" .. sign_time,
+        "q-header-list=",
+        "q-url-param-list=",
+        "q-signature=" .. signature,
+    }
+
+    return concat_tab(arr, '&')
+end
+
+local function send_cls_request(host, topic, secret_id, secret_key, pb_data)
+    local http_new = http:new()
+    http_new:set_timeouts(cls_conn_timeout, cls_send_timeout, cls_read_timeout)
+
+    clear_tab(headers_cache)
+    headers_cache["Host"] = host
+    headers_cache["Content-Type"] = "application/x-protobuf"
+    headers_cache["Authorization"] = sign(secret_id, secret_key, cls_api_path)
+
+    -- TODO: support lz4/zstd compress
+    params_cache.method = "POST"
+    params_cache.body = pb_data
+
+    local cls_url = "http://" .. host .. cls_api_path .. "?topic_id=" .. topic
+    core.log.debug("CLS request URL: ", cls_url)
+
+    local res, err = http_new:request_uri(cls_url, params_cache)
+    if not res then
+        return false, err
+    end
+
+    if res.status ~= 200 then
+        err = fmt("got wrong status: %s, headers: %s, body, %s", res.status, json.encode(res.headers), res.body)
+        -- 413, 404, 401, 403 are not retryable
+        if res.status == 413 or res.status == 404 or res.status == 401 or res.status == 403 then
+            core.log.err(err, ", not retryable")
+            return true
+        end
+
+        return false, err
+    end
+
+    core.log.debug("CLS report success")
+    return true
+end
+
+-- normalized log data for CLS API
+local function normalize_log(log)
+    local normalized_log = {}
+    local log_size = 4 -- empty obj alignment
+    for k, v in pairs(log) do
+        local v_type = type(v)
+        local field = { key = k, value = "" }
+        if v_type == "string" then
+            field["value"] = v
+        elseif v_type == "number" then
+            field["value"] = tostring(v)
+        elseif v_type == "table" then
+            field["value"] = json_encode(v)
+        else
+            field["value"] = tostring(v)
+            core.log.warn("unexpected type " .. v_type .. " for field " .. k)
+        end
+        if #field.value > MAX_SINGLE_VALUE_SIZE then
+            core.log.warn(field.key, " value size over ", MAX_SINGLE_VALUE_SIZE, " , truncated")
+            field.value = field.value:sub(1, MAX_SINGLE_VALUE_SIZE)
+        end
+        insert_tab(normalized_log, field)
+        log_size = log_size + #field.key + #field.value
+    end
+    return normalized_log, log_size
+end
+
+local function send_to_cls(secret_id, secret_key, host, topic_id, logs)
+    clear_tab(log_group_list)
+    local now = ngx_now() * 1000
+
+    local total_size = 0
+    local format_logs = new_tab(#logs, 0)
+    -- sums of all value in a LogGroup should be no more than 5MB
+    for i = 1, #logs, 1 do
+        local contents, log_size = normalize_log(logs[i])
+        if log_size > MAX_LOG_GROUP_VALUE_SIZE then
+            core.log.error("size of log is over 5MB, dropped")
+            goto continue
+        end
+        total_size = total_size + log_size
+        if total_size > MAX_LOG_GROUP_VALUE_SIZE then
+            insert_tab(log_group_list, {
+                logs = format_logs,
+                source = host_ip,
+            })
+            format_logs = new_tab(#logs - i, 0)
+            total_size = 0
+            local data = assert(pb.encode("cls.LogGroupList", log_group_list_pb))
+            send_cls_request(host, topic_id, secret_id, secret_key, data)
+            clear_tab(log_group_list)
+        end
+        insert_tab(format_logs, {
+            time = now,
+            contents = contents,
+        })
+        :: continue ::
+    end
+
+    insert_tab(log_group_list, {
+        logs = format_logs,
+        source = host_ip,
+    })
+    local data = assert(pb.encode("cls.LogGroupList", log_group_list_pb))
+    return send_cls_request(host, topic_id, secret_id, secret_key, data)
+end
+
+return {
+    send_to_cls = send_to_cls
+}
diff --git a/apisix/plugins/tencent-cloud-cls/cls.pb b/apisix/plugins/tencent-cloud-cls/cls.pb
new file mode 100644
index 000000000000..41c84cb31e35
--- /dev/null
+++ b/apisix/plugins/tencent-cloud-cls/cls.pb
@@ -0,0 +1,20 @@
+
+�
+	cls.protocls"~
+Log
+time
 (B0
Rtime,
+contents (2.cls.Log.ContentRcontents1
+Content
+key
 (	Rkey
+value (	Rvalue"0
+LogTag
+key
 (	Rkey
+value (	Rvalue"�

+LogGroup
+logs
 (2.cls.LogRlogs 
+contextFlow 
(	RcontextFlow
+filename 
(	Rfilename
+source 
(	Rsource%
+logTags (2.cls.LogTagRlogTags"A
+LogGroupList1
+logGroupList
 (2
.cls.LogGroupRlogGroupList
\ No newline at end of file
diff --git a/apisix/plugins/tencent-cloud-cls/cls.proto b/apisix/plugins/tencent-cloud-cls/cls.proto
new file mode 100644
index 000000000000..ffa3f6bc24d0
--- /dev/null
+++ b/apisix/plugins/tencent-cloud-cls/cls.proto
@@ -0,0 +1,34 @@
+syntax = "proto2";
+
+package cls;
+
+message Log
+{
+  required int64 time = 1 [jstype = JS_STRING];
+  message Content
+  {
+    required string key = 1;
+    required string value = 2;
+  }
+  repeated Content contents = 2;
+}
+
+message LogTag
+{
+  required string key = 1;
+  required string value = 2;
+}
+
+message LogGroup
+{
+  repeated Log logs = 1;
+  optional string contextFlow = 2;
+  optional string filename = 3;
+  optional string source = 4;
+  repeated LogTag logTags = 5;
+}
+
+message LogGroupList
+{
+  repeated LogGroup logGroupList = 1;
+}

From 3b370ef7b6745e82b0772006d42e87027c2b8055 Mon Sep 17 00:00:00 2001
From: xinchenyuan <xinchenyuan@tencent.com>
Date: Wed, 3 Aug 2022 14:55:49 +0800
Subject: [PATCH 02/24] add tencent-cloud-cls

---
 apisix/plugins/tencent-cloud-cls.lua         | 17 ++++++++
 apisix/plugins/tencent-cloud-cls/cls-sdk.lua | 18 ++++++++-
 apisix/plugins/tencent-cloud-cls/cls.proto   | 42 ++++++++++++++------
 3 files changed, 63 insertions(+), 14 deletions(-)

diff --git a/apisix/plugins/tencent-cloud-cls.lua b/apisix/plugins/tencent-cloud-cls.lua
index e63091f7dad3..8137b177556a 100644
--- a/apisix/plugins/tencent-cloud-cls.lua
+++ b/apisix/plugins/tencent-cloud-cls.lua
@@ -1,3 +1,20 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements.  See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License.  You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
 local core = require("apisix.core")
 local log_util = require("apisix.utils.log-util")
 local bp_manager_mod = require("apisix.utils.batch-processor-manager")
diff --git a/apisix/plugins/tencent-cloud-cls/cls-sdk.lua b/apisix/plugins/tencent-cloud-cls/cls-sdk.lua
index 7b65fe037e4c..a6a77bb187d4 100644
--- a/apisix/plugins/tencent-cloud-cls/cls-sdk.lua
+++ b/apisix/plugins/tencent-cloud-cls/cls-sdk.lua
@@ -1,4 +1,20 @@
--- https://cloud.tencent.com/document/api/614/16873
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements.  See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License.  You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
 local pb = require "pb"
 assert(pb.loadfile("apisix/plugins/tencent-cloud-cls/cls.pb"))
 local http = require("resty.http")
diff --git a/apisix/plugins/tencent-cloud-cls/cls.proto b/apisix/plugins/tencent-cloud-cls/cls.proto
index ffa3f6bc24d0..0029be11ba58 100644
--- a/apisix/plugins/tencent-cloud-cls/cls.proto
+++ b/apisix/plugins/tencent-cloud-cls/cls.proto
@@ -1,34 +1,50 @@
-syntax = "proto2";
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
 
+// https://cloud.tencent.com/document/product/614/59470
 package cls;
 
 message Log
 {
-  required int64 time = 1 [jstype = JS_STRING];
   message Content
   {
-    required string key = 1;
-    required string value = 2;
+    required string key   = 1; // 每组字段的 key
+    required string value = 2; // 每组字段的 value
   }
-  repeated Content contents = 2;
+  required int64   time     = 1; // 时间戳，UNIX时间格式
+  repeated Content contents = 2; // 一条日志里的多个kv组合
 }
 
 message LogTag
 {
-  required string key = 1;
-  required string value = 2;
+  required string key       = 1;
+  required string value     = 2;
 }
 
 message LogGroup
 {
-  repeated Log logs = 1;
-  optional string contextFlow = 2;
-  optional string filename = 3;
-  optional string source = 4;
-  repeated LogTag logTags = 5;
+  repeated Log    logs        = 1; // 多条日志合成的日志数组
+  optional string contextFlow = 2; // 目前暂无效用
+  optional string filename    = 3; // 日志文件名
+  optional string source      = 4; // 日志来源，一般使用机器IP
+  repeated LogTag logTags     = 5;
 }
 
 message LogGroupList
 {
-  repeated LogGroup logGroupList = 1;
+  repeated LogGroup logGroupList = 1; // 日志组列表
 }

From 69e952b1534b4fa04585cb77eecbbf4f47f54472 Mon Sep 17 00:00:00 2001
From: xinchenyuan <xinchenyuan@tencent.com>
Date: Wed, 3 Aug 2022 19:47:01 +0800
Subject: [PATCH 03/24] add ut

add ut
---
 apisix/plugins/tencent-cloud-cls.lua         |   2 +-
 apisix/plugins/tencent-cloud-cls/cls-sdk.lua |   2 +-
 conf/config-default.yaml                     |   1 +
 t/plugin/tencent-cloud-cls.t                 | 131 +++++++++++++++++++
 4 files changed, 134 insertions(+), 2 deletions(-)
 create mode 100644 t/plugin/tencent-cloud-cls.t

diff --git a/apisix/plugins/tencent-cloud-cls.lua b/apisix/plugins/tencent-cloud-cls.lua
index 8137b177556a..a0c8ad9127fe 100644
--- a/apisix/plugins/tencent-cloud-cls.lua
+++ b/apisix/plugins/tencent-cloud-cls.lua
@@ -42,7 +42,7 @@ local schema = {
 
 local _M = {
     version = 0.1,
-    priority = 413,
+    priority = 397,
     name = plugin_name,
     schema = batch_processor_manager:wrap_schema(schema),
 }
diff --git a/apisix/plugins/tencent-cloud-cls/cls-sdk.lua b/apisix/plugins/tencent-cloud-cls/cls-sdk.lua
index a6a77bb187d4..979935d6ab39 100644
--- a/apisix/plugins/tencent-cloud-cls/cls-sdk.lua
+++ b/apisix/plugins/tencent-cloud-cls/cls-sdk.lua
@@ -126,7 +126,7 @@ local function send_cls_request(host, topic, secret_id, secret_key, pb_data)
         err = fmt("got wrong status: %s, headers: %s, body, %s", res.status, json.encode(res.headers), res.body)
         -- 413, 404, 401, 403 are not retryable
         if res.status == 413 or res.status == 404 or res.status == 401 or res.status == 403 then
-            core.log.err(err, ", not retryable")
+            core.log.error(err, ", not retryable")
             return true
         end
 
diff --git a/conf/config-default.yaml b/conf/config-default.yaml
index 0c088d144785..970a635c971c 100644
--- a/conf/config-default.yaml
+++ b/conf/config-default.yaml
@@ -463,6 +463,7 @@ plugins:                          # plugin list (sorted by priority)
   - udp-logger                     # priority: 400
   - file-logger                    # priority: 399
   - clickhouse-logger              # priority: 398
+  - tencent-cloud-cls              # priority: 397
   #- log-rotate                    # priority: 100
   # <- recommend to use priority (0, 100) for your custom plugins
   - example-plugin                 # priority: 0
diff --git a/t/plugin/tencent-cloud-cls.t b/t/plugin/tencent-cloud-cls.t
new file mode 100644
index 000000000000..ab8c2f3253b5
--- /dev/null
+++ b/t/plugin/tencent-cloud-cls.t
@@ -0,0 +1,131 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+use t::APISIX 'no_plan';
+
+log_level('debug');
+repeat_each(1);
+no_long_string();
+no_root_location();
+
+add_block_preprocessor(sub {
+    my ($block) = @_;
+
+    if ((!defined $block->error_log) && (!defined $block->no_error_log)) {
+        $block->set_value("no_error_log", "[error]");
+    }
+
+    if (!defined $block->request) {
+        $block->set_value("request", "GET /t");
+    }
+
+});
+
+run_tests;
+
+__DATA__
+
+=== TEST 1: schema check
+--- config
+    location /t {
+        content_by_lua_block {
+            local plugin = require("apisix.plugins.tencent-cloud-cls")
+            local ok, err = plugin.check_schema({
+                cls_host = "ap-guangzhou.cls.tencentyun.com",
+                cls_topic = "143b5d70-139b-4aec-b54e-bb97756916de",
+                secret_id = "secret_id",
+                secret_key = "secret_key",
+            })
+            if not ok then
+                ngx.say(err)
+            end
+
+            ngx.say("done")
+        }
+    }
+--- response_body
+done
+
+
+
+=== TEST 2: cls config missing
+--- config
+    location /t {
+        content_by_lua_block {
+            local plugin = require("apisix.plugins.tencent-cloud-cls")
+            local ok, err = plugin.check_schema({
+                cls_host = "ap-guangzhou.cls.tencentyun.com",
+                cls_topic = "143b5d70-139b-4aec-b54e-bb97756916de",
+                secret_id = "secret_id",
+            })
+            if not ok then
+                ngx.say(err)
+            end
+
+            ngx.say("done")
+        }
+    }
+--- response_body
+property "secret_key" is required
+done
+
+
+
+=== TEST 3: add plugin
+--- config
+    location /t {
+        content_by_lua_block {
+            local t = require("lib.test_admin").test
+            local code, body = t('/apisix/admin/routes/1',
+                 ngx.HTTP_PUT,
+                 [[{
+                        "plugins": {
+                            "tencent-cloud-cls": {
+                                "cls_host": "ap-guangzhou.cls.tencentyun.com",
+                                "cls_topic": "143b5d70-139b-4aec-b54e-bb97756916de",
+                                "secret_id": "secret_id",
+                                "secret_key": "secret_key"
+                            }
+                        },
+                        "upstream": {
+                            "nodes": {
+                                "127.0.0.1:1982": 1
+                            },
+                            "type": "roundrobin"
+                        },
+                        "uri": "/opentracing"
+                }]]
+                )
+
+            if code >= 300 then
+                ngx.status = code
+            end
+            ngx.say(body)
+        }
+    }
+--- response_body
+passed
+
+
+
+=== TEST 4: access local server
+--- request
+GET /opentracing
+--- response_body
+opentracing
+--- error_log
+Batch Processor[tencent-cloud-cls] successfully processed the entries
+--- wait: 0.5

From f57cc30318a716c901bace68d697604e311a9e19 Mon Sep 17 00:00:00 2001
From: xinchenyuan <xinchenyuan@tencent.com>
Date: Thu, 4 Aug 2022 09:11:38 +0800
Subject: [PATCH 04/24] fix lint

fix lint

fix lint
---
 apisix/plugins/tencent-cloud-cls.lua         |  3 ++-
 apisix/plugins/tencent-cloud-cls/cls-sdk.lua | 14 +++++++++++---
 2 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/apisix/plugins/tencent-cloud-cls.lua b/apisix/plugins/tencent-cloud-cls.lua
index a0c8ad9127fe..9aaa7d68d96c 100644
--- a/apisix/plugins/tencent-cloud-cls.lua
+++ b/apisix/plugins/tencent-cloud-cls.lua
@@ -77,7 +77,8 @@ function _M.log(conf, ctx)
     end
 
     local process = function(entries)
-        return cls_sdk.send_to_cls(conf.secret_id, conf.secret_key, conf.cls_host, conf.cls_topic, entries)
+        return cls_sdk.send_to_cls(conf.secret_id, conf.secret_key,
+                                   conf.cls_host, conf.cls_topic, entries)
     end
 
     batch_processor_manager:add_entry_to_new_processor(conf, entry, ctx, process)
diff --git a/apisix/plugins/tencent-cloud-cls/cls-sdk.lua b/apisix/plugins/tencent-cloud-cls/cls-sdk.lua
index 979935d6ab39..faa1a7adf854 100644
--- a/apisix/plugins/tencent-cloud-cls/cls-sdk.lua
+++ b/apisix/plugins/tencent-cloud-cls/cls-sdk.lua
@@ -16,6 +16,7 @@
 --
 
 local pb = require "pb"
+local assert = assert
 assert(pb.loadfile("apisix/plugins/tencent-cloud-cls/cls.pb"))
 local http = require("resty.http")
 local socket = require("socket")
@@ -31,10 +32,15 @@ local ngx_sha1_bin = ngx.sha1_bin
 local ngx_hmac_sha1 = ngx.hmac_sha1
 
 local fmt = string.format
+local table = table
 local concat_tab = table.concat
 local clear_tab = table.clear
 local new_tab = table.new
 local insert_tab = table.insert
+local ipairs = ipairs
+local pairs = pairs
+local type = type
+local tostring = tostring
 
 local MAX_SINGLE_VALUE_SIZE = 1 * 1024 * 1024
 local MAX_LOG_GROUP_VALUE_SIZE = 5 * 1024 * 1024 -- 5MB
@@ -55,7 +61,7 @@ local function get_ip(hostname)
     local _, resolved = socket.dns.toip(hostname)
     local ListTab = {}
     for _, v in ipairs(resolved.ip) do
-        table.insert(ListTab, v)
+        insert_tab(ListTab, v)
     end
     return ListTab
 end
@@ -80,7 +86,8 @@ local function sign(secret_id, secret_key)
     local format_params = ""
     local format_headers = ""
     local sign_algorithm = "sha1"
-    local http_request_info = fmt("%s\n%s\n%s\n%s\n", method, cls_api_path, format_params, format_headers)
+    local http_request_info = fmt("%s\n%s\n%s\n%s\n",
+                                  method, cls_api_path, format_params, format_headers)
     local cur_time = ngx_time()
     local sign_time = fmt("%d;%d", cur_time, cur_time + auth_expire_time)
     local string_to_sign = fmt("%s\n%s\n%s\n", sign_algorithm, sign_time, sha1(http_request_info))
@@ -123,7 +130,8 @@ local function send_cls_request(host, topic, secret_id, secret_key, pb_data)
     end
 
     if res.status ~= 200 then
-        err = fmt("got wrong status: %s, headers: %s, body, %s", res.status, json.encode(res.headers), res.body)
+        err = fmt("got wrong status: %s, headers: %s, body, %s",
+                  res.status, json.encode(res.headers), res.body)
         -- 413, 404, 401, 403 are not retryable
         if res.status == 413 or res.status == 404 or res.status == 401 or res.status == 403 then
             core.log.error(err, ", not retryable")

From 136e2129d732c52b5087f9b23710a86fbf0842bf Mon Sep 17 00:00:00 2001
From: xinchenyuan <xinchenyuan@tencent.com>
Date: Thu, 4 Aug 2022 09:54:58 +0800
Subject: [PATCH 05/24] add global tag

---
 apisix/plugins/tencent-cloud-cls.lua | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/apisix/plugins/tencent-cloud-cls.lua b/apisix/plugins/tencent-cloud-cls.lua
index 9aaa7d68d96c..599ec13ff179 100644
--- a/apisix/plugins/tencent-cloud-cls.lua
+++ b/apisix/plugins/tencent-cloud-cls.lua
@@ -20,8 +20,9 @@ local log_util = require("apisix.utils.log-util")
 local bp_manager_mod = require("apisix.utils.batch-processor-manager")
 local cls_sdk = require("apisix.plugins.tencent-cloud-cls.cls-sdk")
 local random = math.random
-local ngx = ngx
 math.randomseed(ngx.time() + ngx.worker.pid())
+local ngx = ngx
+local pairs = pairs
 
 local plugin_name = "tencent-cloud-cls"
 local batch_processor_manager = bp_manager_mod.new(plugin_name)
@@ -36,6 +37,7 @@ local schema = {
         sample_rate = { type = "integer", minimum = 1, maximum = 100, default = 100 },
         include_req_body = { type = "boolean", default = false },
         include_resp_body = { type = "boolean", default = false },
+        global_tag = { type = "object" },
     },
     required = { "cls_host", "cls_topic", "secret_id", "secret_key" }
 }
@@ -71,6 +73,11 @@ function _M.log(conf, ctx)
     if not entry.route_id then
         entry.route_id = "no-matched"
     end
+    if conf.global_tag ~= nil then
+        for k, v in pairs(conf.global_tag) do
+            entry[k] = v
+        end
+    end
 
     if batch_processor_manager:add_entry(conf, entry) then
         return

From 4e0e8ffaef2f92102ad6f73d07df2c869ff9f957 Mon Sep 17 00:00:00 2001
From: xinchenyuan <xinchenyuan@tencent.com>
Date: Thu, 4 Aug 2022 14:50:27 +0800
Subject: [PATCH 06/24] add tencent-cloud-cls

---
 apisix/plugins/tencent-cloud-cls/cls-sdk.lua | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/apisix/plugins/tencent-cloud-cls/cls-sdk.lua b/apisix/plugins/tencent-cloud-cls/cls-sdk.lua
index faa1a7adf854..ab4f0d40c558 100644
--- a/apisix/plugins/tencent-cloud-cls/cls-sdk.lua
+++ b/apisix/plugins/tencent-cloud-cls/cls-sdk.lua
@@ -22,6 +22,7 @@ local http = require("resty.http")
 local socket = require("socket")
 local str_util = require("resty.string")
 local core = require("apisix.core")
+local core_gethostname = require("apisix.core.utils").gethostname
 local json = core.json
 local json_encode = json.encode
 
@@ -66,7 +67,7 @@ local function get_ip(hostname)
     return ListTab
 end
 
-local host_ip = tostring(unpack(get_ip(socket.dns.gethostname())))
+local host_ip = tostring(unpack(get_ip(core_gethostname())))
 local log_group_list = {}
 local log_group_list_pb = {
     logGroupList = log_group_list,

From e969b5e287d3075e5881cb29b0b2babf05f14257 Mon Sep 17 00:00:00 2001
From: xinchenyuan <xinchenyuan@tencent.com>
Date: Fri, 5 Aug 2022 09:34:01 +0800
Subject: [PATCH 07/24] add custom log format

---
 apisix/plugins/tencent-cloud-cls.lua | 56 ++++++++++++++++++++++++----
 1 file changed, 49 insertions(+), 7 deletions(-)

diff --git a/apisix/plugins/tencent-cloud-cls.lua b/apisix/plugins/tencent-cloud-cls.lua
index 599ec13ff179..e2a69aa38b2e 100644
--- a/apisix/plugins/tencent-cloud-cls.lua
+++ b/apisix/plugins/tencent-cloud-cls.lua
@@ -24,6 +24,7 @@ math.randomseed(ngx.time() + ngx.worker.pid())
 local ngx = ngx
 local pairs = pairs
 
+
 local plugin_name = "tencent-cloud-cls"
 local batch_processor_manager = bp_manager_mod.new(plugin_name)
 local schema = {
@@ -42,38 +43,78 @@ local schema = {
     required = { "cls_host", "cls_topic", "secret_id", "secret_key" }
 }
 
+
+local metadata_schema = {
+    type = "object",
+    properties = {
+        log_format = log_util.metadata_schema_log_format,
+    },
+}
+
+
 local _M = {
     version = 0.1,
     priority = 397,
     name = plugin_name,
     schema = batch_processor_manager:wrap_schema(schema),
+    metadata_schema = metadata_schema,
 }
 
-function _M.check_schema(conf)
-    return core.schema.check(schema, conf)
+
+function _M.check_schema(conf, schema_type)
+    if schema_type == core.schema.TYPE_METADATA then
+        return core.schema.check(metadata_schema, conf)
+    end
+
+    local ok, err = core.schema.check(schema, conf)
+    if not ok then
+        return nil, err
+    end
+    return log_util.check_log_schema(conf)
 end
 
-function _M.body_filter(conf, ctx)
+
+function _M.access(conf, ctx)
     -- sample if set
     if conf.sample_rate < 100 and random(1, 100) > conf.sample_rate then
         core.log.debug("not sampled")
         return
     end
-    log_util.collect_body(conf, ctx)
     ctx.cls_sample = true
 end
 
+
+function _M.body_filter(conf, ctx)
+    if ctx.cls_sample then
+        log_util.collect_body(conf, ctx)
+    end
+end
+
+
 function _M.log(conf, ctx)
     -- sample if set
-    if ctx.cls_sample == nil then
+    if not ctx.cls_sample then
         core.log.debug("not sampled")
         return
     end
-    local entry = log_util.get_full_log(ngx, conf)
+    local metadata = plugin.plugin_metadata(plugin_name)
+    core.log.info("metadata: ", core.json.delay_encode(metadata))
+
+    local entry
+
+    if metadata and metadata.value.log_format
+            and core.table.nkeys(metadata.value.log_format) > 0
+    then
+        entry = log_util.get_custom_format_log(ctx, metadata.value.log_format)
+    else
+        entry = log_util.get_full_log(ngx, conf)
+    end
+
     if not entry.route_id then
         entry.route_id = "no-matched"
     end
-    if conf.global_tag ~= nil then
+
+    if conf.global_tag then
         for k, v in pairs(conf.global_tag) do
             entry[k] = v
         end
@@ -91,4 +132,5 @@ function _M.log(conf, ctx)
     batch_processor_manager:add_entry_to_new_processor(conf, entry, ctx, process)
 end
 
+
 return _M

From b9ee01cfbdbea22ff04019999f2ea117d93512c9 Mon Sep 17 00:00:00 2001
From: xinchenyuan <xinchenyuan@tencent.com>
Date: Fri, 5 Aug 2022 10:02:41 +0800
Subject: [PATCH 08/24] change random

---
 apisix/plugins/tencent-cloud-cls.lua         | 24 +++++++++++---------
 apisix/plugins/tencent-cloud-cls/cls-sdk.lua | 15 ++++++++----
 2 files changed, 23 insertions(+), 16 deletions(-)

diff --git a/apisix/plugins/tencent-cloud-cls.lua b/apisix/plugins/tencent-cloud-cls.lua
index e2a69aa38b2e..2a6190b49e4f 100644
--- a/apisix/plugins/tencent-cloud-cls.lua
+++ b/apisix/plugins/tencent-cloud-cls.lua
@@ -19,8 +19,7 @@ local core = require("apisix.core")
 local log_util = require("apisix.utils.log-util")
 local bp_manager_mod = require("apisix.utils.batch-processor-manager")
 local cls_sdk = require("apisix.plugins.tencent-cloud-cls.cls-sdk")
-local random = math.random
-math.randomseed(ngx.time() + ngx.worker.pid())
+local math = math
 local ngx = ngx
 local pairs = pairs
 
@@ -35,7 +34,12 @@ local schema = {
         -- https://console.cloud.tencent.com/capi
         secret_id = { type = "string" },
         secret_key = { type = "string" },
-        sample_rate = { type = "integer", minimum = 1, maximum = 100, default = 100 },
+        sample_ratio = {
+            type = "number",
+            minimum = 0.00001,
+            maximum = 1,
+            default = 1
+        },
         include_req_body = { type = "boolean", default = false },
         include_resp_body = { type = "boolean", default = false },
         global_tag = { type = "object" },
@@ -76,11 +80,13 @@ end
 
 function _M.access(conf, ctx)
     -- sample if set
-    if conf.sample_rate < 100 and random(1, 100) > conf.sample_rate then
-        core.log.debug("not sampled")
+    ctx.cls_sample = false
+    if conf.sample_ratio == 1 or math.random() < conf.sample_ratio then
+        core.log.debug("cls sampled")
+        ctx.cls_sample = true
         return
     end
-    ctx.cls_sample = true
+    core.log.debug("cls not sampled")
 end
 
 
@@ -94,7 +100,7 @@ end
 function _M.log(conf, ctx)
     -- sample if set
     if not ctx.cls_sample then
-        core.log.debug("not sampled")
+        core.log.debug("cls not sampled, skip log")
         return
     end
     local metadata = plugin.plugin_metadata(plugin_name)
@@ -110,10 +116,6 @@ function _M.log(conf, ctx)
         entry = log_util.get_full_log(ngx, conf)
     end
 
-    if not entry.route_id then
-        entry.route_id = "no-matched"
-    end
-
     if conf.global_tag then
         for k, v in pairs(conf.global_tag) do
             entry[k] = v
diff --git a/apisix/plugins/tencent-cloud-cls/cls-sdk.lua b/apisix/plugins/tencent-cloud-cls/cls-sdk.lua
index ab4f0d40c558..2a53abcefde6 100644
--- a/apisix/plugins/tencent-cloud-cls/cls-sdk.lua
+++ b/apisix/plugins/tencent-cloud-cls/cls-sdk.lua
@@ -25,13 +25,11 @@ local core = require("apisix.core")
 local core_gethostname = require("apisix.core.utils").gethostname
 local json = core.json
 local json_encode = json.encode
-
 local ngx = ngx
 local ngx_time = ngx.time
 local ngx_now = ngx.now
 local ngx_sha1_bin = ngx.sha1_bin
 local ngx_hmac_sha1 = ngx.hmac_sha1
-
 local fmt = string.format
 local table = table
 local concat_tab = table.concat
@@ -58,13 +56,14 @@ local params_cache = {
     headers = headers_cache,
 }
 
+
 local function get_ip(hostname)
     local _, resolved = socket.dns.toip(hostname)
-    local ListTab = {}
+    local ip_list = {}
     for _, v in ipairs(resolved.ip) do
-        insert_tab(ListTab, v)
+        insert_tab(ip_list, v)
     end
-    return ListTab
+    return ip_list
 end
 
 local host_ip = tostring(unpack(get_ip(core_gethostname())))
@@ -73,14 +72,17 @@ local log_group_list_pb = {
     logGroupList = log_group_list,
 }
 
+
 local function sha1(msg)
     return str_util.to_hex(ngx_sha1_bin(msg))
 end
 
+
 local function sha1_hmac(key, msg)
     return str_util.to_hex(ngx_hmac_sha1(key, msg))
 end
 
+
 -- sign algorithm https://cloud.tencent.com/document/product/614/12445
 local function sign(secret_id, secret_key)
     local method = "post"
@@ -109,6 +111,7 @@ local function sign(secret_id, secret_key)
     return concat_tab(arr, '&')
 end
 
+
 local function send_cls_request(host, topic, secret_id, secret_key, pb_data)
     local http_new = http:new()
     http_new:set_timeouts(cls_conn_timeout, cls_send_timeout, cls_read_timeout)
@@ -146,6 +149,7 @@ local function send_cls_request(host, topic, secret_id, secret_key, pb_data)
     return true
 end
 
+
 -- normalized log data for CLS API
 local function normalize_log(log)
     local normalized_log = {}
@@ -173,6 +177,7 @@ local function normalize_log(log)
     return normalized_log, log_size
 end
 
+
 local function send_to_cls(secret_id, secret_key, host, topic_id, logs)
     clear_tab(log_group_list)
     local now = ngx_now() * 1000

From fd30d497efe484c130859b72b6b92a350c92b801 Mon Sep 17 00:00:00 2001
From: xinchenyuan <xinchenyuan@tencent.com>
Date: Tue, 9 Aug 2022 20:45:00 +0800
Subject: [PATCH 09/24] refa error & proto file load

refa error & proto file load
---
 apisix/plugins/tencent-cloud-cls.lua         |   9 +-
 apisix/plugins/tencent-cloud-cls/cls-sdk.lua | 115 ++++++++++++++++---
 apisix/plugins/tencent-cloud-cls/cls.pb      |  20 ----
 apisix/plugins/tencent-cloud-cls/cls.proto   |  50 --------
 4 files changed, 107 insertions(+), 87 deletions(-)
 delete mode 100644 apisix/plugins/tencent-cloud-cls/cls.pb
 delete mode 100644 apisix/plugins/tencent-cloud-cls/cls.proto

diff --git a/apisix/plugins/tencent-cloud-cls.lua b/apisix/plugins/tencent-cloud-cls.lua
index 2a6190b49e4f..5c624cca3b44 100644
--- a/apisix/plugins/tencent-cloud-cls.lua
+++ b/apisix/plugins/tencent-cloud-cls.lua
@@ -19,6 +19,7 @@ local core = require("apisix.core")
 local log_util = require("apisix.utils.log-util")
 local bp_manager_mod = require("apisix.utils.batch-processor-manager")
 local cls_sdk = require("apisix.plugins.tencent-cloud-cls.cls-sdk")
+local plugin = require("apisix.plugin")
 local math = math
 local ngx = ngx
 local pairs = pairs
@@ -127,8 +128,12 @@ function _M.log(conf, ctx)
     end
 
     local process = function(entries)
-        return cls_sdk.send_to_cls(conf.secret_id, conf.secret_key,
-                                   conf.cls_host, conf.cls_topic, entries)
+        local sdk, err = cls_sdk.new(conf.cls_host, conf.cls_topic, conf.secret_id, conf.secret_key)
+        if err then
+            core.log.error("init sdk failed err:", err)
+            return false, err
+        end
+        return sdk:send_to_cls(entries)
     end
 
     batch_processor_manager:add_entry_to_new_processor(conf, entry, ctx, process)
diff --git a/apisix/plugins/tencent-cloud-cls/cls-sdk.lua b/apisix/plugins/tencent-cloud-cls/cls-sdk.lua
index 2a53abcefde6..17c9ee52c768 100644
--- a/apisix/plugins/tencent-cloud-cls/cls-sdk.lua
+++ b/apisix/plugins/tencent-cloud-cls/cls-sdk.lua
@@ -16,8 +16,7 @@
 --
 
 local pb = require "pb"
-local assert = assert
-assert(pb.loadfile("apisix/plugins/tencent-cloud-cls/cls.pb"))
+local protoc = require("protoc").new()
 local http = require("resty.http")
 local socket = require("socket")
 local str_util = require("resty.string")
@@ -40,6 +39,8 @@ local ipairs = ipairs
 local pairs = pairs
 local type = type
 local tostring = tostring
+local setmetatable = setmetatable
+local pcall = pcall
 
 local MAX_SINGLE_VALUE_SIZE = 1 * 1024 * 1024
 local MAX_LOG_GROUP_VALUE_SIZE = 5 * 1024 * 1024 -- 5MB
@@ -112,9 +113,15 @@ local function sign(secret_id, secret_key)
 end
 
 
-local function send_cls_request(host, topic, secret_id, secret_key, pb_data)
-    local http_new = http:new()
-    http_new:set_timeouts(cls_conn_timeout, cls_send_timeout, cls_read_timeout)
+local function send_cls_request(host, topic, secret_id, secret_key, pb_obj)
+    local ok, pb_data = pcall(pb.encode, "cls.LogGroupList", pb_obj)
+    if not ok or not pb_data then
+        core.log.error("failed to encode LogGroupList, err: ", pb_data)
+        return false, pb_data
+    end
+
+    local client = http:new()
+    client:set_timeouts(cls_conn_timeout, cls_send_timeout, cls_read_timeout)
 
     clear_tab(headers_cache)
     headers_cache["Host"] = host
@@ -128,7 +135,7 @@ local function send_cls_request(host, topic, secret_id, secret_key, pb_data)
     local cls_url = "http://" .. host .. cls_api_path .. "?topic_id=" .. topic
     core.log.debug("CLS request URL: ", cls_url)
 
-    local res, err = http_new:request_uri(cls_url, params_cache)
+    local res, err = client:request_uri(cls_url, params_cache)
     if not res then
         return false, err
     end
@@ -178,13 +185,88 @@ local function normalize_log(log)
 end
 
 
-local function send_to_cls(secret_id, secret_key, host, topic_id, logs)
+local _M = { version = 0.1 }
+local mt = { __index = _M }
+
+local pb_state
+local function init_pb_state()
+    pb.state(nil)
+    protoc.reload()
+    local cls_sdk_protoc = protoc.new()
+    if not cls_sdk_protoc.loaded["tencent-cloud-cls/cls.proto"] then
+        -- https://www.tencentcloud.com/document/product/614/42787
+        local ok, err = pcall(cls_sdk_protoc.load, cls_sdk_protoc, [[
+package cls;
+
+message Log
+{
+  message Content
+  {
+    required string key   = 1; // Key of each field group
+    required string value = 2; // Value of each field group
+  }
+  required int64   time     = 1; // Unix timestamp
+  repeated Content contents = 2; // Multiple key-value pairs in one log
+}
+
+message LogTag
+{
+  required string key       = 1;
+  required string value     = 2;
+}
+
+message LogGroup
+{
+  repeated Log    logs        = 1; // Log array consisting of multiple logs
+  optional string contextFlow = 2; // This parameter does not take effect currently
+  optional string filename    = 3; // Log filename
+  optional string source      = 4; // Log source, which is generally the machine IP
+  repeated LogTag logTags     = 5;
+}
+
+message LogGroupList
+{
+  repeated LogGroup logGroupList = 1; // Log group list
+}
+        ]], "tencent-cloud-cls/cls.proto")
+        if not ok then
+            cls_sdk_protoc:reset()
+            return "failed to load cls.proto: ".. err
+        end
+    end
+    pb_state = pb.state(nil)
+end
+
+
+function _M.new(host, topic, secret_id, secret_key)
+    if not pb_state then
+        local err = init_pb_state()
+        if err then
+            return nil, err
+        end
+    end
+    local self = {
+        host = host,
+        topic = topic,
+        secret_id = secret_id,
+        secret_key = secret_key,
+    }
+    return setmetatable(self, mt)
+end
+
+
+function _M.send_to_cls(self, logs)
     clear_tab(log_group_list)
     local now = ngx_now() * 1000
 
+    -- recovery of stored pb_store
+    pb.state(pb_state)
+
     local total_size = 0
     local format_logs = new_tab(#logs, 0)
-    -- sums of all value in a LogGroup should be no more than 5MB
+    -- sums of all value in all LogGroup should be no more than 5MB
+    -- so send whenever size exceed max size
+    local group_list_start = 1
     for i = 1, #logs, 1 do
         local contents, log_size = normalize_log(logs[i])
         if log_size > MAX_LOG_GROUP_VALUE_SIZE then
@@ -197,10 +279,14 @@ local function send_to_cls(secret_id, secret_key, host, topic_id, logs)
                 logs = format_logs,
                 source = host_ip,
             })
+            local ok, err = send_cls_request(self.host, self.topic,
+                                             self.secret_id, self.secret_key, log_group_list_pb)
+            if not ok then
+                return false, err, group_list_start
+            end
+            group_list_start = i
             format_logs = new_tab(#logs - i, 0)
             total_size = 0
-            local data = assert(pb.encode("cls.LogGroupList", log_group_list_pb))
-            send_cls_request(host, topic_id, secret_id, secret_key, data)
             clear_tab(log_group_list)
         end
         insert_tab(format_logs, {
@@ -214,10 +300,9 @@ local function send_to_cls(secret_id, secret_key, host, topic_id, logs)
         logs = format_logs,
         source = host_ip,
     })
-    local data = assert(pb.encode("cls.LogGroupList", log_group_list_pb))
-    return send_cls_request(host, topic_id, secret_id, secret_key, data)
+    local ok, err =  send_cls_request(self.host, self.topic, self.secret_id,
+                                      self.secret_key, log_group_list_pb)
+    return ok, err, group_list_start
 end
 
-return {
-    send_to_cls = send_to_cls
-}
+return _M
diff --git a/apisix/plugins/tencent-cloud-cls/cls.pb b/apisix/plugins/tencent-cloud-cls/cls.pb
deleted file mode 100644
index 41c84cb31e35..000000000000
--- a/apisix/plugins/tencent-cloud-cls/cls.pb
+++ /dev/null
@@ -1,20 +0,0 @@
-
-�
-	cls.protocls"~
-Log
-time
 (B0
Rtime,
-contents (2.cls.Log.ContentRcontents1
-Content
-key
 (	Rkey
-value (	Rvalue"0
-LogTag
-key
 (	Rkey
-value (	Rvalue"�

-LogGroup
-logs
 (2.cls.LogRlogs 
-contextFlow 
(	RcontextFlow
-filename 
(	Rfilename
-source 
(	Rsource%
-logTags (2.cls.LogTagRlogTags"A
-LogGroupList1
-logGroupList
 (2
.cls.LogGroupRlogGroupList
\ No newline at end of file
diff --git a/apisix/plugins/tencent-cloud-cls/cls.proto b/apisix/plugins/tencent-cloud-cls/cls.proto
deleted file mode 100644
index 0029be11ba58..000000000000
--- a/apisix/plugins/tencent-cloud-cls/cls.proto
+++ /dev/null
@@ -1,50 +0,0 @@
-//
-// Licensed to the Apache Software Foundation (ASF) under one or more
-// contributor license agreements.  See the NOTICE file distributed with
-// this work for additional information regarding copyright ownership.
-// The ASF licenses this file to You under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance with
-// the License.  You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-// https://cloud.tencent.com/document/product/614/59470
-package cls;
-
-message Log
-{
-  message Content
-  {
-    required string key   = 1; // 每组字段的 key
-    required string value = 2; // 每组字段的 value
-  }
-  required int64   time     = 1; // 时间戳，UNIX时间格式
-  repeated Content contents = 2; // 一条日志里的多个kv组合
-}
-
-message LogTag
-{
-  required string key       = 1;
-  required string value     = 2;
-}
-
-message LogGroup
-{
-  repeated Log    logs        = 1; // 多条日志合成的日志数组
-  optional string contextFlow = 2; // 目前暂无效用
-  optional string filename    = 3; // 日志文件名
-  optional string source      = 4; // 日志来源，一般使用机器IP
-  repeated LogTag logTags     = 5;
-}
-
-message LogGroupList
-{
-  repeated LogGroup logGroupList = 1; // 日志组列表
-}

From 6902513f4037496aa1febbd995c89147c5fe21d5 Mon Sep 17 00:00:00 2001
From: xinchenyuan <xinchenyuan@tencent.com>
Date: Mon, 15 Aug 2022 15:10:46 +0800
Subject: [PATCH 10/24] fix ut

---
 t/plugin/tencent-cloud-cls.t | 28 ++++++++++++++++++++++++++--
 1 file changed, 26 insertions(+), 2 deletions(-)

diff --git a/t/plugin/tencent-cloud-cls.t b/t/plugin/tencent-cloud-cls.t
index ab8c2f3253b5..3a1ad634b5b9 100644
--- a/t/plugin/tencent-cloud-cls.t
+++ b/t/plugin/tencent-cloud-cls.t
@@ -32,6 +32,25 @@ add_block_preprocessor(sub {
         $block->set_value("request", "GET /t");
     }
 
+    my $http_config = $block->http_config // <<_EOC_;
+    server {
+        listen 10420;
+        location /structuredlog {
+            content_by_lua_block {
+                ngx.req.read_body()
+                local data = ngx.req.get_body_data()
+                local headers = ngx.req.get_headers()
+                ngx.log(ngx.WARN, "tencent-cloud-cls body: ", data)
+                for k, v in pairs(headers) do
+                    ngx.log(ngx.WARN, "tencent-cloud-cls headers: " .. k .. ":" .. v)
+                end
+                ngx.say("ok")
+            }
+        }
+    }
+_EOC_
+
+    $block->set_value("http_config", $http_config);
 });
 
 run_tests;
@@ -94,10 +113,15 @@ done
                  [[{
                         "plugins": {
                             "tencent-cloud-cls": {
-                                "cls_host": "ap-guangzhou.cls.tencentyun.com",
+                                "cls_host": "127.0.0.1:10420",
                                 "cls_topic": "143b5d70-139b-4aec-b54e-bb97756916de",
                                 "secret_id": "secret_id",
-                                "secret_key": "secret_key"
+                                "secret_key": "secret_key",
+                                "batch_max_size": 1,
+                                "max_retry_count": 1,
+                                "retry_delay": 2,
+                                "buffer_duration": 2,
+                                "inactive_timeout": 2
                             }
                         },
                         "upstream": {

From 0d10153a05776fe3e799b7ec8ed0ee96372db00c Mon Sep 17 00:00:00 2001
From: xinchenyuan <xinchenyuan@tencent.com>
Date: Tue, 16 Aug 2022 18:54:38 +0800
Subject: [PATCH 11/24] fix ci

---
 Makefile | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/Makefile b/Makefile
index 5a575d340c6c..22abbb49389e 100644
--- a/Makefile
+++ b/Makefile
@@ -345,6 +345,9 @@ install: runtime
 	$(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/plugins/zipkin
 	$(ENV_INSTALL) apisix/plugins/zipkin/*.lua $(ENV_INST_LUADIR)/apisix/plugins/zipkin/
 
+	$(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/plugins/tencent-cloud-cls
+	$(ENV_INSTALL) apisix/plugins/tencent-cloud-cls/*.lua $(ENV_INST_LUADIR)/apisix/plugins/tencent-cloud-cls/
+
 	$(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/ssl/router
 	$(ENV_INSTALL) apisix/ssl/router/*.lua $(ENV_INST_LUADIR)/apisix/ssl/router/
 

From b5c1d528fa54f714aea060ebfad73461905c9d56 Mon Sep 17 00:00:00 2001
From: xinchenyuan <xinchenyuan@tencent.com>
Date: Wed, 17 Aug 2022 10:37:18 +0800
Subject: [PATCH 12/24] fix ut

---
 t/admin/plugins.t | 1 +
 1 file changed, 1 insertion(+)

diff --git a/t/admin/plugins.t b/t/admin/plugins.t
index 1d78b2432989..7ec20a35d43c 100644
--- a/t/admin/plugins.t
+++ b/t/admin/plugins.t
@@ -122,6 +122,7 @@ syslog
 udp-logger
 file-logger
 clickhouse-logger
+tencent-cloud-cls
 example-plugin
 aws-lambda
 azure-functions

From c6b6e94c96aa62db0da93b9789e3378ae8ec0164 Mon Sep 17 00:00:00 2001
From: xinchenyuan <xinchenyuan@tencent.com>
Date: Wed, 17 Aug 2022 16:05:07 +0800
Subject: [PATCH 13/24] order

---
 Makefile | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Makefile b/Makefile
index 22abbb49389e..b26ae3e0cbc8 100644
--- a/Makefile
+++ b/Makefile
@@ -342,12 +342,12 @@ install: runtime
 	$(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/plugins/syslog
 	$(ENV_INSTALL) apisix/plugins/syslog/*.lua $(ENV_INST_LUADIR)/apisix/plugins/syslog/
 
-	$(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/plugins/zipkin
-	$(ENV_INSTALL) apisix/plugins/zipkin/*.lua $(ENV_INST_LUADIR)/apisix/plugins/zipkin/
-
 	$(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/plugins/tencent-cloud-cls
 	$(ENV_INSTALL) apisix/plugins/tencent-cloud-cls/*.lua $(ENV_INST_LUADIR)/apisix/plugins/tencent-cloud-cls/
 
+	$(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/plugins/zipkin
+	$(ENV_INSTALL) apisix/plugins/zipkin/*.lua $(ENV_INST_LUADIR)/apisix/plugins/zipkin/
+
 	$(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/ssl/router
 	$(ENV_INSTALL) apisix/ssl/router/*.lua $(ENV_INST_LUADIR)/apisix/ssl/router/
 

From 5d865222ac6910b6816c6d2b69de3ed6ac691810 Mon Sep 17 00:00:00 2001
From: xinchenyuan <xinchenyuan@tencent.com>
Date: Thu, 18 Aug 2022 20:12:47 +0800
Subject: [PATCH 14/24] add some comment

---
 apisix/plugins/tencent-cloud-cls/cls-sdk.lua | 107 +++++++-------
 t/plugin/tencent-cloud-cls.t                 | 139 ++++++++++++++++++-
 2 files changed, 189 insertions(+), 57 deletions(-)

diff --git a/apisix/plugins/tencent-cloud-cls/cls-sdk.lua b/apisix/plugins/tencent-cloud-cls/cls-sdk.lua
index 17c9ee52c768..a0b809a3e033 100644
--- a/apisix/plugins/tencent-cloud-cls/cls-sdk.lua
+++ b/apisix/plugins/tencent-cloud-cls/cls-sdk.lua
@@ -42,6 +42,7 @@ local tostring = tostring
 local setmetatable = setmetatable
 local pcall = pcall
 
+-- api doc https://www.tencentcloud.com/document/product/614/16873
 local MAX_SINGLE_VALUE_SIZE = 1 * 1024 * 1024
 local MAX_LOG_GROUP_VALUE_SIZE = 5 * 1024 * 1024 -- 5MB
 
@@ -113,50 +114,6 @@ local function sign(secret_id, secret_key)
 end
 
 
-local function send_cls_request(host, topic, secret_id, secret_key, pb_obj)
-    local ok, pb_data = pcall(pb.encode, "cls.LogGroupList", pb_obj)
-    if not ok or not pb_data then
-        core.log.error("failed to encode LogGroupList, err: ", pb_data)
-        return false, pb_data
-    end
-
-    local client = http:new()
-    client:set_timeouts(cls_conn_timeout, cls_send_timeout, cls_read_timeout)
-
-    clear_tab(headers_cache)
-    headers_cache["Host"] = host
-    headers_cache["Content-Type"] = "application/x-protobuf"
-    headers_cache["Authorization"] = sign(secret_id, secret_key, cls_api_path)
-
-    -- TODO: support lz4/zstd compress
-    params_cache.method = "POST"
-    params_cache.body = pb_data
-
-    local cls_url = "http://" .. host .. cls_api_path .. "?topic_id=" .. topic
-    core.log.debug("CLS request URL: ", cls_url)
-
-    local res, err = client:request_uri(cls_url, params_cache)
-    if not res then
-        return false, err
-    end
-
-    if res.status ~= 200 then
-        err = fmt("got wrong status: %s, headers: %s, body, %s",
-                  res.status, json.encode(res.headers), res.body)
-        -- 413, 404, 401, 403 are not retryable
-        if res.status == 413 or res.status == 404 or res.status == 401 or res.status == 403 then
-            core.log.error(err, ", not retryable")
-            return true
-        end
-
-        return false, err
-    end
-
-    core.log.debug("CLS report success")
-    return true
-end
-
-
 -- normalized log data for CLS API
 local function normalize_log(log)
     local normalized_log = {}
@@ -193,9 +150,8 @@ local function init_pb_state()
     pb.state(nil)
     protoc.reload()
     local cls_sdk_protoc = protoc.new()
-    if not cls_sdk_protoc.loaded["tencent-cloud-cls/cls.proto"] then
-        -- https://www.tencentcloud.com/document/product/614/42787
-        local ok, err = pcall(cls_sdk_protoc.load, cls_sdk_protoc, [[
+    -- proto file in https://www.tencentcloud.com/document/product/614/42787
+    local ok, err = pcall(cls_sdk_protoc.load, cls_sdk_protoc, [[
 package cls;
 
 message Log
@@ -229,10 +185,9 @@ message LogGroupList
   repeated LogGroup logGroupList = 1; // Log group list
 }
         ]], "tencent-cloud-cls/cls.proto")
-        if not ok then
-            cls_sdk_protoc:reset()
-            return "failed to load cls.proto: ".. err
-        end
+    if not ok then
+        cls_sdk_protoc:reset()
+        return "failed to load cls.proto: ".. err
     end
     pb_state = pb.state(nil)
 end
@@ -255,6 +210,50 @@ function _M.new(host, topic, secret_id, secret_key)
 end
 
 
+function _M.send_cls_request(self, pb_obj)
+    local ok, pb_data = pcall(pb.encode, "cls.LogGroupList", pb_obj)
+    if not ok or not pb_data then
+        core.log.error("failed to encode LogGroupList, err: ", pb_data)
+        return false, pb_data
+    end
+
+    local client = http:new()
+    client:set_timeouts(cls_conn_timeout, cls_send_timeout, cls_read_timeout)
+
+    clear_tab(headers_cache)
+    headers_cache["Host"] = self.host
+    headers_cache["Content-Type"] = "application/x-protobuf"
+    headers_cache["Authorization"] = sign(self.secret_id, self.secret_key, cls_api_path)
+
+    -- TODO: support lz4/zstd compress
+    params_cache.method = "POST"
+    params_cache.body = pb_data
+
+    local cls_url = "http://" .. self.host .. cls_api_path .. "?topic_id=" .. self.topic
+    core.log.debug("CLS request URL: ", cls_url)
+
+    local res, err = client:request_uri(cls_url, params_cache)
+    if not res then
+        return false, err
+    end
+
+    if res.status ~= 200 then
+        err = fmt("got wrong status: %s, headers: %s, body, %s",
+                res.status, json.encode(res.headers), res.body)
+        -- 413, 404, 401, 403 are not retryable
+        if res.status == 413 or res.status == 404 or res.status == 401 or res.status == 403 then
+            core.log.error(err, ", not retryable")
+            return true
+        end
+
+        return false, err
+    end
+
+    core.log.debug("CLS report success")
+    return true
+end
+
+
 function _M.send_to_cls(self, logs)
     clear_tab(log_group_list)
     local now = ngx_now() * 1000
@@ -279,8 +278,7 @@ function _M.send_to_cls(self, logs)
                 logs = format_logs,
                 source = host_ip,
             })
-            local ok, err = send_cls_request(self.host, self.topic,
-                                             self.secret_id, self.secret_key, log_group_list_pb)
+            local ok, err = self:send_cls_request(log_group_list_pb)
             if not ok then
                 return false, err, group_list_start
             end
@@ -300,8 +298,7 @@ function _M.send_to_cls(self, logs)
         logs = format_logs,
         source = host_ip,
     })
-    local ok, err =  send_cls_request(self.host, self.topic, self.secret_id,
-                                      self.secret_key, log_group_list_pb)
+    local ok, err = self:send_cls_request(log_group_list_pb)
     return ok, err, group_list_start
 end
 
diff --git a/t/plugin/tencent-cloud-cls.t b/t/plugin/tencent-cloud-cls.t
index 3a1ad634b5b9..26e478f83d57 100644
--- a/t/plugin/tencent-cloud-cls.t
+++ b/t/plugin/tencent-cloud-cls.t
@@ -48,6 +48,21 @@ add_block_preprocessor(sub {
             }
         }
     }
+    server {
+        listen 10421;
+        location /structuredlog {
+            content_by_lua_block {
+                ngx.req.read_body()
+                local data = ngx.req.get_body_data()
+                local headers = ngx.req.get_headers()
+                ngx.log(ngx.WARN, "tencent-cloud-cls body: ", data)
+                for k, v in pairs(headers) do
+                    ngx.log(ngx.WARN, "tencent-cloud-cls headers: " .. k .. ":" .. v)
+                end
+                ngx.exit(500)
+            }
+        }
+    }
 _EOC_
 
     $block->set_value("http_config", $http_config);
@@ -103,7 +118,59 @@ done
 
 
 
-=== TEST 3: add plugin
+=== TEST 3: add plugin for incorrect server
+--- config
+    location /t {
+        content_by_lua_block {
+            local t = require("lib.test_admin").test
+            local code, body = t('/apisix/admin/routes/1',
+                 ngx.HTTP_PUT,
+                 [[{
+                        "plugins": {
+                            "tencent-cloud-cls": {
+                                "cls_host": "127.0.0.1:10421",
+                                "cls_topic": "143b5d70-139b-4aec-b54e-bb97756916de",
+                                "secret_id": "secret_id",
+                                "secret_key": "secret_key",
+                                "batch_max_size": 1,
+                                "max_retry_count": 1,
+                                "retry_delay": 2,
+                                "buffer_duration": 2,
+                                "inactive_timeout": 2
+                            }
+                        },
+                        "upstream": {
+                            "nodes": {
+                                "127.0.0.1:1982": 1
+                            },
+                            "type": "roundrobin"
+                        },
+                        "uri": "/opentracing"
+                }]]
+                )
+
+            if code >= 300 then
+                ngx.status = code
+            end
+            ngx.say(body)
+        }
+    }
+--- response_body
+passed
+
+
+
+=== TEST 4: incorrect server
+--- request
+GET /opentracing
+--- response_body
+opentracing
+--- error_log
+Batch Processor[tencent-cloud-cls] failed to process entries [1/1]: got wrong status: 500
+--- wait: 0.5
+
+
+=== TEST 5: add plugin
 --- config
     location /t {
         content_by_lua_block {
@@ -145,7 +212,75 @@ passed
 
 
 
-=== TEST 4: access local server
+=== TEST 6: access local server
+--- request
+GET /opentracing
+--- response_body
+opentracing
+--- error_log
+Batch Processor[tencent-cloud-cls] successfully processed the entries
+--- wait: 0.5
+
+
+
+=== TEST 7: verify request
+--- extra_init_by_lua
+    local cls = require("apisix.plugins.tencent-cloud-cls.cls-sdk")
+    cls.send_to_cls = function(self, logs)
+        if (#logs ~= 1) then
+            ngx.log(ngx.ERR, "unexpected logs length: ", #logs)
+            return
+        end
+        return true
+    end
+--- request
+GET /opentracing
+--- response_body
+opentracing
+--- error_log
+Batch Processor[tencent-cloud-cls] successfully processed the entries
+--- wait: 0.5
+
+
+
+=== TEST 8: verify cls api request
+--- extra_init_by_lua
+    local cls = require("apisix.plugins.tencent-cloud-cls.cls-sdk")
+    cls.send_cls_request = function(self, pb_obj)
+        if (#pb_obj.logGroupList ~= 1) then
+            ngx.log(ngx.ERR, "unexpected logGroupList length: ", #pb_obj.logGroupList)
+            return false
+        end
+        local log_group = pb_obj.logGroupList[1]
+        if #log_group.logs ~= 1 then
+            ngx.log(ngx.ERR, "unexpected logs length: ", #log_group.logs)
+            return false
+        end
+        local log = log_group.logs[1]
+        for k, v in ipairs(log.contents) do
+            ngx.log(ngx.ERR, "key:", v.key, ",value:", v.value)
+        end
+        return true
+    end
+--- request
+GET /opentracing
+--- response_body
+opentracing
+--- error_log
+Batch Processor[tencent-cloud-cls] successfully processed the entries
+--- wait: 0.5
+
+
+=== TEST 7: verify request
+--- extra_init_by_lua
+    local cls = require("apisix.plugins.tencent-cloud-cls.cls-sdk")
+    cls.send_to_cls = function(self, logs)
+        if (#logs ~= 1) then
+            ngx.log(ngx.ERR, "unexpected logs length: ", #logs)
+            return
+        end
+        return true
+    end
 --- request
 GET /opentracing
 --- response_body

From f4dd8494b7909cc14a58241ecb7c07f858eea758 Mon Sep 17 00:00:00 2001
From: xinchenyuan <xinchenyuan@tencent.com>
Date: Thu, 18 Aug 2022 20:34:13 +0800
Subject: [PATCH 15/24] refa

refa
---
 apisix/plugins/tencent-cloud-cls/cls-sdk.lua | 22 ++++++++---
 t/plugin/tencent-cloud-cls.t                 | 41 ++++++++++++++------
 2 files changed, 46 insertions(+), 17 deletions(-)

diff --git a/apisix/plugins/tencent-cloud-cls/cls-sdk.lua b/apisix/plugins/tencent-cloud-cls/cls-sdk.lua
index a0b809a3e033..a2be95cc60e6 100644
--- a/apisix/plugins/tencent-cloud-cls/cls-sdk.lua
+++ b/apisix/plugins/tencent-cloud-cls/cls-sdk.lua
@@ -147,7 +147,7 @@ local mt = { __index = _M }
 
 local pb_state
 local function init_pb_state()
-    pb.state(nil)
+    local old_pb_state = pb.state(nil)
     protoc.reload()
     local cls_sdk_protoc = protoc.new()
     -- proto file in https://www.tencentcloud.com/document/product/614/42787
@@ -187,9 +187,10 @@ message LogGroupList
         ]], "tencent-cloud-cls/cls.proto")
     if not ok then
         cls_sdk_protoc:reset()
+        pb.state(old_pb_state)
         return "failed to load cls.proto: ".. err
     end
-    pb_state = pb.state(nil)
+    pb_state = pb.state(old_pb_state)
 end
 
 
@@ -210,7 +211,19 @@ function _M.new(host, topic, secret_id, secret_key)
 end
 
 
+local function do_request_uri(uri, params)
+    local client = http:new()
+    client:set_timeouts(cls_conn_timeout, cls_send_timeout, cls_read_timeout)
+    local res, err = client:request_uri(uri, params)
+    client:close()
+    return res, err
+end
+
+
 function _M.send_cls_request(self, pb_obj)
+    -- recovery of stored pb_store
+    pb.state(pb_state)
+
     local ok, pb_data = pcall(pb.encode, "cls.LogGroupList", pb_obj)
     if not ok or not pb_data then
         core.log.error("failed to encode LogGroupList, err: ", pb_data)
@@ -232,7 +245,7 @@ function _M.send_cls_request(self, pb_obj)
     local cls_url = "http://" .. self.host .. cls_api_path .. "?topic_id=" .. self.topic
     core.log.debug("CLS request URL: ", cls_url)
 
-    local res, err = client:request_uri(cls_url, params_cache)
+    local res, err = do_request_uri(cls_url, params_cache)
     if not res then
         return false, err
     end
@@ -258,9 +271,6 @@ function _M.send_to_cls(self, logs)
     clear_tab(log_group_list)
     local now = ngx_now() * 1000
 
-    -- recovery of stored pb_store
-    pb.state(pb_state)
-
     local total_size = 0
     local format_logs = new_tab(#logs, 0)
     -- sums of all value in all LogGroup should be no more than 5MB
diff --git a/t/plugin/tencent-cloud-cls.t b/t/plugin/tencent-cloud-cls.t
index 26e478f83d57..0d6d2c82aba5 100644
--- a/t/plugin/tencent-cloud-cls.t
+++ b/t/plugin/tencent-cloud-cls.t
@@ -271,20 +271,39 @@ Batch Processor[tencent-cloud-cls] successfully processed the entries
 --- wait: 0.5
 
 
-=== TEST 7: verify request
---- extra_init_by_lua
-    local cls = require("apisix.plugins.tencent-cloud-cls.cls-sdk")
-    cls.send_to_cls = function(self, logs)
-        if (#logs ~= 1) then
-            ngx.log(ngx.ERR, "unexpected logs length: ", #logs)
-            return
-        end
-        return true
-    end
+=== TEST 9: plugin metadata
+--- config
+    location /t {
+        content_by_lua_block {
+            local plugin = require("apisix.plugins.tencent-cloud-cls")
+            local t = require("lib.test_admin").test
+            local code, body = t('/apisix/admin/plugin_metadata/tencent-cloud-cls',
+                 ngx.HTTP_PUT,
+                 [[{
+                        "log_format": {
+                            "host": "$host",
+                            "@timestamp": "$time_iso8601",
+                            "client_ip": "$remote_addr"
+                        }
+                }]]
+                )
+            ngx.say(body)
+        }
+    }
+--- request
+GET /t
+--- response_body
+passed
+--- no_error_log
+[error]
+
+
+
+=== TEST 10: log use log_format
 --- request
 GET /opentracing
 --- response_body
 opentracing
 --- error_log
-Batch Processor[tencent-cloud-cls] successfully processed the entries
+"@timestamp":[true,"time_iso8601"]
 --- wait: 0.5

From 4ed9d202dba4ce1cd5fad68e3d016a3b64ab5d49 Mon Sep 17 00:00:00 2001
From: xinchenyuan <xinchenyuan@tencent.com>
Date: Fri, 19 Aug 2022 11:31:23 +0800
Subject: [PATCH 16/24] fix ut style

---
 t/plugin/tencent-cloud-cls.t | 1 +
 1 file changed, 1 insertion(+)

diff --git a/t/plugin/tencent-cloud-cls.t b/t/plugin/tencent-cloud-cls.t
index 0d6d2c82aba5..83656bea5d0e 100644
--- a/t/plugin/tencent-cloud-cls.t
+++ b/t/plugin/tencent-cloud-cls.t
@@ -271,6 +271,7 @@ Batch Processor[tencent-cloud-cls] successfully processed the entries
 --- wait: 0.5
 
 
+
 === TEST 9: plugin metadata
 --- config
     location /t {

From 3c0ef7cc0b4ace9b87d7d0456662cd7244eec232 Mon Sep 17 00:00:00 2001
From: xinchenyuan <xinchenyuan@tencent.com>
Date: Fri, 19 Aug 2022 13:25:33 +0800
Subject: [PATCH 17/24] fix ut style

---
 t/plugin/tencent-cloud-cls.t | 1 +
 1 file changed, 1 insertion(+)

diff --git a/t/plugin/tencent-cloud-cls.t b/t/plugin/tencent-cloud-cls.t
index 83656bea5d0e..4d1e94d9409c 100644
--- a/t/plugin/tencent-cloud-cls.t
+++ b/t/plugin/tencent-cloud-cls.t
@@ -170,6 +170,7 @@ Batch Processor[tencent-cloud-cls] failed to process entries [1/1]: got wrong st
 --- wait: 0.5
 
 
+
 === TEST 5: add plugin
 --- config
     location /t {

From 3c801226fefc4d27a92c878c6a43cd4666d803a8 Mon Sep 17 00:00:00 2001
From: xinchenyuan <xinchenyuan@tencent.com>
Date: Mon, 22 Aug 2022 09:44:34 +0800
Subject: [PATCH 18/24] update ut

---
 apisix/plugins/tencent-cloud-cls.lua |  1 +
 t/plugin/tencent-cloud-cls.t         | 19 ++++++-------------
 2 files changed, 7 insertions(+), 13 deletions(-)

diff --git a/apisix/plugins/tencent-cloud-cls.lua b/apisix/plugins/tencent-cloud-cls.lua
index 5c624cca3b44..013932cf680c 100644
--- a/apisix/plugins/tencent-cloud-cls.lua
+++ b/apisix/plugins/tencent-cloud-cls.lua
@@ -112,6 +112,7 @@ function _M.log(conf, ctx)
     if metadata and metadata.value.log_format
             and core.table.nkeys(metadata.value.log_format) > 0
     then
+        core.log.debug("using custom format log")
         entry = log_util.get_custom_format_log(ctx, metadata.value.log_format)
     else
         entry = log_util.get_full_log(ngx, conf)
diff --git a/t/plugin/tencent-cloud-cls.t b/t/plugin/tencent-cloud-cls.t
index 4d1e94d9409c..35c2681df664 100644
--- a/t/plugin/tencent-cloud-cls.t
+++ b/t/plugin/tencent-cloud-cls.t
@@ -52,13 +52,6 @@ add_block_preprocessor(sub {
         listen 10421;
         location /structuredlog {
             content_by_lua_block {
-                ngx.req.read_body()
-                local data = ngx.req.get_body_data()
-                local headers = ngx.req.get_headers()
-                ngx.log(ngx.WARN, "tencent-cloud-cls body: ", data)
-                for k, v in pairs(headers) do
-                    ngx.log(ngx.WARN, "tencent-cloud-cls headers: " .. k .. ":" .. v)
-                end
                 ngx.exit(500)
             }
         }
@@ -258,8 +251,9 @@ Batch Processor[tencent-cloud-cls] successfully processed the entries
             return false
         end
         local log = log_group.logs[1]
-        for k, v in ipairs(log.contents) do
-            ngx.log(ngx.ERR, "key:", v.key, ",value:", v.value)
+        if #log.contents == 0 then
+            ngx.log(ngx.ERR, "unexpected contents length: ", #log.contents)
+            return false
         end
         return true
     end
@@ -267,9 +261,8 @@ Batch Processor[tencent-cloud-cls] successfully processed the entries
 GET /opentracing
 --- response_body
 opentracing
---- error_log
-Batch Processor[tencent-cloud-cls] successfully processed the entries
---- wait: 0.5
+--- no_error_log
+[error]
 
 
 
@@ -307,5 +300,5 @@ GET /opentracing
 --- response_body
 opentracing
 --- error_log
-"@timestamp":[true,"time_iso8601"]
+using custom format log
 --- wait: 0.5

From e8ea4967706cf1abf8a79ab1e80e555b3ff084d1 Mon Sep 17 00:00:00 2001
From: xinchenyuan <xinchenyuan@tencent.com>
Date: Mon, 22 Aug 2022 14:15:19 +0800
Subject: [PATCH 19/24] remove secret key console link

---
 apisix/plugins/tencent-cloud-cls.lua | 1 -
 1 file changed, 1 deletion(-)

diff --git a/apisix/plugins/tencent-cloud-cls.lua b/apisix/plugins/tencent-cloud-cls.lua
index 013932cf680c..f0a0fb843c20 100644
--- a/apisix/plugins/tencent-cloud-cls.lua
+++ b/apisix/plugins/tencent-cloud-cls.lua
@@ -32,7 +32,6 @@ local schema = {
     properties = {
         cls_host = { type = "string" },
         cls_topic = { type = "string" },
-        -- https://console.cloud.tencent.com/capi
         secret_id = { type = "string" },
         secret_key = { type = "string" },
         sample_ratio = {

From 4dec98c700d6036d41c625ab6e80b147028a3e70 Mon Sep 17 00:00:00 2001
From: xinchenyuan <xinchenyuan@tencent.com>
Date: Tue, 23 Aug 2022 10:15:05 +0800
Subject: [PATCH 20/24] refa ut

---
 t/plugin/tencent-cloud-cls.t | 40 ++++++++++++++++++++++++++++++++----
 1 file changed, 36 insertions(+), 4 deletions(-)

diff --git a/t/plugin/tencent-cloud-cls.t b/t/plugin/tencent-cloud-cls.t
index 35c2681df664..10f9123e6674 100644
--- a/t/plugin/tencent-cloud-cls.t
+++ b/t/plugin/tencent-cloud-cls.t
@@ -285,8 +285,6 @@ opentracing
             ngx.say(body)
         }
     }
---- request
-GET /t
 --- response_body
 passed
 --- no_error_log
@@ -295,10 +293,44 @@ passed
 
 
 === TEST 10: log use log_format
+--- extra_init_by_lua
+    local cls = require("apisix.plugins.tencent-cloud-cls.cls-sdk")
+    cls.send_cls_request = function(self, pb_obj)
+        if (#pb_obj.logGroupList ~= 1) then
+            ngx.log(ngx.ERR, "unexpected logGroupList length: ", #pb_obj.logGroupList)
+            return false
+        end
+        local log_group = pb_obj.logGroupList[1]
+        if #log_group.logs ~= 1 then
+            ngx.log(ngx.ERR, "unexpected logs length: ", #log_group.logs)
+            return false
+        end
+        local log = log_group.logs[1]
+        if #log.contents == 0 then
+            ngx.log(ngx.ERR, "unexpected contents length: ", #log.contents)
+            return false
+        end
+        local has_host, has_timestamp, has_client_ip = false, false, false
+        for i, tag in ipairs(log.contents) do
+            if tag.key == "host" then
+                has_host = true
+            end
+            if tag.key == "@timestamp" then
+                has_timestamp = true
+            end
+            if tag.key == "client_ip" then
+                has_client_ip = true
+            end
+        end
+        if not(has_host and has_timestamp and has_client_ip) then
+            return false
+        end
+        return true
+    end
 --- request
 GET /opentracing
 --- response_body
 opentracing
---- error_log
-using custom format log
+--- no_error_log
+[error]
 --- wait: 0.5

From 93a98fb8b2fbe9785d95c129ab25cd8f98079b03 Mon Sep 17 00:00:00 2001
From: xinchenyuan <xinchenyuan@tencent.com>
Date: Tue, 23 Aug 2022 10:16:35 +0800
Subject: [PATCH 21/24] refa ut

---
 t/plugin/tencent-cloud-cls.t | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/t/plugin/tencent-cloud-cls.t b/t/plugin/tencent-cloud-cls.t
index 10f9123e6674..14006bbd7e9f 100644
--- a/t/plugin/tencent-cloud-cls.t
+++ b/t/plugin/tencent-cloud-cls.t
@@ -261,8 +261,6 @@ Batch Processor[tencent-cloud-cls] successfully processed the entries
 GET /opentracing
 --- response_body
 opentracing
---- no_error_log
-[error]
 
 
 
@@ -287,8 +285,6 @@ opentracing
     }
 --- response_body
 passed
---- no_error_log
-[error]
 
 
 
@@ -331,6 +327,4 @@ passed
 GET /opentracing
 --- response_body
 opentracing
---- no_error_log
-[error]
 --- wait: 0.5

From 75b93146cfdd0eda44fed5d988557ce69ed0fcc4 Mon Sep 17 00:00:00 2001
From: xinchenyuan <xinchenyuan@tencent.com>
Date: Tue, 23 Aug 2022 10:17:40 +0800
Subject: [PATCH 22/24] refa

---
 apisix/plugins/tencent-cloud-cls/cls-sdk.lua | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/apisix/plugins/tencent-cloud-cls/cls-sdk.lua b/apisix/plugins/tencent-cloud-cls/cls-sdk.lua
index a2be95cc60e6..b3bac663ef68 100644
--- a/apisix/plugins/tencent-cloud-cls/cls-sdk.lua
+++ b/apisix/plugins/tencent-cloud-cls/cls-sdk.lua
@@ -230,9 +230,6 @@ function _M.send_cls_request(self, pb_obj)
         return false, pb_data
     end
 
-    local client = http:new()
-    client:set_timeouts(cls_conn_timeout, cls_send_timeout, cls_read_timeout)
-
     clear_tab(headers_cache)
     headers_cache["Host"] = self.host
     headers_cache["Content-Type"] = "application/x-protobuf"

From 4309dc114c2258e63c44b168887d3a721f2a615f Mon Sep 17 00:00:00 2001
From: xinchenyuan <xinchenyuan@tencent.com>
Date: Tue, 23 Aug 2022 10:23:37 +0800
Subject: [PATCH 23/24] refa

---
 apisix/plugins/tencent-cloud-cls/cls-sdk.lua | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/apisix/plugins/tencent-cloud-cls/cls-sdk.lua b/apisix/plugins/tencent-cloud-cls/cls-sdk.lua
index b3bac663ef68..d2b6e8ad4525 100644
--- a/apisix/plugins/tencent-cloud-cls/cls-sdk.lua
+++ b/apisix/plugins/tencent-cloud-cls/cls-sdk.lua
@@ -222,9 +222,9 @@ end
 
 function _M.send_cls_request(self, pb_obj)
     -- recovery of stored pb_store
-    pb.state(pb_state)
-
+    local old_pb_state = pb.state(pb_state)
     local ok, pb_data = pcall(pb.encode, "cls.LogGroupList", pb_obj)
+    pb_state = pb.state(old_pb_state)
     if not ok or not pb_data then
         core.log.error("failed to encode LogGroupList, err: ", pb_data)
         return false, pb_data

From 4461aaf54b882f8dc5caabcac1f3b8747218866e Mon Sep 17 00:00:00 2001
From: xinchenyuan <xinchenyuan@tencent.com>
Date: Tue, 23 Aug 2022 19:11:18 +0800
Subject: [PATCH 24/24] remove useless log

---
 apisix/plugins/tencent-cloud-cls.lua | 2 --
 1 file changed, 2 deletions(-)

diff --git a/apisix/plugins/tencent-cloud-cls.lua b/apisix/plugins/tencent-cloud-cls.lua
index f0a0fb843c20..b0726e607eae 100644
--- a/apisix/plugins/tencent-cloud-cls.lua
+++ b/apisix/plugins/tencent-cloud-cls.lua
@@ -79,14 +79,12 @@ end
 
 
 function _M.access(conf, ctx)
-    -- sample if set
     ctx.cls_sample = false
     if conf.sample_ratio == 1 or math.random() < conf.sample_ratio then
         core.log.debug("cls sampled")
         ctx.cls_sample = true
         return
     end
-    core.log.debug("cls not sampled")
 end