From 66eb976a130b0bb1bb021d9dbf5249d55a76ec33 Mon Sep 17 00:00:00 2001
From: Bisakh Mondal <bisakhmondal00@gmail.com>
Date: Fri, 29 Oct 2021 20:57:07 +0530
Subject: [PATCH 01/17] datadog plugin

---
 apisix/plugins/datadog.lua | 175 +++++++++++++++++++++++++++++++++++++
 conf/config-default.yaml   |   1 +
 2 files changed, 176 insertions(+)
 create mode 100644 apisix/plugins/datadog.lua

diff --git a/apisix/plugins/datadog.lua b/apisix/plugins/datadog.lua
new file mode 100644
index 000000000000..d0aa060ef2b0
--- /dev/null
+++ b/apisix/plugins/datadog.lua
@@ -0,0 +1,175 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements.  See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License.  You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+local core = require("apisix.core")
+local plugin = require("apisix.plugin")
+local send_statsd = require("apisix.plugins.udp-logger").send_udp_data
+local fetch_info = require("apisix.plugins.prometheus.exporter").parse_info_from_ctx
+local format = string.format
+local concat = table.concat
+local tostring = tostring
+local ngx = ngx
+
+
+local plugin_name = "datadog"
+
+local schema = {
+    type = "object",
+    properties = {
+        sample_rate = {type = "number", default = 1, minimum = 0, maximum = 1},
+        tags = {
+            type = "array",
+            items = {type = "string"},
+            default = {"source:apisix"}
+        }
+    }
+}
+
+local metadata_schema = {
+    type = "object",
+    properties = {
+        host = {type = "string", default = "0.0.0.0"},
+        port = {type = "integer", minimum = 0, default = 8125},
+        namespace = {type = "string", default = "apisix.dev"},
+    },
+}
+
+local _M = {
+    version = 0.1,
+    priority = 495,
+    name = plugin_name,
+    schema = schema,
+    metadata_schema = metadata_schema,
+}
+
+function _M.check_schema(conf, schema_type)
+    if schema_type == core.schema.TYPE_METADATA then
+        return core.schema.check(metadata_schema, conf)
+    end
+    return core.schema.check(schema, conf)
+end
+
+local function generate_tag(sample_rate, tag_arr, route_id, service_id, consumer_name, balancer_ip)
+    local rate, tags = "", ""
+
+    if sample_rate and sample_rate ~= 1 then
+        rate = "|@" .. tostring(sample_rate)
+    end
+
+    if tag_arr and #tag_arr > 0 then
+        tags = "|#" .. concat(tag_arr, ",")
+    end
+
+    if route_id ~= "" then
+        tags = tags .. "route_id:" .. route_id
+    end
+
+    if service_id ~= "" then
+        tags = tags .. "service_id:" .. service_id
+    end
+
+    if consumer_name ~= "" then
+        tags = tags .. "consumer_name:" .. consumer_name
+    end
+    if balancer_ip ~= "" then
+        tags = tags .. "balancer_ip:" .. balancer_ip
+    end
+
+    if tags ~= "" and tags:sub(1, 1) ~= "|" then
+        tags = "|#" .. tags
+    end
+
+    return rate .. tags
+
+end
+
+function _M.log(conf, ctx)
+    core.log.error("conf: ", core.json.delay_encode(conf, true))
+    core.log.error("ctx: ", core.json.delay_encode(ctx, true))
+    local metadata = plugin.plugin_metadata(plugin_name)
+    if not metadata then
+        core.log.error("received nil metadata")
+    end
+
+    local udp_conf = {
+        host = metadata.value.host,
+        port = metadata.value.port
+    }
+
+    local route_id, service_id, consumer_name, balancer_ip = fetch_info(conf, ctx)
+    local prefix = ""
+
+    if metadata.value.namespace ~= "" then
+        prefix = prefix .. "."
+    end
+
+    local suffix = generate_tag(conf.sample_rate, conf.tags,
+                    route_id, service_id, consumer_name, balancer_ip)
+
+    -- request counter
+    local ok, err = send_statsd(udp_conf,
+                        format("%s:%s|%s%s", prefix .. "request.counter", 1, "c", suffix))
+    if not ok then
+        core.log.error("failed to send request_count metric to DogStatsD. err: " .. err)
+    end
+
+
+    -- request latency histogram
+    local latency = (ngx.now() - ngx.req.start_time()) * 1000
+    local ok, err = send_statsd(udp_conf,
+                        format("%s:%s|%s%s", prefix .. "request.latency", latency, "h", suffix))
+    if not ok then
+        core.log.error("failed to send request latency metric to DogStatsD. err: " .. err)
+    end
+
+    -- upstream latency
+    local apisix_latency = latency
+    if ctx.var.upstream_response_time then
+        local upstream_latency = ctx.var.upstream_response_time * 1000
+        local ok, err = send_statsd(udp_conf,
+                format("%s:%s|%s%s", prefix .. "upstream.latency", upstream_latency, "h", suffix))
+        if not ok then
+            core.log.error("failed to send upstream latency metric to DogStatsD. err: " .. err)
+        end
+        apisix_latency =  apisix_latency - upstream_latency
+        if apisix_latency < 0 then
+            apisix_latency = 0
+        end
+    end
+
+    -- apisix_latency
+    local ok, err = send_statsd(udp_conf,
+            format("%s:%s|%s%s", prefix .. "apisix.latency", apisix_latency, "h", suffix))
+    if not ok then
+        core.log.error("failed to send apisix latency metric to DogStatsD. err: " .. err)
+    end
+
+    -- request body size timer
+    local ok, err = send_statsd(udp_conf,
+            format("%s:%s|%s%s", prefix .. "ingress.size", ctx.var.request_length, "ms", suffix))
+    if not ok then
+        core.log.error("failed to send request body size metric to DogStatsD. err: " .. err)
+    end
+
+    -- response body size timer
+    local ok, err = send_statsd(udp_conf,
+            format("%s:%s|%s%s", prefix .. "egress.size", ctx.var.bytes_sent, "ms", suffix))
+    if not ok then
+        core.log.error("failed to send response body size metric to DogStatsD. err: " .. err)
+    end
+end
+
+return _M
diff --git a/conf/config-default.yaml b/conf/config-default.yaml
index 54e760d9b48b..35676eaaffd4 100644
--- a/conf/config-default.yaml
+++ b/conf/config-default.yaml
@@ -341,6 +341,7 @@ plugins:                          # plugin list (sorted by priority)
   #- dubbo-proxy                   # priority: 507
   - grpc-transcode                 # priority: 506
   - prometheus                     # priority: 500
+  - datadog                        # priority: 495
   - echo                           # priority: 412
   - http-logger                    # priority: 410
   - sls-logger                     # priority: 406

From 3acdc46c3c9728d27deb8b4e1170b22e19a67d07 Mon Sep 17 00:00:00 2001
From: Bisakh Mondal <bisakhmondal00@gmail.com>
Date: Fri, 29 Oct 2021 20:57:23 +0530
Subject: [PATCH 02/17] existing apis refactoring

---
 apisix/plugins/prometheus/exporter.lua | 12 ++++++++----
 apisix/plugins/udp-logger.lua          |  8 +++++---
 2 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/apisix/plugins/prometheus/exporter.lua b/apisix/plugins/prometheus/exporter.lua
index b2c1ebcd952a..aa1ab4dccf3a 100644
--- a/apisix/plugins/prometheus/exporter.lua
+++ b/apisix/plugins/prometheus/exporter.lua
@@ -110,10 +110,7 @@ function _M.init()
 
 end
 
-
-function _M.log(conf, ctx)
-    local vars = ctx.var
-
+function _M.parse_info_from_ctx(conf, ctx)
     local route_id = ""
     local balancer_ip = ctx.balancer_ip or ""
     local service_id = ""
@@ -131,6 +128,13 @@ function _M.log(conf, ctx)
             end
         end
     end
+    return route_id, service_id, consumer_name, balancer_ip
+end
+
+function _M.log(conf, ctx)
+    local vars = ctx.var
+
+    local route_id, service_id, consumer_name, balancer_ip = _M.parse_info_from_ctx(conf, ctx)
 
     local matched_uri = ""
     local matched_host = ""
diff --git a/apisix/plugins/udp-logger.lua b/apisix/plugins/udp-logger.lua
index 06f903e6d607..0a279df9ff61 100644
--- a/apisix/plugins/udp-logger.lua
+++ b/apisix/plugins/udp-logger.lua
@@ -53,11 +53,13 @@ function _M.check_schema(conf)
     return core.schema.check(schema, conf)
 end
 
-local function send_udp_data(conf, log_message)
+function _M.send_udp_data(conf, log_message)
     local err_msg
     local res = true
     local sock = udp()
-    sock:settimeout(conf.timeout * 1000)
+    if not conf.timeout then
+        sock:settimeout(conf.timeout * 1000)
+    end
 
     core.log.info("sending a batch logs to ", conf.host, ":", conf.port)
 
@@ -130,7 +132,7 @@ function _M.log(conf, ctx)
             return false, 'error occurred while encoding the data: ' .. err
         end
 
-        return send_udp_data(conf, data)
+        return _M.send_udp_data(conf, data)
     end
 
     local config = {

From bbb29e38f7d6a4652a8aef61d480032fc6b96561 Mon Sep 17 00:00:00 2001
From: Bisakh Mondal <bisakhmondal00@gmail.com>
Date: Sat, 30 Oct 2021 00:11:10 +0530
Subject: [PATCH 03/17] tests

---
 t/admin/plugins.t | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/t/admin/plugins.t b/t/admin/plugins.t
index b5196dd0a042..9e1aab33f3eb 100644
--- a/t/admin/plugins.t
+++ b/t/admin/plugins.t
@@ -40,7 +40,7 @@ __DATA__
 --- request
 GET /apisix/admin/plugins/list
 --- response_body_like eval
-qr/\["real-ip","client-control","ext-plugin-pre-req","zipkin","request-id","fault-injection","serverless-pre-function","batch-requests","cors","ip-restriction","ua-restriction","referer-restriction","uri-blocker","request-validation","openid-connect","authz-casbin","wolf-rbac","ldap-auth","hmac-auth","basic-auth","jwt-auth","key-auth","consumer-restriction","authz-keycloak","proxy-mirror","proxy-cache","proxy-rewrite","api-breaker","limit-conn","limit-count","limit-req","gzip","server-info","traffic-split","redirect","response-rewrite","grpc-transcode","prometheus","echo","http-logger","sls-logger","tcp-logger","kafka-logger","syslog","udp-logger","example-plugin","serverless-post-function","ext-plugin-post-req"\]/
+qr/\["real-ip","client-control","ext-plugin-pre-req","zipkin","request-id","fault-injection","serverless-pre-function","batch-requests","cors","ip-restriction","ua-restriction","referer-restriction","uri-blocker","request-validation","openid-connect","authz-casbin","wolf-rbac","ldap-auth","hmac-auth","basic-auth","jwt-auth","key-auth","consumer-restriction","authz-keycloak","proxy-mirror","proxy-cache","proxy-rewrite","api-breaker","limit-conn","limit-count","limit-req","gzip","server-info","traffic-split","redirect","response-rewrite","grpc-transcode","prometheus","datadog","echo","http-logger","sls-logger","tcp-logger","kafka-logger","syslog","udp-logger","example-plugin","serverless-post-function","ext-plugin-post-req"\]/
 --- no_error_log
 [error]
 

From 960915da86f139fe64afe0759868a6ac26671cd7 Mon Sep 17 00:00:00 2001
From: Bisakh Mondal <bisakhmondal00@gmail.com>
Date: Mon, 1 Nov 2021 10:58:03 +0530
Subject: [PATCH 04/17] nginx tests

---
 apisix/plugins/datadog.lua |  33 ++++----
 t/plugin/datadog.t         | 156 +++++++++++++++++++++++++++++++++++++
 2 files changed, 174 insertions(+), 15 deletions(-)
 create mode 100644 t/plugin/datadog.t

diff --git a/apisix/plugins/datadog.lua b/apisix/plugins/datadog.lua
index d0aa060ef2b0..55ce245b7a9c 100644
--- a/apisix/plugins/datadog.lua
+++ b/apisix/plugins/datadog.lua
@@ -29,22 +29,23 @@ local plugin_name = "datadog"
 local schema = {
     type = "object",
     properties = {
-        sample_rate = {type = "number", default = 1, minimum = 0, maximum = 1},
-        tags = {
-            type = "array",
-            items = {type = "string"},
-            default = {"source:apisix"}
-        }
     }
 }
 
 local metadata_schema = {
     type = "object",
     properties = {
-        host = {type = "string", default = "0.0.0.0"},
-        port = {type = "integer", minimum = 0, default = 8125},
+        host = {type = "string"},
+        port = {type = "integer", minimum = 0},
         namespace = {type = "string", default = "apisix.dev"},
+        sample_rate = {type = "number", default = 1, minimum = 0, maximum = 1},
+        tags = {
+            type = "array",
+            items = {type = "string"},
+            default = {"source:apisix"}
+        }
     },
+    required = {"host", "port"}
 }
 
 local _M = {
@@ -62,7 +63,8 @@ function _M.check_schema(conf, schema_type)
     return core.schema.check(schema, conf)
 end
 
-local function generate_tag(sample_rate, tag_arr, route_id, service_id, consumer_name, balancer_ip)
+local function generate_tag(sample_rate, tag_arr, route_id, service_id,
+        consumer_name, balancer_ip, http_status)
     local rate, tags = "", ""
 
     if sample_rate and sample_rate ~= 1 then
@@ -87,6 +89,9 @@ local function generate_tag(sample_rate, tag_arr, route_id, service_id, consumer
     if balancer_ip ~= "" then
         tags = tags .. "balancer_ip:" .. balancer_ip
     end
+    if http_status then
+        tags = tags .. "http_status:" .. http_status
+    end
 
     if tags ~= "" and tags:sub(1, 1) ~= "|" then
         tags = "|#" .. tags
@@ -97,8 +102,6 @@ local function generate_tag(sample_rate, tag_arr, route_id, service_id, consumer
 end
 
 function _M.log(conf, ctx)
-    core.log.error("conf: ", core.json.delay_encode(conf, true))
-    core.log.error("ctx: ", core.json.delay_encode(ctx, true))
     local metadata = plugin.plugin_metadata(plugin_name)
     if not metadata then
         core.log.error("received nil metadata")
@@ -110,14 +113,14 @@ function _M.log(conf, ctx)
     }
 
     local route_id, service_id, consumer_name, balancer_ip = fetch_info(conf, ctx)
-    local prefix = ""
+    local prefix = metadata.value.namespace
 
-    if metadata.value.namespace ~= "" then
+    if prefix ~= "" then
         prefix = prefix .. "."
     end
 
-    local suffix = generate_tag(conf.sample_rate, conf.tags,
-                    route_id, service_id, consumer_name, balancer_ip)
+    local suffix = generate_tag(metadata.value.sample_rate, metadata.value.tags,
+                    route_id, service_id, consumer_name, balancer_ip, ctx.var.status)
 
     -- request counter
     local ok, err = send_statsd(udp_conf,
diff --git a/t/plugin/datadog.t b/t/plugin/datadog.t
new file mode 100644
index 000000000000..385d30ec010f
--- /dev/null
+++ b/t/plugin/datadog.t
@@ -0,0 +1,156 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+use t::APISIX 'no_plan';
+
+repeat_each(1);
+no_long_string();
+no_root_location();
+run_tests;
+
+__DATA__
+
+=== TEST 1: sanity check metadata
+--- config
+    location /t {
+        content_by_lua_block {
+            local plugin = require("apisix.plugins.datadog")
+            local ok, err = plugin.check_schema({host = "127.0.0.1", port = 8125}, 2)
+            if not ok then
+                ngx.say(err)
+            end
+
+            ngx.say("done")
+        }
+    }
+--- request
+GET /t
+--- response_body
+done
+--- no_error_log
+[error]
+
+
+
+=== TEST 2: missing host inside metadata
+--- config
+    location /t {
+        content_by_lua_block {
+            local plugin = require("apisix.plugins.datadog")
+            local ok, err = plugin.check_schema({port = 8125}, 2)
+            if not ok then
+                ngx.say(err)
+            end
+
+            ngx.say("done")
+        }
+    }
+--- request
+GET /t
+--- response_body
+property "host" is required
+done
+--- no_error_log
+[error]
+
+
+
+=== TEST 3: add plugin
+--- config
+    location /t {
+        content_by_lua_block {
+            local t = require("lib.test_admin").test
+            -- setting the metadata
+            local code, meta_body = t('/apisix/admin/plugin_metadata/datadog',
+                 ngx.HTTP_PUT,
+                 [[{
+                        "host":"127.0.0.1",
+                        "port": 8125
+                }]],
+                [[{
+                    "action": "set",
+                    "node": {
+                        "key": "/apisix/plugin_metadata/datadog",
+                        "value": {
+                            "host": "127.0.0.1",
+                            "namespace": "apisix.dev",
+                            "port": 8125,
+                            "tags": [
+                                "source:apisix"
+                            ],
+                            "sample_rate": 1
+                        }
+                    }
+                }]])
+            
+            if code >= 300 then
+                ngx.status = code
+                ngx.say("fail")
+                return
+            end
+
+            local code, body = t('/apisix/admin/routes/1',
+                 ngx.HTTP_PUT,
+                 [[{
+                        "plugins": {
+                            "datadog": {}
+                        },
+                        "upstream": {
+                            "nodes": {
+                                "127.0.0.1:1982": 1
+                            },
+                            "type": "roundrobin"
+                        },
+                        "uri": "/datadog"
+                }]],
+                [[{
+                    "node": {
+                        "value": {
+                            "plugins": {
+                                "datadog": {}
+                            },
+                            "upstream": {
+                                "nodes": {
+                                    "127.0.0.1:1982": 1
+                                },
+                                "type": "roundrobin"
+                            },
+                            "uri": "/datadog"
+                        },
+                        "key": "/apisix/routes/1"
+                    },
+                    "action": "set"
+                }]]
+                )
+
+            if code >= 300 then
+                ngx.status = code
+                ngx.say("fail")
+                return
+            end
+
+            ngx.print(meta_body .. "\n")
+            ngx.print(body .. "\n")
+
+        }
+    }
+--- request
+GET /t
+--- response_body
+passed
+passed
+--- no_error_log
+[error]

From 2f409ea2eb003c7dee633080329c6bed2c47166d Mon Sep 17 00:00:00 2001
From: Bisakh Mondal <bisakhmondal00@gmail.com>
Date: Mon, 1 Nov 2021 16:15:43 +0530
Subject: [PATCH 05/17] revamped implementation with batch processor: [todo:
 test with mock/real backend]

---
 apisix/plugins/datadog.lua             | 263 ++++++++++++++++---------
 apisix/plugins/prometheus/exporter.lua |  12 +-
 apisix/plugins/udp-logger.lua          |   8 +-
 t/plugin/datadog.t                     |  62 ++----
 4 files changed, 203 insertions(+), 142 deletions(-)

diff --git a/apisix/plugins/datadog.lua b/apisix/plugins/datadog.lua
index 55ce245b7a9c..87349a754de3 100644
--- a/apisix/plugins/datadog.lua
+++ b/apisix/plugins/datadog.lua
@@ -16,36 +16,47 @@
 
 local core = require("apisix.core")
 local plugin = require("apisix.plugin")
-local send_statsd = require("apisix.plugins.udp-logger").send_udp_data
-local fetch_info = require("apisix.plugins.prometheus.exporter").parse_info_from_ctx
+local batch_processor = require("apisix.utils.batch-processor")
+local fetch_log = require("apisix.utils.log-util").get_full_log
+local ngx = ngx
+local udp = ngx.socket.udp
 local format = string.format
 local concat = table.concat
-local tostring = tostring
-local ngx = ngx
-
+local buffers = {}
+local ipairs = ipairs
+local stale_timer_running = false
+local timer_at = ngx.timer.at
 
 local plugin_name = "datadog"
+local defaults = {
+    host = "127.0.0.1",
+    port = 8125,
+    namespace = "apisix",
+    constant_tags = {"source:apisix"}
+}
 
 local schema = {
     type = "object",
     properties = {
+        buffer_duration = {type = "integer", minimum = 1, default = 60},
+        inactive_timeout = {type = "integer", minimum = 1, default = 2},
+        batch_max_size = {type = "integer", minimum = 1, default = 1},
+        max_retry_count = {type = "integer", minimum = 1, default = 3},
     }
 }
 
 local metadata_schema = {
     type = "object",
     properties = {
-        host = {type = "string"},
-        port = {type = "integer", minimum = 0},
-        namespace = {type = "string", default = "apisix.dev"},
-        sample_rate = {type = "number", default = 1, minimum = 0, maximum = 1},
-        tags = {
+        host = {type = "string", default= defaults.host},
+        port = {type = "integer", minimum = 0, default = defaults.port},
+        namespace = {type = "string", default = defaults.namespace},
+        constant_tags = {
             type = "array",
             items = {type = "string"},
-            default = {"source:apisix"}
+            default = defaults.constant_tags
         }
     },
-    required = {"host", "port"}
 }
 
 local _M = {
@@ -63,116 +74,192 @@ function _M.check_schema(conf, schema_type)
     return core.schema.check(schema, conf)
 end
 
-local function generate_tag(sample_rate, tag_arr, route_id, service_id,
-        consumer_name, balancer_ip, http_status)
-    local rate, tags = "", ""
+local function generate_tag(conf, const_tags)
+    local tags = ""
 
-    if sample_rate and sample_rate ~= 1 then
-        rate = "|@" .. tostring(sample_rate)
+    if const_tags and #const_tags > 0 then
+        tags = concat(const_tags, ",") .. ","
     end
 
-    if tag_arr and #tag_arr > 0 then
-        tags = "|#" .. concat(tag_arr, ",")
+    if conf.route_id then
+        tags = tags .. "route_id:" .. conf.route_id .. ","
     end
 
-    if route_id ~= "" then
-        tags = tags .. "route_id:" .. route_id
+    if conf.service_id then
+        tags = tags .. "service_id:" .. conf.service_id .. ","
     end
 
-    if service_id ~= "" then
-        tags = tags .. "service_id:" .. service_id
+    if conf.consumer then
+        tags = tags .. "consumer:" .. conf.consumer .. ","
     end
-
-    if consumer_name ~= "" then
-        tags = tags .. "consumer_name:" .. consumer_name
+    if conf.balancer_ip ~= "" then
+        tags = tags .. "balancer_ip:" .. conf.balancer_ip .. ","
     end
-    if balancer_ip ~= "" then
-        tags = tags .. "balancer_ip:" .. balancer_ip
-    end
-    if http_status then
-        tags = tags .. "http_status:" .. http_status
+    if conf.response.status then
+        tags = tags .. "response_status:" .. conf.response.status .. ","
     end
 
-    if tags ~= "" and tags:sub(1, 1) ~= "|" then
-        tags = "|#" .. tags
+    if tags ~= "" then
+        tags = "|#" .. tags:sub(1, -2)
     end
 
-    return rate .. tags
-
+    return tags
 end
 
-function _M.log(conf, ctx)
-    local metadata = plugin.plugin_metadata(plugin_name)
-    if not metadata then
-        core.log.error("received nil metadata")
+-- remove stale objects from the memory after timer expires
+local function remove_stale_objects(premature)
+    if premature then
+        return
     end
 
-    local udp_conf = {
-        host = metadata.value.host,
-        port = metadata.value.port
-    }
+    for key, batch in ipairs(buffers) do
+        if #batch.entry_buffer.entries == 0 and #batch.batch_to_process == 0 then
+            core.log.warn("removing batch processor stale object, conf: ",
+                          core.json.delay_encode(key))
+            buffers[key] = nil
+        end
+    end
+
+    stale_timer_running = false
+end
 
-    local route_id, service_id, consumer_name, balancer_ip = fetch_info(conf, ctx)
-    local prefix = metadata.value.namespace
+function _M.log(conf, ctx)
 
-    if prefix ~= "" then
-        prefix = prefix .. "."
+    if not stale_timer_running then
+        -- run the timer every 30 mins if any log is present
+        timer_at(1800, remove_stale_objects)
+        stale_timer_running = true
     end
 
-    local suffix = generate_tag(metadata.value.sample_rate, metadata.value.tags,
-                    route_id, service_id, consumer_name, balancer_ip, ctx.var.status)
+    local entry = fetch_log(ngx, {})
+    entry.upstream_latency = ctx.var.upstream_response_time * 1000
+    entry.balancer_ip = ctx.balancer_ip or ""
 
-    -- request counter
-    local ok, err = send_statsd(udp_conf,
-                        format("%s:%s|%s%s", prefix .. "request.counter", 1, "c", suffix))
-    if not ok then
-        core.log.error("failed to send request_count metric to DogStatsD. err: " .. err)
+    local log_buffer = buffers[conf]
+    if log_buffer then
+        log_buffer:push(entry)
+        return
     end
 
+    -- Generate a function to be executed by the batch processor
+    local func = function(entries, batch_max_size)
+        -- Fetching metadata details
+        local metadata = plugin.plugin_metadata(plugin_name)
+        if not metadata then
+            core.log.info("received nil metadata: using metadata defaults: ",
+                                core.json.delay_encode(defaults, true))
+            metadata = {}
+            metadata.value = defaults
+        end
+
+        -- Creating a udp socket
+        local sock = udp()
+        local host, port = metadata.value.host, metadata.value.port
+        core.log.info("sending batch metrics to dogstatsd: ", host, ":", port)
 
-    -- request latency histogram
-    local latency = (ngx.now() - ngx.req.start_time()) * 1000
-    local ok, err = send_statsd(udp_conf,
-                        format("%s:%s|%s%s", prefix .. "request.latency", latency, "h", suffix))
-    if not ok then
-        core.log.error("failed to send request latency metric to DogStatsD. err: " .. err)
-    end
+        local ok, err = sock:setpeername(host, port)
 
-    -- upstream latency
-    local apisix_latency = latency
-    if ctx.var.upstream_response_time then
-        local upstream_latency = ctx.var.upstream_response_time * 1000
-        local ok, err = send_statsd(udp_conf,
-                format("%s:%s|%s%s", prefix .. "upstream.latency", upstream_latency, "h", suffix))
         if not ok then
-            core.log.error("failed to send upstream latency metric to DogStatsD. err: " .. err)
+            return false, "failed to connect to UDP server: host[" .. host
+                        .. "] port[" .. tostring(port) .. "] err: " .. err
         end
-        apisix_latency =  apisix_latency - upstream_latency
-        if apisix_latency < 0 then
-            apisix_latency = 0
+
+        -- Generate prefix & suffix according dogstatsd udp data format.
+        local prefix = metadata.value.namespace
+        if prefix ~= "" then
+            prefix = prefix .. "."
         end
-    end
 
-    -- apisix_latency
-    local ok, err = send_statsd(udp_conf,
-            format("%s:%s|%s%s", prefix .. "apisix.latency", apisix_latency, "h", suffix))
-    if not ok then
-        core.log.error("failed to send apisix latency metric to DogStatsD. err: " .. err)
-    end
+        core.log.info("datadog batch_entry: ", core.json.delay_encode(entries, true))
+        for _, entry in ipairs(entries) do
+            local suffix = generate_tag(entry, metadata.value.tags)
+
+            -- request counter
+            local ok, err = sock:send(format("%s:%s|%s%s", prefix ..
+                                            "request.counter", 1, "c", suffix))
+            if not ok then
+                core.log.error("failed to report request count to dogstatsd server: host[" .. host
+                        .. "] port[" .. tostring(port) .. "] err: " .. err)
+            end
+
+
+            -- request latency histogram
+            local ok, err = sock:send(format("%s:%s|%s%s", prefix ..
+                                        "request.latency", entry.latency, "h", suffix))
+            if not ok then
+                core.log.error("failed to report request latency to dogstatsd server: host["
+                        .. host .. "] port[" .. tostring(port) .. "] err: " .. err)
+            end
+
+            -- upstream latency
+            local apisix_latency = entry.latency
+            if entry.upstream_latency then
+                local ok, err = sock:send(format("%s:%s|%s%s", prefix ..
+                                        "upstream.latency", entry.upstream_latency, "h", suffix))
+                if not ok then
+                    core.log.error("failed to report upstream latency to dogstatsd server: host["
+                                .. host .. "] port[" .. tostring(port) .. "] err: " .. err)
+                end
+                apisix_latency =  apisix_latency - entry.upstream_latency
+                if apisix_latency < 0 then
+                    apisix_latency = 0
+                end
+            end
+
+            -- apisix_latency
+            local ok, err = sock:send(format("%s:%s|%s%s", prefix ..
+                                            "apisix.latency", apisix_latency, "h", suffix))
+            if not ok then
+                core.log.error("failed to report apisix latency to dogstatsd server: host[" .. host
+                        .. "] port[" .. tostring(port) .. "] err: " .. err)
+            end
+
+            -- request body size timer
+            local ok, err = sock:send(format("%s:%s|%s%s", prefix ..
+                                            "ingress.size", entry.request.size, "ms", suffix))
+            if not ok then
+                core.log.error("failed to report req body size to dogstatsd server: host[" .. host
+                        .. "] port[" .. tostring(port) .. "] err: " .. err)
+            end
 
-    -- request body size timer
-    local ok, err = send_statsd(udp_conf,
-            format("%s:%s|%s%s", prefix .. "ingress.size", ctx.var.request_length, "ms", suffix))
-    if not ok then
-        core.log.error("failed to send request body size metric to DogStatsD. err: " .. err)
+            -- response body size timer
+            local ok, err = sock:send(format("%s:%s|%s%s", prefix ..
+                                            "egress.size", entry.response.size, "ms", suffix))
+            if not ok then
+                core.log.error("failed to report response body size to dogstatsd server: host["
+                        .. host .. "] port[" .. tostring(port) .. "] err: " .. err)
+            end
+        end
+
+        -- Releasing the UDP socket desciptor
+        ok, err = sock:close()
+        if not ok then
+            core.log.error("failed to close the UDP connection, host[",
+                            host, "] port[", port, "] ", err)
+        end
     end
 
-    -- response body size timer
-    local ok, err = send_statsd(udp_conf,
-            format("%s:%s|%s%s", prefix .. "egress.size", ctx.var.bytes_sent, "ms", suffix))
-    if not ok then
-        core.log.error("failed to send response body size metric to DogStatsD. err: " .. err)
+    local config = {
+        name = plugin_name,
+        retry_delay = conf.retry_delay,
+        batch_max_size = conf.batch_max_size,
+        max_retry_count = conf.max_retry_count,
+        buffer_duration = conf.buffer_duration,
+        inactive_timeout = conf.inactive_timeout,
+        route_id = ctx.var.route_id,
+        server_addr = ctx.var.server_addr,
+    }
+
+    local err
+    log_buffer, err = batch_processor:new(func, config)
+
+    if not log_buffer then
+        core.log.error("error when creating the batch processor: ", err)
+        return
     end
+
+    buffers[conf] = log_buffer
+    log_buffer:push(entry)
 end
 
 return _M
diff --git a/apisix/plugins/prometheus/exporter.lua b/apisix/plugins/prometheus/exporter.lua
index aa1ab4dccf3a..b2c1ebcd952a 100644
--- a/apisix/plugins/prometheus/exporter.lua
+++ b/apisix/plugins/prometheus/exporter.lua
@@ -110,7 +110,10 @@ function _M.init()
 
 end
 
-function _M.parse_info_from_ctx(conf, ctx)
+
+function _M.log(conf, ctx)
+    local vars = ctx.var
+
     local route_id = ""
     local balancer_ip = ctx.balancer_ip or ""
     local service_id = ""
@@ -128,13 +131,6 @@ function _M.parse_info_from_ctx(conf, ctx)
             end
         end
     end
-    return route_id, service_id, consumer_name, balancer_ip
-end
-
-function _M.log(conf, ctx)
-    local vars = ctx.var
-
-    local route_id, service_id, consumer_name, balancer_ip = _M.parse_info_from_ctx(conf, ctx)
 
     local matched_uri = ""
     local matched_host = ""
diff --git a/apisix/plugins/udp-logger.lua b/apisix/plugins/udp-logger.lua
index 0a279df9ff61..06f903e6d607 100644
--- a/apisix/plugins/udp-logger.lua
+++ b/apisix/plugins/udp-logger.lua
@@ -53,13 +53,11 @@ function _M.check_schema(conf)
     return core.schema.check(schema, conf)
 end
 
-function _M.send_udp_data(conf, log_message)
+local function send_udp_data(conf, log_message)
     local err_msg
     local res = true
     local sock = udp()
-    if not conf.timeout then
-        sock:settimeout(conf.timeout * 1000)
-    end
+    sock:settimeout(conf.timeout * 1000)
 
     core.log.info("sending a batch logs to ", conf.host, ":", conf.port)
 
@@ -132,7 +130,7 @@ function _M.log(conf, ctx)
             return false, 'error occurred while encoding the data: ' .. err
         end
 
-        return _M.send_udp_data(conf, data)
+        return send_udp_data(conf, data)
     end
 
     local config = {
diff --git a/t/plugin/datadog.t b/t/plugin/datadog.t
index 385d30ec010f..917bc5f36575 100644
--- a/t/plugin/datadog.t
+++ b/t/plugin/datadog.t
@@ -19,6 +19,18 @@ use t::APISIX 'no_plan';
 repeat_each(1);
 no_long_string();
 no_root_location();
+no_shuffle();
+
+add_block_preprocessor(sub {
+    my ($block) = @_;
+    if (!$block->request) {
+        $block->set_value("request", "GET /t");
+    }
+    if (!$block->no_error_log && !$block->error_log) {
+        $block->set_value("no_error_log", "[error]\n[alert]");
+    }
+});
+
 run_tests;
 
 __DATA__
@@ -36,39 +48,12 @@ __DATA__
             ngx.say("done")
         }
     }
---- request
-GET /t
---- response_body
-done
---- no_error_log
-[error]
-
-
-
-=== TEST 2: missing host inside metadata
---- config
-    location /t {
-        content_by_lua_block {
-            local plugin = require("apisix.plugins.datadog")
-            local ok, err = plugin.check_schema({port = 8125}, 2)
-            if not ok then
-                ngx.say(err)
-            end
-
-            ngx.say("done")
-        }
-    }
---- request
-GET /t
 --- response_body
-property "host" is required
 done
---- no_error_log
-[error]
 
 
 
-=== TEST 3: add plugin
+=== TEST 2: add plugin
 --- config
     location /t {
         content_by_lua_block {
@@ -81,21 +66,20 @@ done
                         "port": 8125
                 }]],
                 [[{
-                    "action": "set",
                     "node": {
-                        "key": "/apisix/plugin_metadata/datadog",
                         "value": {
+                            "namespace": "apisix",
                             "host": "127.0.0.1",
-                            "namespace": "apisix.dev",
-                            "port": 8125,
-                            "tags": [
+                            "constant_tags": [
                                 "source:apisix"
                             ],
-                            "sample_rate": 1
-                        }
-                    }
+                            "port": 8125
+                        },
+                        "key": "/apisix/plugin_metadata/datadog"
+                    },
+                    "action": "set"
                 }]])
-            
+
             if code >= 300 then
                 ngx.status = code
                 ngx.say("fail")
@@ -147,10 +131,6 @@ done
 
         }
     }
---- request
-GET /t
 --- response_body
 passed
 passed
---- no_error_log
-[error]

From ced7a161fcf8e697a01e6ae353863ad64a7a182a Mon Sep 17 00:00:00 2001
From: Bisakh Mondal <bisakhmondal00@gmail.com>
Date: Mon, 1 Nov 2021 17:53:46 +0530
Subject: [PATCH 06/17] mock testing dogstatsd

---
 apisix/plugins/datadog.lua |  6 +++---
 t/APISIX.pm                |  7 +++++++
 t/lib/mock-dogstatsd.lua   | 40 ++++++++++++++++++++++++++++++++++++++
 t/plugin/datadog.t         | 20 ++++++++++++++-----
 4 files changed, 65 insertions(+), 8 deletions(-)
 create mode 100644 t/lib/mock-dogstatsd.lua

diff --git a/apisix/plugins/datadog.lua b/apisix/plugins/datadog.lua
index 87349a754de3..7926e5b5ad81 100644
--- a/apisix/plugins/datadog.lua
+++ b/apisix/plugins/datadog.lua
@@ -40,8 +40,8 @@ local schema = {
     properties = {
         buffer_duration = {type = "integer", minimum = 1, default = 60},
         inactive_timeout = {type = "integer", minimum = 1, default = 2},
-        batch_max_size = {type = "integer", minimum = 1, default = 1},
-        max_retry_count = {type = "integer", minimum = 1, default = 3},
+        batch_max_size = {type = "integer", minimum = 1, default = 5000},
+        max_retry_count = {type = "integer", minimum = 1, default = 1},
     }
 }
 
@@ -172,7 +172,7 @@ function _M.log(conf, ctx)
 
         core.log.info("datadog batch_entry: ", core.json.delay_encode(entries, true))
         for _, entry in ipairs(entries) do
-            local suffix = generate_tag(entry, metadata.value.tags)
+            local suffix = generate_tag(entry, metadata.value.constant_tags)
 
             -- request counter
             local ok, err = sock:send(format("%s:%s|%s%s", prefix ..
diff --git a/t/APISIX.pm b/t/APISIX.pm
index da9a46e75e84..30f4d3f58ba2 100644
--- a/t/APISIX.pm
+++ b/t/APISIX.pm
@@ -399,6 +399,13 @@ $stream_config
         listen 1985;
         $stream_server_config
     }
+    server {
+        listen 8125;
+        content_by_lua_block {
+            local mock = require("lib.mock-dogstastd")
+            mock.go()
+        }
+    }
 }
 _EOC_
     }
diff --git a/t/lib/mock-dogstatsd.lua b/t/lib/mock-dogstatsd.lua
new file mode 100644
index 000000000000..1f8b7e4ba347
--- /dev/null
+++ b/t/lib/mock-dogstatsd.lua
@@ -0,0 +1,40 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements.  See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License.  You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+local core = require("apisix.core")
+local ngx = ngx
+local socket = ngx.req.socket
+
+local _M = {}
+
+function _M.go()
+    local sock, err = socket()
+    if not sock then
+        core.log.error("failed to get the request socket: ", err)
+        return
+    end
+
+    while true do
+        local data, err = sock:receive('*a')
+        if not data then
+            core.log.error("Socket error: ", err)
+        else
+            core.log.warn(data)
+        end
+    end
+end
+
+return _M
diff --git a/t/plugin/datadog.t b/t/plugin/datadog.t
index 917bc5f36575..49335aafafa0 100644
--- a/t/plugin/datadog.t
+++ b/t/plugin/datadog.t
@@ -54,6 +54,7 @@ done
 
 
 === TEST 2: add plugin
+--- wait: 6
 --- config
     location /t {
         content_by_lua_block {
@@ -90,7 +91,9 @@ done
                  ngx.HTTP_PUT,
                  [[{
                         "plugins": {
-                            "datadog": {}
+                            "datadog": {
+                                "batch_max_size" : 1
+                            }
                         },
                         "upstream": {
                             "nodes": {
@@ -98,13 +101,15 @@ done
                             },
                             "type": "roundrobin"
                         },
-                        "uri": "/datadog"
+                        "uri": "/index.html"
                 }]],
                 [[{
                     "node": {
                         "value": {
                             "plugins": {
-                                "datadog": {}
+                                "datadog": {
+                                    "batch_max_size": 1
+                                }
                             },
                             "upstream": {
                                 "nodes": {
@@ -112,7 +117,7 @@ done
                                 },
                                 "type": "roundrobin"
                             },
-                            "uri": "/datadog"
+                            "uri": "/index.html"
                         },
                         "key": "/apisix/routes/1"
                     },
@@ -126,9 +131,14 @@ done
                 return
             end
 
+            -- local code, _, body4 = t("/index.html", "GET")
+            -- if code >= 300 then
+            --    ngx.status = code
+            --    ngx.say("fail")
+            --    return
+            --end
             ngx.print(meta_body .. "\n")
             ngx.print(body .. "\n")
-
         }
     }
 --- response_body

From 5eaa616c9845a968ba96f5e465d430fa5420e035 Mon Sep 17 00:00:00 2001
From: Bisakh Mondal <bisakhmondal00@gmail.com>
Date: Fri, 5 Nov 2021 11:37:15 +0530
Subject: [PATCH 07/17] fix mock testing

---
 apisix/plugins/datadog.lua                    |  6 +-
 t/APISIX.pm                                   |  7 ---
 ...{mock-dogstatsd.lua => mock_dogstatsd.lua} | 18 +++---
 t/plugin/datadog.t                            | 63 ++++++++++++++++---
 4 files changed, 68 insertions(+), 26 deletions(-)
 rename t/lib/{mock-dogstatsd.lua => mock_dogstatsd.lua} (76%)

diff --git a/apisix/plugins/datadog.lua b/apisix/plugins/datadog.lua
index 7926e5b5ad81..400b2c018126 100644
--- a/apisix/plugins/datadog.lua
+++ b/apisix/plugins/datadog.lua
@@ -81,15 +81,15 @@ local function generate_tag(conf, const_tags)
         tags = concat(const_tags, ",") .. ","
     end
 
-    if conf.route_id then
+    if conf.route_id and conf.route_id ~= "" then
         tags = tags .. "route_id:" .. conf.route_id .. ","
     end
 
-    if conf.service_id then
+    if conf.service_id and conf.service_id ~= "" then
         tags = tags .. "service_id:" .. conf.service_id .. ","
     end
 
-    if conf.consumer then
+    if conf.consumer and conf.consumer ~= "" then
         tags = tags .. "consumer:" .. conf.consumer .. ","
     end
     if conf.balancer_ip ~= "" then
diff --git a/t/APISIX.pm b/t/APISIX.pm
index 30f4d3f58ba2..da9a46e75e84 100644
--- a/t/APISIX.pm
+++ b/t/APISIX.pm
@@ -399,13 +399,6 @@ $stream_config
         listen 1985;
         $stream_server_config
     }
-    server {
-        listen 8125;
-        content_by_lua_block {
-            local mock = require("lib.mock-dogstastd")
-            mock.go()
-        }
-    }
 }
 _EOC_
     }
diff --git a/t/lib/mock-dogstatsd.lua b/t/lib/mock_dogstatsd.lua
similarity index 76%
rename from t/lib/mock-dogstatsd.lua
rename to t/lib/mock_dogstatsd.lua
index 1f8b7e4ba347..49436232aeb6 100644
--- a/t/lib/mock-dogstatsd.lua
+++ b/t/lib/mock_dogstatsd.lua
@@ -22,17 +22,21 @@ local _M = {}
 
 function _M.go()
     local sock, err = socket()
-    if not sock then
-        core.log.error("failed to get the request socket: ", err)
+    if sock then
+        ngx.ctx.sock = sock
+     else
+        sock:send("failed to get the request socket: ", err)
         return
-    end
+     end
+
+    for i = 1, 3 do
+        local data, err = sock:receive()
 
-    while true do
-        local data, err = sock:receive('*a')
         if not data then
-            core.log.error("Socket error: ", err)
+            core.log.error("socket error: ", err)
+            return
         else
-            core.log.warn(data)
+            core.log.warn("message received: ", data)
         end
     end
 end
diff --git a/t/plugin/datadog.t b/t/plugin/datadog.t
index 49335aafafa0..6634ac9ba2f2 100644
--- a/t/plugin/datadog.t
+++ b/t/plugin/datadog.t
@@ -23,6 +23,21 @@ no_shuffle();
 
 add_block_preprocessor(sub {
     my ($block) = @_;
+
+    $block->set_value("stream_conf_enable", 1);
+
+    if (!defined $block->extra_stream_config) {
+        my $stream_config = <<_EOC_;
+    server {
+        listen 8125 udp;
+        content_by_lua_block {
+            require("lib.mock_dogstatsd").go()
+        }
+    }
+_EOC_
+        $block->set_value("extra_stream_config", $stream_config);
+    }
+
     if (!$block->request) {
         $block->set_value("request", "GET /t");
     }
@@ -54,7 +69,6 @@ done
 
 
 === TEST 2: add plugin
---- wait: 6
 --- config
     location /t {
         content_by_lua_block {
@@ -101,7 +115,7 @@ done
                             },
                             "type": "roundrobin"
                         },
-                        "uri": "/index.html"
+                        "uri": "/opentracing"
                 }]],
                 [[{
                     "node": {
@@ -117,7 +131,7 @@ done
                                 },
                                 "type": "roundrobin"
                             },
-                            "uri": "/index.html"
+                            "uri": "/opentracing"
                         },
                         "key": "/apisix/routes/1"
                     },
@@ -131,12 +145,7 @@ done
                 return
             end
 
-            -- local code, _, body4 = t("/index.html", "GET")
-            -- if code >= 300 then
-            --    ngx.status = code
-            --    ngx.say("fail")
-            --    return
-            --end
+
             ngx.print(meta_body .. "\n")
             ngx.print(body .. "\n")
         }
@@ -144,3 +153,39 @@ done
 --- response_body
 passed
 passed
+
+
+
+=== TEST 3: testing behaviour with mock suite
+--- config
+    location /t {
+        content_by_lua_block {
+            local t = require("lib.test_admin").test
+
+            local code, _, body = t("/opentracing", "GET")
+             if code >= 300 then
+                ngx.status = code
+                ngx.say("fail")
+                return
+            end
+            ngx.say(body:sub(1, -2))
+        }
+    }
+--- wait: 3
+--- response_body
+opentracing
+--- grep_error_log eval
+qr/message received: apisix([.\w]+)/
+--- grep_error_log_out
+message received: apisix.request.counter
+message received: apisix.request.latency
+message received: apisix.upstream.latency
+message received: apisix.apisix.latency
+message received: apisix.ingress.size
+message received: apisix.egress.size
+message received: apisix.request.counter
+message received: apisix.request.latency
+message received: apisix.upstream.latency
+message received: apisix.apisix.latency
+message received: apisix.ingress.size
+message received: apisix.egress.size

From 129eb47095ec5525a9010977a40c92c383a8a57e Mon Sep 17 00:00:00 2001
From: Bisakh Mondal <bisakhmondal00@gmail.com>
Date: Fri, 5 Nov 2021 12:51:39 +0530
Subject: [PATCH 08/17] fix error related to batchprocessor

---
 apisix/plugins/datadog.lua |  7 +++++--
 t/lib/mock_dogstatsd.lua   |  7 +++++--
 t/plugin/datadog.t         | 12 ++++--------
 3 files changed, 14 insertions(+), 12 deletions(-)

diff --git a/apisix/plugins/datadog.lua b/apisix/plugins/datadog.lua
index 400b2c018126..ee48a3897669 100644
--- a/apisix/plugins/datadog.lua
+++ b/apisix/plugins/datadog.lua
@@ -24,6 +24,7 @@ local format = string.format
 local concat = table.concat
 local buffers = {}
 local ipairs = ipairs
+local tostring = tostring
 local stale_timer_running = false
 local timer_at = ngx.timer.at
 
@@ -39,7 +40,7 @@ local schema = {
     type = "object",
     properties = {
         buffer_duration = {type = "integer", minimum = 1, default = 60},
-        inactive_timeout = {type = "integer", minimum = 1, default = 2},
+        inactive_timeout = {type = "integer", minimum = 1, default = 5},
         batch_max_size = {type = "integer", minimum = 1, default = 5000},
         max_retry_count = {type = "integer", minimum = 1, default = 1},
     }
@@ -237,8 +238,10 @@ function _M.log(conf, ctx)
             core.log.error("failed to close the UDP connection, host[",
                             host, "] port[", port, "] ", err)
         end
-    end
 
+        -- Returning at the end and ensuring the resource has been released.
+        return true
+    end
     local config = {
         name = plugin_name,
         retry_delay = conf.retry_delay,
diff --git a/t/lib/mock_dogstatsd.lua b/t/lib/mock_dogstatsd.lua
index 49436232aeb6..10dd80cdb402 100644
--- a/t/lib/mock_dogstatsd.lua
+++ b/t/lib/mock_dogstatsd.lua
@@ -29,11 +29,14 @@ function _M.go()
         return
      end
 
-    for i = 1, 3 do
+    while true do
         local data, err = sock:receive()
 
         if not data then
-            core.log.error("socket error: ", err)
+            if err and err ~= "no more data" then
+                core.log.info("socket error, returning: ", err)
+            end
+
             return
         else
             core.log.warn("message received: ", data)
diff --git a/t/plugin/datadog.t b/t/plugin/datadog.t
index 6634ac9ba2f2..2bf3f97ca75c 100644
--- a/t/plugin/datadog.t
+++ b/t/plugin/datadog.t
@@ -106,7 +106,8 @@ done
                  [[{
                         "plugins": {
                             "datadog": {
-                                "batch_max_size" : 1
+                                "batch_max_size" : 1,
+                                "max_retry_count": 1
                             }
                         },
                         "upstream": {
@@ -122,7 +123,8 @@ done
                         "value": {
                             "plugins": {
                                 "datadog": {
-                                    "batch_max_size": 1
+                                    "batch_max_size": 1,
+                                    "max_retry_count": 1
                                 }
                             },
                             "upstream": {
@@ -183,9 +185,3 @@ message received: apisix.upstream.latency
 message received: apisix.apisix.latency
 message received: apisix.ingress.size
 message received: apisix.egress.size
-message received: apisix.request.counter
-message received: apisix.request.latency
-message received: apisix.upstream.latency
-message received: apisix.apisix.latency
-message received: apisix.ingress.size
-message received: apisix.egress.size

From c0decea19d6a1fbbc0510967e0c569f9908296b8 Mon Sep 17 00:00:00 2001
From: Bisakh Mondal <bisakhmondal00@gmail.com>
Date: Fri, 5 Nov 2021 15:57:15 +0530
Subject: [PATCH 09/17] perf: string concat to table concat

---
 apisix/plugins/datadog.lua | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/apisix/plugins/datadog.lua b/apisix/plugins/datadog.lua
index ee48a3897669..460061356124 100644
--- a/apisix/plugins/datadog.lua
+++ b/apisix/plugins/datadog.lua
@@ -76,32 +76,33 @@ function _M.check_schema(conf, schema_type)
 end
 
 local function generate_tag(conf, const_tags)
-    local tags = ""
-
+    local tags
     if const_tags and #const_tags > 0 then
-        tags = concat(const_tags, ",") .. ","
+        tags = core.table.deepcopy(const_tags)
+    else
+        tags = {}
     end
 
     if conf.route_id and conf.route_id ~= "" then
-        tags = tags .. "route_id:" .. conf.route_id .. ","
+        core.table.insert(tags, "route_id:" .. conf.route_id)
     end
 
     if conf.service_id and conf.service_id ~= "" then
-        tags = tags .. "service_id:" .. conf.service_id .. ","
+        core.table.insert(tags, "service_id:" .. conf.service_id)
     end
 
     if conf.consumer and conf.consumer ~= "" then
-        tags = tags .. "consumer:" .. conf.consumer .. ","
+        core.table.insert(tags, "consumer:" .. conf.consumer)
     end
     if conf.balancer_ip ~= "" then
-        tags = tags .. "balancer_ip:" .. conf.balancer_ip .. ","
+        core.table.insert(tags, "balancer_ip:" .. conf.balancer_ip)
     end
     if conf.response.status then
-        tags = tags .. "response_status:" .. conf.response.status .. ","
+        core.table.insert(tags, "response_status:" .. conf.response.status)
     end
 
     if tags ~= "" then
-        tags = "|#" .. tags:sub(1, -2)
+        tags = "|#" .. concat(tags, ',')
     end
 
     return tags

From 45a7f029e7b620d87174f85e1f6401194d727261 Mon Sep 17 00:00:00 2001
From: Bisakh Mondal <bisakhmondal00@gmail.com>
Date: Fri, 5 Nov 2021 15:57:56 +0530
Subject: [PATCH 10/17] new tests

---
 t/lib/mock_dogstatsd.lua |   8 +-
 t/plugin/datadog.t       | 200 ++++++++++++++++++++++++++++++++++++---
 2 files changed, 191 insertions(+), 17 deletions(-)

diff --git a/t/lib/mock_dogstatsd.lua b/t/lib/mock_dogstatsd.lua
index 10dd80cdb402..f4ee675162fa 100644
--- a/t/lib/mock_dogstatsd.lua
+++ b/t/lib/mock_dogstatsd.lua
@@ -22,10 +22,8 @@ local _M = {}
 
 function _M.go()
     local sock, err = socket()
-    if sock then
-        ngx.ctx.sock = sock
-     else
-        sock:send("failed to get the request socket: ", err)
+    if not sock then
+        core.log.error("failed to get the request socket: ", err)
         return
      end
 
@@ -34,7 +32,7 @@ function _M.go()
 
         if not data then
             if err and err ~= "no more data" then
-                core.log.info("socket error, returning: ", err)
+                core.log.error("socket error, returning: ", err)
             end
 
             return
diff --git a/t/plugin/datadog.t b/t/plugin/datadog.t
index 2bf3f97ca75c..7a52c7bdd065 100644
--- a/t/plugin/datadog.t
+++ b/t/plugin/datadog.t
@@ -148,8 +148,8 @@ done
             end
 
 
-            ngx.print(meta_body .. "\n")
-            ngx.print(body .. "\n")
+            ngx.say(meta_body)
+            ngx.say(body)
         }
     }
 --- response_body
@@ -170,18 +170,194 @@ passed
                 ngx.say("fail")
                 return
             end
-            ngx.say(body:sub(1, -2))
+            ngx.print(body)
         }
     }
---- wait: 3
+--- wait: 0.5
 --- response_body
 opentracing
 --- grep_error_log eval
-qr/message received: apisix([.\w]+)/
---- grep_error_log_out
-message received: apisix.request.counter
-message received: apisix.request.latency
-message received: apisix.upstream.latency
-message received: apisix.apisix.latency
-message received: apisix.ingress.size
-message received: apisix.egress.size
+qr/message received: apisix(.+?(?=, ))/
+--- grep_error_log_out eval
+qr/message received: apisix\.request\.counter:1\|c\|#source:apisix,route_id:1,balancer_ip:[\d.]+,response_status:200
+message received: apisix\.request\.latency:[\d.]+\|h\|#source:apisix,route_id:1,balancer_ip:[\d.]+,response_status:200
+message received: apisix\.upstream\.latency:[\d.]+\|h\|#source:apisix,route_id:1,balancer_ip:[\d.]+,response_status:200
+message received: apisix\.apisix\.latency:[\d.]+\|h\|#source:apisix,route_id:1,balancer_ip:[\d.]+,response_status:200
+message received: apisix\.ingress\.size:[\d]+\|ms\|#source:apisix,route_id:1,balancer_ip:[\d.]+,response_status:200
+message received: apisix\.egress\.size:[\d]+\|ms\|#source:apisix,route_id:1,balancer_ip:[\d.]+,response_status:200
+/
+
+
+
+=== TEST 4: testing behaviour with multiple requests
+--- config
+    location /t {
+        content_by_lua_block {
+            local t = require("lib.test_admin").test
+
+            local code, _, body = t("/opentracing", "GET")
+             if code >= 300 then
+                ngx.status = code
+                ngx.say("fail")
+                return
+            end
+            ngx.print(body)
+
+            -- request 2
+            local code, _, body = t("/opentracing", "GET")
+             if code >= 300 then
+                ngx.status = code
+                ngx.say("fail")
+                return
+            end
+            ngx.print(body)
+        }
+    }
+--- wait: 0.5
+--- response_body
+opentracing
+opentracing
+--- grep_error_log eval
+qr/message received: apisix(.+?(?=, ))/
+--- grep_error_log_out eval
+qr/message received: apisix\.request\.counter:1\|c\|#source:apisix,route_id:1,balancer_ip:[\d.]+,response_status:200
+message received: apisix\.request\.latency:[\d.]+\|h\|#source:apisix,route_id:1,balancer_ip:[\d.]+,response_status:200
+message received: apisix\.upstream\.latency:[\d.]+\|h\|#source:apisix,route_id:1,balancer_ip:[\d.]+,response_status:200
+message received: apisix\.apisix\.latency:[\d.]+\|h\|#source:apisix,route_id:1,balancer_ip:[\d.]+,response_status:200
+message received: apisix\.ingress\.size:[\d]+\|ms\|#source:apisix,route_id:1,balancer_ip:[\d.]+,response_status:200
+message received: apisix\.egress\.size:[\d]+\|ms\|#source:apisix,route_id:1,balancer_ip:[\d.]+,response_status:200
+message received: apisix\.request\.counter:1\|c\|#source:apisix,route_id:1,balancer_ip:[\d.]+,response_status:200
+message received: apisix\.request\.latency:[\d.]+\|h\|#source:apisix,route_id:1,balancer_ip:[\d.]+,response_status:200
+message received: apisix\.upstream\.latency:[\d.]+\|h\|#source:apisix,route_id:1,balancer_ip:[\d.]+,response_status:200
+message received: apisix\.apisix\.latency:[\d.]+\|h\|#source:apisix,route_id:1,balancer_ip:[\d.]+,response_status:200
+message received: apisix\.ingress\.size:[\d]+\|ms\|#source:apisix,route_id:1,balancer_ip:[\d.]+,response_status:200
+message received: apisix\.egress\.size:[\d]+\|ms\|#source:apisix,route_id:1,balancer_ip:[\d.]+,response_status:200
+/
+
+
+
+=== TEST 5: testing behaviour with different namespace
+--- config
+    location /t {
+        content_by_lua_block {
+            local t = require("lib.test_admin").test
+
+            -- Change the metadata
+            local code, meta_body = t('/apisix/admin/plugin_metadata/datadog',
+                 ngx.HTTP_PUT,
+                 [[{
+                        "host":"127.0.0.1",
+                        "port": 8125,
+                        "namespace": "mycompany"
+                }]],
+                [[{
+                    "node": {
+                        "value": {
+                            "namespace": "mycompany",
+                            "host": "127.0.0.1",
+                            "constant_tags": [
+                                "source:apisix"
+                            ],
+                            "port": 8125
+                        },
+                        "key": "/apisix/plugin_metadata/datadog"
+                    },
+                    "action": "set"
+                }]])
+
+            if code >= 300 then
+                ngx.status = code
+                ngx.say("fail")
+                return
+            end
+            ngx.say(meta_body)
+
+            local code, _, body = t("/opentracing", "GET")
+             if code >= 300 then
+                ngx.status = code
+                ngx.say("fail")
+                return
+            end
+            ngx.print(body)
+        }
+    }
+--- wait: 0.5
+--- response_body
+passed
+opentracing
+--- grep_error_log eval
+qr/message received: mycompany(.+?(?=, ))/
+--- grep_error_log_out eval
+qr/message received: mycompany\.request\.counter:1\|c\|#source:apisix,route_id:1,balancer_ip:[\d.]+,response_status:200
+message received: mycompany\.request\.latency:[\d.]+\|h\|#source:apisix,route_id:1,balancer_ip:[\d.]+,response_status:200
+message received: mycompany\.upstream\.latency:[\d.]+\|h\|#source:apisix,route_id:1,balancer_ip:[\d.]+,response_status:200
+message received: mycompany\.apisix\.latency:[\d.]+\|h\|#source:apisix,route_id:1,balancer_ip:[\d.]+,response_status:200
+message received: mycompany\.ingress\.size:[\d]+\|ms\|#source:apisix,route_id:1,balancer_ip:[\d.]+,response_status:200
+message received: mycompany\.egress\.size:[\d]+\|ms\|#source:apisix,route_id:1,balancer_ip:[\d.]+,response_status:200
+/
+
+
+
+=== TEST 6: testing behaviour with different namespace
+--- config
+    location /t {
+        content_by_lua_block {
+            local t = require("lib.test_admin").test
+
+            -- Change the metadata
+            local code, meta_body = t('/apisix/admin/plugin_metadata/datadog',
+                 ngx.HTTP_PUT,
+                 [[{
+                        "host":"127.0.0.1",
+                        "port": 8125,
+                        "constant_tags": [
+                                "source:apisix",
+                                "new_tag:must"
+                            ]
+                }]],
+                [[{
+                    "node": {
+                        "value": {
+                            "namespace": "apisix",
+                            "host": "127.0.0.1",
+                            "constant_tags": [
+                                "source:apisix",
+                                "new_tag:must"
+                            ],
+                            "port": 8125
+                        },
+                        "key": "/apisix/plugin_metadata/datadog"
+                    },
+                    "action": "set"
+                }]])
+
+            if code >= 300 then
+                ngx.status = code
+                ngx.say("fail")
+                return
+            end
+            ngx.say(meta_body)
+
+            local code, _, body = t("/opentracing", "GET")
+             if code >= 300 then
+                ngx.status = code
+                ngx.say("fail")
+                return
+            end
+            ngx.print(body)
+        }
+    }
+--- wait: 0.5
+--- response_body
+passed
+opentracing
+--- grep_error_log eval
+qr/message received: apisix(.+?(?=, ))/
+--- grep_error_log_out eval
+qr/message received: apisix\.request\.counter:1\|c\|#source:apisix,new_tag:must,route_id:1,balancer_ip:[\d.]+,response_status:200
+message received: apisix\.request\.latency:[\d.]+\|h\|#source:apisix,new_tag:must,route_id:1,balancer_ip:[\d.]+,response_status:200
+message received: apisix\.upstream\.latency:[\d.]+\|h\|#source:apisix,new_tag:must,route_id:1,balancer_ip:[\d.]+,response_status:200
+message received: apisix\.apisix\.latency:[\d.]+\|h\|#source:apisix,new_tag:must,route_id:1,balancer_ip:[\d.]+,response_status:200
+message received: apisix\.ingress\.size:[\d]+\|ms\|#source:apisix,new_tag:must,route_id:1,balancer_ip:[\d.]+,response_status:200
+message received: apisix\.egress\.size:[\d]+\|ms\|#source:apisix,new_tag:must,route_id:1,balancer_ip:[\d.]+,response_status:200
+/

From 33542c636dc30a666f5f1b1a096b76b37d402f5b Mon Sep 17 00:00:00 2001
From: Bisakh Mondal <bisakhmondal00@gmail.com>
Date: Fri, 5 Nov 2021 16:01:17 +0530
Subject: [PATCH 11/17] code suggestion

---
 t/plugin/datadog.t | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/t/plugin/datadog.t b/t/plugin/datadog.t
index 7a52c7bdd065..bb03e81a3b1a 100644
--- a/t/plugin/datadog.t
+++ b/t/plugin/datadog.t
@@ -54,8 +54,9 @@ __DATA__
 --- config
     location /t {
         content_by_lua_block {
+            local core = require("apisix.core")
             local plugin = require("apisix.plugins.datadog")
-            local ok, err = plugin.check_schema({host = "127.0.0.1", port = 8125}, 2)
+            local ok, err = plugin.check_schema({host = "127.0.0.1", port = 8125}, core.schema.TYPE_METADATA)
             if not ok then
                 ngx.say(err)
             end

From bae5194057b6b5b164b229ad701a2ed6abc8d964 Mon Sep 17 00:00:00 2001
From: Bisakh Mondal <bisakhmondal00@gmail.com>
Date: Fri, 5 Nov 2021 19:03:23 +0530
Subject: [PATCH 12/17] route_name instead of id

---
 apisix/plugins/datadog.lua | 16 +++++++---
 t/plugin/datadog.t         | 64 ++++++++++++++++++++------------------
 2 files changed, 45 insertions(+), 35 deletions(-)

diff --git a/apisix/plugins/datadog.lua b/apisix/plugins/datadog.lua
index 460061356124..f0f6b8500c90 100644
--- a/apisix/plugins/datadog.lua
+++ b/apisix/plugins/datadog.lua
@@ -83,7 +83,10 @@ local function generate_tag(conf, const_tags)
         tags = {}
     end
 
-    if conf.route_id and conf.route_id ~= "" then
+    -- priority on route name, if not found using the route id.
+    if conf.route_name ~= "" then
+        core.table.insert(tags, "route_name:" .. conf.route_name)
+    elseif conf.route_id and conf.route_id ~= "" then
         core.table.insert(tags, "route_id:" .. conf.route_id)
     end
 
@@ -100,12 +103,15 @@ local function generate_tag(conf, const_tags)
     if conf.response.status then
         core.table.insert(tags, "response_status:" .. conf.response.status)
     end
+    if conf.scheme ~= "" then
+        core.table.insert(tags, "scheme:" .. conf.scheme)
+    end
 
-    if tags ~= "" then
-        tags = "|#" .. concat(tags, ',')
+    if #tags > 0 then
+        return "|#" .. concat(tags, ',')
     end
 
-    return tags
+    return ""
 end
 
 -- remove stale objects from the memory after timer expires
@@ -136,6 +142,8 @@ function _M.log(conf, ctx)
     local entry = fetch_log(ngx, {})
     entry.upstream_latency = ctx.var.upstream_response_time * 1000
     entry.balancer_ip = ctx.balancer_ip or ""
+    entry.route_name = ctx.route_name or ""
+    entry.scheme = ctx.upstream_scheme or ""
 
     local log_buffer = buffers[conf]
     if log_buffer then
diff --git a/t/plugin/datadog.t b/t/plugin/datadog.t
index bb03e81a3b1a..c17e10bd6db0 100644
--- a/t/plugin/datadog.t
+++ b/t/plugin/datadog.t
@@ -117,6 +117,7 @@ done
                             },
                             "type": "roundrobin"
                         },
+                        "name": "datadog",
                         "uri": "/opentracing"
                 }]],
                 [[{
@@ -134,7 +135,8 @@ done
                                 },
                                 "type": "roundrobin"
                             },
-                            "uri": "/opentracing"
+                            "uri": "/opentracing",
+                            "name": "datadog"
                         },
                         "key": "/apisix/routes/1"
                     },
@@ -180,12 +182,12 @@ opentracing
 --- grep_error_log eval
 qr/message received: apisix(.+?(?=, ))/
 --- grep_error_log_out eval
-qr/message received: apisix\.request\.counter:1\|c\|#source:apisix,route_id:1,balancer_ip:[\d.]+,response_status:200
-message received: apisix\.request\.latency:[\d.]+\|h\|#source:apisix,route_id:1,balancer_ip:[\d.]+,response_status:200
-message received: apisix\.upstream\.latency:[\d.]+\|h\|#source:apisix,route_id:1,balancer_ip:[\d.]+,response_status:200
-message received: apisix\.apisix\.latency:[\d.]+\|h\|#source:apisix,route_id:1,balancer_ip:[\d.]+,response_status:200
-message received: apisix\.ingress\.size:[\d]+\|ms\|#source:apisix,route_id:1,balancer_ip:[\d.]+,response_status:200
-message received: apisix\.egress\.size:[\d]+\|ms\|#source:apisix,route_id:1,balancer_ip:[\d.]+,response_status:200
+qr/message received: apisix\.request\.counter:1\|c\|#source:apisix,route_name:datadog,balancer_ip:[\d.]+,response_status:200,scheme:http
+message received: apisix\.request\.latency:[\d.]+\|h\|#source:apisix,route_name:datadog,balancer_ip:[\d.]+,response_status:200,scheme:http
+message received: apisix\.upstream\.latency:[\d.]+\|h\|#source:apisix,route_name:datadog,balancer_ip:[\d.]+,response_status:200,scheme:http
+message received: apisix\.apisix\.latency:[\d.]+\|h\|#source:apisix,route_name:datadog,balancer_ip:[\d.]+,response_status:200,scheme:http
+message received: apisix\.ingress\.size:[\d]+\|ms\|#source:apisix,route_name:datadog,balancer_ip:[\d.]+,response_status:200,scheme:http
+message received: apisix\.egress\.size:[\d]+\|ms\|#source:apisix,route_name:datadog,balancer_ip:[\d.]+,response_status:200,scheme:http
 /
 
 
@@ -221,18 +223,18 @@ opentracing
 --- grep_error_log eval
 qr/message received: apisix(.+?(?=, ))/
 --- grep_error_log_out eval
-qr/message received: apisix\.request\.counter:1\|c\|#source:apisix,route_id:1,balancer_ip:[\d.]+,response_status:200
-message received: apisix\.request\.latency:[\d.]+\|h\|#source:apisix,route_id:1,balancer_ip:[\d.]+,response_status:200
-message received: apisix\.upstream\.latency:[\d.]+\|h\|#source:apisix,route_id:1,balancer_ip:[\d.]+,response_status:200
-message received: apisix\.apisix\.latency:[\d.]+\|h\|#source:apisix,route_id:1,balancer_ip:[\d.]+,response_status:200
-message received: apisix\.ingress\.size:[\d]+\|ms\|#source:apisix,route_id:1,balancer_ip:[\d.]+,response_status:200
-message received: apisix\.egress\.size:[\d]+\|ms\|#source:apisix,route_id:1,balancer_ip:[\d.]+,response_status:200
-message received: apisix\.request\.counter:1\|c\|#source:apisix,route_id:1,balancer_ip:[\d.]+,response_status:200
-message received: apisix\.request\.latency:[\d.]+\|h\|#source:apisix,route_id:1,balancer_ip:[\d.]+,response_status:200
-message received: apisix\.upstream\.latency:[\d.]+\|h\|#source:apisix,route_id:1,balancer_ip:[\d.]+,response_status:200
-message received: apisix\.apisix\.latency:[\d.]+\|h\|#source:apisix,route_id:1,balancer_ip:[\d.]+,response_status:200
-message received: apisix\.ingress\.size:[\d]+\|ms\|#source:apisix,route_id:1,balancer_ip:[\d.]+,response_status:200
-message received: apisix\.egress\.size:[\d]+\|ms\|#source:apisix,route_id:1,balancer_ip:[\d.]+,response_status:200
+qr/message received: apisix\.request\.counter:1\|c\|#source:apisix,route_name:datadog,balancer_ip:[\d.]+,response_status:200,scheme:http
+message received: apisix\.request\.latency:[\d.]+\|h\|#source:apisix,route_name:datadog,balancer_ip:[\d.]+,response_status:200,scheme:http
+message received: apisix\.upstream\.latency:[\d.]+\|h\|#source:apisix,route_name:datadog,balancer_ip:[\d.]+,response_status:200,scheme:http
+message received: apisix\.apisix\.latency:[\d.]+\|h\|#source:apisix,route_name:datadog,balancer_ip:[\d.]+,response_status:200,scheme:http
+message received: apisix\.ingress\.size:[\d]+\|ms\|#source:apisix,route_name:datadog,balancer_ip:[\d.]+,response_status:200,scheme:http
+message received: apisix\.egress\.size:[\d]+\|ms\|#source:apisix,route_name:datadog,balancer_ip:[\d.]+,response_status:200,scheme:http
+message received: apisix\.request\.counter:1\|c\|#source:apisix,route_name:datadog,balancer_ip:[\d.]+,response_status:200,scheme:http
+message received: apisix\.request\.latency:[\d.]+\|h\|#source:apisix,route_name:datadog,balancer_ip:[\d.]+,response_status:200,scheme:http
+message received: apisix\.upstream\.latency:[\d.]+\|h\|#source:apisix,route_name:datadog,balancer_ip:[\d.]+,response_status:200,scheme:http
+message received: apisix\.apisix\.latency:[\d.]+\|h\|#source:apisix,route_name:datadog,balancer_ip:[\d.]+,response_status:200,scheme:http
+message received: apisix\.ingress\.size:[\d]+\|ms\|#source:apisix,route_name:datadog,balancer_ip:[\d.]+,response_status:200,scheme:http
+message received: apisix\.egress\.size:[\d]+\|ms\|#source:apisix,route_name:datadog,balancer_ip:[\d.]+,response_status:200,scheme:http
 /
 
 
@@ -289,12 +291,12 @@ opentracing
 --- grep_error_log eval
 qr/message received: mycompany(.+?(?=, ))/
 --- grep_error_log_out eval
-qr/message received: mycompany\.request\.counter:1\|c\|#source:apisix,route_id:1,balancer_ip:[\d.]+,response_status:200
-message received: mycompany\.request\.latency:[\d.]+\|h\|#source:apisix,route_id:1,balancer_ip:[\d.]+,response_status:200
-message received: mycompany\.upstream\.latency:[\d.]+\|h\|#source:apisix,route_id:1,balancer_ip:[\d.]+,response_status:200
-message received: mycompany\.apisix\.latency:[\d.]+\|h\|#source:apisix,route_id:1,balancer_ip:[\d.]+,response_status:200
-message received: mycompany\.ingress\.size:[\d]+\|ms\|#source:apisix,route_id:1,balancer_ip:[\d.]+,response_status:200
-message received: mycompany\.egress\.size:[\d]+\|ms\|#source:apisix,route_id:1,balancer_ip:[\d.]+,response_status:200
+qr/message received: mycompany\.request\.counter:1\|c\|#source:apisix,route_name:datadog,balancer_ip:[\d.]+,response_status:200,scheme:http
+message received: mycompany\.request\.latency:[\d.]+\|h\|#source:apisix,route_name:datadog,balancer_ip:[\d.]+,response_status:200,scheme:http
+message received: mycompany\.upstream\.latency:[\d.]+\|h\|#source:apisix,route_name:datadog,balancer_ip:[\d.]+,response_status:200,scheme:http
+message received: mycompany\.apisix\.latency:[\d.]+\|h\|#source:apisix,route_name:datadog,balancer_ip:[\d.]+,response_status:200,scheme:http
+message received: mycompany\.ingress\.size:[\d]+\|ms\|#source:apisix,route_name:datadog,balancer_ip:[\d.]+,response_status:200,scheme:http
+message received: mycompany\.egress\.size:[\d]+\|ms\|#source:apisix,route_name:datadog,balancer_ip:[\d.]+,response_status:200,scheme:http
 /
 
 
@@ -355,10 +357,10 @@ opentracing
 --- grep_error_log eval
 qr/message received: apisix(.+?(?=, ))/
 --- grep_error_log_out eval
-qr/message received: apisix\.request\.counter:1\|c\|#source:apisix,new_tag:must,route_id:1,balancer_ip:[\d.]+,response_status:200
-message received: apisix\.request\.latency:[\d.]+\|h\|#source:apisix,new_tag:must,route_id:1,balancer_ip:[\d.]+,response_status:200
-message received: apisix\.upstream\.latency:[\d.]+\|h\|#source:apisix,new_tag:must,route_id:1,balancer_ip:[\d.]+,response_status:200
-message received: apisix\.apisix\.latency:[\d.]+\|h\|#source:apisix,new_tag:must,route_id:1,balancer_ip:[\d.]+,response_status:200
-message received: apisix\.ingress\.size:[\d]+\|ms\|#source:apisix,new_tag:must,route_id:1,balancer_ip:[\d.]+,response_status:200
-message received: apisix\.egress\.size:[\d]+\|ms\|#source:apisix,new_tag:must,route_id:1,balancer_ip:[\d.]+,response_status:200
+qr/message received: apisix\.request\.counter:1\|c\|#source:apisix,new_tag:must,route_name:datadog,balancer_ip:[\d.]+,response_status:200,scheme:http
+message received: apisix\.request\.latency:[\d.]+\|h\|#source:apisix,new_tag:must,route_name:datadog,balancer_ip:[\d.]+,response_status:200,scheme:http
+message received: apisix\.upstream\.latency:[\d.]+\|h\|#source:apisix,new_tag:must,route_name:datadog,balancer_ip:[\d.]+,response_status:200,scheme:http
+message received: apisix\.apisix\.latency:[\d.]+\|h\|#source:apisix,new_tag:must,route_name:datadog,balancer_ip:[\d.]+,response_status:200,scheme:http
+message received: apisix\.ingress\.size:[\d]+\|ms\|#source:apisix,new_tag:must,route_name:datadog,balancer_ip:[\d.]+,response_status:200,scheme:http
+message received: apisix\.egress\.size:[\d]+\|ms\|#source:apisix,new_tag:must,route_name:datadog,balancer_ip:[\d.]+,response_status:200,scheme:http
 /

From 9d184718e8fe1c9e2b9e017264883ed0ff5d9437 Mon Sep 17 00:00:00 2001
From: Bisakh Mondal <bisakhmondal00@gmail.com>
Date: Fri, 5 Nov 2021 19:03:33 +0530
Subject: [PATCH 13/17] datadog plugin docs

---
 docs/en/latest/config.json        |   3 +-
 docs/en/latest/plugins/datadog.md | 119 ++++++++++++++++++++++++++++++
 2 files changed, 121 insertions(+), 1 deletion(-)
 create mode 100644 docs/en/latest/plugins/datadog.md

diff --git a/docs/en/latest/config.json b/docs/en/latest/config.json
index 7c88b9c3af7d..11a284b9d1d6 100644
--- a/docs/en/latest/config.json
+++ b/docs/en/latest/config.json
@@ -105,7 +105,8 @@
             "plugins/prometheus",
             "plugins/zipkin",
             "plugins/skywalking",
-            "plugins/node-status"
+            "plugins/node-status",
+            "plugins/datadog"
           ]
         },
         {
diff --git a/docs/en/latest/plugins/datadog.md b/docs/en/latest/plugins/datadog.md
new file mode 100644
index 000000000000..9d2b27986e7a
--- /dev/null
+++ b/docs/en/latest/plugins/datadog.md
@@ -0,0 +1,119 @@
+---
+title: datadog
+---
+
+<!--
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+-->
+
+## Summary
+
+- [Summary](#summary)
+- [Name](#name)
+- [Attributes](#attributes)
+- [Metadata](#metadata)
+- [Exported Metrics](#exported-metrics)
+- [How To Enable](#how-to-enable)
+- [Disable Plugin](#disable-plugin)
+
+## Name
+
+`datadog` is a monitoring plugin built into Apache APISIX for seamless integration with [Datadog](https://www.datadoghq.com/), one of the most used monitoring and observability platform for cloud applications. If enabled, this plugin supports multiple powerful types of metrics capture for every request and response cycle that essentially reflects the behaviour and health of the system.
+
+This plugin pushes its custom metrics to the DogStatsD server, comes bundled with Datadog agent (to learn more about how to install a datadog agent, please visit [here](https://docs.datadoghq.com/agent/) ), over the UDP protocol. DogStatsD basically is an implementation of StatsD protocol which collects the custom metrics for Apache APISIX agent, aggregates it into a single data point and sends it to the configured Datadog server.
+To learn more about DogStatsD, please visit [DogStatsD](https://docs.datadoghq.com/developers/dogstatsd/?tab=hostagent) documentation.
+
+This plugin provides the ability to push metrics as a batch to the external Datadog agent, reusing the same datagram socket. In case if you did not receive the log data, don't worry give it some time. It will automatically send the logs after the timer function expires in our Batch Processor.
+
+For more info on Batch-Processor in Apache APISIX please refer.
+[Batch-Processor](../batch-processor.md)
+
+## Attributes
+
+| Name             | Type   | Requirement  | Default      | Valid | Description                                                                                |
+| -----------      | ------ | -----------  | -------      | ----- | ------------------------------------------------------------                               |
+| batch_max_size   | integer | optional    | 5000         | [1,...] | Max buffer size of each batch                                                            |
+| inactive_timeout | integer | optional    | 5            | [1,...] | Maximum age in seconds when the buffer will be flushed if inactive                       |
+| buffer_duration  | integer | optional    | 60           | [1,...] | Maximum age in seconds of the oldest entry in a batch before the batch must be processed |
+| max_retry_count  | integer | optional    | 1            | [1,...] | Maximum number of retries if one entry fails to reach dogstatsd server                   |
+
+## Metadata
+
+| Name        | Type    | Requirement |     Default        | Valid         | Description                                                            |
+| ----------- | ------  | ----------- |      -------       | -----         | ---------------------------------------------------------------------- |
+| host        | string  | optional    |  "127.0.0.1"       |               | The DogStatsD server host address                                      |
+| port        | integer | optional    |    8125            |               | The DogStatsD server host port                                         |
+| namespace   | string  | optional    |    "apisix"        |               | Prefix for all the custom metrics sent by APISIX agent. Useful for finding entities for metric graph. e.g. (apisix.request.counter)                                        |
+| constant_tags | array | optional    | [ "source:apisix" ] |              | Static tags embedded into generated metrics. Useful for grouping metric over certain signals. |
+
+To know more about how to effectively write tags, please visit [here](https://docs.datadoghq.com/getting_started/tagging/#defining-tags)
+
+## Exported Metrics
+
+Apache APISIX agent, for every request response cycle, export the following metrics to DogStatsD server if the datadog plugin is enabled:
+
+| Metric Name               | StatsD Type   | Description               |
+| -----------               | -----------   | -------                   |
+| Request Counter           | Counter       | No of requests received.   |
+| Request Latency           | Histogram     | Time taken to process the request (in milliseconds). |
+| Upstream latency          | Histogram     | Time taken to proxy the request to the upstream server till a response is received (in milliseconds). |
+| APISIX Latency            | Histogram     | Time taken by APISIX agent to process the request (in milliseconds). |
+| Ingress Size              | Timer         | Request body size in bytes. |
+| Egress Size               | Timer         | Response body size in bytes. |
+
+## How To Enable
+
+The following is an example on how to enable the datadog plugin for a specific route. We are assumming your datadog agent is aready up an running.
+
+```shell
+curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
+{
+      "plugins": {
+            "datadog": {}
+       },
+      "upstream": {
+           "type": "roundrobin",
+           "nodes": {
+               "127.0.0.1:1980": 1
+           }
+      },
+      "uri": "/hello"
+}'
+```
+
+Now any requests to uri `/hello` will generate aforesaid metrics and push it to DogStatsD server of the datadog agent.
+
+## Disable Plugin
+
+Remove the corresponding json configuration in the plugin configuration to disable the `datadog`.
+APISIX plugins are hot-reloaded, therefore no need to restart APISIX.
+
+```shell
+$ curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
+{
+    "methods": ["GET"],
+    "uri": "/hello",
+    "plugins": {},
+    "upstream": {
+        "type": "roundrobin",
+        "nodes": {
+            "127.0.0.1:1980": 1
+        }
+    }
+}'
+```

From f183492210406d16ba64a7711d0967fbd7033804 Mon Sep 17 00:00:00 2001
From: Bisakh Mondal <bisakhmondal00@gmail.com>
Date: Fri, 5 Nov 2021 19:25:30 +0530
Subject: [PATCH 14/17] mispelt fix

---
 docs/en/latest/plugins/datadog.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/en/latest/plugins/datadog.md b/docs/en/latest/plugins/datadog.md
index 9d2b27986e7a..b200c6b1858a 100644
--- a/docs/en/latest/plugins/datadog.md
+++ b/docs/en/latest/plugins/datadog.md
@@ -78,7 +78,7 @@ Apache APISIX agent, for every request response cycle, export the following metr
 
 ## How To Enable
 
-The following is an example on how to enable the datadog plugin for a specific route. We are assumming your datadog agent is aready up an running.
+The following is an example on how to enable the datadog plugin for a specific route. We are assumming your datadog agent is already up an running.
 
 ```shell
 curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '

From e85e9c940f5634f4f4e1012fa8e1e64e56df2696 Mon Sep 17 00:00:00 2001
From: Bisakh Mondal <bisakhmondal00@gmail.com>
Date: Fri, 5 Nov 2021 19:25:45 +0530
Subject: [PATCH 15/17] suggestions

---
 apisix/plugins/datadog.lua        | 2 +-
 docs/en/latest/plugins/datadog.md | 9 +++++++++
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/apisix/plugins/datadog.lua b/apisix/plugins/datadog.lua
index f0f6b8500c90..5e3a24996a12 100644
--- a/apisix/plugins/datadog.lua
+++ b/apisix/plugins/datadog.lua
@@ -78,7 +78,7 @@ end
 local function generate_tag(conf, const_tags)
     local tags
     if const_tags and #const_tags > 0 then
-        tags = core.table.deepcopy(const_tags)
+        tags = core.table.clone(const_tags)
     else
         tags = {}
     end
diff --git a/docs/en/latest/plugins/datadog.md b/docs/en/latest/plugins/datadog.md
index b200c6b1858a..0c6f8072d503 100644
--- a/docs/en/latest/plugins/datadog.md
+++ b/docs/en/latest/plugins/datadog.md
@@ -78,6 +78,15 @@ Apache APISIX agent, for every request response cycle, export the following metr
 
 ## How To Enable
 
+First of all, enable the datadog plugin in the `config.yaml`:
+
+```
+# Add this in config.yaml
+plugins:
+  - ... # plugin you need
+  - datadog
+```
+
 The following is an example on how to enable the datadog plugin for a specific route. We are assumming your datadog agent is already up an running.
 
 ```shell

From 29942551b646c9f8a15ccc4cbf44a35c9a1cc2ec Mon Sep 17 00:00:00 2001
From: Bisakh Mondal <bisakhmondal00@gmail.com>
Date: Mon, 8 Nov 2021 10:58:22 +0530
Subject: [PATCH 16/17] tests for route_id, docs modification

---
 apisix/plugins/datadog.lua        | 30 +++++-----
 docs/en/latest/plugins/datadog.md | 18 +++---
 t/plugin/datadog.t                | 91 +++++++++++++++++++++----------
 3 files changed, 86 insertions(+), 53 deletions(-)

diff --git a/apisix/plugins/datadog.lua b/apisix/plugins/datadog.lua
index 5e3a24996a12..7fe7d3f4e4e7 100644
--- a/apisix/plugins/datadog.lua
+++ b/apisix/plugins/datadog.lua
@@ -75,7 +75,7 @@ function _M.check_schema(conf, schema_type)
     return core.schema.check(schema, conf)
 end
 
-local function generate_tag(conf, const_tags)
+local function generate_tag(entry, const_tags)
     local tags
     if const_tags and #const_tags > 0 then
         tags = core.table.clone(const_tags)
@@ -84,27 +84,27 @@ local function generate_tag(conf, const_tags)
     end
 
     -- priority on route name, if not found using the route id.
-    if conf.route_name ~= "" then
-        core.table.insert(tags, "route_name:" .. conf.route_name)
-    elseif conf.route_id and conf.route_id ~= "" then
-        core.table.insert(tags, "route_id:" .. conf.route_id)
+    if entry.route_name ~= "" then
+        core.table.insert(tags, "route_name:" .. entry.route_name)
+    elseif entry.route_id and entry.route_id ~= "" then
+        core.table.insert(tags, "route_name:" .. entry.route_id)
     end
 
-    if conf.service_id and conf.service_id ~= "" then
-        core.table.insert(tags, "service_id:" .. conf.service_id)
+    if entry.service_id and entry.service_id ~= "" then
+        core.table.insert(tags, "service_id:" .. entry.service_id)
     end
 
-    if conf.consumer and conf.consumer ~= "" then
-        core.table.insert(tags, "consumer:" .. conf.consumer)
+    if entry.consumer and entry.consumer ~= "" then
+        core.table.insert(tags, "consumer:" .. entry.consumer)
     end
-    if conf.balancer_ip ~= "" then
-        core.table.insert(tags, "balancer_ip:" .. conf.balancer_ip)
+    if entry.balancer_ip ~= "" then
+        core.table.insert(tags, "balancer_ip:" .. entry.balancer_ip)
     end
-    if conf.response.status then
-        core.table.insert(tags, "response_status:" .. conf.response.status)
+    if entry.response.status then
+        core.table.insert(tags, "response_status:" .. entry.response.status)
     end
-    if conf.scheme ~= "" then
-        core.table.insert(tags, "scheme:" .. conf.scheme)
+    if entry.scheme ~= "" then
+        core.table.insert(tags, "scheme:" .. entry.scheme)
     end
 
     if #tags > 0 then
diff --git a/docs/en/latest/plugins/datadog.md b/docs/en/latest/plugins/datadog.md
index 0c6f8072d503..0df837539b23 100644
--- a/docs/en/latest/plugins/datadog.md
+++ b/docs/en/latest/plugins/datadog.md
@@ -76,16 +76,18 @@ Apache APISIX agent, for every request response cycle, export the following metr
 | Ingress Size              | Timer         | Request body size in bytes. |
 | Egress Size               | Timer         | Response body size in bytes. |
 
-## How To Enable
+The metrics will be sent to the DogStatsD agent with the following tags:
 
-First of all, enable the datadog plugin in the `config.yaml`:
+> If there is no suitable value for any particular tag, the tag will simply be omitted.
 
-```
-# Add this in config.yaml
-plugins:
-  - ... # plugin you need
-  - datadog
-```
+- **route_name**: Name specified in the route schema definition. If not present, it will fall back to the route id value.
+- **service_id**: If a route has been created with the abstraction of service, the particular service id will be used.
+- **consumer**: If the route has a linked consumer, the consumer Username will be added as a tag.
+- **balancer_ip**: IP of the Upstream balancer that has processed the current request.
+- **response_status**: HTTP response status code.
+- **scheme**: Scheme that has been used to make the request. e.g. HTTP, gRPC, gRPCs etc.
+
+## How To Enable
 
 The following is an example on how to enable the datadog plugin for a specific route. We are assumming your datadog agent is already up an running.
 
diff --git a/t/plugin/datadog.t b/t/plugin/datadog.t
index c17e10bd6db0..b307ee7410a0 100644
--- a/t/plugin/datadog.t
+++ b/t/plugin/datadog.t
@@ -252,20 +252,6 @@ message received: apisix\.egress\.size:[\d]+\|ms\|#source:apisix,route_name:data
                         "host":"127.0.0.1",
                         "port": 8125,
                         "namespace": "mycompany"
-                }]],
-                [[{
-                    "node": {
-                        "value": {
-                            "namespace": "mycompany",
-                            "host": "127.0.0.1",
-                            "constant_tags": [
-                                "source:apisix"
-                            ],
-                            "port": 8125
-                        },
-                        "key": "/apisix/plugin_metadata/datadog"
-                    },
-                    "action": "set"
                 }]])
 
             if code >= 300 then
@@ -301,7 +287,7 @@ message received: mycompany\.egress\.size:[\d]+\|ms\|#source:apisix,route_name:d
 
 
 
-=== TEST 6: testing behaviour with different namespace
+=== TEST 6: testing behaviour with different constant tags
 --- config
     location /t {
         content_by_lua_block {
@@ -317,21 +303,6 @@ message received: mycompany\.egress\.size:[\d]+\|ms\|#source:apisix,route_name:d
                                 "source:apisix",
                                 "new_tag:must"
                             ]
-                }]],
-                [[{
-                    "node": {
-                        "value": {
-                            "namespace": "apisix",
-                            "host": "127.0.0.1",
-                            "constant_tags": [
-                                "source:apisix",
-                                "new_tag:must"
-                            ],
-                            "port": 8125
-                        },
-                        "key": "/apisix/plugin_metadata/datadog"
-                    },
-                    "action": "set"
                 }]])
 
             if code >= 300 then
@@ -364,3 +335,63 @@ message received: apisix\.apisix\.latency:[\d.]+\|h\|#source:apisix,new_tag:must
 message received: apisix\.ingress\.size:[\d]+\|ms\|#source:apisix,new_tag:must,route_name:datadog,balancer_ip:[\d.]+,response_status:200,scheme:http
 message received: apisix\.egress\.size:[\d]+\|ms\|#source:apisix,new_tag:must,route_name:datadog,balancer_ip:[\d.]+,response_status:200,scheme:http
 /
+
+
+
+=== TEST 7: testing behaviour when route_name is missing - must fallback to route_id
+--- config
+    location /t {
+        content_by_lua_block {
+            local t = require("lib.test_admin").test
+
+            local code, body = t('/apisix/admin/routes/1',
+                 ngx.HTTP_PUT,
+                 [[{
+                        "plugins": {
+                            "datadog": {
+                                "batch_max_size" : 1
+                            }
+                        },
+                        "upstream": {
+                            "nodes": {
+                                "127.0.0.1:1982": 1
+                            },
+                            "type": "roundrobin"
+                        },
+                        "uri": "/opentracing"
+                }]]
+            )
+
+            if code >= 300 then
+                ngx.status = code
+                ngx.say("fail")
+                return
+            end
+
+            ngx.say(body)
+
+            -- making a request to the route
+            local code, _, body = t("/opentracing", "GET")
+             if code >= 300 then
+                ngx.status = code
+                ngx.say("fail")
+                return
+            end
+
+            ngx.print(body)
+        }
+    }
+--- response_body
+passed
+opentracing
+--- wait: 0.5
+--- grep_error_log eval
+qr/message received: apisix(.+?(?=, ))/
+--- grep_error_log_out eval
+qr/message received: apisix\.request\.counter:1\|c\|#source:apisix,new_tag:must,route_name:1,balancer_ip:[\d.]+,response_status:200,scheme:http
+message received: apisix\.request\.latency:[\d.]+\|h\|#source:apisix,new_tag:must,route_name:1,balancer_ip:[\d.]+,response_status:200,scheme:http
+message received: apisix\.upstream\.latency:[\d.]+\|h\|#source:apisix,new_tag:must,route_name:1,balancer_ip:[\d.]+,response_status:200,scheme:http
+message received: apisix\.apisix\.latency:[\d.]+\|h\|#source:apisix,new_tag:must,route_name:1,balancer_ip:[\d.]+,response_status:200,scheme:http
+message received: apisix\.ingress\.size:[\d]+\|ms\|#source:apisix,new_tag:must,route_name:1,balancer_ip:[\d.]+,response_status:200,scheme:http
+message received: apisix\.egress\.size:[\d]+\|ms\|#source:apisix,new_tag:must,route_name:1,balancer_ip:[\d.]+,response_status:200,scheme:http
+/

From 3bd09cbd589d504d5c2da6a34f020ba5ba302379 Mon Sep 17 00:00:00 2001
From: Bisakh Mondal <bisakhmondal00@gmail.com>
Date: Mon, 8 Nov 2021 23:15:45 +0530
Subject: [PATCH 17/17] note for duplicate route names

---
 docs/en/latest/plugins/datadog.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/en/latest/plugins/datadog.md b/docs/en/latest/plugins/datadog.md
index 0df837539b23..508ae1c5a2e8 100644
--- a/docs/en/latest/plugins/datadog.md
+++ b/docs/en/latest/plugins/datadog.md
@@ -81,6 +81,7 @@ The metrics will be sent to the DogStatsD agent with the following tags:
 > If there is no suitable value for any particular tag, the tag will simply be omitted.
 
 - **route_name**: Name specified in the route schema definition. If not present, it will fall back to the route id value.
+  - Note: If multiple routes have the same name duplicated, we suggest you to visualize graphs on the Datadog dashboard over multiple tags that could compositely pinpoint a particular route/service. If it's still insufficient for your needs, feel free to drop a feature request at [apisix/issues](https://github.com/apache/apisix/issues).
 - **service_id**: If a route has been created with the abstraction of service, the particular service id will be used.
 - **consumer**: If the route has a linked consumer, the consumer Username will be added as a tag.
 - **balancer_ip**: IP of the Upstream balancer that has processed the current request.