[fluentd] add type tag support to fluentd check

Now we can use either fluentd plugin_id or type as a tag.
[yann@datadoghq.com] rebase current work to solve merge conflicts

checks.d/fluentd.py

@@ -29,6 +29,7 @@ def check(self, instance):
         try:
             url = instance.get('monitor_agent_url')
             plugin_ids = instance.get('plugin_ids', [])
+            tag_by = instance.get('tag_by', 'plugin_id')
 
             parsed_url = urlparse.urlparse(url)
             monitor_agent_host = parsed_url.hostname
@@ -40,11 +41,13 @@ def check(self, instance):
             status = r.json()
 
             for p in status['plugins']:
+                tag = "%s:%s" % (tag_by, p.get(tag_by))
                 for m in self.GAUGES:
                     if p.get(m) is None:
                         continue
-                    if p.get('plugin_id') in plugin_ids:
-                        self.gauge('fluentd.%s' % (m), p.get(m), ["plugin_id:%s" % p.get('plugin_id')])
+                    # Filter unspecified plugins to keep backward compatibility.
+                    if tag_by == 'type' or len(plugin_ids) == 0 or p.get('plugin_id') in plugin_ids:
+                        self.gauge('fluentd.%s' % (m), p.get(m), [tag])
         except Exception, e:
             msg = "No stats could be retrieved from %s : %s" % (url, str(e))
             self.service_check(self.SERVICE_CHECK_NAME, AgentCheck.CRITICAL, tags=service_check_tags, message=msg)

conf.d/fluentd.yaml.example

@@ -2,7 +2,14 @@ init_config:
 
 instances:
     # For every instance, you have an `monitor_agent_url` 
-    # and (optionally) a list of tags.
+    # Use fluend 'plugin_id' as a tag. 
+    # You need to specify plugin id in fluentd configuration file.
+    -  monitor_agent_url: http://example.com:24220/api/plugins.json
+       tag_by: plugin_id
+    # Use fluend plugin 'type' as a tag. 
+    -  monitor_agent_url: http://example.org:24220/api/plugins.json
+       tag_by: type
+    # For backward compatibilty, you can specify plugin_ids that are used as tags.
     -  monitor_agent_url: http://example.com:24220/api/plugins.json
        plugin_ids:
          - plg1

tests/checks/integration/test_fluentd.py

@@ -1,5 +1,6 @@
 import unittest
 import logging
+import re
 from nose.plugins.attrib import attr
 from types import ListType
 logger = logging.getLogger(__file__)
@@ -70,3 +71,54 @@ def test_fluentd_exception(self):
 
         check = load_check('fluentd', config, agentConfig)
         self.assertRaises(Exception, check.run())
+
+    def test_fluentd_with_tag_by_type(self):
+        config = {
+            "init_config": {
+            },
+            "instances": [
+                {
+                    "monitor_agent_url": "http://localhost:24220/api/plugins.json",
+                    "tag_by": "type",
+                }
+            ]
+        }
+
+        agentConfig = {
+            'version': '0.1',
+            'api_key': 'toto'
+        }
+
+        check = load_check('fluentd', config, agentConfig)
+        check.run()
+        metrics = check.get_metrics()
+        for m in metrics:
+            self.assertEquals(m[3]['tags'], ['type:forward'])
+
+    def test_fluentd_with_tag_by_plugin_id(self):
+        config = {
+            "init_config": {
+            },
+            "instances": [
+                {
+                    "monitor_agent_url": "http://localhost:24220/api/plugins.json",
+                    "tag_by": "plugin_id",
+                }
+            ]
+        }
+
+        agentConfig = {
+            'version': '0.1',
+            'api_key': 'toto'
+        }
+
+        check = load_check('fluentd', config, agentConfig)
+        check.run()
+        metrics = check.get_metrics()
+        p = re.compile('plugin_id:plg[12]')
+        for m in metrics:
+            self.assertEquals(len(m[3]['tags']), 1)
+            self.assertTrue(p.match(m[3]['tags'][0]))
+
+if __name__ == '__main__':
+    unittest.main()

tests/checks/mock/test_etcd.py

@@ -0,0 +1,69 @@
+# stdlib
+import unittest
+
+# project
+from tests.checks.common import AgentCheckTest
+
+
+class TestEtcd(AgentCheckTest):
+    CHECK_NAME = "etcd"
+
+    def __init__(self, *args, **kwargs):
+        AgentCheckTest.__init__(self, *args, **kwargs)
+        self.config = {"instances": [{"url": "http://localhost:4001"}]}
+
+    # FIXME: not really an integration test, should be pretty easy
+    # to spin up a cluster to test that.
+    def test_followers(self):
+        mock = {
+            "followers": {
+                "etcd-node1": {
+                    "counts": {
+                        "fail": 1212,
+                        "success": 4163176
+                    },
+                    "latency": {
+                        "average": 2.7206299430775007,
+                        "current": 1.486487,
+                        "maximum": 2018.410279,
+                        "minimum": 1.011763,
+                        "standardDeviation": 6.246990702203536
+                    }
+                },
+                "etcd-node3": {
+                    "counts": {
+                        "fail": 1378,
+                        "success": 4164598
+                    },
+                    "latency": {
+                        "average": 2.707100125761001,
+                        "current": 1.666258,
+                        "maximum": 1409.054765,
+                        "minimum": 0.998415,
+                        "standardDeviation": 5.910089773061448
+                    }
+                }
+            },
+            "leader": "etcd-node2"
+        }
+
+        mocks = {
+            '_get_leader_metrics': lambda u, t: mock
+        }
+
+        self.run_check_twice(self.config, mocks=mocks)
+
+        common_leader_tags = ['url:http://localhost:4001', 'etcd_state:leader']
+        follower_tags = [
+            common_leader_tags[:] + ['follower:etcd-node1'],
+            common_leader_tags[:] + ['follower:etcd-node3'],
+        ]
+
+        for fol_tags in follower_tags:
+            self.assertMetric('etcd.leader.counts.fail', count=1, tags=fol_tags)
+            self.assertMetric('etcd.leader.counts.success', count=1, tags=fol_tags)
+            self.assertMetric('etcd.leader.latency.avg', count=1, tags=fol_tags)
+            self.assertMetric('etcd.leader.latency.min', count=1, tags=fol_tags)
+            self.assertMetric('etcd.leader.latency.max', count=1, tags=fol_tags)
+            self.assertMetric('etcd.leader.latency.stddev', count=1, tags=fol_tags)
+            self.assertMetric('etcd.leader.latency.current', count=1, tags=fol_tags)