[ci] add rake test:coverage

It displays the test coverage of `checks.d`, specifying if it's a mocked
test.

Rakefile

@@ -81,6 +81,9 @@ namespace :test do
   task 'profile:pstats' => ['test:profile'] do
     sh 'python -m pstats stats.dat'
   end
+
+  desc 'Display test coverage for checks'
+  task 'coverage' => 'ci:default:coverage'
 end
 
 desc 'Lint the code through pylint'

ci/default.rb

@@ -4,6 +4,37 @@
   namespace :default do |flavor|
     task :before_install => ['ci:common:before_install']
 
+    task :coverage do
+      ci_dir = File.dirname(__FILE__)
+      checks_dir = File.join(ci_dir, '..', 'checks.d')
+      tests_checks_dir = File.join(ci_dir, '..', 'tests', 'checks')
+      mock_dir = File.join(tests_checks_dir, 'mock')
+      integration_dir = File.join(tests_checks_dir, 'integration')
+      untested, mocked, perfects = [], [], []
+      Dir.glob(File.join(checks_dir, '*.py')).each do |check|
+        check_name = /((\w|_)+).py$/.match(check)[1]
+        if File.exist?(File.join(integration_dir, "test_#{check_name}.py"))
+          perfects.push(check_name)
+        elsif File.exist?(File.join(mock_dir, "test_#{check_name}.py"))
+          mocked.push(check_name)
+        else
+          untested.push(check_name)
+        end
+      end
+      total_checks = (untested + mocked + perfects).length
+      unless untested.empty?
+        puts "Untested checks (#{untested.length}/#{total_checks})".red
+        puts '-----------------------'.red
+        untested.each { |check_name| puts check_name.red }
+        puts ''
+      end
+      unless mocked.empty?
+        puts "Mocked tests (#{mocked.length}/#{total_checks})".yellow
+        puts '--------------------'.yellow
+        mocked.each { |check_name| puts check_name.yellow }
+      end
+    end
+
     task :install => ['ci:common:install']
 
     task :before_script => ['ci:common:before_script']
@@ -12,7 +43,7 @@
       sh %(find . -name '*.py' -not \\( -path '*.cache*' -or -path '*embedded*' -or -path '*venv*' \\) | xargs -n 1 pylint --rcfile=./.pylintrc)
     end
 
-    task :script => ['ci:common:script', :lint] do
+    task :script => ['ci:common:script', :coverage, :lint] do
       Rake::Task['ci:common:run_tests'].invoke('default')
     end
 

ez_setup.py

@@ -100,9 +100,9 @@ def do_download():
     try:
         import pkg_resources
     except ImportError:
-        return do_download()       
+        return do_download()
     try:
-        pkg_resources.require("setuptools>="+version); return
+        pkg_resources.require("setuptools>="+version); return  #pylint: disable=E1102
     except pkg_resources.VersionConflict, e:
         if was_imported:
             print >>sys.stderr, (
@@ -225,7 +225,7 @@ def main(argv, version=DEFAULT_VERSION):
     req = "setuptools>="+version
     import pkg_resources
     try:
-        pkg_resources.require(req)
+        pkg_resources.require(req) #pylint: disable=E1102
     except pkg_resources.VersionConflict:
         try:
             from setuptools.command.easy_install import main

tests/checks/common.py

@@ -109,7 +109,7 @@ def get_check(name, config_str):
                                  agentConfig=agentConfig)
 
 
-class Resources(object):
+class Fixtures(object):
 
     @staticmethod
     def integration_name():
@@ -124,16 +124,16 @@ def integration_name():
 
     @staticmethod
     def directory():
-        return os.path.join(os.path.dirname(__file__), 'resources',
-                            Resources.integration_name())
+        return os.path.join(os.path.dirname(__file__), 'fixtures',
+                            Fixtures.integration_name())
 
     @staticmethod
     def file(file_name):
-        return os.path.join(Resources.directory(), file_name)
+        return os.path.join(Fixtures.directory(), file_name)
 
     @staticmethod
     def read_file(file_name):
-        return open(Resources.file(file_name)).read()
+        return open(Fixtures.file(file_name)).read()
 
 
 class AgentCheckTest(unittest.TestCase):

tests/checks/integration/test_cassandra.py

@@ -6,7 +6,7 @@
 from dogstatsd import Server
 from util import PidFile
 from jmxfetch import JMXFetch
-from tests.checks.common import Resources
+from tests.checks.common import Fixtures
 STATSD_PORT = 8121
 
 
@@ -42,7 +42,7 @@ def setUp(self):
         self.t1 = threading.Thread(target=self.server.start)
         self.t1.start()
 
-        confd_path = Resources.directory()
+        confd_path = Fixtures.directory()
 
         self.jmx_daemon = JMXFetch(confd_path, {'dogstatsd_port': STATSD_PORT})
         self.t2 = threading.Thread(target=self.jmx_daemon.run)

tests/checks/integration/test_nginx.py

@@ -1,7 +1,7 @@
 import unittest
 from nose.plugins.attrib import attr
 
-from tests.checks.common import load_check, Resources
+from tests.checks.common import load_check, Fixtures
 
 
 @attr(requires='nginx')
@@ -45,8 +45,8 @@ def test_nginx(self):
             self.assertEquals(set(can_connect[i]['tags']), set(['host:localhost', 'port:44441']), service_checks)
 
     def test_nginx_plus(self):
-        test_data = Resources.read_file('nginx_plus_in.json')
-        expected = eval(Resources.read_file('nginx_plus_out.python'))
+        test_data = Fixtures.read_file('nginx_plus_in.json')
+        expected = eval(Fixtures.read_file('nginx_plus_out.python'))
         nginx = load_check('nginx', self.config, self.agent_config)
         parsed = nginx.parse_json(test_data)
         parsed.sort()

tests/checks/integration/test_varnish.py

@@ -2,12 +2,12 @@
 from nose.plugins.attrib import attr
 
 # project
-from tests.checks.common import AgentCheckTest, Resources
+from tests.checks.common import AgentCheckTest, Fixtures
 from utils.shell import which
 
 
 def mocked_varnishstatoutput(cmd):
-    return Resources.read_file('dump.xml')
+    return Fixtures.read_file('dump.xml')
 
 
 COMMON_METRICS = [

tests/checks/mock/test_cacti.py

@@ -1,4 +1,4 @@
-from tests.checks.common import get_check, Resources
+from tests.checks.common import get_check, Fixtures
 
 import unittest
 import os
@@ -15,7 +15,7 @@
         mysql_user: root
         rrd_path:   /tmp/cacti_test/rrds
         rrd_whitelist: %s
-""" % Resources.file('whitelist.txt')
+""" % Fixtures.file('whitelist.txt')
 
 
 class TestCacti(unittest.TestCase):

tests/checks/mock/test_ganglia.py

@@ -12,13 +12,13 @@
 from cStringIO import StringIO
 
 from checks.ganglia import Ganglia
-from tests.checks.common import Resources
+from tests.checks.common import Fixtures
 
 
 class TestGanglia(unittest.TestCase):
     def testSpeed(self):
         # Pretend to be gmetad and serve a large piece of content
-        original_file = Resources.file('ganglia.txt')
+        original_file = Fixtures.file('ganglia.txt')
         server = subprocess.Popen("nc -l 8651 < %s" % original_file, shell=True)
         # Wait for 1 second
         time.sleep(1)
@@ -30,7 +30,7 @@ def testSpeed(self):
         # p = pstats.Stats(pfile.name)
         # p.sort_stats('time').print_stats()
         parsed = StringIO(g.check({'ganglia_host': 'localhost', 'ganglia_port': 8651}))
-        original = Resources.file('ganglia.txt')
+        original = Fixtures.file('ganglia.txt')
         x1 = tree.parse(parsed)
         x2 = tree.parse(original)
         # Cursory test

tests/checks/mock/test_go_expvar.py

@@ -3,7 +3,7 @@
 import simplejson as json
 import time
 
-from tests.checks.common import AgentCheckTest, Resources
+from tests.checks.common import AgentCheckTest, Fixtures
 
 
 def _get_data_mock(url):
@@ -53,7 +53,7 @@ class TestGoExpVar(AgentCheckTest):
 
     def __init__(self, *args, **kwargs):
         AgentCheckTest.__init__(self, *args, **kwargs)
-        self._expvar_url = Resources.file('expvar_output')
+        self._expvar_url = Fixtures.file('expvar_output')
         self.mock_config = {
             "instances": [{
                 "expvar_url": self._expvar_url,

tests/checks/mock/test_nagios.py

@@ -3,14 +3,14 @@
 import time
 
 # project
-from tests.checks.common import AgentCheckTest, Resources
+from tests.checks.common import AgentCheckTest, Fixtures
 
 
 class NagiosTestCase(AgentCheckTest):
     CHECK_NAME = 'nagios'
-    NAGIOS_TEST_LOG = Resources.file('nagios.log')
-    NAGIOS_TEST_HOST = Resources.file('host-perfdata')
-    NAGIOS_TEST_SVC = Resources.file('service-perfdata')
+    NAGIOS_TEST_LOG = Fixtures.file('nagios.log')
+    NAGIOS_TEST_HOST = Fixtures.file('host-perfdata')
+    NAGIOS_TEST_SVC = Fixtures.file('service-perfdata')
     NAGIOS_TEST_HOST_TEMPLATE = "[HOSTPERFDATA]\t$TIMET$\t$HOSTNAME$\t$HOSTEXECUTIONTIME$\t$HOSTOUTPUT$\t$HOSTPERFDATA$"
     NAGIOS_TEST_SVC_TEMPLATE = "[SERVICEPERFDATA]\t$TIMET$\t$HOSTNAME$\t$SERVICEDESC$\t$SERVICEEXECUTIONTIME$\t$SERVICELATENCY$\t$SERVICEOUTPUT$\t$SERVICEPERFDATA$"
 

tests/checks/mock/test_riakcs.py

@@ -1,5 +1,5 @@
 import unittest
-from tests.checks.common import load_check, Resources, AgentCheckTest
+from tests.checks.common import load_check, Fixtures, AgentCheckTest
 from mock import Mock
 from socket import error
 from checks import AgentCheck
@@ -17,16 +17,16 @@ def __init__(self, *args, **kwargs):
             self.check = load_check(self.CHECK_NAME, self.config, {})
             self.check._connect = Mock(return_value=(None, None, ["aggregation_key:localhost:8080"]))
             self.check._get_stats = Mock(return_value=self.check.load_json(
-                Resources.read_file('riakcs_in.json')))
+                Fixtures.read_file('riakcs_in.json')))
 
         def test_parser(self):
-            input_json = Resources.read_file('riakcs_in.json')
-            output_python = Resources.read_file('riakcs_out.python')
+            input_json = Fixtures.read_file('riakcs_in.json')
+            output_python = Fixtures.read_file('riakcs_out.python')
             self.assertEquals(self.check.load_json(input_json), eval(output_python))
 
         def test_metrics(self):
             self.run_check(self.config)
-            expected = eval(Resources.read_file('riakcs_metrics.python'))
+            expected = eval(Fixtures.read_file('riakcs_metrics.python'))
             for m in expected:
                 self.assertMetric(m[0], m[2], m[3].get('tags', []))
 

tests/checks/mock/test_solr.py

@@ -8,7 +8,7 @@
 import os
 from jmxfetch import JMXFetch
 
-from tests.checks.common import Resources
+from tests.checks.common import Fixtures
 
 STATSD_PORT = 8127
 
@@ -45,7 +45,7 @@ def setUp(self):
         self.t1 = threading.Thread(target=self.server.start)
         self.t1.start()
 
-        confd_path = Resources.directory()
+        confd_path = Fixtures.directory()
         self.jmx_daemon = JMXFetch(confd_path, {'dogstatsd_port': STATSD_PORT})
         self.t2 = threading.Thread(target=self.jmx_daemon.run)
         self.t2.start()

tests/core/test_config.py

@@ -14,7 +14,7 @@ def testWhiteSpaceConfig(self):
         """Leading whitespace confuse ConfigParser
         """
         agentConfig = get_config(cfg_path=os.path.join(os.path.dirname(os.path.realpath(__file__)),
-                                                       'resources', 'badconfig.conf'))
+                                                       'fixtures', 'badconfig.conf'))
         self.assertEquals(agentConfig["dd_url"], "https://app.datadoghq.com")
         self.assertEquals(agentConfig["api_key"], "1234")
         self.assertEquals(agentConfig["nagios_log"], "/var/log/nagios3/nagios.log")

tests/core/test_flare.py

@@ -18,7 +18,7 @@ def get_mocked_version():
 def get_mocked_temp():
     return os.path.join(
         os.path.dirname(os.path.realpath(__file__)),
-        'resources',
+        'fixtures',
         'flare'
     )
 

tests/core/test_modules.py

@@ -7,7 +7,7 @@
 
 log = logging.getLogger('datadog.test')
 
-TARGET_MODULE = 'tests.core.resources.target_module'
+TARGET_MODULE = 'tests.core.fixtures.target_module'
 default_target = 'DEFAULT'
 specified_target = 'SPECIFIED'
 has_been_mutated = False