Packaging clean-up + SSL warning in http_check

.gitignore

@@ -22,4 +22,5 @@ stats.dat
 conf.d/*.yaml
 !conf.d/network.yaml
 packaging/build/
-packaging/root/
+packaging/root/
+.vagrant/*

Vagrantfile

@@ -0,0 +1,45 @@
+# Vagrantfile API/syntax version. Don't touch unless you know what you're doing!
+VAGRANTFILE_API_VERSION = "2"
+
+Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
+  # All Vagrant configuration is done here. The most common configuration
+  # options are documented and commented below. For a complete reference,
+  # please see the online documentation at vagrantup.com.
+
+  # Debian 7 box
+  config.vm.define "debian" do |deb|
+    deb.vm.box = "debagent"
+    deb.vm.box_url = "https://dl.dropboxusercontent.com/s/xymcvez85i29lym/vagrant-debian-wheezy64.box"
+    deb.vm.synced_folder ".", "/src"
+    deb.vm.provider :virtualbox do |vb|
+      # Use VBoxManage to customize the VM. For example to change memory:
+      vb.customize ["modifyvm", :id, "--memory", "512"]
+    end
+
+    # Manual set-up
+    deb.vm.provision "shell", inline: "sudo apt-get update"
+    deb.vm.provision "shell", inline: "sudo apt-get -y install ruby"
+    deb.vm.provision "shell", inline: "sudo apt-get -y install ruby-dev"
+    deb.vm.provision "shell", inline: "sudo apt-get -y install python"
+    deb.vm.provision "shell", inline: "sudo gem install --no-ri --no-rdoc fpm"
+  end
+
+  # Centos 6 box
+  config.vm.define "redhat" do |rh|
+    rh.vm.box = "rhagent"
+    rh.vm.box_url = "https://github.com/2creatives/vagrant-centos/releases/download/v6.5.1/centos65-x86_64-20131205.box"
+    rh.vm.synced_folder ".", "/src"
+    rh.vm.provider :virtualbox do |vb|
+      # Use VBoxManage to customize the VM. For example to change memory:
+      vb.customize ["modifyvm", :id, "--memory", "512"]
+    end
+
+    # Manual set-up
+    rh.vm.provision "shell", inline: "sudo yum -y update"
+    rh.vm.provision "shell", inline: "sudo yum -y install ruby"
+    rh.vm.provision "shell", inline: "sudo yum -y install ruby-devel"
+    rh.vm.provision "shell", inline: "sudo yum -y install rubygems"
+    rh.vm.provision "shell", inline: "sudo gem install --no-ri --no-rdoc fpm"
+    rh.vm.provision "shell", inline: "sudo yum -y localinstall http://yum.datadoghq.com/rpm/supervisor-3.0-0.5.a10.el6.noarch.rpm"
+  end
+end

checks.d/http_check.py

@@ -18,15 +18,18 @@ def _load_conf(self, instance):
         if url is None:
             raise Exception("Bad configuration. You must specify a url")
         include_content = instance.get('include_content', False)
-        return url, username, password, timeout, include_content, headers, response_time, tags
+        ssl = instance.get('disable_ssl_validation', True)
+        return url, username, password, timeout, include_content, headers, response_time, tags, ssl
 
     def _check(self, instance):
-        addr, username, password, timeout, include_content, headers, response_time, tags = self._load_conf(instance)
+        addr, username, password, timeout, include_content, headers, response_time, tags, disable_ssl_validation = self._load_conf(instance)
         content = ''
         start = time.time()
         try:
             self.log.debug("Connecting to %s" % addr)
-            h = Http(timeout=timeout, disable_ssl_certificate_validation=True)
+            if disable_ssl_validation:
+                self.warning("Skipping SSL certificate validation for %s based on configuration" % addr)
+            h = Http(timeout=timeout, disable_ssl_certificate_validation=disable_ssl_validation)
             if username is not None and password is not None:
                 h.add_credentials(username, password)
             resp, content = h.request(addr, "GET", headers=headers)

conf.d/http_check.yaml.example

@@ -38,6 +38,14 @@ instances:
 
 #        collect_response_time: true
 
+         # The (optional) disable_ssl_validation will instruct the check
+         # to skip the validation of the SSL certificate of the URL being tested.
+         # This is mostly useful when checking SSL connections signed with
+         # certificates that are not themselves signed by a public authority.
+         # When true, the check logs a warning in collector.log
+
+#        disable_ssl_validation: true
+
          # The (optional) headers parameter allows you to send extra headers
          # with the request. This is useful for explicitly specifying the host
          # header or perhaps adding headers for authorisation purposes. Note

graphite.py

@@ -1,9 +1,8 @@
-import sys, os, re, struct
+import struct
 import logging
 import cPickle as pickle
 
 from tornado.ioloop import IOLoop
-from tornado.iostream import IOStream
 
 log = logging.getLogger(__name__)
 
@@ -17,7 +16,9 @@
 class GraphiteServer(TCPServer):
 
     def __init__(self, app, hostname, io_loop=None, ssl_options=None, **kwargs):
-        log.info('Graphite listener is started')
+        log.warn('Graphite listener is started -- if you do not need graphite, turn it off in datadog.conf.')
+        log.warn('Graphite relay uses pickle to transport messages. Pickle is not secured against remote execution exploits.')
+        log.warn('See http://blog.nelhage.com/2011/03/exploiting-pickle/ for more details')
         self.app = app
         self.hostname = hostname
         TCPServer.__init__(self, io_loop=io_loop, ssl_options=ssl_options, **kwargs)
@@ -37,9 +38,9 @@ def __init__(self, stream, address, app, hostname):
         self.stream.set_close_callback(self._on_close)
         self.stream.read_bytes(4, self._on_read_header)
 
-    def _on_read_header(self,data):
+    def _on_read_header(self, data):
         try:
-            size = struct.unpack("!L",data)[0]
+            size = struct.unpack("!L", data)[0]
             log.debug("Receiving a string of size:" + str(size))
             self.stream.read_bytes(size, self._on_read_line)
         except Exception, e:
@@ -84,12 +85,12 @@ def _processMetric(self, metric, datapoint):
             send the datapoint to datadog"""
 
         log.debug("New metric: %s, values: %s" % (metric, datapoint))
-        (metric,host,device) = self._parseMetric(metric)
+        (metric, host, device) = self._parseMetric(metric)
         if metric is not None:    
-            self._postMetric(metric,host,device, datapoint)
+            self._postMetric(metric, host, device, datapoint)
             log.info("Posted metric: %s, host: %s, device: %s" % (metric, host, device))
 
-    def _decode(self,data):
+    def _decode(self, data):
 
         try:
             datapoints = pickle.loads(data)
@@ -99,12 +100,12 @@ def _decode(self,data):
 
         for (metric, datapoint) in datapoints:
             try:
-                datapoint = ( float(datapoint[0]), float(datapoint[1]) )
+                datapoint = (float(datapoint[0]), float(datapoint[1]))
             except Exception, e:
                 log.error(e)
                 continue
 
-            self._processMetric(metric,datapoint) 
+            self._processMetric(metric, datapoint)
 
         self.stream.read_bytes(4, self._on_read_header)
 

packaging/Makefile

@@ -99,9 +99,10 @@ $(FPM_BUILD) -t rpm \
 -d "python-uuid" \
 -d "sysstat" \
 --replaces "datadog-agent-lib < $(VERSION)" \
---pre-install datadog-agent-base-rpm/pre_install \
---post-install datadog-agent-base-rpm/post_install \
---pre-uninstall datadog-agent-base-rpm/pre_uninstall \
+--pre-install    datadog-agent-base-rpm/pre_install \
+--post-install   datadog-agent-base-rpm/post_install \
+--pre-uninstall  datadog-agent-base-rpm/pre_uninstall \
+--post-uninstall datadog-agent-base-rpm/post_uninstall \
 .
 
 datadog_agent_rpm: clean install_full_rpm
@@ -119,6 +120,7 @@ $(FPM_BUILD) -t rpm \
 --pre-install	    datadog-agent-rpm/preinst \
 --post-install	    datadog-agent-rpm/postinst \
 --pre-uninstall	    datadog-agent-rpm/prerm \
+--post-uninstall    datadog-agent-rpm/postrm \
 --config-files "/etc/dd-agent/supervisor.conf" \
 .
 

packaging/datadog-agent-base-rpm/post_uninstall

@@ -0,0 +1,3 @@
+getent passwd dd-agent > /dev/null && userdel dd-agent
+getent group dd-agent >/dev/null && groupdel dd-agent
+exit 0

packaging/datadog-agent-base-rpm/pre_install

@@ -1,5 +1,5 @@
 getent group dd-agent >/dev/null || groupadd -r dd-agent
 getent passwd dd-agent >/dev/null || \
-    useradd -r -M -g dd-agent -d /tmp -s /bin/sh \
+    useradd -r -M -g dd-agent -d /usr/share/datadog/agent -s /bin/false \
     -c "Datadog Agent" dd-agent
 exit 0

packaging/datadog-agent-deb/postinst

@@ -5,7 +5,8 @@ set -e
 case "$1" in
     configure)
         update-rc.d datadog-agent defaults
-        adduser --system dd-agent --shell /bin/sh --no-create-home --quiet
+        adduser --system dd-agent --disabled-login --shell /bin/false --no-create-home --quiet
+        usermod -d /usr/share/datadog/agent dd-agent
         chown root:root /etc/init.d/datadog-agent
         chown -R dd-agent:root /etc/dd-agent
         chown -R dd-agent:root /var/log/datadog

packaging/datadog-agent-deb/postrm

@@ -3,6 +3,7 @@ set -e
 
 if [ "$1" = purge ]; then
     update-rc.d datadog-agent remove
+    deluser dd-agent
 fi
 
 exit 0

packaging/datadog-agent-rpm/postrm

@@ -0,0 +1,3 @@
+getent passwd dd-agent >/dev/null && userdel  dd-agent
+getent group  dd-agent >/dev/null && groupdel dd-agent
+exit 0

packaging/datadog-agent-rpm/preinst

@@ -3,7 +3,7 @@
 
 getent group dd-agent >/dev/null || groupadd -r dd-agent
 getent passwd dd-agent >/dev/null || \
-    useradd -r -M -g dd-agent -d /tmp -s /bin/sh \
+    useradd -r -M -g dd-agent -d /usr/share/datadog/agent -s /bin/false \
     -c "Datadog Agent" dd-agent
 
 grep -q "datadog" /etc/supervisord.conf

tests/test_ganglia.py

@@ -9,6 +9,8 @@
 import tempfile
 from util import json, md5
 import time
+import xml.etree.ElementTree as tree
+from cStringIO import StringIO
 
 from checks.ganglia import Ganglia
 
@@ -27,7 +29,13 @@ def testSpeed(self):
         # profile.runctx("g.check({'ganglia_host': 'localhost', 'ganglia_port': 8651})", {}, {"g": g}, pfile.name)
         # p = pstats.Stats(pfile.name)
         # p.sort_stats('time').print_stats()
-        self.assertEquals(md5(g.check({'ganglia_host': 'localhost', 'ganglia_port': 8651})).hexdigest(), md5(open(TEST_FN).read()).hexdigest())
+        parsed = StringIO(g.check({'ganglia_host': 'localhost', 'ganglia_port': 8651}))
+        original = open(TEST_FN)
+        x1 = tree.parse(parsed)
+        x2 = tree.parse(original)
+        # Cursory test
+        self.assertEquals([c.tag for c in x1.getroot()], [c.tag for c in x2.getroot()])
+
 
 if __name__ == '__main__':
     unittest.main()

util.py

@@ -239,7 +239,6 @@ def get_tags():
             EC2_tags = None
             pass
 
-
         try:
             if socket_to is None:
                 socket_to = 3