diff --git a/checks.d/apache.py b/checks.d/apache.py index 58a23820d7..9dece7b8e9 100644 --- a/checks.d/apache.py +++ b/checks.d/apache.py @@ -1,11 +1,12 @@ # stdlib -import urllib2 import urlparse # project from util import headers from checks import AgentCheck -from checks.utils import add_basic_auth + +# 3rd party +import requests class Apache(AgentCheck): """Tracks basic connection/requests/workers metrics @@ -37,9 +38,10 @@ def check(self, instance): url = self.assumed_url.get(instance['apache_status_url'], instance['apache_status_url']) tags = instance.get('tags', []) - req = urllib2.Request(url, None, headers(self.agentConfig)) + + auth = None if 'apache_user' in instance and 'apache_password' in instance: - add_basic_auth(req, instance['apache_user'], instance['apache_password']) + auth = (instance['apache_user'], instance['apache_password']) # Submit a service check for status page availability. parsed_url = urlparse.urlparse(url) @@ -48,7 +50,9 @@ def check(self, instance): service_check_name = 'apache.can_connect' service_check_tags = ['host:%s' % apache_host, 'port:%s' % apache_port] try: - request = urllib2.urlopen(req) + r = requests.get(url, auth=auth, headers=headers(self.agentConfig)) + r.raise_for_status() + except Exception: self.service_check(service_check_name, AgentCheck.CRITICAL, tags=service_check_tags) @@ -57,10 +61,10 @@ def check(self, instance): self.service_check(service_check_name, AgentCheck.OK, tags=service_check_tags) - response = request.read() + response = r.content metric_count = 0 # Loop through and extract the numerical values - for line in response.split('\n'): + for line in response.splitlines(): values = line.split(': ') if len(values) == 2: # match metric, value = values diff --git a/checks.d/couch.py b/checks.d/couch.py index e24ac18107..e2acd96060 100644 --- a/checks.d/couch.py +++ b/checks.d/couch.py @@ -1,13 +1,10 @@ -# stdlib -import urllib2 - # project from util import headers -from checks.utils import add_basic_auth from checks import AgentCheck # 3rd party import simplejson as json +import requests class CouchDb(AgentCheck): """Extracts stats from CouchDB via its REST API @@ -37,15 +34,15 @@ def _create_metric(self, data, tags=None): def _get_stats(self, url, instance): "Hit a given URL and return the parsed json" self.log.debug('Fetching Couchdb stats at url: %s' % url) - req = urllib2.Request(url, None, headers(self.agentConfig)) + auth = None if 'user' in instance and 'password' in instance: - add_basic_auth(req, instance['user'], instance['password']) + auth = (instance['user'], instance['password']) - # Do the request, log any errors - request = urllib2.urlopen(req) - response = request.read() - return json.loads(response) + r = requests.get(url, auth=auth, headers=headers(self.agentConfig), + timeout=10) + r.raise_for_status() + return r.json() def check(self, instance): server = instance.get('server', None) @@ -67,9 +64,9 @@ def get_data(self, server, instance): service_check_tags = ['instance:%s' % server] try: overall_stats = self._get_stats(url, instance) - except urllib2.URLError as e: + except requests.exceptions.HTTPError as e: self.service_check(self.SERVICE_CHECK_NAME, AgentCheck.CRITICAL, - tags=service_check_tags, message=str(e.reason)) + tags=service_check_tags, message=str(e.message)) raise except Exception as e: self.service_check(self.SERVICE_CHECK_NAME, AgentCheck.CRITICAL, diff --git a/checks.d/couchbase.py b/checks.d/couchbase.py index e95d0197fe..cb40f6aac3 100644 --- a/checks.d/couchbase.py +++ b/checks.d/couchbase.py @@ -1,18 +1,14 @@ # stdlib -import urllib2 import re import sys -# exceptions -from urllib2 import HTTPError - # project from util import headers from checks import AgentCheck -from checks.utils import add_basic_auth # 3rd party import simplejson as json +import requests #Constants COUCHBASE_STATS_PATH = '/pools/default' @@ -51,14 +47,17 @@ def _create_metrics(self, data, tags=None): def _get_stats(self, url, instance): """ Hit a given URL and return the parsed json. """ self.log.debug('Fetching Couchbase stats at url: %s' % url) - req = urllib2.Request(url, None, headers(self.agentConfig)) - if 'user' in instance and 'password' in instance: - add_basic_auth(req, instance['user'], instance['password']) timeout = float(instance.get('timeout', DEFAULT_TIMEOUT)) - request = urllib2.urlopen(req, timeout=timeout) - response = request.read() - return json.loads(response) + + auth = None + if 'user' in instance and 'password' in instance: + auth = (instance['user'], instance['password']) + + r = requests.get(url, auth=auth, headers=headers(self.agentConfig), + timeout=timeout) + r.raise_for_status() + return r.json() def check(self, instance): server = instance.get('server', None) @@ -92,9 +91,9 @@ def get_data(self, server, instance): # No overall stats? bail out now if overall_stats is None: raise Exception("No data returned from couchbase endpoint: %s" % url) - except urllib2.URLError as e: + except requests.exceptions.HTTPError as e: self.service_check(self.SERVICE_CHECK_NAME, AgentCheck.CRITICAL, - tags=service_check_tags, message=str(e.reason)) + tags=service_check_tags, message=str(e.message)) raise except Exception as e: self.service_check(self.SERVICE_CHECK_NAME, AgentCheck.CRITICAL, @@ -128,7 +127,7 @@ def get_data(self, server, instance): try: bucket_stats = self._get_stats(url, instance) - except HTTPError: + except requests.exceptions.HTTPError: url_backup = '%s/pools/nodes/buckets/%s/stats' % (server, bucket_name) bucket_stats = self._get_stats(url_backup, instance) diff --git a/checks.d/fluentd.py b/checks.d/fluentd.py index 67fc434567..bfb64447fd 100644 --- a/checks.d/fluentd.py +++ b/checks.d/fluentd.py @@ -1,6 +1,5 @@ # stdlib from collections import defaultdict -import urllib2 import urlparse # project @@ -9,6 +8,7 @@ # 3rd party import simplejson as json +import requests class Fluentd(AgentCheck): SERVICE_CHECK_NAME = 'fluentd.is_ok' @@ -35,9 +35,9 @@ def check(self, instance): monitor_agent_port = parsed_url.port or 24220 service_check_tags = ['fluentd_host:%s' % monitor_agent_host, 'fluentd_port:%s' % monitor_agent_port] - req = urllib2.Request(url, None, headers(self.agentConfig)) - res = urllib2.urlopen(req).read() - status = json.loads(res) + r = requests.get(url, headers=headers(self.agentConfig)) + r.raise_for_status() + status = r.json() for p in status['plugins']: for m in self.GAUGES: diff --git a/checks.d/haproxy.py b/checks.d/haproxy.py index c7b51ed94e..6856bbde0b 100644 --- a/checks.d/haproxy.py +++ b/checks.d/haproxy.py @@ -1,5 +1,4 @@ # stdlib -import urllib2 import time from collections import defaultdict @@ -7,6 +6,9 @@ from checks import AgentCheck from util import headers +# 3rd party +import requests + STATS_URL = "/;csv;norefresh" EVENT_TYPE = SOURCE_TYPE_NAME = 'haproxy' @@ -81,20 +83,16 @@ def _fetch_data(self, url, username, password): ''' Hit a given URL and return the parsed json ''' # Try to fetch data from the stats URL - passman = urllib2.HTTPPasswordMgrWithDefaultRealm() - passman.add_password(None, url, username, password) - authhandler = urllib2.HTTPBasicAuthHandler(passman) - opener = urllib2.build_opener(authhandler) - urllib2.install_opener(opener) + auth = (username, password) url = "%s%s" % (url, STATS_URL) self.log.debug("HAProxy Fetching haproxy search data from: %s" % url) - req = urllib2.Request(url, None, headers(self.agentConfig)) - request = urllib2.urlopen(req) - response = request.read() - # Split the data by line - return response.split('\n') + r = requests.get(url, auth=auth, headers=headers(self.agentConfig)) + r.raise_for_status() + + return r.content.splitlines() + def _process_data( self, data, collect_aggregates_only, process_events, url=None, diff --git a/checks.d/http_check.py b/checks.d/http_check.py index c13e33f3a2..33f69d4e8c 100644 --- a/checks.d/http_check.py +++ b/checks.d/http_check.py @@ -7,8 +7,8 @@ from urlparse import urlparse # 3rd party -from httplib2 import Http, HttpLib2Error import tornado +import requests # project from checks.network_checks import NetworkCheck, Status, EventType @@ -72,10 +72,15 @@ def _check(self, instance): self.log.debug("Connecting to %s" % addr) if disable_ssl_validation and urlparse(addr)[0] == "https": self.warning("Skipping SSL certificate validation for %s based on configuration" % addr) - h = Http(timeout=timeout, disable_ssl_certificate_validation=disable_ssl_validation) + + auth = None if username is not None and password is not None: - h.add_credentials(username, password) - resp, content = h.request(addr, "GET", headers=headers) + auth = (username, password) + + r = requests.get(addr, auth=auth,timeout=timeout, headers=headers, + verify=not disable_ssl_validation) + r.raise_for_status() + except socket.timeout, e: length = int((time.time() - start) * 1000) @@ -86,7 +91,17 @@ def _check(self, instance): "%s. Connection failed after %s ms" % (str(e), length) )) - except HttpLib2Error, e: + except requests.exceptions.HTTPError, r: + length = int((time.time() - start) * 1000) + self.log.info("%s is DOWN, error code: %s" % (addr, str(r.status_code))) + + content = r.content if include_content else '' + + service_checks.append(( + self.SC_STATUS, Status.DOWN, (r.status_code, r.reason, content or '') + )) + + except requests.exceptions.ConnectionError, e: length = int((time.time() - start) * 1000) self.log.info("%s is DOWN, error: %s. Connection failed after %s ms" % (addr, str(e), length)) service_checks.append(( @@ -119,18 +134,10 @@ def _check(self, instance): self.gauge('network.http.response_time', running_time, tags=tags_list) if not service_checks: - if resp is not None and int(resp.status) >= 400: - self.log.info("%s is DOWN, error code: %s" % (addr, str(resp.status))) - if not include_content: - content = '' - service_checks.append(( - self.SC_STATUS, Status.DOWN, (resp.status, resp.reason, content or '') - )) - else: - self.log.debug("%s is UP" % addr) - service_checks.append(( - self.SC_STATUS, Status.UP, "UP" - )) + self.log.debug("%s is UP" % addr) + service_checks.append(( + self.SC_STATUS, Status.UP, "UP" + )) if ssl_expire and urlparse(addr)[0] == "https": status, msg = self.check_cert_expiration(instance) diff --git a/checks.d/kyototycoon.py b/checks.d/kyototycoon.py index 2978e010c7..1128463742 100644 --- a/checks.d/kyototycoon.py +++ b/checks.d/kyototycoon.py @@ -1,11 +1,13 @@ # stdlib import re -import urllib2 from collections import defaultdict # project from checks import AgentCheck +# 3rd party +import requests + db_stats = re.compile(r'^db_(\d)+$') whitespace = re.compile(r'\s') @@ -61,11 +63,13 @@ def check(self, instance): if name is not None: service_check_tags.append('instance:%s' % name) + try: - response = urllib2.urlopen(url) - except urllib2.URLError as e: + r = requests.get(url) + r.raise_for_status() + except requests.exceptions.HTTPError as e: self.service_check(self.SERVICE_CHECK_NAME, AgentCheck.CRITICAL, - tags=service_check_tags, message=str(e.reason)) + tags=service_check_tags, message=str(e.message)) raise except Exception as e: self.service_check(self.SERVICE_CHECK_NAME, AgentCheck.CRITICAL, @@ -74,10 +78,10 @@ def check(self, instance): else: self.service_check(self.SERVICE_CHECK_NAME, AgentCheck.OK) - body = response.read() + body = r.content - totals = defaultdict(lambda: 0) - for line in body.split('\n'): + totals = defaultdict(int) + for line in body.splitlines(): if '\t' not in line: continue diff --git a/checks.d/lighttpd.py b/checks.d/lighttpd.py index 6120027d9d..1c3986bd13 100644 --- a/checks.d/lighttpd.py +++ b/checks.d/lighttpd.py @@ -1,12 +1,12 @@ # stdlib -import urllib2 import urlparse # project from util import headers from checks import AgentCheck -from checks.utils import add_basic_auth +# 3rd party +import requests class Lighttpd(AgentCheck): """Tracks basic connection/requests/workers metrics @@ -71,9 +71,11 @@ def check(self, instance): tags = instance.get('tags', []) self.log.debug("Connecting to %s" % url) - req = urllib2.Request(url, None, headers(self.agentConfig)) + + auth = None if 'user' in instance and 'password' in instance: - add_basic_auth(req, instance['user'], instance['password']) + auth = (instance['user'], instance['password']) + # Submit a service check for status page availability. parsed_url = urlparse.urlparse(url) @@ -82,7 +84,8 @@ def check(self, instance): service_check_name = 'lighthttpd.can_connect' service_check_tags = ['host:%s' % lighttpd_url, 'port:%s' % lighttpd_port] try: - request = urllib2.urlopen(req) + r = requests.get(url, auth=auth, headers=headers(self.agentConfig)) + r.raise_for_status() except Exception: self.service_check(service_check_name, AgentCheck.CRITICAL, tags=service_check_tags) @@ -91,9 +94,9 @@ def check(self, instance): self.service_check(service_check_name, AgentCheck.OK, tags=service_check_tags) - headers_resp = request.info().headers + headers_resp = r.headers server_version = self._get_server_version(headers_resp) - response = request.read() + response = r.content metric_count = 0 # Loop through and extract the numerical values @@ -138,17 +141,15 @@ def check(self, instance): raise Exception("No metrics were fetched for this instance. Make sure that %s is the proper url." % instance['lighttpd_status_url']) def _get_server_version(self, headers): - for h in headers: - if "Server:" not in h: - continue - try: - version = int(h.split('/')[1][0]) - except Exception, e: - self.log.debug("Error while trying to get server version %s" % str(e)) - version = "Unknown" - self.log.debug("Lighttpd server version is %s" % version) - return version - - self.log.debug("Lighttpd server version is Unknown") - return "Unknown" + if "server" not in headers: + self.log.debug("Lighttpd server version is Unknown") + return "Unknown" + v = headers['server'] + try: + version = int(v.split('/')[1][0]) + except Exception, e: + self.log.debug("Error while trying to get server version %s" % str(e)) + version = "Unknown" + self.log.debug("Lighttpd server version is %s" % version) + return version diff --git a/checks.d/mesos.py b/checks.d/mesos.py index 2986c2a575..94387f3ee9 100644 --- a/checks.d/mesos.py +++ b/checks.d/mesos.py @@ -1,7 +1,6 @@ # stdlib import time from hashlib import md5 -import urllib2 # project from checks import AgentCheck diff --git a/checks.d/nginx.py b/checks.d/nginx.py index 3f0208ca2e..3f99208bc2 100644 --- a/checks.d/nginx.py +++ b/checks.d/nginx.py @@ -1,12 +1,11 @@ # stdlib import re -import urllib2 +import requests import urlparse # project from util import headers from checks import AgentCheck -from checks.utils import add_basic_auth # 3rd party import simplejson as json @@ -51,9 +50,10 @@ def check(self, instance): def _get_data(self, instance): url = instance.get('nginx_status_url') - req = urllib2.Request(url, None, headers(self.agentConfig)) + + auth = None if 'user' in instance and 'password' in instance: - add_basic_auth(req, instance['user'], instance['password']) + auth = (instance['user'], instance['password']) # Submit a service check for status page availability. parsed_url = urlparse.urlparse(url) @@ -62,7 +62,8 @@ def _get_data(self, instance): service_check_name = 'nginx.can_connect' service_check_tags = ['host:%s' % nginx_host, 'port:%s' % nginx_port] try: - response = urllib2.urlopen(req) + r = requests.get(url, auth=auth, headers=headers(self.agentConfig)) + r.raise_for_status() except Exception: self.service_check(service_check_name, AgentCheck.CRITICAL, tags=service_check_tags) @@ -71,9 +72,9 @@ def _get_data(self, instance): self.service_check(service_check_name, AgentCheck.OK, tags=service_check_tags) - body = response.read() - resp_headers = response.info() - return body, resp_headers.get('Content-Type', 'text/plain') + body = r.content + resp_headers = r.headers + return body, resp_headers.get('content-type', 'text/plain') @classmethod def parse_text(cls, raw, tags): diff --git a/checks.d/rabbitmq.py b/checks.d/rabbitmq.py index 41e383f468..5caca980b2 100644 --- a/checks.d/rabbitmq.py +++ b/checks.d/rabbitmq.py @@ -1,16 +1,16 @@ # stdlib -import urllib -import urllib2 import urlparse import time import re import pprint +import urllib # project from checks import AgentCheck # 3rd party import simplejson as json +import requests EVENT_TYPE = SOURCE_TYPE_NAME = 'rabbitmq' QUEUE_TYPE = 'queues' @@ -125,37 +125,35 @@ def _get_config(self, instance): if type(filter_objects) != list: raise TypeError("{0} / {0}_regexes parameter must be a list".format(object_type)) - # setup urllib2 for Basic Auth - auth_handler = urllib2.HTTPBasicAuthHandler() - auth_handler.add_password(realm='RabbitMQ Management', uri=base_url, user=username, passwd=password) - opener = urllib2.build_opener(auth_handler) - urllib2.install_opener(opener) + auth = (username, password) - return base_url, max_detailed, specified + return base_url, max_detailed, specified, auth def check(self, instance): - base_url, max_detailed, specified = self._get_config(instance) + base_url, max_detailed, specified, auth = self._get_config(instance) # Generate metrics from the status API. - self.get_stats(instance, base_url, QUEUE_TYPE, max_detailed[QUEUE_TYPE], specified[QUEUE_TYPE]) - self.get_stats(instance, base_url, NODE_TYPE, max_detailed[NODE_TYPE], specified[NODE_TYPE]) + self.get_stats(instance, base_url, QUEUE_TYPE, max_detailed[QUEUE_TYPE], specified[QUEUE_TYPE], auth=auth) + self.get_stats(instance, base_url, NODE_TYPE, max_detailed[NODE_TYPE], specified[NODE_TYPE], auth=auth) # Generate a service check from the aliveness API. vhosts = instance.get('vhosts') - self._check_aliveness(base_url, vhosts) + self._check_aliveness(base_url, vhosts, auth=auth) - def _get_data(self, url): + def _get_data(self, url, auth=None): try: - data = json.loads(urllib2.urlopen(url).read()) - except urllib2.URLError, e: + r = requests.get(url, auth=auth) + r.raise_for_status() + data = r.json() + except requests.exceptions.HTTPError as e: raise Exception('Cannot open RabbitMQ API url: %s %s' % (url, str(e))) except ValueError, e: raise Exception('Cannot parse JSON response from API url: %s %s' % (url, str(e))) return data - def get_stats(self, instance, base_url, object_type, max_detailed, filters): + def get_stats(self, instance, base_url, object_type, max_detailed, filters, auth=None): """ instance: the check instance base_url: the url of the rabbitmq management api (e.g. http://localhost:15672/api) @@ -164,7 +162,7 @@ def get_stats(self, instance, base_url, object_type, max_detailed, filters): filters: explicit or regexes filters of specified queues or nodes (specified in the yaml file) """ - data = self._get_data(urlparse.urljoin(base_url, object_type)) + data = self._get_data(urlparse.urljoin(base_url, object_type), auth=auth) explicit_filters = list(filters['explicit']) # Make a copy of this list as we will remove items from it at each iteration regex_filters = filters['regexes'] @@ -277,7 +275,7 @@ def alert(self, base_url, max_detailed, size, object_type): self.event(event) - def _check_aliveness(self, base_url, vhosts=None): + def _check_aliveness(self, base_url, vhosts=None, auth=None): """ Check the aliveness API against all or a subset of vhosts. The API will return {"status": "ok"} and a 200 response code in the case that the check passes. @@ -287,7 +285,7 @@ def _check_aliveness(self, base_url, vhosts=None): if not vhosts: # Fetch a list of _all_ vhosts from the API. vhosts_url = urlparse.urljoin(base_url, 'vhosts') - vhosts_response = self._get_data(vhosts_url) + vhosts_response = self._get_data(vhosts_url, auth=auth) vhosts = [v['name'] for v in vhosts_response] for vhost in vhosts: @@ -297,7 +295,7 @@ def _check_aliveness(self, base_url, vhosts=None): aliveness_url = urlparse.urljoin(base_url, path) message = None try: - aliveness_response = self._get_data(aliveness_url) + aliveness_response = self._get_data(aliveness_url, auth=auth) message = u"Response from aliveness API: %s" % aliveness_response if aliveness_response.get('status') == 'ok': status = AgentCheck.OK diff --git a/checks/network_checks.py b/checks/network_checks.py index 13b874a57b..68de69114a 100644 --- a/checks/network_checks.py +++ b/checks/network_checks.py @@ -156,7 +156,6 @@ def _process_results(self): self.nb_failures = 0 self.restart_pool() continue - self.report_as_service_check(sc_name, status, instance, msg) # FIXME: 5.3, this has been deprecated before, get rid of events diff --git a/checks/utils.py b/checks/utils.py index 64c2e8eef4..79a1dae721 100644 --- a/checks/utils.py +++ b/checks/utils.py @@ -19,15 +19,6 @@ def median(vals): return vals[int(len(vals) / 2)] -def add_basic_auth(request, username, password): - """ A helper to add basic authentication to a urllib2 request. We do this - across a variety of checks so it's good to have this in one place. - """ - auth_str = base64.encodestring('%s:%s' % (username, password)).strip() - request.add_header('Authorization', 'Basic %s' % auth_str) - return request - - class TailFile(object): CRC_SIZE = 16 diff --git a/tests/test_haproxy.py b/tests/test_haproxy.py index b26af76e04..c6a4515fc3 100644 --- a/tests/test_haproxy.py +++ b/tests/test_haproxy.py @@ -1,11 +1,10 @@ import unittest import subprocess import time -import urllib2 import tempfile import os import logging - +import requests from util import get_hostname from tests.common import load_check, kill_subprocess from nose.plugins.attrib import attr @@ -22,14 +21,12 @@ def _wait(self, url): while True: try: STATS_URL = ";csv;norefresh" - passman = urllib2.HTTPPasswordMgrWithDefaultRealm() - passman.add_password(None, url, "datadog", "isdevops") - authhandler = urllib2.HTTPBasicAuthHandler(passman) - opener = urllib2.build_opener(authhandler) - urllib2.install_opener(opener) + auth = ("datadog", "isdevops") url = "%s%s" % (url,STATS_URL) - req = urllib2.Request(url) - request = urllib2.urlopen(req) + + r = requests.get(url, auth=auth) + r.raise_for_status() + break except Exception: time.sleep(0.5)