From 73f58b0f0068ba59ba2a7ae2df832759f52bea43 Mon Sep 17 00:00:00 2001 From: Quentin Madec Date: Mon, 9 Mar 2015 17:59:24 -0400 Subject: [PATCH 1/6] [core] add flare feature (contact support) - Create a Flare class to collect all relevant files (logs & configuration files), and then upload them to datadog, which will transfer them to support - Put all code relative to the configcheck in a function for reusability - add the flare option to init.d script --- agent.py | 302 ++++++++++++++++++++++++++-- config.py | 8 +- packaging/debian/datadog-agent.init | 16 +- 3 files changed, 298 insertions(+), 28 deletions(-) diff --git a/agent.py b/agent.py index 531185585c..fe6d1e1dfa 100755 --- a/agent.py +++ b/agent.py @@ -22,16 +22,26 @@ import sys import time import glob +import tarfile +import subprocess +import json +import tempfile +import re +import atexit # Custom modules from checks.collector import Collector -from checks.check_status import CollectorStatus -from config import get_config, get_system_stats, get_parsed_args, load_check_directory, get_confd_path, check_yaml, get_logging_config +from checks.check_status import CollectorStatus, DogstatsdStatus, ForwarderStatus +from config import get_config, get_system_stats, get_parsed_args,\ + load_check_directory, get_logging_config, check_yaml,\ + get_config, get_config_path, get_confd_path from daemon import Daemon, AgentSupervisor from emitter import http_emitter from util import Watchdog, PidFile, EC2, get_os, get_hostname from jmxfetch import JMXFetch +# 3p +import requests # Constants PID_NAME = "dd-agent" @@ -196,6 +206,266 @@ def _do_restart(self): self.collector.stop() sys.exit(AgentSupervisor.RESTART_EXIT_STATUS) +def configcheck(): + osname = get_os() + all_valid = True + for conf_path in glob.glob(os.path.join(get_confd_path(osname), "*.yaml")): + basename = os.path.basename(conf_path) + try: + check_yaml(conf_path) + except Exception, e: + all_valid = False + print "%s contains errors:\n %s" % (basename, e) + else: + print "%s is valid" % basename + if all_valid: + print "All yaml files passed. You can now run the Datadog agent." + return 0 + else: + print("Fix the invalid yaml files above in order to start the Datadog agent. " + "A useful external tool for yaml parsing can be found at " + "http://yaml-online-parser.appspot.com/") + return 1 + +class Flare(object): + """ + Compress all important logs and configuration files for debug, + and then send them to Datadog (which transfers them to Support) + """ + + DATADOG_SUPPORT_URL = '/zendesk/flare' + PASSWORD_REGEX = re.compile('( *(\w|_)*pass(word)?:).+') + COMMENT_REGEX = re.compile('^ *#.*') + APIKEY_REGEX = re.compile('^api_key:') + + def __init__(self, cmdline=False, case_id=None): + self._case_id = case_id + self._cmdline = cmdline + self._init_tarfile() + self._save_logs_path(get_logging_config()) + config = get_config() + self._api_key = config.get('api_key') + self._url = "{0}{1}".format(config.get('dd_url'), self.DATADOG_SUPPORT_URL) + self._hostname = get_hostname(config) + self._prefix = "datadog-{0}".format(self._hostname) + + def collect(self): + if not self._api_key: + raise Exception('No api_key found') + self._print("Collecting logs and configuration files:") + + self._add_logs_tar() + self._add_conf_tar() + self._print(" * datadog-agent configcheck output") + self._add_command_output_tar('configcheck.log', configcheck) + self._print(" * datadog-agent status output") + self._add_command_output_tar('status.log', self._supervisor_status) + self._print(" * datadog-agent info output") + self._add_command_output_tar('info.log', self._info_all) + + self._print("Saving all files to {0}".format(self._tar_path)) + self._tar.close() + + # Upload the tar file + def upload(self, confirmation=True): + # Ask for confirmation first + if confirmation: + self._ask_for_confirmation() + + email = self._ask_for_email() + + self._print("Uploading {0} to Datadog Support".format(self._tar_path)) + url = self._url + if self._case_id: + url = "{0}/{1}".format(self._url, str(self._case_id)) + files = {'flare_file': open(self._tar_path, 'rb')} + data = { + 'api_key': self._api_key, + 'case_id': self._case_id, + 'hostname': self._hostname, + 'email': email + } + r = requests.post(url, files=files, data=data) + self._analyse_result(r) + + # Start by creating the tar file which will contain everything + def _init_tarfile(self): + # Default temp path + self._tar_path = os.path.join(tempfile.gettempdir(), 'datadog-agent.tar.bz2') + + if os.path.exists(self._tar_path): + os.remove(self._tar_path) + self._tar = tarfile.open(self._tar_path, 'w:bz2') + + # Save logs file paths + def _save_logs_path(self, config): + prefix = '' + if get_os() == 'windows': + prefix = 'windows_' + self._collector_log = config.get('{0}collector_log_file'.format(prefix)) + self._forwarder_log = config.get('{0}forwarder_log_file'.format(prefix)) + self._dogstatsd_log = config.get('{0}dogstatsd_log_file'.format(prefix)) + self._jmxfetch_log = config.get('jmxfetch_log_file') + + # Add logs to the tarfile + def _add_logs_tar(self): + self._add_log_file_tar(self._collector_log) + self._add_log_file_tar(self._forwarder_log) + self._add_log_file_tar(self._dogstatsd_log) + self._add_log_file_tar(self._jmxfetch_log) + self._add_log_file_tar( + "{0}/*supervisord.log*".format(os.path.dirname(self._collector_log)) + ) + + def _add_log_file_tar(self, file_path): + for f in glob.glob('{0}*'.format(file_path)): + self._print(" * {0}".format(f)) + self._tar.add( + f, + os.path.join(self._prefix, 'log', os.path.basename(f)) + ) + + # Collect all conf + def _add_conf_tar(self): + conf_path = get_config_path() + self._print(" * {0}".format(conf_path)) + self._tar.add( + self._strip_comment(conf_path), + os.path.join(self._prefix, 'etc', 'datadog.conf') + ) + + if get_os() != 'windows': + supervisor_path = os.path.join( + os.path.dirname(get_config_path()), + 'supervisor.conf' + ) + self._print(" * {0}".format(supervisor_path)) + self._tar.add( + self._strip_comment(supervisor_path), + os.path.join(self._prefix, 'etc', 'supervisor.conf') + ) + + for file_path in glob.glob(os.path.join(get_confd_path(), '*.yaml')): + self._add_clean_confd(file_path) + + # Return path to a temp file without comment + def _strip_comment(self, file_path): + _, temp_path = tempfile.mkstemp(prefix='dd') + atexit.register(os.remove, temp_path) + temp_file = open(temp_path, 'w') + orig_file = open(file_path, 'r').read() + + for line in orig_file.splitlines(True): + if not self.COMMENT_REGEX.match(line) and not self.APIKEY_REGEX.match(line): + temp_file.write(line) + temp_file.close() + + return temp_path + + # Remove password before collecting the file + def _add_clean_confd(self, file_path): + basename = os.path.basename(file_path) + + temp_path, password_found = self._strip_password(file_path) + self._print(" * {0}{1}".format(file_path, password_found)) + self._tar.add( + temp_path, + os.path.join(self._prefix, 'etc', 'conf.d', basename) + ) + + # Return path to a temp file without password and comment + def _strip_password(self, file_path): + _, temp_path = tempfile.mkstemp(prefix='dd') + atexit.register(os.remove, temp_path) + temp_file = open(temp_path, 'w') + orig_file = open(file_path, 'r').read() + password_found = '' + for line in orig_file.splitlines(True): + if self.PASSWORD_REGEX.match(line): + line = re.sub(self.PASSWORD_REGEX, r'\1 ********', line) + password_found = ' - this file contains a password which '\ + 'has been removed in the version collected' + if not self.COMMENT_REGEX.match(line): + temp_file.write(line) + temp_file.close() + + return temp_path, password_found + + # Add output of the command to the tarfile + def _add_command_output_tar(self, name, command): + temp_file = os.path.join(tempfile.gettempdir(), name) + if os.path.exists(temp_file): + os.remove(temp_file) + backup = sys.stdout + sys.stdout = open(temp_file, 'w') + command() + sys.stdout.close() + sys.stdout = backup + self._tar.add(temp_file, os.path.join(self._prefix, name)) + os.remove(temp_file) + + # Print supervisor status (and nothing on windows) + def _supervisor_status(self): + if get_os == 'windows': + print 'Windows - status not implemented' + else: + print '/etc/init.d/datadog-agent status' + self._print_output_command(['/etc/init.d/datadog-agent', 'status']) + print 'supervisorctl status' + self._print_output_command(['/opt/datadog-agent/bin/supervisorctl', + '-c', '/etc/dd-agent/supervisor.conf', + 'status']) + + # Print output of command + def _print_output_command(self, command): + try: + status = subprocess.check_output(command, stderr=subprocess.STDOUT) + except subprocess.CalledProcessError, e: + status = 'Not able to get status, exit number {0}, exit ouput:\n'\ + '{1}'.format(str(e.returncode), e.output) + print status + + # Print info of all agent components + def _info_all(self): + CollectorStatus.print_latest_status(verbose=True) + DogstatsdStatus.print_latest_status(verbose=True) + ForwarderStatus.print_latest_status(verbose=True) + + # Function to ask for confirmation before upload + def _ask_for_confirmation(self): + print '{0} is going to be uploaded to Datadog.'.format(self._tar_path) + print 'Do you want to continue [Y/n]?', + choice = raw_input().lower() + if choice not in ['yes', 'y', '']: + print 'Aborting... (you can still use {0})'.format(self._tar_path) + sys.exit(1) + + # Ask for email if needed + def _ask_for_email(self): + if self._case_id: + return None + print 'Please enter your email:', + return raw_input().lower() + + # Print output (success/error) of the request + def _analyse_result(self, resp): + if resp.status_code == 200: + self._print("Your logs were successfully uploaded. For future reference,"\ + " your internal case id is {0}".format(json.loads(resp.text)['case_id'])) + elif resp.status_code == 400: + raise Exception('Your request is incorrect, error {0}'.format(resp.text)) + elif resp.status_code == 500: + raise Exception('An error has occurred while uploading: {0}'.format(resp.text)) + else: + raise Exception('An unknown error has occured, please email support directly') + + # Print to the console or to the log + def _print(self, output): + if self._cmdline: + print output + else: + log.info(output) + def main(): options, args = get_parsed_args() agentConfig = get_config(options=options) @@ -212,6 +482,7 @@ def main(): 'check', 'configcheck', 'jmx', + 'flare', ] if len(args) < 1: @@ -296,25 +567,7 @@ def parent_func(): agent.start_event = False check.stop() elif 'configcheck' == command or 'configtest' == command: - osname = get_os() - all_valid = True - for conf_path in glob.glob(os.path.join(get_confd_path(osname), "*.yaml")): - basename = os.path.basename(conf_path) - try: - check_yaml(conf_path) - except Exception, e: - all_valid = False - print "%s contains errors:\n %s" % (basename, e) - else: - print "%s is valid" % basename - if all_valid: - print "All yaml files passed. You can now run the Datadog agent." - return 0 - else: - print("Fix the invalid yaml files above in order to start the Datadog agent. " - "A useful external tool for yaml parsing can be found at " - "http://yaml-online-parser.appspot.com/") - return 1 + configcheck() elif 'jmx' == command: from jmxfetch import JMX_LIST_COMMANDS, JMXFetch @@ -342,6 +595,13 @@ def parent_func(): agent.start_event = False print "Couldn't find any valid JMX configuration in your conf.d directory: %s" % confd_directory print "Have you enabled any JMX check ?" print "If you think it's not normal please get in touch with Datadog Support" + + elif 'flare' == command: + case_id = int(args[1]) if len(args) > 1 else None + f = Flare(True, case_id) + f.collect() + f.upload() + return 0 diff --git a/config.py b/config.py index 9db8d92294..fad18582b7 100644 --- a/config.py +++ b/config.py @@ -627,7 +627,9 @@ def get_proxy(agentConfig, use_system_settings=False): return None -def get_confd_path(osname): +def get_confd_path(osname=None): + if not osname: + osname = get_os() bad_path = '' if osname == 'windows': try: @@ -651,7 +653,9 @@ def get_confd_path(osname): raise PathNotFound(bad_path) -def get_checksd_path(osname): +def get_checksd_path(osname=None): + if not osname: + osname = get_os() if osname == 'windows': return _windows_checksd_path() else: diff --git a/packaging/debian/datadog-agent.init b/packaging/debian/datadog-agent.init index 5bde22a25b..3387926ec3 100644 --- a/packaging/debian/datadog-agent.init +++ b/packaging/debian/datadog-agent.init @@ -82,13 +82,13 @@ case "$1" in log_daemon_msg "Resuming starting process." fi - + log_daemon_msg "Starting $DESC (using supervisord)" "$NAME" PATH=$SYSTEM_PATH start-stop-daemon --start --quiet --oknodo --exec $SUPERVISORD_PATH -- -c $SUPERVISOR_FILE --pidfile $SUPERVISOR_PIDFILE if [ $? -ne 0 ]; then log_end_msg 1 fi - + # check if the agent is running once per second for 10 seconds retries=10 while [ $retries -gt 1 ]; do @@ -108,10 +108,10 @@ case "$1" in exit 1 ;; stop) - + log_daemon_msg "Stopping $DESC (stopping supervisord)" "$NAME" start-stop-daemon --stop --retry 30 --quiet --oknodo --pidfile $SUPERVISOR_PIDFILE - + log_end_msg $? ;; @@ -154,9 +154,15 @@ case "$1" in exit $? ;; + flare) + shift + su $AGENTUSER -c "$AGENTPATH flare $@" + exit $? + ;; + *) N=/etc/init.d/$NAME - echo "Usage: $N {start|stop|restart|info|status|configcheck|configtest|jmx}" + echo "Usage: $N {start|stop|restart|info|status|configcheck|configtest|jmx|flare}" exit 1 ;; esac From 2febe47e14a42226bd786b306a9154f52a5e95e9 Mon Sep 17 00:00:00 2001 From: Quentin Madec Date: Thu, 12 Mar 2015 18:52:20 -0400 Subject: [PATCH 2/6] [core] move flare to utils/ - create a directory utils - move configcheck and flare into utils - address various comments #1422 - add pip freeze output --- agent.py | 299 ++++--------------------------------------- util.py | 8 ++ utils/__init__.py | 0 utils/flare.py | 313 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 343 insertions(+), 277 deletions(-) create mode 100644 utils/__init__.py create mode 100644 utils/flare.py diff --git a/agent.py b/agent.py index fe6d1e1dfa..40c4433b05 100755 --- a/agent.py +++ b/agent.py @@ -21,27 +21,29 @@ import signal import sys import time -import glob -import tarfile -import subprocess -import json -import tempfile -import re -import atexit # Custom modules from checks.collector import Collector -from checks.check_status import CollectorStatus, DogstatsdStatus, ForwarderStatus -from config import get_config, get_system_stats, get_parsed_args,\ - load_check_directory, get_logging_config, check_yaml,\ - get_config, get_config_path, get_confd_path -from daemon import Daemon, AgentSupervisor +from checks.check_status import CollectorStatus +from config import ( + get_confd_path, + get_config, + get_logging_config, + get_parsed_args, + get_system_stats, + load_check_directory, +) +from daemon import AgentSupervisor, Daemon from emitter import http_emitter -from util import Watchdog, PidFile, EC2, get_os, get_hostname from jmxfetch import JMXFetch - -# 3p -import requests +from util import ( + EC2, + get_hostname, + get_os, + PidFile, + Watchdog, +) +from utils.flare import configcheck, Flare # Constants PID_NAME = "dd-agent" @@ -206,266 +208,6 @@ def _do_restart(self): self.collector.stop() sys.exit(AgentSupervisor.RESTART_EXIT_STATUS) -def configcheck(): - osname = get_os() - all_valid = True - for conf_path in glob.glob(os.path.join(get_confd_path(osname), "*.yaml")): - basename = os.path.basename(conf_path) - try: - check_yaml(conf_path) - except Exception, e: - all_valid = False - print "%s contains errors:\n %s" % (basename, e) - else: - print "%s is valid" % basename - if all_valid: - print "All yaml files passed. You can now run the Datadog agent." - return 0 - else: - print("Fix the invalid yaml files above in order to start the Datadog agent. " - "A useful external tool for yaml parsing can be found at " - "http://yaml-online-parser.appspot.com/") - return 1 - -class Flare(object): - """ - Compress all important logs and configuration files for debug, - and then send them to Datadog (which transfers them to Support) - """ - - DATADOG_SUPPORT_URL = '/zendesk/flare' - PASSWORD_REGEX = re.compile('( *(\w|_)*pass(word)?:).+') - COMMENT_REGEX = re.compile('^ *#.*') - APIKEY_REGEX = re.compile('^api_key:') - - def __init__(self, cmdline=False, case_id=None): - self._case_id = case_id - self._cmdline = cmdline - self._init_tarfile() - self._save_logs_path(get_logging_config()) - config = get_config() - self._api_key = config.get('api_key') - self._url = "{0}{1}".format(config.get('dd_url'), self.DATADOG_SUPPORT_URL) - self._hostname = get_hostname(config) - self._prefix = "datadog-{0}".format(self._hostname) - - def collect(self): - if not self._api_key: - raise Exception('No api_key found') - self._print("Collecting logs and configuration files:") - - self._add_logs_tar() - self._add_conf_tar() - self._print(" * datadog-agent configcheck output") - self._add_command_output_tar('configcheck.log', configcheck) - self._print(" * datadog-agent status output") - self._add_command_output_tar('status.log', self._supervisor_status) - self._print(" * datadog-agent info output") - self._add_command_output_tar('info.log', self._info_all) - - self._print("Saving all files to {0}".format(self._tar_path)) - self._tar.close() - - # Upload the tar file - def upload(self, confirmation=True): - # Ask for confirmation first - if confirmation: - self._ask_for_confirmation() - - email = self._ask_for_email() - - self._print("Uploading {0} to Datadog Support".format(self._tar_path)) - url = self._url - if self._case_id: - url = "{0}/{1}".format(self._url, str(self._case_id)) - files = {'flare_file': open(self._tar_path, 'rb')} - data = { - 'api_key': self._api_key, - 'case_id': self._case_id, - 'hostname': self._hostname, - 'email': email - } - r = requests.post(url, files=files, data=data) - self._analyse_result(r) - - # Start by creating the tar file which will contain everything - def _init_tarfile(self): - # Default temp path - self._tar_path = os.path.join(tempfile.gettempdir(), 'datadog-agent.tar.bz2') - - if os.path.exists(self._tar_path): - os.remove(self._tar_path) - self._tar = tarfile.open(self._tar_path, 'w:bz2') - - # Save logs file paths - def _save_logs_path(self, config): - prefix = '' - if get_os() == 'windows': - prefix = 'windows_' - self._collector_log = config.get('{0}collector_log_file'.format(prefix)) - self._forwarder_log = config.get('{0}forwarder_log_file'.format(prefix)) - self._dogstatsd_log = config.get('{0}dogstatsd_log_file'.format(prefix)) - self._jmxfetch_log = config.get('jmxfetch_log_file') - - # Add logs to the tarfile - def _add_logs_tar(self): - self._add_log_file_tar(self._collector_log) - self._add_log_file_tar(self._forwarder_log) - self._add_log_file_tar(self._dogstatsd_log) - self._add_log_file_tar(self._jmxfetch_log) - self._add_log_file_tar( - "{0}/*supervisord.log*".format(os.path.dirname(self._collector_log)) - ) - - def _add_log_file_tar(self, file_path): - for f in glob.glob('{0}*'.format(file_path)): - self._print(" * {0}".format(f)) - self._tar.add( - f, - os.path.join(self._prefix, 'log', os.path.basename(f)) - ) - - # Collect all conf - def _add_conf_tar(self): - conf_path = get_config_path() - self._print(" * {0}".format(conf_path)) - self._tar.add( - self._strip_comment(conf_path), - os.path.join(self._prefix, 'etc', 'datadog.conf') - ) - - if get_os() != 'windows': - supervisor_path = os.path.join( - os.path.dirname(get_config_path()), - 'supervisor.conf' - ) - self._print(" * {0}".format(supervisor_path)) - self._tar.add( - self._strip_comment(supervisor_path), - os.path.join(self._prefix, 'etc', 'supervisor.conf') - ) - - for file_path in glob.glob(os.path.join(get_confd_path(), '*.yaml')): - self._add_clean_confd(file_path) - - # Return path to a temp file without comment - def _strip_comment(self, file_path): - _, temp_path = tempfile.mkstemp(prefix='dd') - atexit.register(os.remove, temp_path) - temp_file = open(temp_path, 'w') - orig_file = open(file_path, 'r').read() - - for line in orig_file.splitlines(True): - if not self.COMMENT_REGEX.match(line) and not self.APIKEY_REGEX.match(line): - temp_file.write(line) - temp_file.close() - - return temp_path - - # Remove password before collecting the file - def _add_clean_confd(self, file_path): - basename = os.path.basename(file_path) - - temp_path, password_found = self._strip_password(file_path) - self._print(" * {0}{1}".format(file_path, password_found)) - self._tar.add( - temp_path, - os.path.join(self._prefix, 'etc', 'conf.d', basename) - ) - - # Return path to a temp file without password and comment - def _strip_password(self, file_path): - _, temp_path = tempfile.mkstemp(prefix='dd') - atexit.register(os.remove, temp_path) - temp_file = open(temp_path, 'w') - orig_file = open(file_path, 'r').read() - password_found = '' - for line in orig_file.splitlines(True): - if self.PASSWORD_REGEX.match(line): - line = re.sub(self.PASSWORD_REGEX, r'\1 ********', line) - password_found = ' - this file contains a password which '\ - 'has been removed in the version collected' - if not self.COMMENT_REGEX.match(line): - temp_file.write(line) - temp_file.close() - - return temp_path, password_found - - # Add output of the command to the tarfile - def _add_command_output_tar(self, name, command): - temp_file = os.path.join(tempfile.gettempdir(), name) - if os.path.exists(temp_file): - os.remove(temp_file) - backup = sys.stdout - sys.stdout = open(temp_file, 'w') - command() - sys.stdout.close() - sys.stdout = backup - self._tar.add(temp_file, os.path.join(self._prefix, name)) - os.remove(temp_file) - - # Print supervisor status (and nothing on windows) - def _supervisor_status(self): - if get_os == 'windows': - print 'Windows - status not implemented' - else: - print '/etc/init.d/datadog-agent status' - self._print_output_command(['/etc/init.d/datadog-agent', 'status']) - print 'supervisorctl status' - self._print_output_command(['/opt/datadog-agent/bin/supervisorctl', - '-c', '/etc/dd-agent/supervisor.conf', - 'status']) - - # Print output of command - def _print_output_command(self, command): - try: - status = subprocess.check_output(command, stderr=subprocess.STDOUT) - except subprocess.CalledProcessError, e: - status = 'Not able to get status, exit number {0}, exit ouput:\n'\ - '{1}'.format(str(e.returncode), e.output) - print status - - # Print info of all agent components - def _info_all(self): - CollectorStatus.print_latest_status(verbose=True) - DogstatsdStatus.print_latest_status(verbose=True) - ForwarderStatus.print_latest_status(verbose=True) - - # Function to ask for confirmation before upload - def _ask_for_confirmation(self): - print '{0} is going to be uploaded to Datadog.'.format(self._tar_path) - print 'Do you want to continue [Y/n]?', - choice = raw_input().lower() - if choice not in ['yes', 'y', '']: - print 'Aborting... (you can still use {0})'.format(self._tar_path) - sys.exit(1) - - # Ask for email if needed - def _ask_for_email(self): - if self._case_id: - return None - print 'Please enter your email:', - return raw_input().lower() - - # Print output (success/error) of the request - def _analyse_result(self, resp): - if resp.status_code == 200: - self._print("Your logs were successfully uploaded. For future reference,"\ - " your internal case id is {0}".format(json.loads(resp.text)['case_id'])) - elif resp.status_code == 400: - raise Exception('Your request is incorrect, error {0}'.format(resp.text)) - elif resp.status_code == 500: - raise Exception('An error has occurred while uploading: {0}'.format(resp.text)) - else: - raise Exception('An unknown error has occured, please email support directly') - - # Print to the console or to the log - def _print(self, output): - if self._cmdline: - print output - else: - log.info(output) - def main(): options, args = get_parsed_args() agentConfig = get_config(options=options) @@ -600,7 +342,10 @@ def parent_func(): agent.start_event = False case_id = int(args[1]) if len(args) > 1 else None f = Flare(True, case_id) f.collect() - f.upload() + try: + f.upload() + except Exception, e: + print 'The upload failed:\n{0}'.format(str(e)) return 0 diff --git a/util.py b/util.py index 4d210f04cf..f765234cba 100644 --- a/util.py +++ b/util.py @@ -566,6 +566,10 @@ def is_darwin(name=None): name = name or sys.platform return 'darwin' in name + @staticmethod + def is_mac(name=None): + Platform.is_darwin(name) + @staticmethod def is_freebsd(name=None): name = name or sys.platform @@ -601,6 +605,10 @@ def is_win32(name=None): name = name or sys.platform return name == "win32" + @staticmethod + def is_windows(name=None): + return Platform.is_win32(name) + """ Iterable Recipes """ diff --git a/utils/__init__.py b/utils/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/utils/flare.py b/utils/flare.py new file mode 100644 index 0000000000..26d04aad9e --- /dev/null +++ b/utils/flare.py @@ -0,0 +1,313 @@ +import atexit +import glob +import logging +import os.path +import re +import simplejson as json +import subprocess +import sys +import tarfile +import tempfile +from time import strftime + +# DD imports +from checks.check_status import CollectorStatus, DogstatsdStatus, ForwarderStatus +from config import ( + check_yaml, + get_confd_path, + get_config, + get_config_path, + get_logging_config, + get_os, +) +from util import ( + get_hostname, + Platform, +) + +# 3p +import requests + + +# Globals +log = logging.getLogger('flare') + +def configcheck(): + osname = get_os() + all_valid = True + for conf_path in glob.glob(os.path.join(get_confd_path(osname), "*.yaml")): + basename = os.path.basename(conf_path) + try: + check_yaml(conf_path) + except Exception, e: + all_valid = False + print "%s contains errors:\n %s" % (basename, e) + else: + print "%s is valid" % basename + if all_valid: + print "All yaml files passed. You can now run the Datadog agent." + return 0 + else: + print("Fix the invalid yaml files above in order to start the Datadog agent. " + "A useful external tool for yaml parsing can be found at " + "http://yaml-online-parser.appspot.com/") + return 1 + +class Flare(object): + """ + Compress all important logs and configuration files for debug, + and then send them to Datadog (which transfers them to Support) + """ + + DATADOG_SUPPORT_URL = '/zendesk/flare' + PASSWORD_REGEX = re.compile('( *(\w|_)*pass(word)?:).+') + COMMENT_REGEX = re.compile('^ *#.*') + APIKEY_REGEX = re.compile('^api_key: *\w+(\w{5})$') + REPLACE_APIKEY = r'api_key: *************************\1' + COMPRESSED_FILE = 'datadog-agent-{0}.tar.bz2' + # We limit to 10MB arbitrary + MAX_UPLOAD_SIZE = 10485000 + + + def __init__(self, cmdline=False, case_id=None): + self._case_id = case_id + self._cmdline = cmdline + self._init_tarfile() + self._save_logs_path() + config = get_config() + self._api_key = config.get('api_key') + self._url = "{0}{1}".format(config.get('dd_url'), self.DATADOG_SUPPORT_URL) + self._hostname = get_hostname(config) + self._prefix = "datadog-{0}".format(self._hostname) + + # Collect all conf and logs files and compress them + def collect(self): + if not self._api_key: + raise Exception('No api_key found') + log.info("Collecting logs and configuration files:") + + self._add_logs_tar() + self._add_conf_tar() + log.info(" * datadog-agent configcheck output") + self._add_command_output_tar('configcheck.log', configcheck) + log.info(" * datadog-agent status output") + self._add_command_output_tar('status.log', self._supervisor_status) + log.info(" * datadog-agent info output") + self._add_command_output_tar('info.log', self._info_all) + log.info(" * pip freeze") + self._add_command_output_tar('freeze.log', self._pip_freeze) + + log.info("Saving all files to {0}".format(self._tar_path)) + self._tar.close() + + # Upload the tar file + def upload(self, confirmation=True): + self._check_size() + + if confirmation: + self._ask_for_confirmation() + + email = self._ask_for_email() + + log.info("Uploading {0} to Datadog Support".format(self._tar_path)) + url = self._url + if self._case_id: + url = '{0}/{1}'.format(self._url, str(self._case_id)) + files = {'flare_file': open(self._tar_path, 'rb')} + data = { + 'api_key': self._api_key, + 'case_id': self._case_id, + 'hostname': self._hostname, + 'email': email + } + r = requests.post(url, files=files, data=data) + self._analyse_result(r) + + # Start by creating the tar file which will contain everything + def _init_tarfile(self): + # Default temp path + self._tar_path = os.path.join( + tempfile.gettempdir(), + self.COMPRESSED_FILE.format(strftime("%Y-%m-%d-%H-%M-%S")) + ) + + if os.path.exists(self._tar_path): + os.remove(self._tar_path) + self._tar = tarfile.open(self._tar_path, 'w:bz2') + + # Save logs file paths + def _save_logs_path(self): + prefix = '' + if Platform.is_windows(): + prefix = 'windows_' + config = get_logging_config() + self._collector_log = config.get('{0}collector_log_file'.format(prefix)) + self._forwarder_log = config.get('{0}forwarder_log_file'.format(prefix)) + self._dogstatsd_log = config.get('{0}dogstatsd_log_file'.format(prefix)) + self._jmxfetch_log = config.get('jmxfetch_log_file') + + # Add logs to the tarfile + def _add_logs_tar(self): + self._add_log_file_tar(self._collector_log) + self._add_log_file_tar(self._forwarder_log) + self._add_log_file_tar(self._dogstatsd_log) + self._add_log_file_tar(self._jmxfetch_log) + self._add_log_file_tar( + "{0}/*supervisord.log*".format(os.path.dirname(self._collector_log)) + ) + + def _add_log_file_tar(self, file_path): + for f in glob.glob('{0}*'.format(file_path)): + log.info(" * {0}".format(f)) + self._tar.add( + f, + os.path.join(self._prefix, 'log', os.path.basename(f)) + ) + + # Collect all conf + def _add_conf_tar(self): + conf_path = get_config_path() + log.info(" * {0}".format(conf_path)) + self._tar.add( + self._strip_comment(conf_path), + os.path.join(self._prefix, 'etc', 'datadog.conf') + ) + + if not Platform.is_windows(): + supervisor_path = os.path.join( + os.path.dirname(get_config_path()), + 'supervisor.conf' + ) + log.info(" * {0}".format(supervisor_path)) + self._tar.add( + self._strip_comment(supervisor_path), + os.path.join(self._prefix, 'etc', 'supervisor.conf') + ) + + for file_path in glob.glob(os.path.join(get_confd_path(), '*.yaml')): + self._add_clean_confd(file_path) + + # Return path to a temp file without comment + def _strip_comment(self, file_path): + _, temp_path = tempfile.mkstemp(prefix='dd') + atexit.register(os.remove, temp_path) + temp_file = open(temp_path, 'w') + orig_file = open(file_path, 'r').read() + + for line in orig_file.splitlines(True): + if not self.COMMENT_REGEX.match(line): + temp_file.write(re.sub(self.APIKEY_REGEX, self.REPLACE_APIKEY, line)) + temp_file.close() + + return temp_path + + # Remove password before collecting the file + def _add_clean_confd(self, file_path): + basename = os.path.basename(file_path) + + temp_path, password_found = self._strip_password(file_path) + log.info(" * {0}{1}".format(file_path, password_found)) + self._tar.add( + temp_path, + os.path.join(self._prefix, 'etc', 'conf.d', basename) + ) + + # Return path to a temp file without password and comment + def _strip_password(self, file_path): + _, temp_path = tempfile.mkstemp(prefix='dd') + atexit.register(os.remove, temp_path) + temp_file = open(temp_path, 'w') + orig_file = open(file_path, 'r').read() + password_found = '' + for line in orig_file.splitlines(True): + if self.PASSWORD_REGEX.match(line): + line = re.sub(self.PASSWORD_REGEX, r'\1 ********', line) + password_found = ' - this file contains a password which '\ + 'has been removed in the version collected' + if not self.COMMENT_REGEX.match(line): + temp_file.write(line) + temp_file.close() + + return temp_path, password_found + + # Add output of the command to the tarfile + def _add_command_output_tar(self, name, command): + temp_file = os.path.join(tempfile.gettempdir(), name) + if os.path.exists(temp_file): + os.remove(temp_file) + backup = sys.stdout + sys.stdout = open(temp_file, 'w') + command() + sys.stdout.close() + sys.stdout = backup + self._tar.add(temp_file, os.path.join(self._prefix, name)) + os.remove(temp_file) + + # Print supervisor status (and nothing on windows) + def _supervisor_status(self): + if Platform.is_windows(): + print 'Windows - status not implemented' + else: + print '/etc/init.d/datadog-agent status' + self._print_output_command(['/etc/init.d/datadog-agent', 'status']) + print 'supervisorctl status' + self._print_output_command(['/opt/datadog-agent/bin/supervisorctl', + '-c', '/etc/dd-agent/supervisor.conf', + 'status']) + + # Print output of command + def _print_output_command(self, command): + try: + status = subprocess.check_output(command, stderr=subprocess.STDOUT) + except subprocess.CalledProcessError, e: + status = 'Not able to get ouput, exit number {0}, exit ouput:\n'\ + '{1}'.format(str(e.returncode), e.output) + print status + + # Print info of all agent components + def _info_all(self): + CollectorStatus.print_latest_status(verbose=True) + DogstatsdStatus.print_latest_status(verbose=True) + ForwarderStatus.print_latest_status(verbose=True) + + # Run a pip freeze + def _pip_freeze(self): + try: + import pip + pip.main(['freeze']) + except ImportError: + print 'Unable to import pip' + + # Check if the file is not too big before upload + def _check_size(self): + if os.path.getsize(self._tar_path) > self.MAX_UPLOAD_SIZE: + log.info('{0} won\'t be uploaded, its size is too important.\n'\ + 'You can send it directly to support by mail.') + sys.exit(1) + + # Function to ask for confirmation before upload + def _ask_for_confirmation(self): + print '{0} is going to be uploaded to Datadog.'.format(self._tar_path) + choice = raw_input('Do you want to continue [Y/n]? ').lower() + if choice not in ['yes', 'y', '']: + print 'Aborting (you can still use {0})'.format(self._tar_path) + sys.exit(1) + + # Ask for email if needed + def _ask_for_email(self): + if self._case_id: + return None + return raw_input('Please enter your email: ').lower() + + # Print output (success/error) of the request + def _analyse_result(self, resp): + if resp.status_code == 200: + log.info("Your logs were successfully uploaded. For future reference,"\ + " your internal case id is {0}".format(json.loads(resp.text)['case_id'])) + elif resp.status_code == 400: + raise Exception('Your request is incorrect: {0}'.format(resp.text)) + elif resp.status_code == 500: + raise Exception('An error has occurred while uploading: {0}'.format(resp).text) + else: + raise Exception('An unknown error has occured: {0}\n'\ + 'Please contact support by email'.format(resp.text)) From a26b9ec973aed17676b827373e3e4197eccecdea Mon Sep 17 00:00:00 2001 From: Quentin Madec Date: Mon, 16 Mar 2015 10:47:58 -0400 Subject: [PATCH 3/6] [flare] send api_key in query parameter & rename support URL into something more generic --- utils/flare.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/flare.py b/utils/flare.py index 26d04aad9e..6b1795d7c9 100644 --- a/utils/flare.py +++ b/utils/flare.py @@ -59,7 +59,7 @@ class Flare(object): and then send them to Datadog (which transfers them to Support) """ - DATADOG_SUPPORT_URL = '/zendesk/flare' + DATADOG_SUPPORT_URL = '/support/flare' PASSWORD_REGEX = re.compile('( *(\w|_)*pass(word)?:).+') COMMENT_REGEX = re.compile('^ *#.*') APIKEY_REGEX = re.compile('^api_key: *\w+(\w{5})$') @@ -113,9 +113,9 @@ def upload(self, confirmation=True): url = self._url if self._case_id: url = '{0}/{1}'.format(self._url, str(self._case_id)) + url = "{0}?api_key={1}".format(url, self._api_key) files = {'flare_file': open(self._tar_path, 'rb')} data = { - 'api_key': self._api_key, 'case_id': self._case_id, 'hostname': self._hostname, 'email': email From 18ef936fa713f4287bba483089c3b9bdec54d495 Mon Sep 17 00:00:00 2001 From: Quentin Madec Date: Tue, 17 Mar 2015 14:45:48 -0400 Subject: [PATCH 4/6] [flare] use new endpoints Create a function in config to get new endpoints, and use it also in the forwarder --- config.py | 20 ++++++++++++++++++++ ddagent.py | 27 +++------------------------ utils/flare.py | 24 +++++++++++++++--------- 3 files changed, 38 insertions(+), 33 deletions(-) diff --git a/config.py b/config.py index fad18582b7..15d2abef84 100644 --- a/config.py +++ b/config.py @@ -17,6 +17,7 @@ from socket import gaierror from optparse import OptionParser, Values from cStringIO import StringIO +from urlparse import urlparse # project @@ -57,6 +58,10 @@ ] DEFAULT_CHECKS = ("network", "ntp") +LEGACY_DATADOG_URLS = [ + "app.datadoghq.com", + "app.datad0g.com", +] class PathNotFound(Exception): pass @@ -93,6 +98,21 @@ def get_parsed_args(): def get_version(): return AGENT_VERSION + +# Return url endpoint, here because needs access to version number +def get_url_endpoint(default_url, endpoint_type='app'): + parsed_url = urlparse(default_url) + if parsed_url.netloc not in LEGACY_DATADOG_URLS: + return default_url + + subdomain = parsed_url.netloc.split(".")[0] + + # Replace https://app.datadoghq.com in https://5-2-0-app.agent.datadoghq.com + return default_url.replace(subdomain, + "{0}-{1}.agent".format( + get_version().replace(".", "-"), + endpoint_type)) + def skip_leading_wsp(f): "Works on a file, returns a file-like object" return StringIO("\n".join(map(string.strip, f.readlines()))) diff --git a/ddagent.py b/ddagent.py index 81abba6ccd..7b2d1d10cb 100755 --- a/ddagent.py +++ b/ddagent.py @@ -28,7 +28,6 @@ from hashlib import md5 from datetime import datetime, timedelta from socket import gaierror, error as socket_error -from urlparse import urlparse # Tornado import tornado.httpserver @@ -40,7 +39,7 @@ # agent import from util import Watchdog, get_uuid, get_hostname, json, get_tornado_ioloop from emitter import http_emitter -from config import get_config, get_version +from config import get_config, get_url_endpoint, get_version from checks.check_status import ForwarderStatus from transaction import Transaction, TransactionManager import modules @@ -73,11 +72,6 @@ THROTTLING_DELAY = timedelta(microseconds=1000000/2) # 2 msg/second -LEGACY_DATADOG_URLS = [ - "app.datadoghq.com", - "app.datad0g.com", -] - class EmitterThread(threading.Thread): def __init__(self, *args, **kwargs): @@ -193,23 +187,8 @@ def __init__(self, data, headers): def __sizeof__(self): return sys.getsizeof(self._data) - @classmethod - def get_url_endpoint(cls, endpoint): - default_url = cls._application._agentConfig[endpoint] - parsed_url = urlparse(default_url) - if parsed_url.netloc not in LEGACY_DATADOG_URLS: - return default_url - - subdomain = parsed_url.netloc.split(".")[0] - - # Replace https://app.datadoghq.com in https://5-2-0-app.agent.datadoghq.com - return default_url.replace(subdomain, - "{0}-{1}.agent".format( - get_version().replace(".", "-"), - subdomain)) - def get_url(self, endpoint): - endpoint_base_url = self.get_url_endpoint(endpoint) + endpoint_base_url = get_url_endpoint(self._application._agentConfig[endpoint]) api_key = self._application._agentConfig.get('api_key') if api_key: return endpoint_base_url + '/intake?api_key=%s' % api_key @@ -285,7 +264,7 @@ def on_response(self, response): class APIMetricTransaction(MetricTransaction): def get_url(self, endpoint): - endpoint_base_url = self.get_url_endpoint(endpoint) + endpoint_base_url = get_url_endpoint(self._application._agentConfig[endpoint]) config = self._application._agentConfig api_key = config['api_key'] url = endpoint_base_url + '/api/v1/series/?api_key=' + api_key diff --git a/utils/flare.py b/utils/flare.py index 6b1795d7c9..a8dab2e302 100644 --- a/utils/flare.py +++ b/utils/flare.py @@ -19,6 +19,7 @@ get_config_path, get_logging_config, get_os, + get_url_endpoint, ) from util import ( get_hostname, @@ -28,7 +29,6 @@ # 3p import requests - # Globals log = logging.getLogger('flare') @@ -76,7 +76,10 @@ def __init__(self, cmdline=False, case_id=None): self._save_logs_path() config = get_config() self._api_key = config.get('api_key') - self._url = "{0}{1}".format(config.get('dd_url'), self.DATADOG_SUPPORT_URL) + self._url = "{0}{1}".format( + get_url_endpoint(config.get('dd_url'), endpoint_type='flare'), + self.DATADOG_SUPPORT_URL + ) self._hostname = get_hostname(config) self._prefix = "datadog-{0}".format(self._hostname) @@ -296,18 +299,21 @@ def _ask_for_confirmation(self): # Ask for email if needed def _ask_for_email(self): if self._case_id: - return None + return '' return raw_input('Please enter your email: ').lower() # Print output (success/error) of the request def _analyse_result(self, resp): - if resp.status_code == 200: + if resp.status_code in range(200, 203): log.info("Your logs were successfully uploaded. For future reference,"\ - " your internal case id is {0}".format(json.loads(resp.text)['case_id'])) - elif resp.status_code == 400: - raise Exception('Your request is incorrect: {0}'.format(resp.text)) - elif resp.status_code == 500: - raise Exception('An error has occurred while uploading: {0}'.format(resp).text) + " your internal case id is {0}".format( + json.loads(resp.text)['case_id'])) + elif resp.status_code in range(400, 405): + raise Exception('Your request is incorrect: {0}'.format( + json.loads(resp.text)['error'])) + elif resp.status_code in range(500, 506): + raise Exception('An error has occurred while uploading: {0}'.format( + json.loads(resp.text)['error'])) else: raise Exception('An unknown error has occured: {0}\n'\ 'Please contact support by email'.format(resp.text)) From 9b12fb8d04dedf3a5d5b557ad9fac9d9c2a9d87c Mon Sep 17 00:00:00 2001 From: Quentin Madec Date: Wed, 18 Mar 2015 10:47:19 -0400 Subject: [PATCH 5/6] [flare] add tests and some cleanup - test the upload part (mocking requests.post ) - test the existence of the endpoint (marked as failing for now) - clean configcheck --- tests/flare/datadog-agent-1.tar.bz2 | Bin 0 -> 14 bytes tests/test_flare.py | 102 ++++++++++++++++++++++++++++ utils/flare.py | 35 +++++----- 3 files changed, 121 insertions(+), 16 deletions(-) create mode 100644 tests/flare/datadog-agent-1.tar.bz2 create mode 100644 tests/test_flare.py diff --git a/tests/flare/datadog-agent-1.tar.bz2 b/tests/flare/datadog-agent-1.tar.bz2 new file mode 100644 index 0000000000000000000000000000000000000000..b56f3b974d6a345462b5a64b15a84c9b23bb40ec GIT binary patch literal 14 TcmZ>Y%CHnKa Date: Wed, 18 Mar 2015 15:44:17 -0400 Subject: [PATCH 6/6] [flare] add source install compatibility - add flare option to centos and source install commands - rewrite the analyse_result part (dealing first with our custom 400 errors, then with standard errors, and then with success) - fix endpoint test --- packaging/centos/datadog-agent.init | 7 ++ packaging/datadog-agent/source/agent | 8 ++ tests/test_flare.py | 8 +- utils/flare.py | 105 +++++++++++++++++---------- 4 files changed, 87 insertions(+), 41 deletions(-) diff --git a/packaging/centos/datadog-agent.init b/packaging/centos/datadog-agent.init index 3f7b489edd..b55e1d0265 100644 --- a/packaging/centos/datadog-agent.init +++ b/packaging/centos/datadog-agent.init @@ -186,6 +186,13 @@ case "$1" in su $AGENTUSER -c "$AGENTPATH jmx $@" exit $? ;; + + flare) + shift + su $AGENTUSER -c "$AGENTPATH flare $@" + exit $? + ;; + *) echo "Usage: $0 {start|stop|restart|info|status|configcheck|configtest|jmx}" exit 2 diff --git a/packaging/datadog-agent/source/agent b/packaging/datadog-agent/source/agent index 14032836e4..ba7bee479a 100755 --- a/packaging/datadog-agent/source/agent +++ b/packaging/datadog-agent/source/agent @@ -135,6 +135,14 @@ case $action in exit $? ;; + + flare) + shift + python agent/agent.py flare $@ + exit $? + ;; + + *) echo "Usage: $0 {start|stop|restart|info|status|configcheck|check|jmx}" exit 2 diff --git a/tests/test_flare.py b/tests/test_flare.py index 504f7c22a9..595c90879c 100644 --- a/tests/test_flare.py +++ b/tests/test_flare.py @@ -26,6 +26,12 @@ def __init__(self, status_code=200): self.status_code = status_code self.text = '{"case_id":1337}' + def json(self): + return {'case_id': 1337} + + def raise_for_status(self): + return None + class FlareTest(unittest.TestCase): @mock.patch('utils.flare.strftime', side_effect=mocked_strftime) @@ -99,4 +105,4 @@ def test_endpoint(self, mock_config, mock_temp, mock_stfrtime): f.upload(confirmation=False) raise Exception('Should fail before') except Exception, e: - self.assertEqual(str(e), "Invalid inputs: {'email': None}") + self.assertEqual(str(e), "Your request is incorrect: Invalid inputs: {'email': None}") diff --git a/utils/flare.py b/utils/flare.py index 359f2db200..90824e40ad 100644 --- a/utils/flare.py +++ b/utils/flare.py @@ -3,7 +3,6 @@ import logging import os.path import re -import simplejson as json import subprocess import sys import tarfile @@ -65,7 +64,7 @@ class Flare(object): COMPRESSED_FILE = 'datadog-agent-{0}.tar.bz2' # We limit to 10MB arbitrary MAX_UPLOAD_SIZE = 10485000 - TIMEOUT = 15 + TIMEOUT = 30 def __init__(self, cmdline=False, case_id=None): @@ -193,13 +192,11 @@ def _add_conf_tar(self): def _strip_comment(self, file_path): _, temp_path = tempfile.mkstemp(prefix='dd') atexit.register(os.remove, temp_path) - temp_file = open(temp_path, 'w') - orig_file = open(file_path, 'r').read() - - for line in orig_file.splitlines(True): - if not self.COMMENT_REGEX.match(line): - temp_file.write(re.sub(self.APIKEY_REGEX, self.REPLACE_APIKEY, line)) - temp_file.close() + with open(temp_path, 'w') as temp_file: + with open(file_path, 'r') as orig_file: + for line in orig_file.readlines(): + if not self.COMMENT_REGEX.match(line): + temp_file.write(re.sub(self.APIKEY_REGEX, self.REPLACE_APIKEY, line)) return temp_path @@ -218,17 +215,16 @@ def _add_clean_confd(self, file_path): def _strip_password(self, file_path): _, temp_path = tempfile.mkstemp(prefix='dd') atexit.register(os.remove, temp_path) - temp_file = open(temp_path, 'w') - orig_file = open(file_path, 'r').read() - password_found = '' - for line in orig_file.splitlines(True): - if self.PASSWORD_REGEX.match(line): - line = re.sub(self.PASSWORD_REGEX, r'\1 ********', line) - password_found = ' - this file contains a password which '\ - 'has been removed in the version collected' - if not self.COMMENT_REGEX.match(line): - temp_file.write(line) - temp_file.close() + with open(temp_path, 'w') as temp_file: + with open(file_path, 'r') as orig_file: + password_found = '' + for line in orig_file.readlines(): + if self.PASSWORD_REGEX.match(line): + line = re.sub(self.PASSWORD_REGEX, r'\1 ********', line) + password_found = ' - this file contains a password which '\ + 'has been removed in the version collected' + if not self.COMMENT_REGEX.match(line): + temp_file.write(line) return temp_path, password_found @@ -250,13 +246,45 @@ def _supervisor_status(self): if Platform.is_windows(): print 'Windows - status not implemented' else: - print '/etc/init.d/datadog-agent status' - self._print_output_command(['/etc/init.d/datadog-agent', 'status']) - print 'supervisorctl status' - self._print_output_command(['/opt/datadog-agent/bin/supervisorctl', - '-c', '/etc/dd-agent/supervisor.conf', + agent_exec = self._get_path_agent_exec() + print '{0} status'.format(agent_exec) + self._print_output_command([agent_exec, 'status']) + supervisor_exec = self._get_path_supervisor_exec() + print '{0} status'.format(supervisor_exec) + self._print_output_command([supervisor_exec, + '-c', self._get_path_supervisor_conf(), 'status']) + # Find the agent exec (package or source) + def _get_path_agent_exec(self): + agent_exec = '/etc/init.d/datadog-agent' + if not os.path.isfile(agent_exec): + agent_exec = os.path.join( + os.path.dirname(os.path.realpath(__file__)), + '../../bin/agent' + ) + return agent_exec + + # Find the supervisor exec (package or source) + def _get_path_supervisor_exec(self): + supervisor_exec = '/opt/datadog-agent/bin/supervisorctl' + if not os.path.isfile(supervisor_exec): + supervisor_exec = os.path.join( + os.path.dirname(os.path.realpath(__file__)), + '../../venv/bin/supervisorctl' + ) + return supervisor_exec + + # Find the supervisor conf (package or source) + def _get_path_supervisor_conf(self): + supervisor_conf = '/etc/init.d/datadog-agent' + if not os.path.isfile(supervisor_conf): + supervisor_conf = os.path.join( + os.path.dirname(os.path.realpath(__file__)), + '../../supervisord/supervisord.conf' + ) + return supervisor_conf + # Print output of command def _print_output_command(self, command): try: @@ -303,20 +331,17 @@ def _ask_for_email(self): # Print output (success/error) of the request def _analyse_result(self): + # First catch our custom explicit 400 + if self._resp.status_code == 400: + raise Exception('Your request is incorrect: {0}'.format(self._resp.json()['error'])) + # Then raise potential 500 and 404 + self._resp.raise_for_status() try: - json_resp = json.loads(self._resp.text) + json_resp = self._resp.json() + # Failed parsing except ValueError, e: - raise Exception('An unknown error has occured: {0}\n'\ - 'Please contact support by email'.format(self._resp.text)) - if self._resp.status_code in range(200, 203): - log.info("Your logs were successfully uploaded. For future reference,"\ - " your internal case id is {0}".format(json_resp['case_id'])) - elif self._resp.status_code in range(400, 405): - raise Exception('Your request is incorrect: {0}'.format(json_resp['error'])) - elif self._resp.status_code in range(500, 506): - raise Exception('An error has occurred while uploading: {0}'.format( - json_resp['error'])) - else: - raise Exception('An unknown error has occured: {0} - {1}\n'\ - 'Please contact support by email'.format( - self._resp.status_code, self._resp.text)) + raise Exception('An unknown error has occured - '\ + 'Please contact support by email') + # Finally, correct + log.info("Your logs were successfully uploaded. For future reference,"\ + " your internal case id is {0}".format(json_resp['case_id']))