Skip to content

Commit

Permalink
Merge pull request #1422 from DataDog/quentin/flare
Browse files Browse the repository at this point in the history
[core] add flare feature (contact support)
  • Loading branch information
remh committed Mar 18, 2015
2 parents 03d7716 + 8b10531 commit 68d7e4a
Show file tree
Hide file tree
Showing 11 changed files with 547 additions and 55 deletions.
53 changes: 29 additions & 24 deletions agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,17 +21,29 @@
import signal
import sys
import time
import glob

# Custom modules
from checks.collector import Collector
from checks.check_status import CollectorStatus
from config import get_config, get_system_stats, get_parsed_args, load_check_directory, get_confd_path, check_yaml, get_logging_config
from daemon import Daemon, AgentSupervisor
from config import (
get_confd_path,
get_config,
get_logging_config,
get_parsed_args,
get_system_stats,
load_check_directory,
)
from daemon import AgentSupervisor, Daemon
from emitter import http_emitter
from util import Watchdog, PidFile, EC2, get_os, get_hostname
from jmxfetch import JMXFetch

from util import (
EC2,
get_hostname,
get_os,
PidFile,
Watchdog,
)
from utils.flare import configcheck, Flare

# Constants
PID_NAME = "dd-agent"
Expand Down Expand Up @@ -212,6 +224,7 @@ def main():
'check',
'configcheck',
'jmx',
'flare',
]

if len(args) < 1:
Expand Down Expand Up @@ -296,25 +309,7 @@ def parent_func(): agent.start_event = False
check.stop()

elif 'configcheck' == command or 'configtest' == command:
osname = get_os()
all_valid = True
for conf_path in glob.glob(os.path.join(get_confd_path(osname), "*.yaml")):
basename = os.path.basename(conf_path)
try:
check_yaml(conf_path)
except Exception, e:
all_valid = False
print "%s contains errors:\n %s" % (basename, e)
else:
print "%s is valid" % basename
if all_valid:
print "All yaml files passed. You can now run the Datadog agent."
return 0
else:
print("Fix the invalid yaml files above in order to start the Datadog agent. "
"A useful external tool for yaml parsing can be found at "
"http://yaml-online-parser.appspot.com/")
return 1
configcheck()

elif 'jmx' == command:
from jmxfetch import JMX_LIST_COMMANDS, JMXFetch
Expand Down Expand Up @@ -342,6 +337,16 @@ def parent_func(): agent.start_event = False
print "Couldn't find any valid JMX configuration in your conf.d directory: %s" % confd_directory
print "Have you enabled any JMX check ?"
print "If you think it's not normal please get in touch with Datadog Support"

elif 'flare' == command:
case_id = int(args[1]) if len(args) > 1 else None
f = Flare(True, case_id)
f.collect()
try:
f.upload()
except Exception, e:
print 'The upload failed:\n{0}'.format(str(e))

return 0


Expand Down
28 changes: 26 additions & 2 deletions config.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
from socket import gaierror
from optparse import OptionParser, Values
from cStringIO import StringIO
from urlparse import urlparse

# project

Expand Down Expand Up @@ -57,6 +58,10 @@
]

DEFAULT_CHECKS = ("network", "ntp")
LEGACY_DATADOG_URLS = [
"app.datadoghq.com",
"app.datad0g.com",
]

class PathNotFound(Exception):
pass
Expand Down Expand Up @@ -93,6 +98,21 @@ def get_parsed_args():
def get_version():
return AGENT_VERSION


# Return url endpoint, here because needs access to version number
def get_url_endpoint(default_url, endpoint_type='app'):
parsed_url = urlparse(default_url)
if parsed_url.netloc not in LEGACY_DATADOG_URLS:
return default_url

subdomain = parsed_url.netloc.split(".")[0]

# Replace https://app.datadoghq.com in https://5-2-0-app.agent.datadoghq.com
return default_url.replace(subdomain,
"{0}-{1}.agent".format(
get_version().replace(".", "-"),
endpoint_type))

def skip_leading_wsp(f):
"Works on a file, returns a file-like object"
return StringIO("\n".join(map(string.strip, f.readlines())))
Expand Down Expand Up @@ -627,7 +647,9 @@ def get_proxy(agentConfig, use_system_settings=False):
return None


def get_confd_path(osname):
def get_confd_path(osname=None):
if not osname:
osname = get_os()
bad_path = ''
if osname == 'windows':
try:
Expand All @@ -651,7 +673,9 @@ def get_confd_path(osname):
raise PathNotFound(bad_path)


def get_checksd_path(osname):
def get_checksd_path(osname=None):
if not osname:
osname = get_os()
if osname == 'windows':
return _windows_checksd_path()
else:
Expand Down
27 changes: 3 additions & 24 deletions ddagent.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@
from hashlib import md5
from datetime import datetime, timedelta
from socket import gaierror, error as socket_error
from urlparse import urlparse

# Tornado
import tornado.httpserver
Expand All @@ -40,7 +39,7 @@
# agent import
from util import Watchdog, get_uuid, get_hostname, json, get_tornado_ioloop
from emitter import http_emitter
from config import get_config, get_version
from config import get_config, get_url_endpoint, get_version
from checks.check_status import ForwarderStatus
from transaction import Transaction, TransactionManager
import modules
Expand Down Expand Up @@ -73,11 +72,6 @@

THROTTLING_DELAY = timedelta(microseconds=1000000/2) # 2 msg/second

LEGACY_DATADOG_URLS = [
"app.datadoghq.com",
"app.datad0g.com",
]

class EmitterThread(threading.Thread):

def __init__(self, *args, **kwargs):
Expand Down Expand Up @@ -193,23 +187,8 @@ def __init__(self, data, headers):
def __sizeof__(self):
return sys.getsizeof(self._data)

@classmethod
def get_url_endpoint(cls, endpoint):
default_url = cls._application._agentConfig[endpoint]
parsed_url = urlparse(default_url)
if parsed_url.netloc not in LEGACY_DATADOG_URLS:
return default_url

subdomain = parsed_url.netloc.split(".")[0]

# Replace https://app.datadoghq.com in https://5-2-0-app.agent.datadoghq.com
return default_url.replace(subdomain,
"{0}-{1}.agent".format(
get_version().replace(".", "-"),
subdomain))

def get_url(self, endpoint):
endpoint_base_url = self.get_url_endpoint(endpoint)
endpoint_base_url = get_url_endpoint(self._application._agentConfig[endpoint])
api_key = self._application._agentConfig.get('api_key')
if api_key:
return endpoint_base_url + '/intake?api_key=%s' % api_key
Expand Down Expand Up @@ -285,7 +264,7 @@ def on_response(self, response):
class APIMetricTransaction(MetricTransaction):

def get_url(self, endpoint):
endpoint_base_url = self.get_url_endpoint(endpoint)
endpoint_base_url = get_url_endpoint(self._application._agentConfig[endpoint])
config = self._application._agentConfig
api_key = config['api_key']
url = endpoint_base_url + '/api/v1/series/?api_key=' + api_key
Expand Down
7 changes: 7 additions & 0 deletions packaging/centos/datadog-agent.init
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,13 @@ case "$1" in
su $AGENTUSER -c "$AGENTPATH jmx $@"
exit $?
;;

flare)
shift
su $AGENTUSER -c "$AGENTPATH flare $@"
exit $?
;;

*)
echo "Usage: $0 {start|stop|restart|info|status|configcheck|configtest|jmx}"
exit 2
Expand Down
8 changes: 8 additions & 0 deletions packaging/datadog-agent/source/agent
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,14 @@ case $action in
exit $?
;;


flare)
shift
python agent/agent.py flare $@
exit $?
;;


*)
echo "Usage: $0 {start|stop|restart|info|status|configcheck|check|jmx}"
exit 2
Expand Down
16 changes: 11 additions & 5 deletions packaging/debian/datadog-agent.init
Original file line number Diff line number Diff line change
Expand Up @@ -82,13 +82,13 @@ case "$1" in
log_daemon_msg "Resuming starting process."
fi


log_daemon_msg "Starting $DESC (using supervisord)" "$NAME"
PATH=$SYSTEM_PATH start-stop-daemon --start --quiet --oknodo --exec $SUPERVISORD_PATH -- -c $SUPERVISOR_FILE --pidfile $SUPERVISOR_PIDFILE
if [ $? -ne 0 ]; then
log_end_msg 1
fi

# check if the agent is running once per second for 10 seconds
retries=10
while [ $retries -gt 1 ]; do
Expand All @@ -108,10 +108,10 @@ case "$1" in
exit 1
;;
stop)

log_daemon_msg "Stopping $DESC (stopping supervisord)" "$NAME"
start-stop-daemon --stop --retry 30 --quiet --oknodo --pidfile $SUPERVISOR_PIDFILE

log_end_msg $?

;;
Expand Down Expand Up @@ -154,9 +154,15 @@ case "$1" in
exit $?
;;

flare)
shift
su $AGENTUSER -c "$AGENTPATH flare $@"
exit $?
;;

*)
N=/etc/init.d/$NAME
echo "Usage: $N {start|stop|restart|info|status|configcheck|configtest|jmx}"
echo "Usage: $N {start|stop|restart|info|status|configcheck|configtest|jmx|flare}"
exit 1
;;
esac
Expand Down
Binary file added tests/flare/datadog-agent-1.tar.bz2
Binary file not shown.
108 changes: 108 additions & 0 deletions tests/test_flare.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
import unittest
import os.path
import mock
from utils.flare import Flare

def get_mocked_config():
return {
'api_key': 'APIKEY',
'dd_url': 'https://app.datadoghq.com',
}

def get_mocked_version():
return '6.6.6'

def get_mocked_temp():
return os.path.join(
os.path.dirname(os.path.realpath(__file__)),
'flare'
)

def mocked_strftime(t):
return '1'

class FakeResponse(object):
def __init__(self, status_code=200):
self.status_code = status_code
self.text = '{"case_id":1337}'

def json(self):
return {'case_id': 1337}

def raise_for_status(self):
return None

class FlareTest(unittest.TestCase):

@mock.patch('utils.flare.strftime', side_effect=mocked_strftime)
@mock.patch('tempfile.gettempdir', side_effect=get_mocked_temp)
@mock.patch('config.get_version', side_effect=get_mocked_version)
@mock.patch('utils.flare.get_config', side_effect=get_mocked_config)
def test_init(self, mock_config, mock_version, mock_tempdir, mock_strftime):
f = Flare(case_id=1337)
conf = mock_config()
self.assertEqual(f._case_id, 1337)
self.assertEqual(f._api_key, conf['api_key'])
self.assertEqual(f._url, 'https://6-6-6-flare.agent.datadoghq.com/support/flare')
self.assertEqual(f._tar_path, os.path.join(get_mocked_temp(), "datadog-agent-1.tar.bz2"))

@mock.patch('utils.flare.requests.post', return_value=FakeResponse())
@mock.patch('config.get_version', side_effect=get_mocked_version)
@mock.patch('utils.flare.strftime', side_effect=mocked_strftime)
@mock.patch('tempfile.gettempdir', side_effect=get_mocked_temp)
@mock.patch('utils.flare.get_config', side_effect=get_mocked_config)
def test_upload_with_case(self, mock_config, mock_tempdir, mock_stfrtime, mock_version, mock_requests):
f = Flare(case_id=1337)

assert not mock_requests.called
f.upload(confirmation=False)
assert mock_requests.called
args, kwargs = mock_requests.call_args_list[0]
self.assertEqual(
args,
('https://6-6-6-flare.agent.datadoghq.com/support/flare/1337?api_key=APIKEY',)
)
self.assertEqual(
kwargs['files']['flare_file'].name,
os.path.join(get_mocked_temp(), "datadog-agent-1.tar.bz2")
)
self.assertEqual(kwargs['data']['case_id'], 1337)
self.assertEqual(kwargs['data']['email'], '')
assert kwargs['data']['hostname']

@mock.patch('utils.flare.requests.post', return_value=FakeResponse())
@mock.patch('config.get_version', side_effect=get_mocked_version)
@mock.patch('utils.flare.strftime', side_effect=mocked_strftime)
@mock.patch('tempfile.gettempdir', side_effect=get_mocked_temp)
@mock.patch('utils.flare.get_config', side_effect=get_mocked_config)
def test_upload_no_case(self, mock_config, mock_tempdir, mock_stfrtime, mock_version, mock_requests):
f = Flare()
f._ask_for_email = lambda: 'test@example.com'

assert not mock_requests.called
f.upload(confirmation=False)
assert mock_requests.called
args, kwargs = mock_requests.call_args_list[0]
self.assertEqual(
args,
('https://6-6-6-flare.agent.datadoghq.com/support/flare?api_key=APIKEY',)
)
self.assertEqual(
kwargs['files']['flare_file'].name,
os.path.join(get_mocked_temp(), "datadog-agent-1.tar.bz2")
)
self.assertEqual(kwargs['data']['case_id'], None)
self.assertEqual(kwargs['data']['email'], 'test@example.com')
assert kwargs['data']['hostname']

@mock.patch('utils.flare.strftime', side_effect=mocked_strftime)
@mock.patch('tempfile.gettempdir', side_effect=get_mocked_temp)
@mock.patch('utils.flare.get_config', side_effect=get_mocked_config)
def test_endpoint(self, mock_config, mock_temp, mock_stfrtime):
f = Flare()
f._ask_for_email = lambda: None
try:
f.upload(confirmation=False)
raise Exception('Should fail before')
except Exception, e:
self.assertEqual(str(e), "Your request is incorrect: Invalid inputs: {'email': None}")
Loading

0 comments on commit 68d7e4a

Please sign in to comment.