From 8aa516c9153307e49acb3554d532c897c9c858c4 Mon Sep 17 00:00:00 2001 From: Jaime Fullaondo Date: Tue, 4 Jul 2017 15:41:30 +0200 Subject: [PATCH 1/4] [auto-discovery][jmx] a jmxfetch restart should prompt AD check reload. --- agent.py | 10 +++++++++- jmxfetch.py | 12 ++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/agent.py b/agent.py index b6edb1768b..e26cf73349 100755 --- a/agent.py +++ b/agent.py @@ -48,7 +48,7 @@ ) from daemon import AgentSupervisor, Daemon from emitter import http_emitter -from jmxfetch import get_jmx_checks +from jmxfetch import get_jmx_checks, JMXFetch # utils from utils.cloud_metadata import EC2 @@ -103,6 +103,7 @@ def __init__(self, pidfile, autorestart, start_event=True, in_developer_mode=Fal self.sd_backend = None self.supervisor_proxy = None self.sd_pipe = None + self.last_jmx_piped = None def _handle_sigterm(self, signum, frame): """Handles SIGTERM and SIGINT, which gracefully stops the agent.""" @@ -342,6 +343,12 @@ def run(self, config=None): try: self.sd_backend.reload_check_configs = get_config_store( self._agentConfig).crawl_config_template() + + # JMXFetch restarts should prompt reload + jmx_launch = JMXFetch._get_jmx_launchtime() + if self.last_jmx_piped and self.last_jmx_piped < jmx_launch: + self.sd_backend.reload_check_configs = True + except Exception as e: log.warn('Something went wrong while looking for config template changes: %s' % str(e)) @@ -454,6 +461,7 @@ def _submit_jmx_service_discovery(self, jmx_sd_configs): # JMX will unblock when it reads on the other end. os.write(self.sd_pipe, buffer) os.write(self.sd_pipe, SD_CONFIG_TERM) + self.last_jmx_piped = time.time() except Exception as e: log.exception("unable to submit YAML via pipe: %s", e) else: diff --git a/jmxfetch.py b/jmxfetch.py index f44b707339..bc9622c4ce 100644 --- a/jmxfetch.py +++ b/jmxfetch.py @@ -70,6 +70,8 @@ 'list_limited_attributes': "List attributes that do match one of your instances configuration but that are not being collected because it would exceed the number of metrics that can be collected", JMX_COLLECT_COMMAND: "Start the collection of metrics based on your current configuration and display them in the console"} +JMX_LAUNCH_FILE = 'jmx.launch' + LINK_TO_DOC = "See http://docs.datadoghq.com/integrations/java/ for more information" @@ -336,6 +338,16 @@ def _start(self, path_to_java, java_run_opts, jmx_checks, command, reporter, too log.exception("Couldn't launch JMXFetch") raise + @staticmethod + def _get_jmx_launchtime(): + fpath = os.path.join(get_jmx_pipe_path(), JMX_LAUNCH_FILE) + try: + _stat = os.stat(fpath) + except OSError as e: + raise e + + return _stat.st_atime + @staticmethod def _is_jmx_check(check_config, check_name, checks_list): init_config = check_config.get('init_config', {}) or {} From ecfac975cdbf6020783a01f9ef2cc33c19119e07 Mon Sep 17 00:00:00 2001 From: Jaime Fullaondo Date: Tue, 4 Jul 2017 15:53:36 +0200 Subject: [PATCH 2/4] [auto-discovery][jmx] handle missing launch file exception --- agent.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/agent.py b/agent.py index e26cf73349..1a2c075c3f 100755 --- a/agent.py +++ b/agent.py @@ -345,9 +345,12 @@ def run(self, config=None): self._agentConfig).crawl_config_template() # JMXFetch restarts should prompt reload - jmx_launch = JMXFetch._get_jmx_launchtime() - if self.last_jmx_piped and self.last_jmx_piped < jmx_launch: - self.sd_backend.reload_check_configs = True + try: + jmx_launch = JMXFetch._get_jmx_launchtime() + if self.last_jmx_piped and self.last_jmx_piped < jmx_launch: + self.sd_backend.reload_check_configs = True + except OSError as e: + log.debug("could not stat JMX lunch file: %s", e) except Exception as e: log.warn('Something went wrong while looking for config template changes: %s' % str(e)) From fd650c5f81d1b01cdfa6a929f6624b27764ee944 Mon Sep 17 00:00:00 2001 From: Jaime Fullaondo Date: Mon, 17 Jul 2017 17:11:45 +0200 Subject: [PATCH 3/4] [auto-discovery][jmxfetch] use modify time instead. --- jmxfetch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jmxfetch.py b/jmxfetch.py index bc9622c4ce..63775a4973 100644 --- a/jmxfetch.py +++ b/jmxfetch.py @@ -346,7 +346,7 @@ def _get_jmx_launchtime(): except OSError as e: raise e - return _stat.st_atime + return _stat.st_mtime @staticmethod def _is_jmx_check(check_config, check_name, checks_list): From 6840937d9ce0ee4dc42195bc7ff7c4c382c9df0a Mon Sep 17 00:00:00 2001 From: Jaime Fullaondo Date: Thu, 20 Jul 2017 12:10:49 +0200 Subject: [PATCH 4/4] [auto-discovery][jmx] refactor to handle re-submissions in a cleaner manner. --- agent.py | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/agent.py b/agent.py index 1a2c075c3f..8e36fb87b8 100755 --- a/agent.py +++ b/agent.py @@ -219,6 +219,11 @@ def info(cls, verbose=None): logging.getLogger().setLevel(logging.ERROR) return CollectorStatus.print_latest_status(verbose=verbose) + def sd_pipe_jmx_configs(self, hostname): + jmx_sd_configs = generate_jmx_configs(self._agentConfig, hostname) + if jmx_sd_configs: + self._submit_jmx_service_discovery(jmx_sd_configs) + def run(self, config=None): """Main loop of the collector""" @@ -259,7 +264,7 @@ def run(self, config=None): if self._agentConfig.get('service_discovery'): self.sd_backend = get_sd_backend(self._agentConfig) - if _is_affirmative(self._agentConfig.get('sd_jmx_enable', False)): + if self.sd_backend and _is_affirmative(self._agentConfig.get('sd_jmx_enable', False)): pipe_path = get_jmx_pipe_path() if Platform.is_windows(): pipe_name = pipe_path.format(pipename=SD_PIPE_NAME) @@ -281,9 +286,7 @@ def run(self, config=None): # Load JMX configs if available if self._jmx_service_discovery_enabled: - jmx_sd_configs = generate_jmx_configs(self._agentConfig, hostname) - if jmx_sd_configs: - self._submit_jmx_service_discovery(jmx_sd_configs) + self.sd_pipe_jmx_configs(hostname) # Initialize the Collector self.collector = Collector(self._agentConfig, emitters, systemStats, hostname) @@ -327,6 +330,16 @@ def run(self, config=None): else: self.reload_configs() + # JMXFetch restarts should prompt re-piping *all* JMX configs + if self._jmx_service_discovery_enabled and \ + (not self.reload_configs_flag or isinstance(self.reload_configs_flag, set)): + try: + jmx_launch = JMXFetch._get_jmx_launchtime() + if self.last_jmx_piped and self.last_jmx_piped < jmx_launch: + self.sd_pipe_jmx_configs(hostname) + except Exception as e: + log.debug("could not stat JMX lunch file: %s", e) + # Do the work. Pass `configs_reloaded` to let the collector know if it needs to # look for the AgentMetrics check and pop it out. self.collector.run(checksd=self._checksd, @@ -343,15 +356,6 @@ def run(self, config=None): try: self.sd_backend.reload_check_configs = get_config_store( self._agentConfig).crawl_config_template() - - # JMXFetch restarts should prompt reload - try: - jmx_launch = JMXFetch._get_jmx_launchtime() - if self.last_jmx_piped and self.last_jmx_piped < jmx_launch: - self.sd_backend.reload_check_configs = True - except OSError as e: - log.debug("could not stat JMX lunch file: %s", e) - except Exception as e: log.warn('Something went wrong while looking for config template changes: %s' % str(e))