From 5d0a8dfaab57ef4ccd886c3fc7bf8a80f27ae10b Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Thu, 18 Jul 2013 13:42:33 -0400 Subject: [PATCH 1/3] add DirectoryCheck and sample config file --- checks.d/directory.py | 67 +++++++++++++++++++++++++++++++++++ conf.d/directory.yaml.example | 20 +++++++++++ 2 files changed, 87 insertions(+) create mode 100644 checks.d/directory.py create mode 100644 conf.d/directory.yaml.example diff --git a/checks.d/directory.py b/checks.d/directory.py new file mode 100644 index 0000000000..12bccc59d8 --- /dev/null +++ b/checks.d/directory.py @@ -0,0 +1,67 @@ +from fnmatch import fnmatch +from os import stat, walk +from os.path import abspath, exists, join +import time + +from checks import AgentCheck + + +class DirectoryCheck(AgentCheck): + """This check is for monitoring and reporting metrics on the files for a provided directory + + WARNING: the user/group that dd-agent runs as must have access to stat the files in the desired directory + + Config options: + "directory" - string, the directory to gather stats for. required + "name" - string, the name to use when tagging the metrics. defaults to the "directory" + "pattern" - string, the `fnmatch` pattern to use when reading the "directory"'s files. default "*" + "recursive" - boolean, when true the stats will recurse into directories. default False + """ + def check(self, instance): + if "directory" not in instance: + raise Exception('DirectoryCheck: missing "directory" in config') + + directory = instance["directory"] + abs_directory = abspath(directory) + name = instance.get("name") or directory + pattern = instance.get("pattern") or "*" + recursive = instance.get("recursive") or False + + if not exists(abs_directory): + raise Exception("DirectoryCheck: the directory ({0}) does not exist".format(abs_directory)) + + self._get_stats(abs_directory, name, pattern, recursive) + + def _get_stats(self, directory, name, pattern, recursive): + tags = ["name:{0}".format(name)] + directory_bytes = 0 + directory_files = 0 + for root, dirs, files in walk(directory): + for filename in files: + filename = join(root, filename) + # check if it passes our filter + if not fnmatch(filename, pattern): + continue + try: + file_stat = stat(filename) + + except OSError, ose: + self.warning("DirectoryCheck: could not stat file {0} - {1}".format(filename, ose)) + else: + directory_files += 1 + directory_bytes += file_stat.st_size + # file specific metrics + self.histogram("directory.file.bytes", file_stat.st_size, tags=tags) + self.histogram("directory.file.modified_sec_ago", time.time() - file_stat.st_mtime, tags=tags) + self.histogram("directory.file.created_sec_ago", time.time() - file_stat.st_ctime, tags=tags) + + # os.walk gives us all sub-directories and their files + # if we do not want to do this recursively and just want + # the top level directory we gave it, then break + if not recursive: + break + + # number of files + self.gauge("directory.files", directory_files, tags=tags) + # total file size + self.gauge("directory.bytes", directory_bytes, tags=tags) diff --git a/conf.d/directory.yaml.example b/conf.d/directory.yaml.example new file mode 100644 index 0000000000..a55888eef6 --- /dev/null +++ b/conf.d/directory.yaml.example @@ -0,0 +1,20 @@ +init_config: + +instances: + # This config is for the DirectoryCheck which is used to report metrics + # for the files in a given directory + # + # For each instance you *must* provide at least a "directory" + # + # WARNING: the user/group that dd-agent runs as must have access to stat the files in the desired directory + # + # The following options are available: + # "directory" - string, the directory to gather stats for. required + # "name" - string, the name to use when tagging the metrics. defaults to the "directory" + # "pattern" - string, the `fnmatch` pattern to use when reading the "directory"'s files. default "*" + # "recursive" - boolean, when true the stats will recurse into directories. default False + + - directory: "/tmp" + name: "tmp" + pattern: "*.log" + recursive: True From 683c552a54a784b8840730c1a53779f7b701b068 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Thu, 18 Jul 2013 16:00:23 -0400 Subject: [PATCH 2/3] use % style formatting --- checks.d/directory.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/checks.d/directory.py b/checks.d/directory.py index 12bccc59d8..abfcce8098 100644 --- a/checks.d/directory.py +++ b/checks.d/directory.py @@ -28,12 +28,12 @@ def check(self, instance): recursive = instance.get("recursive") or False if not exists(abs_directory): - raise Exception("DirectoryCheck: the directory ({0}) does not exist".format(abs_directory)) + raise Exception("DirectoryCheck: the directory (%s) does not exist" % abs_directory) self._get_stats(abs_directory, name, pattern, recursive) def _get_stats(self, directory, name, pattern, recursive): - tags = ["name:{0}".format(name)] + tags = ["name:%s" % name] directory_bytes = 0 directory_files = 0 for root, dirs, files in walk(directory): @@ -46,7 +46,7 @@ def _get_stats(self, directory, name, pattern, recursive): file_stat = stat(filename) except OSError, ose: - self.warning("DirectoryCheck: could not stat file {0} - {1}".format(filename, ose)) + self.warning("DirectoryCheck: could not stat file %s - %s" % (filename, ose)) else: directory_files += 1 directory_bytes += file_stat.st_size From cd8ad835ef3b9566665b9d260367af8d41823370 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Fri, 19 Jul 2013 11:20:21 -0400 Subject: [PATCH 3/3] prefix directory check metric names with system.disk --- checks.d/directory.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/checks.d/directory.py b/checks.d/directory.py index abfcce8098..0d4651587a 100644 --- a/checks.d/directory.py +++ b/checks.d/directory.py @@ -51,9 +51,9 @@ def _get_stats(self, directory, name, pattern, recursive): directory_files += 1 directory_bytes += file_stat.st_size # file specific metrics - self.histogram("directory.file.bytes", file_stat.st_size, tags=tags) - self.histogram("directory.file.modified_sec_ago", time.time() - file_stat.st_mtime, tags=tags) - self.histogram("directory.file.created_sec_ago", time.time() - file_stat.st_ctime, tags=tags) + self.histogram("system.disk.directory.file.bytes", file_stat.st_size, tags=tags) + self.histogram("system.disk.directory.file.modified_sec_ago", time.time() - file_stat.st_mtime, tags=tags) + self.histogram("system.disk.directory.file.created_sec_ago", time.time() - file_stat.st_ctime, tags=tags) # os.walk gives us all sub-directories and their files # if we do not want to do this recursively and just want @@ -62,6 +62,6 @@ def _get_stats(self, directory, name, pattern, recursive): break # number of files - self.gauge("directory.files", directory_files, tags=tags) + self.gauge("system.disk.directory.files", directory_files, tags=tags) # total file size - self.gauge("directory.bytes", directory_bytes, tags=tags) + self.gauge("system.disk.directory.bytes", directory_bytes, tags=tags)