Merge pull request #57 from DataDog/olivielpeau/domains-wildcard

Wildcard support on domains

src/main/java/org/datadog/jmxfetch/Filter.java

@@ -4,16 +4,19 @@
 import java.util.ArrayList;
 import java.util.Set;
 import java.lang.ClassCastException;
+import java.util.regex.Pattern;
 
 
 class Filter {
     LinkedHashMap<String, Object> filter;
+    Pattern domainRegex;
+    ArrayList<Pattern> beanRegexes = null;
 
     /**
      * A simple class to manipulate include/exclude filter elements more easily
      * A filter may contain:
-     * - A domain (key: 'domain')
-     * - Bean names (key: 'bean' or 'bean_name')
+     * - A domain (key: 'domain') or a domain regex (key: 'domain_regex')
+     * - Bean names (key: 'bean' or 'bean_name') or bean regexes (key: 'bean_regex')
      * - Attributes (key: 'attribute')
      * - Additional bean parameters (other keys)
      */
@@ -23,7 +26,7 @@ public Filter(Object filter) {
         LinkedHashMap<String, Object> castFilter;
         if (filter != null) {
             castFilter = (LinkedHashMap<String, Object>) filter;
-        } else{
+        } else {
             castFilter = new LinkedHashMap<String, Object>();
         }
         this.filter = castFilter;
@@ -78,10 +81,48 @@ public ArrayList<String> getBeanNames() {
         return toStringArrayList(beanNames);
     }
 
+    private static ArrayList<Pattern> toPatternArrayList(final Object toCast) {
+        ArrayList<Pattern> patternArrayList = new ArrayList<Pattern>();
+        ArrayList<String> stringArrayList = toStringArrayList(toCast);
+        for (String string : stringArrayList) {
+            patternArrayList.add(Pattern.compile(string));
+        }
+
+        return patternArrayList;
+    }
+
+    public ArrayList<Pattern> getBeanRegexes() {
+        // Return bean regexes as an ArrayList of Pattern whether it's defined as
+        // a list or not
+
+        if (this.beanRegexes == null) {
+            if (filter.get("bean_regex") == null){
+                this.beanRegexes = new ArrayList<Pattern>();
+            } else {
+                final Object beanRegexNames = filter.get("bean_regex");
+                this.beanRegexes = toPatternArrayList(beanRegexNames);
+            }
+        }
+
+        return this.beanRegexes;
+    }
+
     public String getDomain() {
         return (String) filter.get("domain");
     }
 
+    public Pattern getDomainRegex() {
+        if (this.filter.get("domain_regex") == null) {
+            return null;
+        }
+
+        if (this.domainRegex == null) {
+            this.domainRegex = Pattern.compile((String) this.filter.get("domain_regex"));
+        }
+
+        return this.domainRegex;
+    }
+
     public Object getAttribute() {
         return filter.get("attribute");
     }

src/main/java/org/datadog/jmxfetch/JMXAttribute.java

@@ -10,6 +10,7 @@
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.regex.Pattern;
 
 import javax.management.AttributeNotFoundException;
 import javax.management.InstanceNotFoundException;
@@ -23,7 +24,7 @@
 public abstract class JMXAttribute {
 
     private final static Logger LOGGER = Logger.getLogger(Instance.class.getName());
-    private static final List<String> EXCLUDED_BEAN_PARAMS = Arrays.asList("domain", "bean_name", "bean", "attribute");
+    private static final List<String> EXCLUDED_BEAN_PARAMS = Arrays.asList("domain", "domain_regex", "bean_name", "bean", "bean_regex", "attribute");
     private static final String FIRST_CAP_PATTERN = "(.)([A-Z][a-z]+)";
     private static final String ALL_CAP_PATTERN = "([a-z0-9])([A-Z])";
     private static final String METRIC_REPLACEMENT = "([^a-zA-Z0-9_.]+)|(^[^a-zA-Z]+)";
@@ -149,39 +150,18 @@ Object getJmxValue() throws AttributeNotFoundException, InstanceNotFoundExceptio
 
     boolean matchDomain(Configuration conf) {
         String includeDomain = conf.getInclude().getDomain();
-        return includeDomain == null || includeDomain.equals(domain);
+        Pattern includeDomainRegex = conf.getInclude().getDomainRegex();
+
+        return (includeDomain == null || includeDomain.equals(domain))
+            && (includeDomainRegex == null || includeDomainRegex.matcher(domain).matches());
     }
 
     boolean excludeMatchDomain(Configuration conf) {
         String excludeDomain = conf.getExclude().getDomain();
-        return excludeDomain != null && excludeDomain.equals(domain);
-    }
+        Pattern excludeDomainRegex = conf.getExclude().getDomainRegex();
 
-    boolean excludeMatchBean(Configuration conf) {
-       Filter exclude = conf.getExclude();
-       ArrayList<String> beanNames = exclude.getBeanNames();
-
-       if(beanNames.contains(beanName)){
-    	   return true;
-       }
-
-       for (String bean_attr : exclude.keySet()) {
-           if (EXCLUDED_BEAN_PARAMS.contains(bean_attr)) {
-               continue;
-           }
-
-           if (beanParameters.get(bean_attr) == null) {
-               continue;
-           }
-
-           ArrayList<String> beanValues = exclude.getParameterValues(bean_attr);
-           for (String beanVal : beanValues) {
-               if (beanParameters.get(bean_attr).equals(beanVal)) {
-                   return true;
-               }
-           }
-       }
-        return false;
+        return excludeDomain != null  && excludeDomain.equals(domain)
+            || excludeDomainRegex != null && excludeDomainRegex.matcher(domain).matches();
     }
 
     Object convertMetricValue(Object metricValue) {
@@ -227,7 +207,22 @@ Object convertMetricValue(Object metricValue) {
         }
     }
 
-    boolean matchBean(Configuration configuration) {
+    private boolean matchBeanRegex(Filter filter, boolean matchIfNoRegex) {
+        ArrayList<Pattern> beanRegexes = filter.getBeanRegexes();
+        if (beanRegexes.isEmpty()) {
+            return matchIfNoRegex;
+        }
+
+        for (Pattern beanRegex : beanRegexes) {
+            if(beanRegex.matcher(beanName).matches()) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    private boolean matchBeanName(Configuration configuration) {
         boolean matchBeanAttr = true;
         Filter include = configuration.getInclude();
 
@@ -261,6 +256,41 @@ boolean matchBean(Configuration configuration) {
         return matchBeanAttr;
     }
 
+    private boolean excludeMatchBeanName(Configuration conf) {
+        Filter exclude = conf.getExclude();
+        ArrayList<String> beanNames = exclude.getBeanNames();
+
+        if(beanNames.contains(beanName)){
+            return true;
+        }
+
+        for (String bean_attr : exclude.keySet()) {
+            if (EXCLUDED_BEAN_PARAMS.contains(bean_attr)) {
+                continue;
+            }
+
+            if (beanParameters.get(bean_attr) == null) {
+                continue;
+            }
+
+            ArrayList<String> beanValues = exclude.getParameterValues(bean_attr);
+            for (String beanVal : beanValues) {
+                if (beanParameters.get(bean_attr).equals(beanVal)) {
+                    return true;
+                }
+            }
+        }
+        return false;
+    }
+
+    boolean matchBean(Configuration configuration) {
+        return matchBeanName(configuration) && matchBeanRegex(configuration.getInclude(), true);
+    }
+
+    boolean excludeMatchBean(Configuration configuration) {
+        return excludeMatchBeanName(configuration) || matchBeanRegex(configuration.getExclude(), false);
+    }
+
     @SuppressWarnings("unchecked")
     HashMap<Object, Object> getValueConversions() {
         if (valueConversions == null) {

src/test/java/org/datadog/jmxfetch/TestApp.java

@@ -3,7 +3,6 @@
 import com.beust.jcommander.JCommander;
 
 import org.apache.log4j.Level;
-import org.datadog.jmxfetch.Status;
 import org.datadog.jmxfetch.reporter.Reporter;
 import org.datadog.jmxfetch.reporter.ConsoleReporter;
 import org.datadog.jmxfetch.util.CustomLogger;
@@ -135,6 +134,35 @@ public void testDomainExclude() throws Exception {
         mbs.unregisterMBean(excludeMe);
     }
 
+    @Test
+    public void testDomainRegex() throws Exception {
+        // We expose a few metrics through JMX
+        MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+
+        ObjectName includeObjectName1 = new ObjectName("org.datadog.jmxfetch.includeme:type=AType");
+        ObjectName includeObjectName2 = new ObjectName("org.datadog.jmxfetch.includeme.too:type=AType");
+        ObjectName excludeObjectName = new ObjectName("org.datadog.jmxfetch.includeme.not.me:type=AType");
+        SimpleTestJavaApp testApp = new SimpleTestJavaApp();
+        mbs.registerMBean(testApp, includeObjectName1);
+        mbs.registerMBean(testApp, includeObjectName2);
+        mbs.registerMBean(testApp, excludeObjectName);
+
+        // Initializing application
+        AppConfig appConfig = new AppConfig();
+        App app = initApp("jmx_domain_regex.yaml", appConfig);
+
+        // Collecting metrics
+        app.doIteration();
+        LinkedList<HashMap<String, Object>> metrics = ((ConsoleReporter) appConfig.getReporter()).getMetrics();
+
+        // First filter 15 = 13 metrics from java.lang + 3 metrics explicitly defined - 1 implicitly defined in exclude section
+        assertEquals(15, metrics.size());
+
+        mbs.unregisterMBean(includeObjectName1);
+        mbs.unregisterMBean(includeObjectName2);
+        mbs.unregisterMBean(excludeObjectName);
+    }
+
     @Test
     public void testParameterMatch() throws Exception {
         // Do not match beans which do not contain types specified in the conf
@@ -224,6 +252,62 @@ public void testListBeansInclude() throws Exception {
         mbs.unregisterMBean(includeMe);
     }
 
+    @Test
+    public void testListBeansRegexInclude() throws Exception {
+        // We expose a few metrics through JMX
+        MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+        ObjectName includeMe = new ObjectName("org.datadog.jmxfetch.test:type=IncludeMe");
+        ObjectName includeMeToo = new ObjectName("org.datadog.jmxfetch.test:type=IncludeMeToo");
+        ObjectName notIncludeMe = new ObjectName("org.datadog.jmxfetch.test:type=RightType");
+        SimpleTestJavaApp testApp = new SimpleTestJavaApp();
+        mbs.registerMBean(testApp, includeMe);
+        mbs.registerMBean(testApp, includeMeToo);
+        mbs.registerMBean(testApp, notIncludeMe);
+
+        // Initializing application
+        AppConfig appConfig = new AppConfig();
+        App app = initApp("jmx_list_beans_regex_include.yaml", appConfig);
+
+        // Collecting metrics
+        app.doIteration();
+        LinkedList<HashMap<String, Object>> metrics = ((ConsoleReporter) appConfig.getReporter()).getMetrics();
+
+        // First filter 15 = 13 metrics from java.lang + 2 metrics explicitly defined
+        assertEquals(15, metrics.size());
+
+        mbs.unregisterMBean(includeMe);
+        mbs.unregisterMBean(includeMeToo);
+        mbs.unregisterMBean(notIncludeMe);
+    }
+
+    @Test
+    public void testListBeansRegexExclude() throws Exception {
+        // We expose a few metrics through JMX
+        MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+        ObjectName includeMe = new ObjectName("org.datadog.jmxfetch.test:type=IncludeMe");
+        ObjectName excludeMe = new ObjectName("org.datadog.jmxfetch.test:type=ExcludeMe,scope=InScope");
+        ObjectName excludeMeToo = new ObjectName("org.datadog.jmxfetch.test:scope=OutOfScope");
+        SimpleTestJavaApp testApp = new SimpleTestJavaApp();
+        mbs.registerMBean(testApp, includeMe);
+        mbs.registerMBean(testApp, excludeMe);
+        mbs.registerMBean(testApp, excludeMeToo);
+
+        // Initializing application
+        AppConfig appConfig = new AppConfig();
+        App app = initApp("jmx_list_beans_regex_exclude.yaml", appConfig);
+
+        // Collecting metrics
+        app.doIteration();
+        LinkedList<HashMap<String, Object>> metrics = ((ConsoleReporter) appConfig.getReporter()).getMetrics();
+
+        // First filter 14 = 13 metrics from java.lang + 1 metrics explicitly defined
+        assertEquals(14, metrics.size());
+
+        mbs.unregisterMBean(includeMe);
+        mbs.unregisterMBean(excludeMe);
+        mbs.unregisterMBean(excludeMeToo);
+    }
+
     @Test
     public void testListBeansExclude() throws Exception {
         // We expose a few metrics through JMX

src/test/resources/jmx_domain_regex.yaml

@@ -0,0 +1,14 @@
+init_config:
+
+instances:
+    -   process_name_regex: .*surefire.*
+        name: jmx_test_instance
+        conf:
+            - include:
+                domain_regex: .*includeme.*
+                attribute:
+                    ShouldBe100:
+                        metric_type: gauge
+                        alias: this.is.100
+              exclude:
+                domain_regex: .*\.me$

src/test/resources/jmx_list_beans_regex_exclude.yaml

@@ -0,0 +1,18 @@
+init_config:
+
+instances:
+    -   process_name_regex: .*surefire.*
+        name: jmx_test_instance
+        tags:
+            env: stage
+            newTag: test
+        conf:
+            - include:
+                attribute:
+                    ShouldBe100:
+                        metric_type: gauge
+                        alias: this.is.100
+              exclude:
+                bean_regex:
+                     - .*[,:]type=ExcludeMe.*
+                     - .*[,:]scope=Out.*

src/test/resources/jmx_list_beans_regex_include.yaml

@@ -0,0 +1,17 @@
+init_config:
+
+instances:
+    -   process_name_regex: .*surefire.*
+        name: jmx_test_instance
+        tags:
+            env: stage
+            newTag: test
+        conf:
+            - include:
+               bean_regex:
+                    - .*type=\w*WrongType
+                    - .*type=Include.*
+               attribute:
+                    ShouldBe100:
+                        metric_type: gauge
+                        alias: this.is.100