moved tokenizer filter to a converter

fixes #177

graylog2-plugin-interfaces/src/main/java/org/graylog2/plugin/Message.java

@@ -153,6 +153,10 @@ public String getSource() {
     }
 
     public void addField(String key, Object value) {
+        if (value == null) {
+            return;
+        }
+
         // Don't accept protected keys. (some are allowed though lol)
         if (RESERVED_FIELDS.contains(key) && !RESERVED_SETTABLE_FIELDS.contains(key)) {
             return;
@@ -167,6 +171,10 @@ public void addField(String key, Object value) {
     }      
 
     public void addFields(Map<String, String> fields) {
+        if(fields == null) {
+            return;
+        }
+
         for (Map.Entry<String, String> field : fields.entrySet()) {
             addField(field.getKey(), field.getValue());
         }

graylog2-plugin-interfaces/src/main/java/org/graylog2/plugin/inputs/Converter.java

@@ -37,6 +37,7 @@ public enum Type {
         SPLIT_AND_COUNT,
         SYSLOG_PRI_LEVEL,
         SYSLOG_PRI_FACILITY,
+        TOKENIZER,
         IP_ANONYMIZER
     }
 
@@ -57,5 +58,6 @@ public Map<String, Object> getConfig() {
     }
 
     public abstract Object convert(String value);
+    public abstract boolean buildsMultipleFields();
 
 }

graylog2-plugin-interfaces/src/main/java/org/graylog2/plugin/inputs/Extractor.java

@@ -184,10 +184,12 @@ public void runConverters(GraylogServer server, Message msg) {
                     continue;
                 }
 
-                String value = (String) msg.getFields().get(targetField);
-
-                msg.removeField(targetField);
-                msg.addField(targetField, converter.convert(value));
+                if (!converter.buildsMultipleFields()) {
+                    msg.removeField(targetField);
+                    msg.addField(targetField, converter.convert((String) msg.getFields().get(targetField)));
+                } else {
+                    msg.addFields((Map<String, String>) converter.convert((String) msg.getFields().get(targetField)));
+                }
             } catch (Exception e) {
                 this.converterExceptions.incrementAndGet();
                 LOG.error("Could not apply converter [{}].", converter.getType(), e);

graylog2-server/src/main/java/org/graylog2/Configuration.java

@@ -143,9 +143,6 @@ public class Configuration {
     @Parameter("rules_file")
     private String droolsRulesFile;
 
-    @Parameter(value = "enable_tokenizer_filter", required = true)
-    private boolean enableTokenizerFilter = true;
-
     @Parameter(value = "enable_graphite_output", required = false)
     private boolean enableGraphiteOutput = false;
 
@@ -355,10 +352,6 @@ public List<ServerAddress> getMongoReplicaSet() {
         return replicaServers;
     }
 
-    public boolean isEnableTokenizerFilter() {
-        return enableTokenizerFilter;
-    }
-
     public boolean isEnableGraphiteOutput() {
         return enableGraphiteOutput;
     }

graylog2-server/src/main/java/org/graylog2/Main.java

@@ -235,7 +235,6 @@ public static void main(String[] args) {
         // Register message filters. (Order is important here)
         server.registerFilter(new ExtractorFilter());
         server.registerFilter(new BlacklistFilter());
-        if (configuration.isEnableTokenizerFilter()) { server.registerFilter(new TokenizerFilter()); }
         server.registerFilter(new StreamMatcherFilter());
         server.registerFilter(new RewriteFilter());
 

graylog2-server/src/main/java/org/graylog2/inputs/converters/ConverterFactory.java

@@ -45,6 +45,8 @@ public static Converter factory(Converter.Type type, Map<String, Object> config)
                 return new SyslogPriFacilityConverter(config);
             case IP_ANONYMIZER:
                 return new IPAnonymizerConverter(config);
+            case TOKENIZER:
+                return new TokenizerConverter(config);
             default:
                 throw new NoSuchConverterException();
         }

graylog2-server/src/main/java/org/graylog2/inputs/converters/DateConverter.java

@@ -52,4 +52,9 @@ public Object convert(String value) {
         return DateTime.parse(value, DateTimeFormat.forPattern(dateFormat));
     }
 
+    @Override
+    public boolean buildsMultipleFields() {
+        return false;
+    }
+
 }

graylog2-server/src/main/java/org/graylog2/inputs/converters/HashConverter.java

@@ -43,4 +43,9 @@ public Object convert(String value) {
         return DigestUtils.md5Hex(value);
     }
 
+    @Override
+    public boolean buildsMultipleFields() {
+        return false;
+    }
+
 }

graylog2-server/src/main/java/org/graylog2/inputs/converters/IPAnonymizerConverter.java

@@ -45,4 +45,9 @@ public Object convert(String value) {
         return p.matcher(value).replaceAll(REPLACEMENT);
     }
 
+    @Override
+    public boolean buildsMultipleFields() {
+        return false;
+    }
+
 }

graylog2-server/src/main/java/org/graylog2/inputs/converters/NumericConverter.java

@@ -48,4 +48,9 @@ public Object convert(String value) {
         return result;
     }
 
+    @Override
+    public boolean buildsMultipleFields() {
+        return false;
+    }
+
 }

graylog2-server/src/main/java/org/graylog2/inputs/converters/SplitAndCountConverter.java

@@ -58,4 +58,9 @@ public Object convert(String value) {
         return value.split(splitByEscaped).length;
     }
 
+    @Override
+    public boolean buildsMultipleFields() {
+        return false;
+    }
+
 }

graylog2-server/src/main/java/org/graylog2/inputs/converters/SyslogPriFacilityConverter.java

@@ -50,4 +50,9 @@ public Object convert(String value) {
         return Tools.syslogFacilityToReadable(SyslogPriUtilities.facilityFromPriority(priority));
     }
 
+    @Override
+    public boolean buildsMultipleFields() {
+        return false;
+    }
+
 }

graylog2-server/src/main/java/org/graylog2/inputs/converters/SyslogPriLevelConverter.java

@@ -49,4 +49,9 @@ public Object convert(String value) {
         return SyslogPriUtilities.levelFromPriority(priority);
     }
 
+    @Override
+    public boolean buildsMultipleFields() {
+        return false;
+    }
+
 }

graylog2-server/src/main/java/org/graylog2/inputs/converters/TokenizerConverter.java

@@ -0,0 +1,88 @@
+/**
+ * Copyright 2013 Lennart Koopmann <lennart@torch.sh>
+ *
+ * This file is part of Graylog2.
+ *
+ * Graylog2 is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Graylog2 is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Graylog2.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+package org.graylog2.inputs.converters;
+
+import com.google.common.base.CharMatcher;
+import com.google.common.collect.Maps;
+import org.graylog2.plugin.inputs.Converter;
+
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * @author Lennart Koopmann <lennart@torch.sh>
+ */
+public class TokenizerConverter extends Converter {
+
+    private static final Pattern p = Pattern.compile("[a-zA-Z0-9_-]*");
+    private static final Pattern kvPattern = Pattern.compile("\\s?=\\s?");
+    private static final Pattern spacePattern = Pattern.compile(" ");
+    private static final Pattern quotedValuePattern = Pattern.compile("([a-zA-Z0-9_-]+=\"[^\"]+\")");
+    private static final CharMatcher QUOTE_MATCHER = CharMatcher.is('"').precomputed();
+    private static final CharMatcher EQUAL_SIGN_MATCHER = CharMatcher.is('=').precomputed();
+
+    public TokenizerConverter(Map<String, Object> config) {
+        super(Type.TOKENIZER, config);
+    }
+
+    @Override
+    public Object convert(String value) {
+        if (value == null || value.isEmpty()) {
+            return value;
+        }
+
+        Map<String, String> fields = Maps.newHashMap();
+
+        if (value.contains("=")) {
+            final String nmsg = kvPattern.matcher(value).replaceAll("=");
+            if (nmsg.contains("=\"")) {
+                Matcher m = quotedValuePattern.matcher(nmsg);
+                while (m.find()) {
+                    String[] kv = m.group(1).split("=");
+                    if (kv.length == 2 && p.matcher(kv[0]).matches()) {
+                        fields.put(kv[0].trim(), QUOTE_MATCHER.removeFrom(kv[1]).trim());
+                    }
+                }
+            } else {
+                final String[] parts = spacePattern.split(nmsg);
+                if (parts != null) {
+                    for (String part : parts) {
+                        if (part.contains("=") && EQUAL_SIGN_MATCHER.countIn(part) == 1) {
+                            String[] kv = part.split("=");
+                            if (kv.length == 2 && p.matcher(kv[0]).matches() && !fields.containsKey(kv[0])) {
+                                fields.put(kv[0].trim(), kv[1].trim());
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        return fields;
+    }
+
+    @Override
+    public boolean buildsMultipleFields() {
+        return true;
+    }
+
+
+}