From f8003fac101f583d47b1342602f495a6271fb886 Mon Sep 17 00:00:00 2001 From: Konrad Merz Date: Thu, 28 Feb 2019 11:38:17 +0100 Subject: [PATCH 1/7] Bring back pom.xml --- pom.xml | 273 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 273 insertions(+) create mode 100644 pom.xml diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..0274073 --- /dev/null +++ b/pom.xml @@ -0,0 +1,273 @@ + + 4.0.0 + + org.graylog2.repackaged + grok + 0.1.19-graylog-SNAPSHOT + jar + Grok + Simple API that allows you to easily parse logs and other files + http://maven.apache.org + + + org.sonatype.oss + oss-parent + 7 + + + + + Apache License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0 + repo + + + + + + anthony-corbacho + Anthony Corbacho + corbacho.anthony@gmail.com + Thekraken + http://release.thekraken.io + https://github.com/anthonycorbacho + + + + + scm:git:git@github.com:Graylog2/java-grok.git + scm:git:git@github.com:Graylog2/java-grok.git + scm:git:git@github.com:Graylog2/java-grok.git + + + + Travis + https://travis-ci.org/thekrakken/java-grok + + + + Github + https://github.com/thekrakken/java-grok/issues + + + + 4.12 + 3.9.1 + 3.7 + 24.0-jre + + 1.8 + + + + + junit + junit + ${junit.version} + test + + + org.assertj + assertj-core + ${assertj.version} + test + + + + org.apache.commons + commons-lang3 + ${common.version} + compile + + + + com.google.guava + guava + ${guava.version} + test + + + + org.slf4j + slf4j-api + 1.7.21 + compile + + + + + + + maven-compiler-plugin + 3.1 + + ${java.version} + ${java.version} + + + + + maven-javadoc-plugin + 2.9.1 + + + + + aggregate + + aggregate + + site + + + + + + + + maven-scm-plugin + 1.8.1 + + developerConnection + branch-0.1 + branch + + + + + maven-enforcer-plugin + 1.3.1 + + + enforce + + + + + true + + + enforce + + + + + + + + + + maven-dependency-plugin + 2.8 + + + maven-surefire-plugin + 2.16 + + -Xmx2g -Xms1g + + + + maven-assembly-plugin + 2.4 + + + org.codehaus.mojo + exec-maven-plugin + 1.2.1 + + + + org.apache.maven.plugins + maven-clean-plugin + 2.4.1 + + + + drivers + false + + + + + + + + + + + build-distr + + false + + + + + + maven-surefire-plugin + + true + + + + maven-assembly-plugin + + + make-assembly + package + + single + + + + + + + + + + + publish-distr + + false + + + + + maven-surefire-plugin + + true + + + + + + + + release-sign-artifacts + + + performRelease + true + + + + + + maven-gpg-plugin + 1.4 + + + sign-artifacts + verify + + sign + + + + + + + + + From 94c7b39753fc37a24c3dea881357137edfca43f7 Mon Sep 17 00:00:00 2001 From: Konrad Merz Date: Thu, 28 Feb 2019 11:44:22 +0100 Subject: [PATCH 2/7] Bring back 'named-regexp' dependency --- pom.xml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pom.xml b/pom.xml index 0274073..947c9f8 100644 --- a/pom.xml +++ b/pom.xml @@ -93,6 +93,13 @@ 1.7.21 compile + + com.github.tony19 + named-regexp + ${named.regex.version} + compile + + From c7f9e5d793f34923f6628fa9b4f2fb5822c481a3 Mon Sep 17 00:00:00 2001 From: Konrad Merz Date: Thu, 28 Feb 2019 13:23:35 +0100 Subject: [PATCH 3/7] Replace java.util.regex.* with com.google.code.regexp.* --- pom.xml | 1 + src/main/java/io/krakens/grok/api/Converter.java | 2 +- src/main/java/io/krakens/grok/api/Discovery.java | 4 ++-- src/main/java/io/krakens/grok/api/Grok.java | 5 +++-- src/main/java/io/krakens/grok/api/GrokCompiler.java | 8 +++++--- src/main/java/io/krakens/grok/api/GrokUtils.java | 5 +++-- src/main/java/io/krakens/grok/api/Match.java | 4 +++- src/test/java/io/krakens/grok/api/BasicTest.java | 7 +++++++ src/test/java/io/krakens/grok/api/GrokTest.java | 5 +++-- 9 files changed, 28 insertions(+), 13 deletions(-) diff --git a/pom.xml b/pom.xml index 947c9f8..59669a7 100644 --- a/pom.xml +++ b/pom.xml @@ -55,6 +55,7 @@ 3.9.1 3.7 24.0-jre + 0.2.3 1.8 diff --git a/src/main/java/io/krakens/grok/api/Converter.java b/src/main/java/io/krakens/grok/api/Converter.java index 22065dd..a6226e1 100644 --- a/src/main/java/io/krakens/grok/api/Converter.java +++ b/src/main/java/io/krakens/grok/api/Converter.java @@ -15,9 +15,9 @@ import java.util.List; import java.util.Map; import java.util.function.Function; -import java.util.regex.Pattern; import java.util.stream.Collectors; +import com.google.code.regexp.Pattern; /** * Convert String argument to the right type. * diff --git a/src/main/java/io/krakens/grok/api/Discovery.java b/src/main/java/io/krakens/grok/api/Discovery.java index 94a34f7..fad2fc8 100644 --- a/src/main/java/io/krakens/grok/api/Discovery.java +++ b/src/main/java/io/krakens/grok/api/Discovery.java @@ -8,11 +8,11 @@ import java.util.Map; import java.util.Map.Entry; import java.util.TreeMap; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import org.apache.commons.lang3.StringUtils; +import com.google.code.regexp.Matcher; +import com.google.code.regexp.Pattern; /** * {@code Discovery} try to find the best pattern for the given string. diff --git a/src/main/java/io/krakens/grok/api/Grok.java b/src/main/java/io/krakens/grok/api/Grok.java index 6729167..158ad84 100644 --- a/src/main/java/io/krakens/grok/api/Grok.java +++ b/src/main/java/io/krakens/grok/api/Grok.java @@ -5,11 +5,12 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import io.krakens.grok.api.Converter.IConverter; +import com.google.code.regexp.Matcher; +import com.google.code.regexp.Pattern; + import org.apache.commons.lang3.StringUtils; /** diff --git a/src/main/java/io/krakens/grok/api/GrokCompiler.java b/src/main/java/io/krakens/grok/api/GrokCompiler.java index 13813b9..739e637 100644 --- a/src/main/java/io/krakens/grok/api/GrokCompiler.java +++ b/src/main/java/io/krakens/grok/api/GrokCompiler.java @@ -15,8 +15,10 @@ import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.regex.Matcher; -import java.util.regex.Pattern; + +import com.google.code.regexp.Matcher; +import com.google.code.regexp.Pattern; + import io.krakens.grok.api.exception.GrokException; @@ -158,7 +160,7 @@ public Grok compile(final String pattern, ZoneId defaultTimeZone, boolean namedO } iterationLeft--; - Set namedGroups = GrokUtils.getNameGroups(GrokUtils.GROK_PATTERN.pattern()); + Set namedGroups = GrokUtils.getNameGroups(GrokUtils.GROK_PATTERN.toString()); Matcher matcher = GrokUtils.GROK_PATTERN.matcher(namedRegex); // Match %{Foo:bar} -> pattern name and subname // Match %{Foo=regex} -> add new regex definition diff --git a/src/main/java/io/krakens/grok/api/GrokUtils.java b/src/main/java/io/krakens/grok/api/GrokUtils.java index 0cd0f62..a3bab71 100644 --- a/src/main/java/io/krakens/grok/api/GrokUtils.java +++ b/src/main/java/io/krakens/grok/api/GrokUtils.java @@ -4,8 +4,9 @@ import java.util.LinkedHashSet; import java.util.Map; import java.util.Set; -import java.util.regex.Matcher; -import java.util.regex.Pattern; + +import com.google.code.regexp.Matcher; +import com.google.code.regexp.Pattern; /** diff --git a/src/main/java/io/krakens/grok/api/Match.java b/src/main/java/io/krakens/grok/api/Match.java index 1cbef15..7dfa46d 100644 --- a/src/main/java/io/krakens/grok/api/Match.java +++ b/src/main/java/io/krakens/grok/api/Match.java @@ -8,7 +8,9 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.regex.Matcher; + +import com.google.code.regexp.Matcher; + import io.krakens.grok.api.Converter.IConverter; import io.krakens.grok.api.exception.GrokException; diff --git a/src/test/java/io/krakens/grok/api/BasicTest.java b/src/test/java/io/krakens/grok/api/BasicTest.java index 77ce928..d77f5b5 100644 --- a/src/test/java/io/krakens/grok/api/BasicTest.java +++ b/src/test/java/io/krakens/grok/api/BasicTest.java @@ -127,4 +127,11 @@ public void test007_testLoadPatternFromReader() throws IOException, GrokExceptio assertEquals("(?€)", grok.getNamedRegex()); } + @Test + public void test008_namedGroupPatternWithUnderscore() throws GrokException { + compiler.register("test", "hello world"); + Grok grok = compiler.compile("%{test}"); + assertEquals("(?hello world)", grok.getNamedRegex()); + } + } diff --git a/src/test/java/io/krakens/grok/api/GrokTest.java b/src/test/java/io/krakens/grok/api/GrokTest.java index 0277c9d..de4d714 100644 --- a/src/test/java/io/krakens/grok/api/GrokTest.java +++ b/src/test/java/io/krakens/grok/api/GrokTest.java @@ -21,11 +21,12 @@ import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import io.krakens.grok.api.exception.GrokException; +import com.google.code.regexp.Matcher; +import com.google.code.regexp.Pattern; + import com.google.common.collect.ImmutableMap; import com.google.common.io.Resources; import org.assertj.core.api.Assertions; From 29719bce1888edb2936f2fb52c7e46f0e9a431df Mon Sep 17 00:00:00 2001 From: Konrad Merz Date: Thu, 28 Feb 2019 13:48:57 +0100 Subject: [PATCH 4/7] Add test to ensure named underscores work in the future --- src/main/resources/patterns/patterns | 3 +++ src/test/java/io/krakens/grok/api/GrokTest.java | 9 +++++++++ 2 files changed, 12 insertions(+) diff --git a/src/main/resources/patterns/patterns b/src/main/resources/patterns/patterns index 52235d8..2eb7233 100644 --- a/src/main/resources/patterns/patterns +++ b/src/main/resources/patterns/patterns @@ -106,3 +106,6 @@ COMMONAPACHELOG_DATATYPED %{IPORHOST:clientip} %{USER:ident;boolean} %{USER:auth # Log Levels LOGLEVEL ([A|a]lert|ALERT|[T|t]race|TRACE|[D|d]ebug|DEBUG|[N|n]otice|NOTICE|[I|i]nfo|INFO|[W|w]arn?(?:ing)?|WARN?(?:ING)?|[E|e]rr?(?:or)?|ERR?(?:OR)?|[C|c]rit?(?:ical)?|CRIT?(?:ICAL)?|[F|f]atal|FATAL|[S|s]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?) + +# NamedGroup with underscore +NAMEDGROUPWITHUNDERSCORE (?test) diff --git a/src/test/java/io/krakens/grok/api/GrokTest.java b/src/test/java/io/krakens/grok/api/GrokTest.java index de4d714..3bff5a8 100644 --- a/src/test/java/io/krakens/grok/api/GrokTest.java +++ b/src/test/java/io/krakens/grok/api/GrokTest.java @@ -672,4 +672,13 @@ public void testTimeZone() { instant = (Instant) grok.match(dateWithTimeZone).capture().get("timestamp"); assertEquals(ZonedDateTime.parse(dateWithTimeZone, dtf.withZone(ZoneOffset.ofHours(8))).toInstant(), instant); } + + @Test + public void testNamedGroupWithUnderscore() { + String grokPatternName = "NAMEDGROUPWITHUNDERSCORE"; + String testString = "test"; + Grok grok = compiler.compile("%{" + grokPatternName + "}"); + String result = (String) grok.match(testString).capture().get(grokPatternName); + assertEquals("test", result); + } } From 6b2502b04a1ee8018201cd575bba1d73de1c4994 Mon Sep 17 00:00:00 2001 From: Konrad Merz Date: Mon, 25 Mar 2019 13:10:27 +0100 Subject: [PATCH 5/7] Fix test8 in BasicTest The test originally assumed that name0 is the result with underscore. I have no idea why I thought this. The test now will test that a pattern with underscore will be returned proper after compilation --- src/test/java/io/krakens/grok/api/BasicTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/io/krakens/grok/api/BasicTest.java b/src/test/java/io/krakens/grok/api/BasicTest.java index d77f5b5..7779222 100644 --- a/src/test/java/io/krakens/grok/api/BasicTest.java +++ b/src/test/java/io/krakens/grok/api/BasicTest.java @@ -129,9 +129,9 @@ public void test007_testLoadPatternFromReader() throws IOException, GrokExceptio @Test public void test008_namedGroupPatternWithUnderscore() throws GrokException { - compiler.register("test", "hello world"); + compiler.register("test", "(?test)"); Grok grok = compiler.compile("%{test}"); - assertEquals("(?hello world)", grok.getNamedRegex()); + assertEquals("(?(?test))", grok.getNamedRegex()); } } From 344082c60ba0ca60a8162e142e167e006fe939ce Mon Sep 17 00:00:00 2001 From: Konrad Merz Date: Mon, 25 Mar 2019 13:18:31 +0100 Subject: [PATCH 6/7] Revert usage of namedGroups from Grok and Match --- src/main/java/io/krakens/grok/api/GrokCompiler.java | 3 +-- src/main/java/io/krakens/grok/api/GrokUtils.java | 9 --------- src/main/java/io/krakens/grok/api/Match.java | 2 +- 3 files changed, 2 insertions(+), 12 deletions(-) diff --git a/src/main/java/io/krakens/grok/api/GrokCompiler.java b/src/main/java/io/krakens/grok/api/GrokCompiler.java index 739e637..e4b7f07 100644 --- a/src/main/java/io/krakens/grok/api/GrokCompiler.java +++ b/src/main/java/io/krakens/grok/api/GrokCompiler.java @@ -160,13 +160,12 @@ public Grok compile(final String pattern, ZoneId defaultTimeZone, boolean namedO } iterationLeft--; - Set namedGroups = GrokUtils.getNameGroups(GrokUtils.GROK_PATTERN.toString()); Matcher matcher = GrokUtils.GROK_PATTERN.matcher(namedRegex); // Match %{Foo:bar} -> pattern name and subname // Match %{Foo=regex} -> add new regex definition if (matcher.find()) { continueIteration = true; - Map group = GrokUtils.namedGroups(matcher, namedGroups); + Map group = matcher.namedGroups(); if (group.get("definition") != null) { patternDefinitions.put(group.get("pattern"), group.get("definition")); group.put("name", group.get("name") + "=" + group.get("definition")); diff --git a/src/main/java/io/krakens/grok/api/GrokUtils.java b/src/main/java/io/krakens/grok/api/GrokUtils.java index a3bab71..9e1cb48 100644 --- a/src/main/java/io/krakens/grok/api/GrokUtils.java +++ b/src/main/java/io/krakens/grok/api/GrokUtils.java @@ -44,13 +44,4 @@ public static Set getNameGroups(String regex) { } return namedGroups; } - - public static Map namedGroups(Matcher matcher, Set groupNames) { - Map namedGroups = new LinkedHashMap<>(); - for (String groupName : groupNames) { - String groupValue = matcher.group(groupName); - namedGroups.put(groupName, groupValue); - } - return namedGroups; - } } diff --git a/src/main/java/io/krakens/grok/api/Match.java b/src/main/java/io/krakens/grok/api/Match.java index 7dfa46d..eb77d56 100644 --- a/src/main/java/io/krakens/grok/api/Match.java +++ b/src/main/java/io/krakens/grok/api/Match.java @@ -129,7 +129,7 @@ private Map capture(boolean flattened ) throws GrokException { // _capture.put("LINE", this.line); // _capture.put("LENGTH", this.line.length() +""); - Map mappedw = GrokUtils.namedGroups(this.match, this.grok.namedGroups); + Map mappedw = this.match.namedGroups(); mappedw.forEach((key, valueString) -> { String id = this.grok.getNamedRegexCollectionById(key); From 16a2d9aae845bc0e641efdedbc7afc9557281b79 Mon Sep 17 00:00:00 2001 From: Konrad Merz Date: Mon, 25 Mar 2019 13:29:48 +0100 Subject: [PATCH 7/7] Fix namedGroups field of Grok.java --- src/main/java/io/krakens/grok/api/GrokUtils.java | 2 +- src/test/java/io/krakens/grok/api/BasicTest.java | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/main/java/io/krakens/grok/api/GrokUtils.java b/src/main/java/io/krakens/grok/api/GrokUtils.java index 9e1cb48..a2fb4f4 100644 --- a/src/main/java/io/krakens/grok/api/GrokUtils.java +++ b/src/main/java/io/krakens/grok/api/GrokUtils.java @@ -34,7 +34,7 @@ public class GrokUtils { + "\\}"); public static final Pattern NAMED_REGEX = Pattern - .compile("\\(\\?<([a-zA-Z][a-zA-Z0-9]*)>"); + .compile("\\(\\?<([a-zA-Z][a-zA-Z0-9_]*)>"); public static Set getNameGroups(String regex) { Set namedGroups = new LinkedHashSet<>(); diff --git a/src/test/java/io/krakens/grok/api/BasicTest.java b/src/test/java/io/krakens/grok/api/BasicTest.java index 7779222..819fcd7 100644 --- a/src/test/java/io/krakens/grok/api/BasicTest.java +++ b/src/test/java/io/krakens/grok/api/BasicTest.java @@ -14,7 +14,9 @@ import java.io.StringReader; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.regex.PatternSyntaxException; import io.krakens.grok.api.exception.GrokException; @@ -132,6 +134,20 @@ public void test008_namedGroupPatternWithUnderscore() throws GrokException { compiler.register("test", "(?test)"); Grok grok = compiler.compile("%{test}"); assertEquals("(?(?test))", grok.getNamedRegex()); + Set expectedNamedGroups = new HashSet<>(2); + expectedNamedGroups.add("name0"); + expectedNamedGroups.add("test_field"); + assertEquals(expectedNamedGroups, grok.namedGroups); } + @Test + public void test009_namedGroupPattern() throws GrokException { + compiler.register("test", "(?test)"); + Grok grok = compiler.compile("%{test}"); + assertEquals("(?(?test))", grok.getNamedRegex()); + Set expectedNamedGroups = new HashSet<>(2); + expectedNamedGroups.add("name0"); + expectedNamedGroups.add("testfield"); + assertEquals(expectedNamedGroups, grok.namedGroups); + } }