Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Bring back support for grok patterns containing named groups with underscore #2

Merged
merged 7 commits into from
Mar 25, 2019
Merged

Bring back support for grok patterns containing named groups with underscore #2

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
f8003fa
Bring back pom.xml
kmerz Feb 28, 2019
94c7b39
Bring back 'named-regexp' dependency
kmerz Feb 28, 2019
c7f9e5d
Replace java.util.regex.* with com.google.code.regexp.*
kmerz Feb 28, 2019
29719bc
Add test to ensure named underscores work in the future
kmerz Feb 28, 2019
6b2502b
Fix test8 in BasicTest
kmerz Mar 25, 2019
344082c
Revert usage of namedGroups from Grok and Match
kmerz Mar 25, 2019
16a2d9a
Fix namedGroups field of Grok.java
kmerz Mar 25, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
281 changes: 281 additions & 0 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,281 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>

<groupId>org.graylog2.repackaged</groupId>
<artifactId>grok</artifactId>
<version>0.1.19-graylog-SNAPSHOT</version>
<packaging>jar</packaging>
<name>Grok</name>
<description>Simple API that allows you to easily parse logs and other files</description>
<url>http://maven.apache.org</url>

<parent>
<groupId>org.sonatype.oss</groupId>
<artifactId>oss-parent</artifactId>
<version>7</version>
</parent>

<licenses>
<license>
<name>Apache License, Version 2.0</name>
<url>http://www.apache.org/licenses/LICENSE-2.0</url>
<distribution>repo</distribution>
</license>
</licenses>

<developers>
<developer>
<id>anthony-corbacho</id>
<name>Anthony Corbacho</name>
<email>corbacho.anthony@gmail.com</email>
<organization>Thekraken</organization>
<organizationUrl>http://release.thekraken.io</organizationUrl>
<url>https://github.com/anthonycorbacho</url>
</developer>
</developers>

<scm>
<connection>scm:git:git@github.com:Graylog2/java-grok.git</connection>
<developerConnection>scm:git:git@github.com:Graylog2/java-grok.git</developerConnection>
<url>scm:git:git@github.com:Graylog2/java-grok.git</url>
</scm>

<ciManagement>
<system>Travis</system>
<url>https://travis-ci.org/thekrakken/java-grok</url>
</ciManagement>

<issueManagement>
<system>Github</system>
<url>https://github.com/thekrakken/java-grok/issues</url>
</issueManagement>

<properties>
<junit.version>4.12</junit.version>
<assertj.version>3.9.1</assertj.version>
<common.version>3.7</common.version>
<guava.version>24.0-jre</guava.version>
<named.regex.version>0.2.3</named.regex.version>
<!-- maven -->
<java.version>1.8</java.version>
</properties>

<dependencies>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>${junit.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId>
<version>${assertj.version}</version>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>${common.version}</version>
<scope>compile</scope>
</dependency>

<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${guava.version}</version>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.7.21</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>com.github.tony19</groupId>
<artifactId>named-regexp</artifactId>
<version>${named.regex.version}</version>
<scope>compile</scope>
</dependency>

</dependencies>

<build>
<plugins>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.1</version>
<configuration>
<source>${java.version}</source>
<target>${java.version}</target>
</configuration>
</plugin>

<plugin>
<artifactId>maven-javadoc-plugin</artifactId>
<version>2.9.1</version>
<configuration><!-- Default configuration for all reports -->
</configuration>
<executions>
<execution>
<id>aggregate</id>
<goals>
<goal>aggregate</goal>
</goals>
<phase>site</phase>
<configuration><!-- Specific configuration for the aggregate report -->
</configuration>
</execution>
</executions>
</plugin>

<plugin>
<artifactId>maven-scm-plugin</artifactId>
<version>1.8.1</version>
<configuration>
<connectionType>developerConnection</connectionType>
<scmVersion>branch-0.1</scmVersion>
<scmVersionType>branch</scmVersionType>
</configuration>
</plugin>

<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<version>1.3.1</version>
<executions>
<execution>
<id>enforce</id>
<configuration>
<rules>
<DependencyConvergence />
</rules>
<failFast>true</failFast>
</configuration>
<goals>
<goal>enforce</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>

<pluginManagement>
<plugins>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<version>2.8</version>
</plugin>
<plugin>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.16</version>
<configuration combine.children="append">
<argLine>-Xmx2g -Xms1g</argLine>
</configuration>
</plugin>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<version>2.4</version>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
<version>1.2.1</version>
</plugin>

<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-clean-plugin</artifactId>
<version>2.4.1</version>
<configuration>
<filesets>
<fileset>
<directory>drivers</directory>
<followSymlinks>false</followSymlinks>
</fileset>
</filesets>
</configuration>
</plugin>
</plugins>
</pluginManagement>
</build>

<profiles>
<profile>
<id>build-distr</id>
<activation>
<activeByDefault>false</activeByDefault>
</activation>
<build>
<pluginManagement>
<plugins>
<plugin>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<skipTests>true</skipTests>
</configuration>
</plugin>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</pluginManagement>
</build>
</profile>

<profile>
<id>publish-distr</id>
<activation>
<activeByDefault>false</activeByDefault>
</activation>
<build>
<plugins>
<plugin>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<skipTests>true</skipTests>
</configuration>
</plugin>
</plugins>
</build>
</profile>

<profile>
<id>release-sign-artifacts</id>
<activation>
<property>
<name>performRelease</name>
<value>true</value>
</property>
</activation>
<build>
<plugins>
<plugin>
<artifactId>maven-gpg-plugin</artifactId>
<version>1.4</version>
<executions>
<execution>
<id>sign-artifacts</id>
<phase>verify</phase>
<goals>
<goal>sign</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project>
2 changes: 1 addition & 1 deletion src/main/java/io/krakens/grok/api/Converter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,9 @@
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

import com.google.code.regexp.Pattern;
/**
* Convert String argument to the right type.
*
Expand Down
4 changes: 2 additions & 2 deletions src/main/java/io/krakens/grok/api/Discovery.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,11 @@
import java.util.Map;
import java.util.Map.Entry;
import java.util.TreeMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.lang3.StringUtils;

import com.google.code.regexp.Matcher;
import com.google.code.regexp.Pattern;

/**
* {@code Discovery} try to find the best pattern for the given string.
Expand Down
5 changes: 3 additions & 2 deletions src/main/java/io/krakens/grok/api/Grok.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,12 @@
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import io.krakens.grok.api.Converter.IConverter;

import com.google.code.regexp.Matcher;
import com.google.code.regexp.Pattern;

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The commit that changed to java.util.regex in the upstream repository (thekrakken@bd098c3) changed the following in this file:

image

With your change we are still calling GrokUtils.namedGroups(m, m.group()); instead of the previous m.namedGroups(). Is that correct or do we need to use m.namedGroups() again?

import org.apache.commons.lang3.StringUtils;

/**
Expand Down
8 changes: 5 additions & 3 deletions src/main/java/io/krakens/grok/api/GrokCompiler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,10 @@
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import com.google.code.regexp.Matcher;
import com.google.code.regexp.Pattern;


import io.krakens.grok.api.exception.GrokException;

Expand Down Expand Up @@ -158,7 +160,7 @@ public Grok compile(final String pattern, ZoneId defaultTimeZone, boolean namedO
}
iterationLeft--;

Set<String> namedGroups = GrokUtils.getNameGroups(GrokUtils.GROK_PATTERN.pattern());
Set<String> namedGroups = GrokUtils.getNameGroups(GrokUtils.GROK_PATTERN.toString());
Matcher matcher = GrokUtils.GROK_PATTERN.matcher(namedRegex);
// Match %{Foo:bar} -> pattern name and subname
// Match %{Foo=regex} -> add new regex definition
Expand Down
5 changes: 3 additions & 2 deletions src/main/java/io/krakens/grok/api/GrokUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,9 @@
import java.util.LinkedHashSet;
import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import com.google.code.regexp.Matcher;
import com.google.code.regexp.Pattern;


/**
Expand Down
4 changes: 3 additions & 1 deletion src/main/java/io/krakens/grok/api/Match.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,9 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;

import com.google.code.regexp.Matcher;

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same as above, do we need to use m.namedGroups() here?

-    Map<String, String> mappedw = this.match.namedGroups();
+    Map<String, String> mappedw = GrokUtils.namedGroups(this.match,this.subject);


import io.krakens.grok.api.Converter.IConverter;
import io.krakens.grok.api.exception.GrokException;
Expand Down
3 changes: 3 additions & 0 deletions src/main/resources/patterns/patterns
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,3 +106,6 @@ COMMONAPACHELOG_DATATYPED %{IPORHOST:clientip} %{USER:ident;boolean} %{USER:auth

# Log Levels
LOGLEVEL ([A|a]lert|ALERT|[T|t]race|TRACE|[D|d]ebug|DEBUG|[N|n]otice|NOTICE|[I|i]nfo|INFO|[W|w]arn?(?:ing)?|WARN?(?:ING)?|[E|e]rr?(?:or)?|ERR?(?:OR)?|[C|c]rit?(?:ical)?|CRIT?(?:ICAL)?|[F|f]atal|FATAL|[S|s]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?)

# NamedGroup with underscore
NAMEDGROUPWITHUNDERSCORE (?<test_field>test)
7 changes: 7 additions & 0 deletions src/test/java/io/krakens/grok/api/BasicTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,4 +127,11 @@ public void test007_testLoadPatternFromReader() throws IOException, GrokExceptio
assertEquals("(?<name0>€)", grok.getNamedRegex());
}

@Test
public void test008_namedGroupPatternWithUnderscore() throws GrokException {
compiler.register("test", "hello world");
Grok grok = compiler.compile("%{test}");
assertEquals("(?<name_0>hello world)", grok.getNamedRegex());
}

}
Loading