diff --git a/CHANGELOG.md b/CHANGELOG.md index b7ebc675f000a..e434afd65c7b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Publish transport-grpc-spi exposing QueryBuilderProtoConverter and QueryBuilderProtoConverterRegistry ([#18949](https://github.com/opensearch-project/OpenSearch/pull/18949)) - Support system generated search pipeline. ([#19128](https://github.com/opensearch-project/OpenSearch/pull/19128)) - Add `epoch_micros` date format ([#14669](https://github.com/opensearch-project/OpenSearch/issues/14669)) +- Grok processor supports capturing multiple values for same field name ([#18799](https://github.com/opensearch-project/OpenSearch/pull/18799) ### Changed - Add CompletionStage variants to methods in the Client Interface and default to ActionListener impl ([#18998](https://github.com/opensearch-project/OpenSearch/pull/18998)) diff --git a/libs/grok/src/main/java/org/opensearch/grok/Grok.java b/libs/grok/src/main/java/org/opensearch/grok/Grok.java index aa5b1a936b99d..02e5f5bd54ff2 100644 --- a/libs/grok/src/main/java/org/opensearch/grok/Grok.java +++ b/libs/grok/src/main/java/org/opensearch/grok/Grok.java @@ -99,17 +99,28 @@ public final class Grok { private final Regex compiledExpression; private final MatcherWatchdog matcherWatchdog; private final List captureConfig; + private final boolean captureAllMatches; public Grok(Map patternBank, String grokPattern, Consumer logCallBack) { - this(patternBank, grokPattern, true, MatcherWatchdog.noop(), logCallBack); + this(patternBank, grokPattern, true, MatcherWatchdog.noop(), logCallBack, false); } public Grok(Map patternBank, String grokPattern, MatcherWatchdog matcherWatchdog, Consumer logCallBack) { - this(patternBank, grokPattern, true, matcherWatchdog, logCallBack); + this(patternBank, grokPattern, true, matcherWatchdog, logCallBack, false); + } + + public Grok( + Map patternBank, + String grokPattern, + MatcherWatchdog matcherWatchdog, + Consumer logCallBack, + boolean captureAllMatches + ) { + this(patternBank, grokPattern, true, matcherWatchdog, logCallBack, captureAllMatches); } Grok(Map patternBank, String grokPattern, boolean namedCaptures, Consumer logCallBack) { - this(patternBank, grokPattern, namedCaptures, MatcherWatchdog.noop(), logCallBack); + this(patternBank, grokPattern, namedCaptures, MatcherWatchdog.noop(), logCallBack, false); } private Grok( @@ -117,11 +128,13 @@ private Grok( String grokPattern, boolean namedCaptures, MatcherWatchdog matcherWatchdog, - Consumer logCallBack + Consumer logCallBack, + boolean captureAllMatches ) { this.patternBank = patternBank; this.namedCaptures = namedCaptures; this.matcherWatchdog = matcherWatchdog; + this.captureAllMatches = captureAllMatches; validatePatternBank(); @@ -395,7 +408,7 @@ public boolean match(byte[] utf8Bytes, int offset, int length, GrokCaptureExtrac if (result == Matcher.FAILED) { return false; } - extracter.extract(utf8Bytes, offset, matcher.getEagerRegion()); + extracter.extract(utf8Bytes, offset, matcher.getEagerRegion(), captureAllMatches); return true; } diff --git a/libs/grok/src/main/java/org/opensearch/grok/GrokCaptureExtracter.java b/libs/grok/src/main/java/org/opensearch/grok/GrokCaptureExtracter.java index b6d881de4fac1..9e052584cef63 100644 --- a/libs/grok/src/main/java/org/opensearch/grok/GrokCaptureExtracter.java +++ b/libs/grok/src/main/java/org/opensearch/grok/GrokCaptureExtracter.java @@ -57,14 +57,29 @@ static class MapExtracter extends GrokCaptureExtracter { result = captureConfig.isEmpty() ? emptyMap() : new HashMap<>(); fieldExtracters = new ArrayList<>(captureConfig.size()); for (GrokCaptureConfig config : captureConfig) { - fieldExtracters.add(config.objectExtracter(v -> result.put(config.name(), v))); + fieldExtracters.add(config.objectExtracter(v -> { + String fieldName = config.name(); + Object existing = result.get(fieldName); + if (existing == null) { + result.put(fieldName, v); + } else if (existing instanceof List) { + @SuppressWarnings("unchecked") + List list = (List) existing; + list.add(v); + } else { + List list = new ArrayList<>(); + list.add(existing); + list.add(v); + result.put(fieldName, list); + } + })); } } @Override - void extract(byte[] utf8Bytes, int offset, Region region) { + void extract(byte[] utf8Bytes, int offset, Region region, boolean captureAllMatches) { for (GrokCaptureExtracter extracter : fieldExtracters) { - extracter.extract(utf8Bytes, offset, region); + extracter.extract(utf8Bytes, offset, region, captureAllMatches); } } @@ -73,5 +88,5 @@ Map result() { } } - abstract void extract(byte[] utf8Bytes, int offset, Region region); + abstract void extract(byte[] utf8Bytes, int offset, Region region, boolean captureAllMatches); } diff --git a/libs/grok/src/main/java/org/opensearch/grok/GrokCaptureType.java b/libs/grok/src/main/java/org/opensearch/grok/GrokCaptureType.java index 0bd3bb47e55df..1868d441446a8 100644 --- a/libs/grok/src/main/java/org/opensearch/grok/GrokCaptureType.java +++ b/libs/grok/src/main/java/org/opensearch/grok/GrokCaptureType.java @@ -104,13 +104,16 @@ static GrokCaptureType fromString(String str) { protected final GrokCaptureExtracter rawExtracter(int[] backRefs, Consumer emit) { return new GrokCaptureExtracter() { @Override - void extract(byte[] utf8Bytes, int offset, Region region) { + void extract(byte[] utf8Bytes, int offset, Region region, boolean captureAllMatches) { for (int number : backRefs) { if (region.getBeg(number) >= 0) { int matchOffset = offset + region.getBeg(number); int matchLength = region.getEnd(number) - region.getBeg(number); emit.accept(new String(utf8Bytes, matchOffset, matchLength, StandardCharsets.UTF_8)); - return; // Capture only the first value. + // return the first match value if captureAllMatches is false, else continue to capture all values + if (!captureAllMatches) { + return; + } } } } diff --git a/libs/grok/src/test/java/org/opensearch/grok/GrokTests.java b/libs/grok/src/test/java/org/opensearch/grok/GrokTests.java index 8476d541aa46e..f3e587aa9dd8c 100644 --- a/libs/grok/src/test/java/org/opensearch/grok/GrokTests.java +++ b/libs/grok/src/test/java/org/opensearch/grok/GrokTests.java @@ -739,6 +739,87 @@ public void testJavaFilePatternWithSpaces() { assertThat(grok.match("Test Class.java"), is(true)); } + public void testMultipleCapturesWithSameFieldName() { + // Test that multiple captures with the same field name are collected into a list + BiConsumer scheduler = getLongRunnableBiConsumer(); + // Pattern with repeated capture groups for the same field + Grok grok = new Grok( + Grok.BUILTIN_PATTERNS, + "%{IP:ipaddress} %{IP:ipaddress}", + MatcherWatchdog.newInstance(10, 200, System::currentTimeMillis, scheduler), + logger::warn, + true + ); + + // Input with two different IP addresses + Map matches = grok.captures("192.168.1.1 192.168.1.2"); + + assertNotNull("Should have matches", matches); + Object ipaddress = matches.get("ipaddress"); + assertTrue("Should be a List", ipaddress instanceof List); + + @SuppressWarnings("unchecked") + List ipList = (List) ipaddress; + assertEquals("Should have 2 elements", 2, ipList.size()); + assertEquals("First IP should match", "192.168.1.1", ipList.get(0)); + assertEquals("Second IP should match", "192.168.1.2", ipList.get(1)); + } + + public void testMultipleCapturesWithSameFieldNameDifferentTypes() { + BiConsumer scheduler = getLongRunnableBiConsumer(); + // Pattern with repeated capture groups for the same field with different types + Grok grok = new Grok( + Grok.BUILTIN_PATTERNS, + "%{NUMBER:value:int} %{NUMBER:value:long}", + MatcherWatchdog.newInstance(10, 200, System::currentTimeMillis, scheduler), + logger::warn, + true + ); + + // Input with two different numbers + Map matches = grok.captures("123 456"); + + assertNotNull("Should have matches", matches); + Object value = matches.get("value"); + assertTrue("Should be a List", value instanceof List); + + @SuppressWarnings("unchecked") + List valueList = (List) value; + assertEquals("Should have 2 elements", 2, valueList.size()); + assertEquals("First value should be an Integer", Integer.valueOf(123), valueList.get(0)); + assertEquals("Second value should be a Long", Long.valueOf(456), valueList.get(1)); + } + + public void testMultipleCapturesWithSameFieldNameInComplexPattern() { + // Test a more complex pattern with multiple captures of the same field + BiConsumer scheduler = getLongRunnableBiConsumer(); + + // Pattern with multiple fields, one of which appears multiple times + Grok grok = new Grok( + Grok.BUILTIN_PATTERNS, + "%{WORD:name} has IPs: %{IP:ip}, %{IP:ip} and %{IP:ip}", + MatcherWatchdog.newInstance(10, 200, System::currentTimeMillis, scheduler), + logger::warn, + true + ); + + // Input with a name and three IPs + Map matches = grok.captures("Server has IPs: 192.168.1.1, 192.168.1.2 and 192.168.1.3"); + + assertNotNull("Should have matches", matches); + assertEquals("Name should match", "Server", matches.get("name")); + + Object ip = matches.get("ip"); + assertTrue("IP should be a List", ip instanceof List); + + @SuppressWarnings("unchecked") + List ipList = (List) ip; + assertEquals("Should have 3 IPs", 3, ipList.size()); + assertEquals("First IP should match", "192.168.1.1", ipList.get(0)); + assertEquals("Second IP should match", "192.168.1.2", ipList.get(1)); + assertEquals("Third IP should match", "192.168.1.3", ipList.get(2)); + } + public void testLogCallBack() { AtomicReference message = new AtomicReference<>(); Grok grok = new Grok(Grok.BUILTIN_PATTERNS, ".*\\[.*%{SPACE}*\\].*", message::set); @@ -747,6 +828,23 @@ public void testLogCallBack() { assertThat(message.get(), containsString("regular expression has redundant nested repeat operator")); } + private static BiConsumer getLongRunnableBiConsumer() { + AtomicBoolean run = new AtomicBoolean(true); + return (delay, command) -> { + try { + Thread.sleep(delay); + } catch (InterruptedException e) { + throw new AssertionError(e); + } + Thread t = new Thread(() -> { + if (run.get()) { + command.run(); + } + }); + t.start(); + }; + } + private void assertGrokedField(String fieldName) { String line = "foo"; Grok grok = new Grok(Grok.BUILTIN_PATTERNS, "%{WORD:" + fieldName + "}", logger::warn); diff --git a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/GrokProcessor.java b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/GrokProcessor.java index a2fe199c24d0a..892878b6dcb2a 100644 --- a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/GrokProcessor.java +++ b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/GrokProcessor.java @@ -58,6 +58,7 @@ public final class GrokProcessor extends AbstractProcessor { private final Grok grok; private final boolean traceMatch; private final boolean ignoreMissing; + private final boolean captureAllMatches; GrokProcessor( String tag, @@ -67,14 +68,16 @@ public final class GrokProcessor extends AbstractProcessor { String matchField, boolean traceMatch, boolean ignoreMissing, + boolean captureAllMatches, MatcherWatchdog matcherWatchdog ) { super(tag, description); this.matchField = matchField; this.matchPatterns = matchPatterns; - this.grok = new Grok(patternBank, combinePatterns(matchPatterns, traceMatch), matcherWatchdog, logger::debug); + this.grok = new Grok(patternBank, combinePatterns(matchPatterns, traceMatch), matcherWatchdog, logger::debug, captureAllMatches); this.traceMatch = traceMatch; this.ignoreMissing = ignoreMissing; + this.captureAllMatches = captureAllMatches; // Joni warnings are only emitted on an attempt to match, and the warning emitted for every call to match which is too verbose // so here we emit a warning (if there is one) to the logfile at warn level on construction / processor creation. new Grok(patternBank, combinePatterns(matchPatterns, traceMatch), matcherWatchdog, logger::warn).match("___nomatch___"); @@ -130,6 +133,10 @@ List getMatchPatterns() { return matchPatterns; } + boolean isCaptureAllMatches() { + return captureAllMatches; + } + static String combinePatterns(List patterns, boolean traceMatch) { String combinedPattern; if (patterns.size() > 1) { @@ -176,6 +183,7 @@ public GrokProcessor create( List matchPatterns = ConfigurationUtils.readList(TYPE, processorTag, config, "patterns"); boolean traceMatch = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "trace_match", false); boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false); + boolean captureAllMatches = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "capture_all_matches", false); if (matchPatterns.isEmpty()) { throw newConfigurationException(TYPE, processorTag, "patterns", "List of patterns must not be empty"); @@ -195,6 +203,7 @@ public GrokProcessor create( matchField, traceMatch, ignoreMissing, + captureAllMatches, matcherWatchdog ); } catch (Exception e) { diff --git a/modules/ingest-common/src/test/java/org/opensearch/ingest/common/GrokProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/opensearch/ingest/common/GrokProcessorFactoryTests.java index 397ffeb4e6493..b6542885abf24 100644 --- a/modules/ingest-common/src/test/java/org/opensearch/ingest/common/GrokProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/opensearch/ingest/common/GrokProcessorFactoryTests.java @@ -134,4 +134,19 @@ public void testCreateWithInvalidPatternDefinition() throws Exception { equalTo("[patterns] Invalid regex pattern found in: [%{MY_PATTERN:name}!]. premature end of char-class") ); } + + public void testBuildWithCaptureAllMatches() throws Exception { + GrokProcessor.Factory factory = new GrokProcessor.Factory(Collections.emptyMap(), MatcherWatchdog.noop()); + + Map config = new HashMap<>(); + config.put("field", "_field"); + config.put("patterns", Collections.singletonList("(?\\w+)")); + config.put("capture_all_matches", true); + String processorTag = randomAlphaOfLength(10); + GrokProcessor processor = factory.create(null, processorTag, null, config); + assertThat(processor.getTag(), equalTo(processorTag)); + assertThat(processor.getMatchField(), equalTo("_field")); + assertThat(processor.getGrok(), notNullValue()); + assertThat(processor.isCaptureAllMatches(), is(true)); + } } diff --git a/modules/ingest-common/src/test/java/org/opensearch/ingest/common/GrokProcessorTests.java b/modules/ingest-common/src/test/java/org/opensearch/ingest/common/GrokProcessorTests.java index 1ed6feeb7143b..271f4cd08fe1c 100644 --- a/modules/ingest-common/src/test/java/org/opensearch/ingest/common/GrokProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/opensearch/ingest/common/GrokProcessorTests.java @@ -40,10 +40,12 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import static org.opensearch.ingest.IngestDocumentMatcher.assertIngestDocument; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; public class GrokProcessorTests extends OpenSearchTestCase { @@ -59,6 +61,7 @@ public void testMatch() throws Exception { fieldName, false, false, + false, MatcherWatchdog.noop() ); processor.execute(doc); @@ -77,6 +80,7 @@ public void testIgnoreCase() throws Exception { fieldName, false, false, + false, MatcherWatchdog.noop() ); processor.execute(doc); @@ -95,6 +99,7 @@ public void testNoMatch() { fieldName, false, false, + false, MatcherWatchdog.noop() ); Exception e = expectThrows(Exception.class, () -> processor.execute(doc)); @@ -115,6 +120,7 @@ public void testNoMatchingPatternName() { fieldName, false, false, + false, MatcherWatchdog.noop() ) ); @@ -134,6 +140,7 @@ public void testMatchWithoutCaptures() throws Exception { fieldName, false, false, + false, MatcherWatchdog.noop() ); processor.execute(doc); @@ -152,6 +159,7 @@ public void testNullField() { fieldName, false, false, + false, MatcherWatchdog.noop() ); Exception e = expectThrows(Exception.class, () -> processor.execute(doc)); @@ -171,6 +179,7 @@ public void testNullFieldWithIgnoreMissing() throws Exception { fieldName, false, true, + false, MatcherWatchdog.noop() ); processor.execute(ingestDocument); @@ -189,6 +198,7 @@ public void testNotStringField() { fieldName, false, false, + false, MatcherWatchdog.noop() ); Exception e = expectThrows(Exception.class, () -> processor.execute(doc)); @@ -207,6 +217,7 @@ public void testNotStringFieldWithIgnoreMissing() { fieldName, false, true, + false, MatcherWatchdog.noop() ); Exception e = expectThrows(Exception.class, () -> processor.execute(doc)); @@ -224,6 +235,7 @@ public void testMissingField() { fieldName, false, false, + false, MatcherWatchdog.noop() ); Exception e = expectThrows(Exception.class, () -> processor.execute(doc)); @@ -242,6 +254,7 @@ public void testMissingFieldWithIgnoreMissing() throws Exception { fieldName, false, true, + false, MatcherWatchdog.noop() ); processor.execute(ingestDocument); @@ -264,6 +277,7 @@ public void testMultiplePatternsWithMatchReturn() throws Exception { fieldName, false, false, + false, MatcherWatchdog.noop() ); processor.execute(doc); @@ -288,6 +302,7 @@ public void testSetMetadata() throws Exception { fieldName, true, false, + false, MatcherWatchdog.noop() ); processor.execute(doc); @@ -311,6 +326,7 @@ public void testTraceWithOnePattern() throws Exception { fieldName, true, false, + false, MatcherWatchdog.noop() ); processor.execute(doc); @@ -350,6 +366,7 @@ public void testCombineSamePatternNameAcrossPatterns() throws Exception { fieldName, randomBoolean(), randomBoolean(), + false, MatcherWatchdog.noop() ); processor.execute(doc); @@ -371,6 +388,7 @@ public void testFirstWinNamedCapture() throws Exception { fieldName, randomBoolean(), randomBoolean(), + false, MatcherWatchdog.noop() ); processor.execute(doc); @@ -392,10 +410,106 @@ public void testUnmatchedNamesNotIncludedInDocument() throws Exception { fieldName, randomBoolean(), randomBoolean(), + false, MatcherWatchdog.noop() ); processor.execute(doc); assertFalse(doc.hasField("first")); assertThat(doc.getFieldValue("second", String.class), equalTo("3")); } + + public void testCaptureAllMatchesWithSameFieldName() throws Exception { + String fieldName = RandomDocumentPicks.randomFieldName(random()); + IngestDocument doc = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>()); + doc.setFieldValue(fieldName, "1 2 3"); + Map patternBank = new HashMap<>(); + patternBank.put("NUMBER", "\\d"); + + // Create processor with captureAllMatches=true + GrokProcessor processor = new GrokProcessor( + randomAlphaOfLength(10), + null, + patternBank, + Collections.singletonList("%{NUMBER:num} %{NUMBER:num} %{NUMBER:num}"), + fieldName, + false, + false, + true, + MatcherWatchdog.noop() + ); + + processor.execute(doc); + + // Verify that 'num' field contains a list of all matches + Object numField = doc.getFieldValue("num", Object.class); + assertThat(numField, instanceOf(List.class)); + + @SuppressWarnings("unchecked") + List numList = (List) numField; + assertEquals(3, numList.size()); + assertEquals("1", numList.get(0)); + assertEquals("2", numList.get(1)); + assertEquals("3", numList.get(2)); + + fieldName = RandomDocumentPicks.randomFieldName(random()); + doc = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>()); + doc.setFieldValue(fieldName, "192.168.1.1 172.16.0.1"); + patternBank = new HashMap<>(); + patternBank.put( + "IP", + "(? ipList = (List) ipField; + assertEquals(2, ipList.size()); + assertEquals("192.168.1.1", ipList.get(0)); + assertEquals("172.16.0.1", ipList.get(1)); + } + + public void testCaptureAllMatchesDisabled() throws Exception { + String fieldName = RandomDocumentPicks.randomFieldName(random()); + IngestDocument doc = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>()); + doc.setFieldValue(fieldName, "1 2 3"); + Map patternBank = new HashMap<>(); + patternBank.put("NUMBER", "\\d"); + + // Create processor with captureAllMatches=false (default behavior) + GrokProcessor processor = new GrokProcessor( + randomAlphaOfLength(10), + null, + patternBank, + Collections.singletonList("%{NUMBER:num} %{NUMBER:num} %{NUMBER:num}"), + fieldName, + false, + false, + false, + MatcherWatchdog.noop() + ); + + processor.execute(doc); + + // Verify that only the first match is captured + String numValue = doc.getFieldValue("num", String.class); + assertEquals("1", numValue); + } } diff --git a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/121_grok_capture_all_matches.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/121_grok_capture_all_matches.yml new file mode 100644 index 0000000000000..a53aca0dc707a --- /dev/null +++ b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/121_grok_capture_all_matches.yml @@ -0,0 +1,164 @@ +--- +teardown: + - do: + ingest.delete_pipeline: + id: "my_pipeline" + ignore: 404 + +--- +"Test Grok Pipeline with capture_all_matches disabled (default)": + - do: + ingest.put_pipeline: + id: "my_pipeline" + body: > + { + "description": "_description", + "processors": [ + { + "grok" : { + "field" : "field1", + "patterns" : ["%{IP:ipAddress} %{IP:ipAddress} %{IP:ipAddress}"] + } + } + ] + } + - match: { acknowledged: true } + + - do: + index: + index: test + id: 1 + pipeline: "my_pipeline" + body: {field1: "192.168.1.1 10.0.0.1 172.16.0.1"} + + - do: + get: + index: test + id: 1 + - match: { _source.ipAddress: "192.168.1.1" } + +--- +"Test Grok Pipeline with capture_all_matches enabled": + - do: + ingest.put_pipeline: + id: "my_pipeline" + body: > + { + "description": "_description", + "processors": [ + { + "grok" : { + "field" : "field1", + "patterns" : ["%{IP:ipAddress} %{IP:ipAddress} %{IP:ipAddress}"], + "capture_all_matches": true + } + } + ] + } + - match: { acknowledged: true } + + - do: + index: + index: test + id: 1 + pipeline: "my_pipeline" + body: {field1: "192.168.1.1 10.0.0.1 172.16.0.1"} + + - do: + get: + index: test + id: 1 + - length: { _source.ipAddress: 3 } + - match: { _source.ipAddress.0: "192.168.1.1" } + - match: { _source.ipAddress.1: "10.0.0.1" } + - match: { _source.ipAddress.2: "172.16.0.1" } + + +--- +"Test Grok Pipeline with capture_all_matches and multiple patterns": + - do: + ingest.put_pipeline: + id: "my_pipeline" + body: > + { + "description": "_description", + "processors": [ + { + "grok" : { + "field" : "field1", + "patterns" : [ + "User %{WORD:username} logged in from %{IP:ipAddress}", + "IP: %{IP:ipAddress}" + ], + "capture_all_matches": true + } + } + ] + } + - match: { acknowledged: true } + + - do: + index: + index: test + id: 1 + pipeline: "my_pipeline" + body: {field1: "User john logged in from 192.168.1.1"} + + - do: + get: + index: test + id: 1 + - match: { _source.username: "john" } + - match: { _source.ipAddress: "192.168.1.1" } + + - do: + index: + index: test + id: 2 + pipeline: "my_pipeline" + body: {field1: "IP: 10.0.0.1"} + + - do: + get: + index: test + id: 2 + - match: { _source.ipAddress: "10.0.0.1" } + +--- +"Test Grok Pipeline with capture_all_matches and custom pattern": + - do: + ingest.put_pipeline: + id: "my_pipeline" + body: > + { + "description": "_description", + "processors": [ + { + "grok" : { + "field" : "field1", + "patterns" : ["Tags: %{TAGS:tag}, %{TAGS:tag}, %{TAGS:tag}"], + "pattern_definitions" : { + "TAGS" : "[a-z0-9]+" + }, + "capture_all_matches": true + } + } + ] + } + - match: { acknowledged: true } + + - do: + index: + index: test + id: 1 + pipeline: "my_pipeline" + body: {field1: "Tags: foo, bar, baz"} + + - do: + get: + index: test + id: 1 + - match: { _source.tag.0: "foo" } + - match: { _source.tag.1: "bar" } + - match: { _source.tag.2: "baz" } + - length: { _source.tag: 3 }