From 5971781325b72df033c0d06e546d3d976ab66aad Mon Sep 17 00:00:00 2001 From: Andrew Stucki Date: Mon, 8 Feb 2021 16:10:34 -0500 Subject: [PATCH 1/9] Make NetworkDirectionProcessor more robust --- .../processors/network-direction.asciidoc | 7 +- .../ingest/common/IngestCommonPlugin.java | 2 +- .../common/NetworkDirectionProcessor.java | 75 ++++++++++++++----- ...NetworkDirectionProcessorFactoryTests.java | 8 +- .../NetworkDirectionProcessorTests.java | 45 +++++++++-- .../xpack/ingest/IngestPlugin.java | 28 +++++++ 6 files changed, 134 insertions(+), 31 deletions(-) create mode 100644 x-pack/plugin/ingest/src/main/java/org/elasticsearch/xpack/ingest/IngestPlugin.java diff --git a/docs/reference/ingest/processors/network-direction.asciidoc b/docs/reference/ingest/processors/network-direction.asciidoc index 21e91cf3e41f9..12f1d5a3d8bc6 100644 --- a/docs/reference/ingest/processors/network-direction.asciidoc +++ b/docs/reference/ingest/processors/network-direction.asciidoc @@ -21,8 +21,9 @@ only the `internal_networks` option must be specified. | `source_ip` | no | `source.ip` | Field containing the source IP address. | `destination_ip` | no | `destination.ip` | Field containing the destination IP address. | `target_field` | no | `network.direction` | Output field for the network direction. -| `internal_networks`| yes | | List of internal networks. Supports IPv4 and -IPv6 addresses and ranges in CIDR notation. Also supports the named ranges listed below. +| `internal_networks`| no | | List of internal networks. Supports IPv4 and +IPv6 addresses and ranges in CIDR notation. Also supports the named ranges listed below. These may be constructed with <>. +| `internal_networks_field`| no | | A field on the given document to read the `internal_networks` configuration from. | `ignore_missing` | no | `true` | If `true` and any required fields are missing, the processor quietly exits without modifying the document. @@ -30,6 +31,8 @@ the processor quietly exits without modifying the document. include::common-options.asciidoc[] |====== +One of either `internal_networks` or `internal_networks_field` must be specified. If `internal_networks_field` is specified, it follows the behavior specified by `ignore_missing`. + [float] [[supported-named-network-ranges]] ===== Supported named network ranges diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java index e91d88b1cb220..845dcede79bec 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java @@ -79,7 +79,7 @@ public Map getProcessors(Processor.Parameters paramet entry(HtmlStripProcessor.TYPE, new HtmlStripProcessor.Factory()), entry(CsvProcessor.TYPE, new CsvProcessor.Factory()), entry(UriPartsProcessor.TYPE, new UriPartsProcessor.Factory()), - entry(NetworkDirectionProcessor.TYPE, new NetworkDirectionProcessor.Factory()), + entry(NetworkDirectionProcessor.TYPE, new NetworkDirectionProcessor.Factory(parameters.scriptService)), entry(CommunityIdProcessor.TYPE, new CommunityIdProcessor.Factory()), entry(FingerprintProcessor.TYPE, new FingerprintProcessor.Factory()) ); diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/NetworkDirectionProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/NetworkDirectionProcessor.java index f3324c4bc77e6..68c5814aa74fb 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/NetworkDirectionProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/NetworkDirectionProcessor.java @@ -14,11 +14,18 @@ import org.elasticsearch.ingest.ConfigurationUtils; import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.ingest.Processor; +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.common.network.InetAddresses; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.script.TemplateScript; +import org.elasticsearch.common.network.CIDRUtils; import java.net.InetAddress; -import java.util.Arrays; +import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Arrays; +import java.util.stream.Collectors; import static org.elasticsearch.ingest.ConfigurationUtils.readBooleanProperty; @@ -48,7 +55,8 @@ public class NetworkDirectionProcessor extends AbstractProcessor { private final String sourceIpField; private final String destinationIpField; private final String targetField; - private final List internalNetworks; + private final List internalNetworks; + private final String internalNetworksField; private final boolean ignoreMissing; NetworkDirectionProcessor( @@ -57,7 +65,8 @@ public class NetworkDirectionProcessor extends AbstractProcessor { String sourceIpField, String destinationIpField, String targetField, - List internalNetworks, + List internalNetworks, + String internalNetworksField, boolean ignoreMissing ) { super(tag, description); @@ -65,6 +74,7 @@ public class NetworkDirectionProcessor extends AbstractProcessor { this.destinationIpField = destinationIpField; this.targetField = targetField; this.internalNetworks = internalNetworks; + this.internalNetworksField = internalNetworksField; this.ignoreMissing = ignoreMissing; } @@ -80,7 +90,7 @@ public String getTargetField() { return targetField; } - public List getInternalNetworks() { + public List getInternalNetworks() { return internalNetworks; } @@ -103,8 +113,17 @@ public IngestDocument execute(IngestDocument ingestDocument) throws Exception { return ingestDocument; } - private String getDirection(IngestDocument d) { - if (internalNetworks == null) { + private String getDirection(IngestDocument d) throws Exception { + List networks = new ArrayList<>(); + + if (internalNetworksField != null) { + @SuppressWarnings("unchecked") + List stringList = d.getFieldValue(internalNetworksField, networks.getClass(), ignoreMissing); + networks.addAll(stringList); + } else if (internalNetworks != null) { + networks = internalNetworks.stream().map(network -> d.renderTemplate(network)).collect(Collectors.toList()); + } + if (networks == null) { return null; } @@ -118,8 +137,8 @@ private String getDirection(IngestDocument d) { return null; } - boolean sourceInternal = isInternal(sourceIpAddrString); - boolean destinationInternal = isInternal(destIpAddrString); + boolean sourceInternal = isInternal(networks, sourceIpAddrString); + boolean destinationInternal = isInternal(networks, destIpAddrString); if (sourceInternal && destinationInternal) { return DIRECTION_INTERNAL; @@ -133,8 +152,8 @@ private String getDirection(IngestDocument d) { return DIRECTION_EXTERNAL; } - private boolean isInternal(String ip) { - for (String network : internalNetworks) { + private boolean isInternal(List networks, String ip) { + for (String network : networks) { if (inNetwork(ip, network)) { return true; } @@ -227,11 +246,15 @@ public String getType() { } public static final class Factory implements Processor.Factory { - + private final ScriptService scriptService; static final String DEFAULT_SOURCE_IP = "source.ip"; static final String DEFAULT_DEST_IP = "destination.ip"; static final String DEFAULT_TARGET = "network.direction"; + public Factory(ScriptService scriptService) { + this.scriptService = scriptService; + } + @Override public NetworkDirectionProcessor create( Map registry, @@ -239,19 +262,37 @@ public NetworkDirectionProcessor create( String description, Map config ) throws Exception { - String sourceIpField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "source_ip", DEFAULT_SOURCE_IP); - String destIpField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "destination_ip", DEFAULT_DEST_IP); - String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", DEFAULT_TARGET); - List internalNetworks = ConfigurationUtils.readList(TYPE, processorTag, config, "internal_networks"); - boolean ignoreMissing = readBooleanProperty(TYPE, processorTag, config, "ignore_missing", true); + final String sourceIpField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "source_ip", DEFAULT_SOURCE_IP); + final String destIpField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "destination_ip", DEFAULT_DEST_IP); + final String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", DEFAULT_TARGET); + final boolean ignoreMissing = readBooleanProperty(TYPE, processorTag, config, "ignore_missing", true); + + final List internalNetworks = ConfigurationUtils.readOptionalList(TYPE, processorTag, config, "internal_networks"); + final String internalNetworksField = ConfigurationUtils.readOptionalStringProperty( + TYPE, + processorTag, + config, + "internal_networks_field" + ); + if (internalNetworks == null && internalNetworksField == null) { + throw new ElasticsearchParseException("either [internal_networks] or [internal_networks_field] must be specified"); + } + + List internalNetworkTemplates = null; + if (internalNetworks != null) { + internalNetworkTemplates = internalNetworks.stream() + .map(n -> ConfigurationUtils.compileTemplate(TYPE, processorTag, "internal_networks", n, scriptService)) + .collect(Collectors.toList()); + } return new NetworkDirectionProcessor( processorTag, description, sourceIpField, destIpField, targetField, - internalNetworks, + internalNetworkTemplates, + internalNetworksField, ignoreMissing ); } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorFactoryTests.java index f8510f7444ac6..4bd63dced1a17 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorFactoryTests.java @@ -10,10 +10,12 @@ import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.ingest.TestTemplateService; import org.junit.Before; import java.util.ArrayList; import java.util.HashMap; +import java.util.Collections; import java.util.List; import java.util.Map; @@ -28,7 +30,7 @@ public class NetworkDirectionProcessorFactoryTests extends ESTestCase { @Before public void init() { - factory = new NetworkDirectionProcessor.Factory(); + factory = new NetworkDirectionProcessor.Factory(TestTemplateService.instance()); } public void testCreate() throws Exception { @@ -52,7 +54,7 @@ public void testCreate() throws Exception { assertThat(networkProcessor.getSourceIpField(), equalTo(sourceIpField)); assertThat(networkProcessor.getDestinationIpField(), equalTo(destIpField)); assertThat(networkProcessor.getTargetField(), equalTo(targetField)); - assertThat(networkProcessor.getInternalNetworks(), equalTo(internalNetworks)); + assertThat(networkProcessor.getInternalNetworks().get(0).newInstance(Collections.emptyMap()).execute(), equalTo("10.0.0.0/8")); assertThat(networkProcessor.getIgnoreMissing(), equalTo(ignoreMissing)); } @@ -63,7 +65,7 @@ public void testRequiredFields() throws Exception { factory.create(null, processorTag, null, config); fail("factory create should have failed"); } catch (ElasticsearchParseException e) { - assertThat(e.getMessage(), equalTo("[internal_networks] required property is missing")); + assertThat(e.getMessage(), equalTo("either [internal_networks] or [internal_networks_field] must be specified")); } } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorTests.java index 96fea8a731425..7120ca9e03462 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorTests.java @@ -10,10 +10,14 @@ import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.ingest.TestTemplateService; +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.ingest.TestTemplateService; import java.util.Arrays; import java.util.HashMap; import java.util.List; +import java.util.ArrayList; import java.util.Map; import static org.elasticsearch.ingest.common.NetworkDirectionProcessor.Factory.DEFAULT_DEST_IP; @@ -49,8 +53,11 @@ private Map buildEvent(String source, String destination) { } public void testNoInternalNetworks() throws Exception { - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> testNetworkDirectionProcessor(buildEvent(), null)); - assertThat(e.getMessage(), containsString("unable to calculate network direction from document")); + ElasticsearchParseException e = expectThrows( + ElasticsearchParseException.class, + () -> testNetworkDirectionProcessor(buildEvent(), null) + ); + assertThat(e.getMessage(), containsString("either [internal_networks] or [internal_networks_field] must be specified")); } public void testNoSource() throws Exception { @@ -130,6 +137,27 @@ private void testNetworkDirectionProcessor(Map source, String[] testNetworkDirectionProcessor(source, internalNetworks, expectedDirection, false); } + public void testReadFromField() throws Exception { + String processorTag = randomAlphaOfLength(10); + Map source = buildEvent("192.168.1.1", "192.168.1.2"); + ArrayList networks = new ArrayList<>(); + networks.add("public"); + source.put("some_field", networks); + + Map config = new HashMap<>(); + config.put("internal_networks_field", "some_field"); + NetworkDirectionProcessor processor = new NetworkDirectionProcessor.Factory(TestTemplateService.instance()).create( + null, + processorTag, + null, + config + ); + IngestDocument input = new IngestDocument(source, Map.of()); + IngestDocument output = processor.execute(input); + String hash = output.getFieldValue(DEFAULT_TARGET, String.class); + assertThat(hash, equalTo("external")); + } + private void testNetworkDirectionProcessor( Map source, String[] internalNetworks, @@ -140,14 +168,15 @@ private void testNetworkDirectionProcessor( if (internalNetworks != null) networks = Arrays.asList(internalNetworks); - var processor = new NetworkDirectionProcessor( + String processorTag = randomAlphaOfLength(10); + Map config = new HashMap<>(); + config.put("internal_networks", networks); + config.put("ignore_missing", ignoreMissing); + NetworkDirectionProcessor processor = new NetworkDirectionProcessor.Factory(TestTemplateService.instance()).create( null, + processorTag, null, - DEFAULT_SOURCE_IP, - DEFAULT_DEST_IP, - DEFAULT_TARGET, - networks, - ignoreMissing + config ); IngestDocument input = new IngestDocument(source, Map.of()); diff --git a/x-pack/plugin/ingest/src/main/java/org/elasticsearch/xpack/ingest/IngestPlugin.java b/x-pack/plugin/ingest/src/main/java/org/elasticsearch/xpack/ingest/IngestPlugin.java new file mode 100644 index 0000000000000..0b8c6fbda12de --- /dev/null +++ b/x-pack/plugin/ingest/src/main/java/org/elasticsearch/xpack/ingest/IngestPlugin.java @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.ingest; + +import org.elasticsearch.ingest.Processor; +import org.elasticsearch.plugins.Plugin; + +import java.util.Map; + +public class IngestPlugin extends Plugin implements org.elasticsearch.plugins.IngestPlugin { + + @Override + public Map getProcessors(Processor.Parameters parameters) { + return Map.of( + UriPartsProcessor.TYPE, + new UriPartsProcessor.Factory(), + NetworkDirectionProcessor.TYPE, + new NetworkDirectionProcessor.Factory(parameters.scriptService), + CommunityIdProcessor.TYPE, + new CommunityIdProcessor.Factory() + ); + } +} From cf6701a840033c7be6fd162142d310c92d3e81ae Mon Sep 17 00:00:00 2001 From: Andrew Stucki Date: Mon, 8 Feb 2021 16:27:56 -0500 Subject: [PATCH 2/9] fix up ignore missing usage --- .../ingest/common/NetworkDirectionProcessor.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/NetworkDirectionProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/NetworkDirectionProcessor.java index 68c5814aa74fb..2bea08b62ae85 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/NetworkDirectionProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/NetworkDirectionProcessor.java @@ -119,13 +119,13 @@ private String getDirection(IngestDocument d) throws Exception { if (internalNetworksField != null) { @SuppressWarnings("unchecked") List stringList = d.getFieldValue(internalNetworksField, networks.getClass(), ignoreMissing); + if (stringList == null) { + return null; + } networks.addAll(stringList); - } else if (internalNetworks != null) { + } else { networks = internalNetworks.stream().map(network -> d.renderTemplate(network)).collect(Collectors.toList()); } - if (networks == null) { - return null; - } String sourceIpAddrString = d.getFieldValue(sourceIpField, String.class, ignoreMissing); if (sourceIpAddrString == null) { From de75aa74e64cd2f18fd8b902b35e3fd209e9ec2c Mon Sep 17 00:00:00 2001 From: Andrew Stucki Date: Tue, 23 Mar 2021 12:40:49 -0400 Subject: [PATCH 3/9] remove file on bad rebase --- .../xpack/ingest/IngestPlugin.java | 28 ------------------- 1 file changed, 28 deletions(-) delete mode 100644 x-pack/plugin/ingest/src/main/java/org/elasticsearch/xpack/ingest/IngestPlugin.java diff --git a/x-pack/plugin/ingest/src/main/java/org/elasticsearch/xpack/ingest/IngestPlugin.java b/x-pack/plugin/ingest/src/main/java/org/elasticsearch/xpack/ingest/IngestPlugin.java deleted file mode 100644 index 0b8c6fbda12de..0000000000000 --- a/x-pack/plugin/ingest/src/main/java/org/elasticsearch/xpack/ingest/IngestPlugin.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.ingest; - -import org.elasticsearch.ingest.Processor; -import org.elasticsearch.plugins.Plugin; - -import java.util.Map; - -public class IngestPlugin extends Plugin implements org.elasticsearch.plugins.IngestPlugin { - - @Override - public Map getProcessors(Processor.Parameters parameters) { - return Map.of( - UriPartsProcessor.TYPE, - new UriPartsProcessor.Factory(), - NetworkDirectionProcessor.TYPE, - new NetworkDirectionProcessor.Factory(parameters.scriptService), - CommunityIdProcessor.TYPE, - new CommunityIdProcessor.Factory() - ); - } -} From 8bafdf53a8663dc4f2588c3a729bd9e5fd572b94 Mon Sep 17 00:00:00 2001 From: Andrew Stucki Date: Tue, 23 Mar 2021 13:30:58 -0400 Subject: [PATCH 4/9] fix checkstyle check --- .../ingest/common/NetworkDirectionProcessorTests.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorTests.java index 7120ca9e03462..3ae2f5cce81cf 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorTests.java @@ -20,8 +20,6 @@ import java.util.ArrayList; import java.util.Map; -import static org.elasticsearch.ingest.common.NetworkDirectionProcessor.Factory.DEFAULT_DEST_IP; -import static org.elasticsearch.ingest.common.NetworkDirectionProcessor.Factory.DEFAULT_SOURCE_IP; import static org.elasticsearch.ingest.common.NetworkDirectionProcessor.Factory.DEFAULT_TARGET; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; From 880e5de57b4b4aad693c40d828834c07014c3bad Mon Sep 17 00:00:00 2001 From: Andrew Stucki Date: Tue, 23 Mar 2021 13:33:25 -0400 Subject: [PATCH 5/9] fix doc ref --- docs/reference/ingest/processors/network-direction.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/ingest/processors/network-direction.asciidoc b/docs/reference/ingest/processors/network-direction.asciidoc index 12f1d5a3d8bc6..4108cd287fee5 100644 --- a/docs/reference/ingest/processors/network-direction.asciidoc +++ b/docs/reference/ingest/processors/network-direction.asciidoc @@ -22,7 +22,7 @@ only the `internal_networks` option must be specified. | `destination_ip` | no | `destination.ip` | Field containing the destination IP address. | `target_field` | no | `network.direction` | Output field for the network direction. | `internal_networks`| no | | List of internal networks. Supports IPv4 and -IPv6 addresses and ranges in CIDR notation. Also supports the named ranges listed below. These may be constructed with <>. +IPv6 addresses and ranges in CIDR notation. Also supports the named ranges listed below. These may be constructed with <>. | `internal_networks_field`| no | | A field on the given document to read the `internal_networks` configuration from. | `ignore_missing` | no | `true` | If `true` and any required fields are missing, the processor quietly exits without modifying the document. From 07c387b50e56df3ad3d972394ae80f225418bccc Mon Sep 17 00:00:00 2001 From: Andrew Stucki Date: Wed, 7 Apr 2021 23:04:16 -0400 Subject: [PATCH 6/9] Address feedback --- .../processors/network-direction.asciidoc | 4 ++-- .../common/NetworkDirectionProcessor.java | 8 +++++-- ...NetworkDirectionProcessorFactoryTests.java | 2 +- .../NetworkDirectionProcessorTests.java | 23 ++++++++++++++++++- 4 files changed, 31 insertions(+), 6 deletions(-) diff --git a/docs/reference/ingest/processors/network-direction.asciidoc b/docs/reference/ingest/processors/network-direction.asciidoc index 4108cd287fee5..47edef4531310 100644 --- a/docs/reference/ingest/processors/network-direction.asciidoc +++ b/docs/reference/ingest/processors/network-direction.asciidoc @@ -21,8 +21,8 @@ only the `internal_networks` option must be specified. | `source_ip` | no | `source.ip` | Field containing the source IP address. | `destination_ip` | no | `destination.ip` | Field containing the destination IP address. | `target_field` | no | `network.direction` | Output field for the network direction. -| `internal_networks`| no | | List of internal networks. Supports IPv4 and -IPv6 addresses and ranges in CIDR notation. Also supports the named ranges listed below. These may be constructed with <>. +| `internal_networks`| yes * | | List of internal networks. Supports IPv4 and +IPv6 addresses and ranges in CIDR notation. Also supports the named ranges listed below. These may be constructed with <>. * Must specify only one of `internal_networks` or `internal_networks_field`. | `internal_networks_field`| no | | A field on the given document to read the `internal_networks` configuration from. | `ignore_missing` | no | `true` | If `true` and any required fields are missing, the processor quietly exits without modifying the document. diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/NetworkDirectionProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/NetworkDirectionProcessor.java index 2bea08b62ae85..a30c02d635f5a 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/NetworkDirectionProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/NetworkDirectionProcessor.java @@ -22,11 +22,12 @@ import java.net.InetAddress; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Map; -import java.util.Arrays; import java.util.stream.Collectors; +import static org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException; import static org.elasticsearch.ingest.ConfigurationUtils.readBooleanProperty; public class NetworkDirectionProcessor extends AbstractProcessor { @@ -276,7 +277,10 @@ public NetworkDirectionProcessor create( ); if (internalNetworks == null && internalNetworksField == null) { - throw new ElasticsearchParseException("either [internal_networks] or [internal_networks_field] must be specified"); + throw newConfigurationException(TYPE, processorTag, "internal_networks", "or [internal_networks_field] must be specified"); + } + if (internalNetworks != null && internalNetworksField != null) { + throw newConfigurationException(TYPE, processorTag, "internal_networks", "and [internal_networks_field] cannot both be used in the same processor"); } List internalNetworkTemplates = null; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorFactoryTests.java index 4bd63dced1a17..1df2978840720 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorFactoryTests.java @@ -65,7 +65,7 @@ public void testRequiredFields() throws Exception { factory.create(null, processorTag, null, config); fail("factory create should have failed"); } catch (ElasticsearchParseException e) { - assertThat(e.getMessage(), equalTo("either [internal_networks] or [internal_networks_field] must be specified")); + assertThat(e.getMessage(), equalTo("[internal_networks] or [internal_networks_field] must be specified")); } } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorTests.java index 3ae2f5cce81cf..941d39591348e 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorTests.java @@ -55,7 +55,7 @@ public void testNoInternalNetworks() throws Exception { ElasticsearchParseException.class, () -> testNetworkDirectionProcessor(buildEvent(), null) ); - assertThat(e.getMessage(), containsString("either [internal_networks] or [internal_networks_field] must be specified")); + assertThat(e.getMessage(), containsString("[internal_networks] or [internal_networks_field] must be specified")); } public void testNoSource() throws Exception { @@ -156,6 +156,27 @@ public void testReadFromField() throws Exception { assertThat(hash, equalTo("external")); } + public void testInternalNetworksAndField() throws Exception { + String processorTag = randomAlphaOfLength(10); + Map source = buildEvent("192.168.1.1", "192.168.1.2"); + ArrayList networks = new ArrayList<>(); + networks.add("public"); + source.put("some_field", networks); + Map config = new HashMap<>(); + config.put("internal_networks_field", "some_field"); + config.put("internal_networks", networks); + ElasticsearchParseException e = expectThrows( + ElasticsearchParseException.class, + () -> new NetworkDirectionProcessor.Factory(TestTemplateService.instance()).create( + null, + processorTag, + null, + config + ) + ); + assertThat(e.getMessage(), containsString("[internal_networks] and [internal_networks_field] cannot both be used in the same processor")); + } + private void testNetworkDirectionProcessor( Map source, String[] internalNetworks, From 5dd499a22fafaf111f2a88f4fc2034772a67d582 Mon Sep 17 00:00:00 2001 From: Andrew Stucki Date: Wed, 7 Apr 2021 23:18:18 -0400 Subject: [PATCH 7/9] fix up checkstyle issues --- .../ingest/common/NetworkDirectionProcessor.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/NetworkDirectionProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/NetworkDirectionProcessor.java index a30c02d635f5a..d190fff2b33e0 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/NetworkDirectionProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/NetworkDirectionProcessor.java @@ -14,11 +14,8 @@ import org.elasticsearch.ingest.ConfigurationUtils; import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.ingest.Processor; -import org.elasticsearch.ElasticsearchParseException; -import org.elasticsearch.common.network.InetAddresses; import org.elasticsearch.script.ScriptService; import org.elasticsearch.script.TemplateScript; -import org.elasticsearch.common.network.CIDRUtils; import java.net.InetAddress; import java.util.ArrayList; @@ -280,7 +277,11 @@ public NetworkDirectionProcessor create( throw newConfigurationException(TYPE, processorTag, "internal_networks", "or [internal_networks_field] must be specified"); } if (internalNetworks != null && internalNetworksField != null) { - throw newConfigurationException(TYPE, processorTag, "internal_networks", "and [internal_networks_field] cannot both be used in the same processor"); + throw newConfigurationException( + TYPE, + processorTag, + "internal_networks", "and [internal_networks_field] cannot both be used in the same processor" + ); } List internalNetworkTemplates = null; From 84e2f9e3e7143d8df7fd700aad6b21dd290fce6c Mon Sep 17 00:00:00 2001 From: Andrew Stucki Date: Wed, 7 Apr 2021 23:34:44 -0400 Subject: [PATCH 8/9] fix up checkstyle issues in tests --- .../ingest/common/NetworkDirectionProcessorTests.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorTests.java index 941d39591348e..7788ba1963c15 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorTests.java @@ -174,7 +174,9 @@ public void testInternalNetworksAndField() throws Exception { config ) ); - assertThat(e.getMessage(), containsString("[internal_networks] and [internal_networks_field] cannot both be used in the same processor")); + assertThat(e.getMessage(), containsString( + "[internal_networks] and [internal_networks_field] cannot both be used in the same processor" + )); } private void testNetworkDirectionProcessor( From 4792947211cc450bc6029f8dee80a5886a16dcb7 Mon Sep 17 00:00:00 2001 From: Andrew Stucki Date: Mon, 12 Apr 2021 12:36:36 -0400 Subject: [PATCH 9/9] Add test for internal_networks_field --- .../common/NetworkDirectionProcessor.java | 4 +++ ...NetworkDirectionProcessorFactoryTests.java | 26 +++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/NetworkDirectionProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/NetworkDirectionProcessor.java index d190fff2b33e0..bbdb1c607d705 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/NetworkDirectionProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/NetworkDirectionProcessor.java @@ -92,6 +92,10 @@ public List getInternalNetworks() { return internalNetworks; } + public String getInternalNetworksField() { + return internalNetworksField; + } + public boolean getIgnoreMissing() { return ignoreMissing; } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorFactoryTests.java index 1df2978840720..47a12625eed5b 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorFactoryTests.java @@ -23,6 +23,7 @@ import static org.elasticsearch.ingest.common.NetworkDirectionProcessor.Factory.DEFAULT_SOURCE_IP; import static org.elasticsearch.ingest.common.NetworkDirectionProcessor.Factory.DEFAULT_TARGET; import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; public class NetworkDirectionProcessorFactoryTests extends ESTestCase { @@ -54,10 +55,35 @@ public void testCreate() throws Exception { assertThat(networkProcessor.getSourceIpField(), equalTo(sourceIpField)); assertThat(networkProcessor.getDestinationIpField(), equalTo(destIpField)); assertThat(networkProcessor.getTargetField(), equalTo(targetField)); + assertThat(networkProcessor.getInternalNetworks().size(), greaterThan(0)); assertThat(networkProcessor.getInternalNetworks().get(0).newInstance(Collections.emptyMap()).execute(), equalTo("10.0.0.0/8")); assertThat(networkProcessor.getIgnoreMissing(), equalTo(ignoreMissing)); } + public void testCreateInternalNetworksField() throws Exception { + Map config = new HashMap<>(); + + String sourceIpField = randomAlphaOfLength(6); + config.put("source_ip", sourceIpField); + String destIpField = randomAlphaOfLength(6); + config.put("destination_ip", destIpField); + String targetField = randomAlphaOfLength(6); + config.put("target_field", targetField); + String internalNetworksField = randomAlphaOfLength(6); + config.put("internal_networks_field", internalNetworksField); + boolean ignoreMissing = randomBoolean(); + config.put("ignore_missing", ignoreMissing); + + String processorTag = randomAlphaOfLength(10); + NetworkDirectionProcessor networkProcessor = factory.create(null, processorTag, null, config); + assertThat(networkProcessor.getTag(), equalTo(processorTag)); + assertThat(networkProcessor.getSourceIpField(), equalTo(sourceIpField)); + assertThat(networkProcessor.getDestinationIpField(), equalTo(destIpField)); + assertThat(networkProcessor.getTargetField(), equalTo(targetField)); + assertThat(networkProcessor.getInternalNetworksField(), equalTo(internalNetworksField)); + assertThat(networkProcessor.getIgnoreMissing(), equalTo(ignoreMissing)); + } + public void testRequiredFields() throws Exception { HashMap config = new HashMap<>(); String processorTag = randomAlphaOfLength(10);