From 550f5417f60e7f07880dc2d6785f84cafa49c046 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Wed, 31 Jul 2019 16:26:57 +0700 Subject: [PATCH] Add support for overwrite parameter in the enrich processor. Similar to how it is supported in the set processor: https://www.elastic.co/guide/en/elasticsearch/reference/current/set-processor.html Relates to #32789 --- .../xpack/enrich/EnrichProcessorFactory.java | 3 +- .../xpack/enrich/ExactMatchProcessor.java | 13 ++++- .../enrich/EnrichProcessorFactoryTests.java | 10 ++++ .../enrich/ExactMatchProcessorTests.java | 48 +++++++++++++++++-- 4 files changed, 67 insertions(+), 7 deletions(-) diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactory.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactory.java index df4bed9c57587..9c5b626a910c3 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactory.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactory.java @@ -36,6 +36,7 @@ public Processor create(Map processorFactories, Strin String enrichKey = ConfigurationUtils.readStringProperty(TYPE, tag, config, "enrich_key", policy.getEnrichKey()); boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, tag, config, "ignore_missing", false); + boolean overrideEnabled = ConfigurationUtils.readBooleanProperty(TYPE, tag, config, "override", true); final List specifications; final List> specificationConfig = ConfigurationUtils.readList(TYPE, tag, config, "enrich_values"); @@ -53,7 +54,7 @@ public Processor create(Map processorFactories, Strin switch (policy.getType()) { case EnrichPolicy.EXACT_MATCH_TYPE: - return new ExactMatchProcessor(tag, client, policyName, enrichKey, ignoreMissing, specifications); + return new ExactMatchProcessor(tag, client, policyName, enrichKey, ignoreMissing, overrideEnabled, specifications); default: throw new IllegalArgumentException("unsupported policy type [" + policy.getType() + "]"); } diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/ExactMatchProcessor.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/ExactMatchProcessor.java index 39f81fac78c92..48bffea168466 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/ExactMatchProcessor.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/ExactMatchProcessor.java @@ -32,6 +32,7 @@ final class ExactMatchProcessor extends AbstractProcessor { private final String policyName; private final String enrichKey; private final boolean ignoreMissing; + private final boolean overrideEnabled; private final List specifications; ExactMatchProcessor(String tag, @@ -39,6 +40,7 @@ final class ExactMatchProcessor extends AbstractProcessor { String policyName, String enrichKey, boolean ignoreMissing, + boolean overrideEnabled, List specifications) { this( tag, @@ -46,6 +48,7 @@ final class ExactMatchProcessor extends AbstractProcessor { policyName, enrichKey, ignoreMissing, + overrideEnabled, specifications ); } @@ -55,12 +58,14 @@ final class ExactMatchProcessor extends AbstractProcessor { String policyName, String enrichKey, boolean ignoreMissing, + boolean overrideEnabled, List specifications) { super(tag); this.searchRunner = searchRunner; this.policyName = policyName; this.enrichKey = enrichKey; this.ignoreMissing = ignoreMissing; + this.overrideEnabled = overrideEnabled; this.specifications = specifications; } @@ -111,7 +116,9 @@ public void execute(IngestDocument ingestDocument, BiConsumer getSpecifications() { return specifications; } diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactoryTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactoryTests.java index 7e9ad952f4385..19ecdf3dc49b8 100644 --- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactoryTests.java +++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactoryTests.java @@ -38,6 +38,11 @@ public void testCreateProcessorInstance() throws Exception { config.put("ignore_missing", keyIgnoreMissing); } + Boolean overrideEnabled = randomBoolean() ? null : randomBoolean(); + if (overrideEnabled != null) { + config.put("override", overrideEnabled); + } + int numRandomValues = randomIntBetween(1, 8); List> randomValues = new ArrayList<>(numRandomValues); for (int i = 0; i < numRandomValues; i++) { @@ -55,6 +60,11 @@ public void testCreateProcessorInstance() throws Exception { assertThat(result.getPolicyName(), equalTo("majestic")); assertThat(result.getEnrichKey(), equalTo("host")); assertThat(result.isIgnoreMissing(), is(keyIgnoreMissing)); + if (overrideEnabled != null) { + assertThat(result.isOverrideEnabled(), is(overrideEnabled)); + } else { + assertThat(result.isOverrideEnabled(), is(true)); + } assertThat(result.getSpecifications().size(), equalTo(numRandomValues)); for (int i = 0; i < numRandomValues; i++) { EnrichSpecification actual = result.getSpecifications().get(i); diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/ExactMatchProcessorTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/ExactMatchProcessorTests.java index 4dc8ae51262d3..839aead0c142a 100644 --- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/ExactMatchProcessorTests.java +++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/ExactMatchProcessorTests.java @@ -33,6 +33,7 @@ import java.io.IOException; import java.io.UncheckedIOException; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.function.BiConsumer; @@ -47,7 +48,7 @@ public class ExactMatchProcessorTests extends ESTestCase { public void testBasics() throws Exception { MockSearchFunction mockSearch = mockedSearchFunction(Map.of("elastic.co", Map.of("globalRank", 451, "tldRank",23, "tld", "co"))); - ExactMatchProcessor processor = new ExactMatchProcessor("_tag", mockSearch, "_name", "domain", false, + ExactMatchProcessor processor = new ExactMatchProcessor("_tag", mockSearch, "_name", "domain", false, true, List.of(new EnrichSpecification("tldRank", "tld_rank"), new EnrichSpecification("tld", "tld"))); IngestDocument ingestDocument = new IngestDocument("_index", "_type", "_id", "_routing", 1L, VersionType.INTERNAL, Map.of("domain", "elastic.co")); @@ -76,7 +77,7 @@ public void testBasics() throws Exception { public void testNoMatch() throws Exception { MockSearchFunction mockSearch = mockedSearchFunction(); - ExactMatchProcessor processor = new ExactMatchProcessor("_tag", mockSearch, "_name", "domain", false, + ExactMatchProcessor processor = new ExactMatchProcessor("_tag", mockSearch, "_name", "domain", false, true, List.of(new EnrichSpecification("tldRank", "tld_rank"), new EnrichSpecification("tld", "tld"))); IngestDocument ingestDocument = new IngestDocument("_index", "_type", "_id", "_routing", 1L, VersionType.INTERNAL, Map.of("domain", "elastic.com")); @@ -106,7 +107,7 @@ public void testNoMatch() throws Exception { public void testSearchFailure() throws Exception { String indexName = ".enrich-_name"; MockSearchFunction mockSearch = mockedSearchFunction(new IndexNotFoundException(indexName)); - ExactMatchProcessor processor = new ExactMatchProcessor("_tag", mockSearch, "_name", "domain", false, + ExactMatchProcessor processor = new ExactMatchProcessor("_tag", mockSearch, "_name", "domain", false, true, List.of(new EnrichSpecification("tldRank", "tld_rank"), new EnrichSpecification("tld", "tld"))); IngestDocument ingestDocument = new IngestDocument("_index", "_type", "_id", "_routing", 1L, VersionType.INTERNAL, Map.of("domain", "elastic.com")); @@ -141,7 +142,7 @@ public void testSearchFailure() throws Exception { public void testIgnoreKeyMissing() throws Exception { { ExactMatchProcessor processor = new ExactMatchProcessor("_tag", mockedSearchFunction(), "_name", "domain", - true, List.of(new EnrichSpecification("tldRank", "tld_rank"), new EnrichSpecification("tld", "tld"))); + true, true, List.of(new EnrichSpecification("tldRank", "tld_rank"), new EnrichSpecification("tld", "tld"))); IngestDocument ingestDocument = new IngestDocument("_index", "_type", "_id", "_routing", 1L, VersionType.INTERNAL, Map.of()); assertThat(ingestDocument.getSourceAndMetadata().size(), equalTo(6)); @@ -152,7 +153,7 @@ public void testIgnoreKeyMissing() throws Exception { } { ExactMatchProcessor processor = new ExactMatchProcessor("_tag", mockedSearchFunction(), "_name", "domain", - false, List.of(new EnrichSpecification("tldRank", "tld_rank"), new EnrichSpecification("tld", "tld"))); + false, true, List.of(new EnrichSpecification("tldRank", "tld_rank"), new EnrichSpecification("tld", "tld"))); IngestDocument ingestDocument = new IngestDocument("_index", "_type", "_id", "_routing", 1L, VersionType.INTERNAL, Map.of()); IngestDocument[] resultHolder = new IngestDocument[1]; Exception[] exceptionHolder = new Exception[1]; @@ -166,6 +167,43 @@ public void testIgnoreKeyMissing() throws Exception { } } + public void testExistingFieldWithOverrideDisabled() throws Exception { + MockSearchFunction mockSearch = mockedSearchFunction(Map.of("elastic.co", Map.of("globalRank", 451, "tldRank",23, "tld", "co"))); + ExactMatchProcessor processor = new ExactMatchProcessor("_tag", mockSearch, "_name", "domain", false, false, + List.of(new EnrichSpecification("tld", "tld"))); + + IngestDocument ingestDocument = new IngestDocument(new HashMap<>(Map.of("domain", "elastic.co", "tld", "tld")), Map.of()); + IngestDocument[] resultHolder = new IngestDocument[1]; + Exception[] exceptionHolder = new Exception[1]; + processor.execute(ingestDocument, (result, e) -> { + resultHolder[0] = result; + exceptionHolder[0] = e; + }); + assertThat(exceptionHolder[0], nullValue()); + assertThat(resultHolder[0].hasField("tld"), equalTo(true)); + assertThat(resultHolder[0].getFieldValue("tld", Object.class), equalTo("tld")); + } + + public void testExistingNullFieldWithOverrideDisabled() throws Exception { + MockSearchFunction mockSearch = mockedSearchFunction(Map.of("elastic.co", Map.of("globalRank", 451, "tldRank",23, "tld", "co"))); + ExactMatchProcessor processor = new ExactMatchProcessor("_tag", mockSearch, "_name", "domain", false, false, + List.of(new EnrichSpecification("tld", "tld"))); + + Map source = new HashMap<>(); + source.put("domain", "elastic.co"); + source.put("tld", null); + IngestDocument ingestDocument = new IngestDocument(source, Map.of()); + IngestDocument[] resultHolder = new IngestDocument[1]; + Exception[] exceptionHolder = new Exception[1]; + processor.execute(ingestDocument, (result, e) -> { + resultHolder[0] = result; + exceptionHolder[0] = e; + }); + assertThat(exceptionHolder[0], nullValue()); + assertThat(resultHolder[0].hasField("tld"), equalTo(true)); + assertThat(resultHolder[0].getFieldValue("tld", Object.class), equalTo(null)); + } + private static final class MockSearchFunction implements BiConsumer> { private final SearchResponse mockResponse; private final SetOnce capturedRequest;