From 5c020adfc72d9dd83c65115eef8b50042ceb33a7 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Tue, 9 Jun 2020 15:25:33 -0700 Subject: [PATCH 1/6] Add optional description parameter to ingest processors. This commit adds an optional field, `description`, to all ingest processors so that users can explain the purpose of the specific processor instance. Closes #56000. --- .../common/AbstractStringProcessor.java | 10 ++-- .../ingest/common/AppendProcessor.java | 8 +-- .../ingest/common/BytesProcessor.java | 8 +-- .../ingest/common/ConvertProcessor.java | 8 +-- .../ingest/common/CsvProcessor.java | 15 +++--- .../ingest/common/DateIndexNameProcessor.java | 10 ++-- .../ingest/common/DateProcessor.java | 9 ++-- .../ingest/common/DissectProcessor.java | 8 +-- .../ingest/common/DotExpanderProcessor.java | 8 +-- .../ingest/common/FailProcessor.java | 8 +-- .../ingest/common/ForEachProcessor.java | 8 +-- .../ingest/common/GrokProcessor.java | 8 +-- .../ingest/common/GsubProcessor.java | 9 ++-- .../ingest/common/HtmlStripProcessor.java | 8 +-- .../ingest/common/JoinProcessor.java | 8 +-- .../ingest/common/JsonProcessor.java | 8 +-- .../ingest/common/KeyValueProcessor.java | 8 +-- .../ingest/common/LowercaseProcessor.java | 8 +-- .../ingest/common/RemoveProcessor.java | 8 +-- .../ingest/common/RenameProcessor.java | 9 ++-- .../ingest/common/ScriptProcessor.java | 9 ++-- .../ingest/common/SetProcessor.java | 11 ++-- .../ingest/common/SortProcessor.java | 8 +-- .../ingest/common/SplitProcessor.java | 9 ++-- .../ingest/common/TrimProcessor.java | 8 +-- .../ingest/common/URLDecodeProcessor.java | 8 +-- .../ingest/common/UppercaseProcessor.java | 8 +-- ...bstractStringProcessorFactoryTestCase.java | 8 +-- .../common/AppendProcessorFactoryTests.java | 10 ++-- .../common/ConvertProcessorFactoryTests.java | 12 ++--- .../common/CsvProcessorFactoryTests.java | 2 +- .../common/DateIndexNameFactoryTests.java | 14 ++--- .../common/DateProcessorFactoryTests.java | 16 +++--- .../common/DissectProcessorFactoryTests.java | 10 ++-- .../DotExpanderProcessorFactoryTests.java | 14 ++--- .../common/FailProcessorFactoryTests.java | 6 +-- .../common/ForEachProcessorFactoryTests.java | 22 ++++---- .../common/GrokProcessorFactoryTests.java | 16 +++--- .../common/GsubProcessorFactoryTests.java | 6 +-- .../common/JoinProcessorFactoryTests.java | 8 +-- .../common/JsonProcessorFactoryTests.java | 10 ++-- .../common/KeyValueProcessorFactoryTests.java | 10 ++-- .../ingest/common/KeyValueProcessorTests.java | 2 +- .../common/RemoveProcessorFactoryTests.java | 8 +-- .../ingest/common/RemoveProcessorTests.java | 4 +- .../common/RenameProcessorFactoryTests.java | 8 +-- .../common/ScriptProcessorFactoryTests.java | 12 ++--- .../common/SetProcessorFactoryTests.java | 12 ++--- .../common/SortProcessorFactoryTests.java | 10 ++-- .../common/SplitProcessorFactoryTests.java | 10 ++-- .../ingest/common/SplitProcessorTests.java | 2 +- .../ingest/geoip/GeoIpProcessor.java | 13 +++-- .../geoip/GeoIpProcessorFactoryTests.java | 32 +++++------ .../ingest/useragent/UserAgentProcessor.java | 10 ++-- .../UserAgentProcessorFactoryTests.java | 16 +++--- .../attachment/AttachmentProcessor.java | 9 ++-- .../AttachmentProcessorFactoryTests.java | 14 ++--- .../action/ingest/AsyncIngestProcessorIT.java | 8 +-- .../elasticsearch/index/FinalPipelineIT.java | 12 ++--- .../elasticsearch/ingest/IngestClientIT.java | 6 ++- .../ingest/AbstractProcessor.java | 13 +++-- .../ingest/CompoundProcessor.java | 5 ++ .../ingest/ConditionalProcessor.java | 9 ++-- .../ingest/ConfigurationUtils.java | 6 ++- .../elasticsearch/ingest/DropProcessor.java | 8 +-- .../elasticsearch/ingest/IngestService.java | 2 +- .../ingest/PipelineProcessor.java | 8 +-- .../org/elasticsearch/ingest/Processor.java | 14 +++-- .../ingest/TrackingResultProcessor.java | 5 ++ .../ingest/SimulateExecutionServiceTests.java | 28 +++++----- .../SimulatePipelineRequestParsingTests.java | 2 +- .../ingest/CompoundProcessorTests.java | 45 +++++++++------- .../ingest/ConditionalProcessorTests.java | 13 +++++ .../ingest/ConfigurationUtilsTests.java | 42 ++++++++++++++- .../elasticsearch/ingest/FakeProcessor.java | 9 +++- .../ingest/IngestServiceTests.java | 53 ++++++++++--------- .../ingest/PipelineProcessorTests.java | 25 +++++---- .../ingest/TrackingResultProcessorTests.java | 32 ++++++----- .../ingest/WrappingProcessorImpl.java | 10 +++- .../ingest/IngestTestPlugin.java | 4 +- .../elasticsearch/ingest/TestProcessor.java | 25 +++++---- .../xpack/enrich/AbstractEnrichProcessor.java | 7 ++- .../xpack/enrich/EnrichProcessorFactory.java | 2 +- .../xpack/enrich/GeoMatchProcessor.java | 6 ++- .../xpack/enrich/MatchProcessor.java | 6 ++- .../enrich/EnrichProcessorFactoryTests.java | 14 ++--- .../inference/ingest/InferenceProcessor.java | 6 ++- ...sportGetTrainedModelsStatsActionTests.java | 2 +- .../InferenceProcessorFactoryTests.java | 18 +++---- .../monitoring/test/MockIngestPlugin.java | 2 +- .../ingest/SetSecurityUserProcessor.java | 10 ++-- .../SetSecurityUserProcessorFactoryTests.java | 10 ++-- .../xpack/spatial/ingest/CircleProcessor.java | 8 +-- .../ingest/CircleProcessorFactoryTests.java | 12 ++--- 94 files changed, 592 insertions(+), 456 deletions(-) diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java index ded75e95be73d..ad3a53c7ef7ef 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java @@ -39,8 +39,8 @@ abstract class AbstractStringProcessor extends AbstractProcessor { private final boolean ignoreMissing; private final String targetField; - AbstractStringProcessor(String tag, String field, boolean ignoreMissing, String targetField) { - super(tag); + AbstractStringProcessor(String tag, String description, boolean ignoreMissing, String targetField, String field) { + super(tag, description); this.field = field; this.ignoreMissing = ignoreMissing; this.targetField = targetField; @@ -106,15 +106,15 @@ protected Factory(String processorType) { @Override public AbstractStringProcessor create(Map registry, String tag, - Map config) throws Exception { + String description, Map config) throws Exception { String field = ConfigurationUtils.readStringProperty(processorType, tag, config, "field"); boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(processorType, tag, config, "ignore_missing", false); String targetField = ConfigurationUtils.readStringProperty(processorType, tag, config, "target_field", field); - return newProcessor(tag, config, field, ignoreMissing, targetField); + return newProcessor(tag, description, config, field, ignoreMissing, targetField); } - protected abstract AbstractStringProcessor newProcessor(String processorTag, Map config, String field, + protected abstract AbstractStringProcessor newProcessor(String processorTag, String description, Map config, String field, boolean ignoreMissing, String targetField); } } diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AppendProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AppendProcessor.java index 058d1bf22d81c..9449f8f7dad8f 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AppendProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AppendProcessor.java @@ -41,8 +41,8 @@ public final class AppendProcessor extends AbstractProcessor { private final TemplateScript.Factory field; private final ValueSource value; - AppendProcessor(String tag, TemplateScript.Factory field, ValueSource value) { - super(tag); + AppendProcessor(String tag, String description, TemplateScript.Factory field, ValueSource value) { + super(tag, description); this.field = field; this.value = value; } @@ -76,12 +76,12 @@ public Factory(ScriptService scriptService) { @Override public AppendProcessor create(Map registry, String processorTag, - Map config) throws Exception { + String description, Map config) throws Exception { String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field"); Object value = ConfigurationUtils.readObject(TYPE, processorTag, config, "value"); TemplateScript.Factory compiledTemplate = ConfigurationUtils.compileTemplate(TYPE, processorTag, "field", field, scriptService); - return new AppendProcessor(processorTag, compiledTemplate, ValueSource.wrap(value, scriptService)); + return new AppendProcessor(processorTag, description, compiledTemplate, ValueSource.wrap(value, scriptService)); } } } diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/BytesProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/BytesProcessor.java index 8de75878f5fe5..57ef3a3b0770d 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/BytesProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/BytesProcessor.java @@ -31,8 +31,8 @@ public final class BytesProcessor extends AbstractStringProcessor { public static final String TYPE = "bytes"; - BytesProcessor(String processorTag, String field, boolean ignoreMissing, String targetField) { - super(processorTag, field, ignoreMissing, targetField); + BytesProcessor(String processorTag, String description, String field, boolean ignoreMissing, String targetField) { + super(processorTag, description, ignoreMissing, targetField, field); } public static long apply(String value) { @@ -56,9 +56,9 @@ public Factory() { } @Override - protected BytesProcessor newProcessor(String tag, Map config, String field, + protected BytesProcessor newProcessor(String tag, String description, Map config, String field, boolean ignoreMissing, String targetField) { - return new BytesProcessor(tag, field, ignoreMissing, targetField); + return new BytesProcessor(tag, description, field, ignoreMissing, targetField); } } } diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java index aca48efe6c11c..623aab05f3aaf 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java @@ -148,8 +148,8 @@ public static Type fromString(String processorTag, String propertyName, String t private final Type convertType; private final boolean ignoreMissing; - ConvertProcessor(String tag, String field, String targetField, Type convertType, boolean ignoreMissing) { - super(tag); + ConvertProcessor(String tag, String description, String field, String targetField, Type convertType, boolean ignoreMissing) { + super(tag, description); this.field = field; this.targetField = targetField; this.convertType = convertType; @@ -205,13 +205,13 @@ public String getType() { public static final class Factory implements Processor.Factory { @Override public ConvertProcessor create(Map registry, String processorTag, - Map config) throws Exception { + String description, Map config) throws Exception { String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field"); String typeProperty = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "type"); String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", field); Type convertType = Type.fromString(processorTag, "type", typeProperty); boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false); - return new ConvertProcessor(processorTag, field, targetField, convertType, ignoreMissing); + return new ConvertProcessor(processorTag, description, field, targetField, convertType, ignoreMissing); } } } diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/CsvProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/CsvProcessor.java index 07ab92d258155..971faf9c0e111 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/CsvProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/CsvProcessor.java @@ -22,6 +22,7 @@ import org.elasticsearch.ingest.AbstractProcessor; import org.elasticsearch.ingest.ConfigurationUtils; import org.elasticsearch.ingest.IngestDocument; +import org.elasticsearch.ingest.Processor; import java.util.List; import java.util.Map; @@ -54,9 +55,9 @@ public final class CsvProcessor extends AbstractProcessor { final boolean ignoreMissing; final Object emptyValue; - CsvProcessor(String tag, String field, String[] headers, boolean trim, char separator, char quote, boolean ignoreMissing, - Object emptyValue) { - super(tag); + CsvProcessor(String tag, String description, String field, String[] headers, boolean trim, char separator, char quote, + boolean ignoreMissing, Object emptyValue) { + super(tag, description); this.field = field; this.headers = headers; this.trim = trim; @@ -89,8 +90,8 @@ public String getType() { public static final class Factory implements org.elasticsearch.ingest.Processor.Factory { @Override - public CsvProcessor create(Map registry, String processorTag, - Map config) { + public CsvProcessor create(Map registry, String processorTag, + String description, Map config) { String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field"); String quote = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "quote", "\""); if (quote.length() != 1) { @@ -110,8 +111,8 @@ public CsvProcessor create(Map> dateFormats; - DateIndexNameProcessor(String tag, String field, List> dateFormats, ZoneId timezone, - TemplateScript.Factory indexNamePrefixTemplate, TemplateScript.Factory dateRoundingTemplate, + DateIndexNameProcessor(String tag, String description, String field, List> dateFormats, + ZoneId timezone, TemplateScript.Factory indexNamePrefixTemplate, TemplateScript.Factory dateRoundingTemplate, TemplateScript.Factory indexNameFormatTemplate) { - super(tag); + super(tag, description); this.field = field; this.timezone = timezone; this.dateFormats = dateFormats; @@ -145,7 +145,7 @@ public Factory(ScriptService scriptService) { @Override public DateIndexNameProcessor create(Map registry, String tag, - Map config) throws Exception { + String description, Map config) throws Exception { String localeString = ConfigurationUtils.readOptionalStringProperty(TYPE, tag, config, "locale"); String timezoneString = ConfigurationUtils.readOptionalStringProperty(TYPE, tag, config, "timezone"); ZoneId timezone = timezoneString == null ? ZoneOffset.UTC : ZoneId.of(timezoneString); @@ -177,7 +177,7 @@ public DateIndexNameProcessor create(Map registry, St String indexNameFormat = ConfigurationUtils.readStringProperty(TYPE, tag, config, "index_name_format", "yyyy-MM-dd"); TemplateScript.Factory indexNameFormatTemplate = ConfigurationUtils.compileTemplate(TYPE, tag, "index_name_format", indexNameFormat, scriptService); - return new DateIndexNameProcessor(tag, field, dateFormats, timezone, indexNamePrefixTemplate, + return new DateIndexNameProcessor(tag, description, field, dateFormats, timezone, indexNamePrefixTemplate, dateRoundingTemplate, indexNameFormatTemplate); } } diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java index 390279bdbb5d7..4fb95a3080c4b 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java @@ -52,9 +52,9 @@ public final class DateProcessor extends AbstractProcessor { private final List formats; private final List, Function>> dateParsers; - DateProcessor(String tag, @Nullable TemplateScript.Factory timezone, @Nullable TemplateScript.Factory locale, + DateProcessor(String tag, String description, @Nullable TemplateScript.Factory timezone, @Nullable TemplateScript.Factory locale, String field, List formats, String targetField) { - super(tag); + super(tag, description); this.timezone = timezone; this.locale = locale; this.field = field; @@ -137,7 +137,7 @@ public Factory(ScriptService scriptService) { } public DateProcessor create(Map registry, String processorTag, - Map config) throws Exception { + String description, Map config) throws Exception { String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field"); String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", DEFAULT_TARGET_FIELD); String timezoneString = ConfigurationUtils.readOptionalStringProperty(TYPE, processorTag, config, "timezone"); @@ -153,7 +153,8 @@ public DateProcessor create(Map registry, String proc "locale", localeString, scriptService); } List formats = ConfigurationUtils.readList(TYPE, processorTag, config, "formats"); - return new DateProcessor(processorTag, compiledTimezoneTemplate, compiledLocaleTemplate, field, formats, targetField); + return new DateProcessor(processorTag, description, compiledTimezoneTemplate, compiledLocaleTemplate, field, formats, + targetField); } } } diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DissectProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DissectProcessor.java index fa51d047e73e3..2ed4349a69730 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DissectProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DissectProcessor.java @@ -37,8 +37,8 @@ public final class DissectProcessor extends AbstractProcessor { final String appendSeparator; final DissectParser dissectParser; - DissectProcessor(String tag, String field, String pattern, String appendSeparator, boolean ignoreMissing) { - super(tag); + DissectProcessor(String tag, String description, String field, String pattern, String appendSeparator, boolean ignoreMissing) { + super(tag, description); this.field = field; this.ignoreMissing = ignoreMissing; this.pattern = pattern; @@ -66,12 +66,12 @@ public String getType() { public static final class Factory implements Processor.Factory { @Override - public DissectProcessor create(Map registry, String processorTag, Map config) { + public DissectProcessor create(Map registry, String processorTag, String description, Map config) { String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field"); String pattern = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "pattern"); String appendSeparator = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "append_separator", ""); boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false); - return new DissectProcessor(processorTag, field, pattern, appendSeparator, ignoreMissing); + return new DissectProcessor(processorTag, description, field, pattern, appendSeparator, ignoreMissing); } } } diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DotExpanderProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DotExpanderProcessor.java index f4215af8390dd..ea198b7b4f842 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DotExpanderProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DotExpanderProcessor.java @@ -33,8 +33,8 @@ public final class DotExpanderProcessor extends AbstractProcessor { private final String path; private final String field; - DotExpanderProcessor(String tag, String path, String field) { - super(tag); + DotExpanderProcessor(String tag, String description, String path, String field) { + super(tag, description); this.path = path; this.field = field; } @@ -97,7 +97,7 @@ public static final class Factory implements Processor.Factory { @Override public Processor create(Map processorFactories, String tag, - Map config) throws Exception { + String description, Map config) throws Exception { String field = ConfigurationUtils.readStringProperty(TYPE, tag, config, "field"); if (field.contains(".") == false) { throw ConfigurationUtils.newConfigurationException(ConfigurationUtils.TAG_KEY, tag, "field", @@ -117,7 +117,7 @@ public Processor create(Map processorFactories, Strin } String path = ConfigurationUtils.readOptionalStringProperty(TYPE, tag, config, "path"); - return new DotExpanderProcessor(tag, path, field); + return new DotExpanderProcessor(tag, description, path, field); } } } diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/FailProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/FailProcessor.java index 0b62fbf72c8e5..0305322f196f5 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/FailProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/FailProcessor.java @@ -38,8 +38,8 @@ public final class FailProcessor extends AbstractProcessor { private final TemplateScript.Factory message; - FailProcessor(String tag, TemplateScript.Factory message) { - super(tag); + FailProcessor(String tag, String description, TemplateScript.Factory message) { + super(tag, description); this.message = message; } @@ -67,11 +67,11 @@ public Factory(ScriptService scriptService) { @Override public FailProcessor create(Map registry, String processorTag, - Map config) throws Exception { + String description, Map config) throws Exception { String message = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "message"); TemplateScript.Factory compiledTemplate = ConfigurationUtils.compileTemplate(TYPE, processorTag, "message", message, scriptService); - return new FailProcessor(processorTag, compiledTemplate); + return new FailProcessor(processorTag, description, compiledTemplate); } } } diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ForEachProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ForEachProcessor.java index d79a64626b611..5583424b5d36a 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ForEachProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ForEachProcessor.java @@ -54,8 +54,8 @@ public final class ForEachProcessor extends AbstractProcessor implements Wrappin private final Processor processor; private final boolean ignoreMissing; - ForEachProcessor(String tag, String field, Processor processor, boolean ignoreMissing) { - super(tag); + ForEachProcessor(String tag, String description, String field, Processor processor, boolean ignoreMissing) { + super(tag, description); this.field = field; this.processor = processor; this.ignoreMissing = ignoreMissing; @@ -134,7 +134,7 @@ public static final class Factory implements Processor.Factory { @Override public ForEachProcessor create(Map factories, String tag, - Map config) throws Exception { + String description, Map config) throws Exception { String field = readStringProperty(TYPE, tag, config, "field"); boolean ignoreMissing = readBooleanProperty(TYPE, tag, config, "ignore_missing", false); Map> processorConfig = readMap(TYPE, tag, config, "processor"); @@ -145,7 +145,7 @@ public ForEachProcessor create(Map factories, String Map.Entry> entry = entries.iterator().next(); Processor processor = ConfigurationUtils.readProcessor(factories, scriptService, entry.getKey(), entry.getValue()); - return new ForEachProcessor(tag, field, processor, ignoreMissing); + return new ForEachProcessor(tag, description, field, processor, ignoreMissing); } } } diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java index ef587d1761dfb..04c30e68c102c 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java @@ -46,9 +46,9 @@ public final class GrokProcessor extends AbstractProcessor { private final boolean traceMatch; private final boolean ignoreMissing; - GrokProcessor(String tag, Map patternBank, List matchPatterns, String matchField, + GrokProcessor(String tag, String description, Map patternBank, List matchPatterns, String matchField, boolean traceMatch, boolean ignoreMissing, MatcherWatchdog matcherWatchdog) { - super(tag); + super(tag, description); this.matchField = matchField; this.matchPatterns = matchPatterns; this.grok = new Grok(patternBank, combinePatterns(matchPatterns, traceMatch), matcherWatchdog, logger::debug); @@ -148,7 +148,7 @@ public Factory(Map builtinPatterns, MatcherWatchdog matcherWatch @Override public GrokProcessor create(Map registry, String processorTag, - Map config) throws Exception { + String description, Map config) throws Exception { String matchField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field"); List matchPatterns = ConfigurationUtils.readList(TYPE, processorTag, config, "patterns"); boolean traceMatch = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "trace_match", false); @@ -164,7 +164,7 @@ public GrokProcessor create(Map registry, String proc } try { - return new GrokProcessor(processorTag, patternBank, matchPatterns, matchField, traceMatch, ignoreMissing, + return new GrokProcessor(processorTag, description, patternBank, matchPatterns, matchField, traceMatch, ignoreMissing, matcherWatchdog); } catch (Exception e) { throw newConfigurationException(TYPE, processorTag, "patterns", diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GsubProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GsubProcessor.java index 9f3e656bba4b6..0521bee160ddf 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GsubProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GsubProcessor.java @@ -36,8 +36,9 @@ public final class GsubProcessor extends AbstractStringProcessor { private final Pattern pattern; private final String replacement; - GsubProcessor(String tag, String field, Pattern pattern, String replacement, boolean ignoreMissing, String targetField) { - super(tag, field, ignoreMissing, targetField); + GsubProcessor(String tag, String description, String field, Pattern pattern, String replacement, boolean ignoreMissing, + String targetField) { + super(tag, description, ignoreMissing, targetField, field); this.pattern = pattern; this.replacement = replacement; } @@ -67,7 +68,7 @@ public Factory() { } @Override - protected GsubProcessor newProcessor(String processorTag, Map config, String field, + protected GsubProcessor newProcessor(String processorTag, String description, Map config, String field, boolean ignoreMissing, String targetField) { String pattern = readStringProperty(TYPE, processorTag, config, "pattern"); String replacement = readStringProperty(TYPE, processorTag, config, "replacement"); @@ -77,7 +78,7 @@ protected GsubProcessor newProcessor(String processorTag, Map co } catch (Exception e) { throw newConfigurationException(TYPE, processorTag, "pattern", "Invalid regex pattern. " + e.getMessage()); } - return new GsubProcessor(processorTag, field, searchPattern, replacement, ignoreMissing, targetField); + return new GsubProcessor(processorTag, description, field, searchPattern, replacement, ignoreMissing, targetField); } } } diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/HtmlStripProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/HtmlStripProcessor.java index b3bd3882868c0..09e84e05ff971 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/HtmlStripProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/HtmlStripProcessor.java @@ -30,8 +30,8 @@ public final class HtmlStripProcessor extends AbstractStringProcessor { public static final String TYPE = "html_strip"; - HtmlStripProcessor(String tag, String field, boolean ignoreMissing, String targetField) { - super(tag, field, ignoreMissing, targetField); + HtmlStripProcessor(String tag, String description, String field, boolean ignoreMissing, String targetField) { + super(tag, description, ignoreMissing, targetField, field); } @Override @@ -66,9 +66,9 @@ public Factory() { } @Override - protected HtmlStripProcessor newProcessor(String tag, Map config, String field, + protected HtmlStripProcessor newProcessor(String tag, String description, Map config, String field, boolean ignoreMissing, String targetField) { - return new HtmlStripProcessor(tag, field, ignoreMissing, targetField); + return new HtmlStripProcessor(tag, description, field, ignoreMissing, targetField); } } } diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JoinProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JoinProcessor.java index f29a688886194..417eaaee0a05a 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JoinProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JoinProcessor.java @@ -40,8 +40,8 @@ public final class JoinProcessor extends AbstractProcessor { private final String separator; private final String targetField; - JoinProcessor(String tag, String field, String separator, String targetField) { - super(tag); + JoinProcessor(String tag, String description, String field, String separator, String targetField) { + super(tag, description); this.field = field; this.separator = separator; this.targetField = targetField; @@ -80,11 +80,11 @@ public String getType() { public static final class Factory implements Processor.Factory { @Override public JoinProcessor create(Map registry, String processorTag, - Map config) throws Exception { + String description, Map config) throws Exception { String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field"); String separator = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "separator"); String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", field); - return new JoinProcessor(processorTag, field, separator, targetField); + return new JoinProcessor(processorTag, description, field, separator, targetField); } } } diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java index 90a648347cdfd..d191a0f984bae 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java @@ -48,8 +48,8 @@ public final class JsonProcessor extends AbstractProcessor { private final String targetField; private final boolean addToRoot; - JsonProcessor(String tag, String field, String targetField, boolean addToRoot) { - super(tag); + JsonProcessor(String tag, String description, String field, String targetField, boolean addToRoot) { + super(tag, description); this.field = field; this.targetField = targetField; this.addToRoot = addToRoot; @@ -124,7 +124,7 @@ public String getType() { public static final class Factory implements Processor.Factory { @Override public JsonProcessor create(Map registry, String processorTag, - Map config) throws Exception { + String description, Map config) throws Exception { String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field"); String targetField = ConfigurationUtils.readOptionalStringProperty(TYPE, processorTag, config, "target_field"); boolean addToRoot = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "add_to_root", false); @@ -138,7 +138,7 @@ public JsonProcessor create(Map registry, String proc targetField = field; } - return new JsonProcessor(processorTag, field, targetField, addToRoot); + return new JsonProcessor(processorTag, description, field, targetField, addToRoot); } } } diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/KeyValueProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/KeyValueProcessor.java index 69c7e9ff75187..50fe063b8cca2 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/KeyValueProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/KeyValueProcessor.java @@ -52,10 +52,10 @@ public final class KeyValueProcessor extends AbstractProcessor { private final boolean ignoreMissing; private final Consumer execution; - KeyValueProcessor(String tag, String field, String fieldSplit, String valueSplit, Set includeKeys, + KeyValueProcessor(String tag, String description, String field, String fieldSplit, String valueSplit, Set includeKeys, Set excludeKeys, String targetField, boolean ignoreMissing, String trimKey, String trimValue, boolean stripBrackets, String prefix) { - super(tag); + super(tag, description); this.field = field; this.targetField = targetField; this.fieldSplit = fieldSplit; @@ -201,7 +201,7 @@ public String getType() { public static class Factory implements Processor.Factory { @Override public KeyValueProcessor create(Map registry, String processorTag, - Map config) throws Exception { + String description, Map config) throws Exception { String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field"); String targetField = ConfigurationUtils.readOptionalStringProperty(TYPE, processorTag, config, "target_field"); String fieldSplit = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field_split"); @@ -223,7 +223,7 @@ public KeyValueProcessor create(Map registry, String } boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false); return new KeyValueProcessor( - processorTag, field, fieldSplit, valueSplit, includeKeys, excludeKeys, targetField, ignoreMissing, + processorTag, description, field, fieldSplit, valueSplit, includeKeys, excludeKeys, targetField, ignoreMissing, trimKey, trimValue, stripBrackets, prefix ); } diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/LowercaseProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/LowercaseProcessor.java index 6c14dbdabba78..ade4c7e9db659 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/LowercaseProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/LowercaseProcessor.java @@ -31,8 +31,8 @@ public final class LowercaseProcessor extends AbstractStringProcessor { public static final String TYPE = "lowercase"; - LowercaseProcessor(String processorTag, String field, boolean ignoreMissing, String targetField) { - super(processorTag, field, ignoreMissing, targetField); + LowercaseProcessor(String processorTag, String description, String field, boolean ignoreMissing, String targetField) { + super(processorTag, description, ignoreMissing, targetField, field); } public static String apply(String value) { @@ -56,9 +56,9 @@ public Factory() { } @Override - protected LowercaseProcessor newProcessor(String tag, Map config, String field, + protected LowercaseProcessor newProcessor(String tag, String description, Map config, String field, boolean ignoreMissing, String targetField) { - return new LowercaseProcessor(tag, field, ignoreMissing, targetField); + return new LowercaseProcessor(tag, description, field, ignoreMissing, targetField); } } } diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RemoveProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RemoveProcessor.java index 6002abb9e67a3..1ba2c88bad0ac 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RemoveProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RemoveProcessor.java @@ -41,8 +41,8 @@ public final class RemoveProcessor extends AbstractProcessor { private final List fields; private final boolean ignoreMissing; - RemoveProcessor(String tag, List fields, boolean ignoreMissing) { - super(tag); + RemoveProcessor(String tag, String description, List fields, boolean ignoreMissing) { + super(tag, description); this.fields = new ArrayList<>(fields); this.ignoreMissing = ignoreMissing; } @@ -81,7 +81,7 @@ public Factory(ScriptService scriptService) { @Override public RemoveProcessor create(Map registry, String processorTag, - Map config) throws Exception { + String description, Map config) throws Exception { final List fields = new ArrayList<>(); final Object field = ConfigurationUtils.readObject(TYPE, processorTag, config, "field"); if (field instanceof List) { @@ -96,7 +96,7 @@ public RemoveProcessor create(Map registry, String pr .map(f -> ConfigurationUtils.compileTemplate(TYPE, processorTag, "field", f, scriptService)) .collect(Collectors.toList()); boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false); - return new RemoveProcessor(processorTag, compiledTemplates, ignoreMissing); + return new RemoveProcessor(processorTag, description, compiledTemplates, ignoreMissing); } } } diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RenameProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RenameProcessor.java index 2abd920048f73..c04d56754a764 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RenameProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RenameProcessor.java @@ -39,8 +39,9 @@ public final class RenameProcessor extends AbstractProcessor { private final TemplateScript.Factory targetField; private final boolean ignoreMissing; - RenameProcessor(String tag, TemplateScript.Factory field, TemplateScript.Factory targetField, boolean ignoreMissing) { - super(tag); + RenameProcessor(String tag, String description, TemplateScript.Factory field, TemplateScript.Factory targetField, + boolean ignoreMissing) { + super(tag, description); this.field = field; this.targetField = targetField; this.ignoreMissing = ignoreMissing; @@ -104,7 +105,7 @@ public Factory(ScriptService scriptService) { @Override public RenameProcessor create(Map registry, String processorTag, - Map config) throws Exception { + String description, Map config) throws Exception { String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field"); TemplateScript.Factory fieldTemplate = ConfigurationUtils.compileTemplate(TYPE, processorTag, "field", field, scriptService); @@ -112,7 +113,7 @@ public RenameProcessor create(Map registry, String pr TemplateScript.Factory targetFieldTemplate = ConfigurationUtils.compileTemplate(TYPE, processorTag, "target_field", targetField, scriptService); boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false); - return new RenameProcessor(processorTag, fieldTemplate, targetFieldTemplate , ignoreMissing); + return new RenameProcessor(processorTag, description, fieldTemplate, targetFieldTemplate , ignoreMissing); } } } diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java index e6ac9e71839ec..ddc5d78be629f 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java @@ -55,11 +55,12 @@ public final class ScriptProcessor extends AbstractProcessor { * Processor that evaluates a script with an ingest document in its context * * @param tag The processor's tag. + * @param description The processor's description. * @param script The {@link Script} to execute. * @param scriptService The {@link ScriptService} used to execute the script. */ - ScriptProcessor(String tag, Script script, ScriptService scriptService) { - super(tag); + ScriptProcessor(String tag, String description, Script script, ScriptService scriptService) { + super(tag, description); this.script = script; this.scriptService = scriptService; } @@ -95,7 +96,7 @@ public Factory(ScriptService scriptService) { @Override public ScriptProcessor create(Map registry, String processorTag, - Map config) throws Exception { + String description, Map config) throws Exception { try (XContentBuilder builder = XContentBuilder.builder(JsonXContent.jsonXContent).map(config); InputStream stream = BytesReference.bytes(builder).streamInput(); XContentParser parser = XContentType.JSON.xContent().createParser(NamedXContentRegistry.EMPTY, @@ -111,7 +112,7 @@ public ScriptProcessor create(Map registry, String pr throw newConfigurationException(TYPE, processorTag, null, e); } - return new ScriptProcessor(processorTag, script, scriptService); + return new ScriptProcessor(processorTag, description, script, scriptService); } } } diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SetProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SetProcessor.java index 0af51e5b895e4..6acc0c45fde10 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SetProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SetProcessor.java @@ -41,12 +41,12 @@ public final class SetProcessor extends AbstractProcessor { private final TemplateScript.Factory field; private final ValueSource value; - SetProcessor(String tag, TemplateScript.Factory field, ValueSource value) { - this(tag, field, value, true); + SetProcessor(String tag, String description, TemplateScript.Factory field, ValueSource value) { + this(tag, description, field, value, true); } - SetProcessor(String tag, TemplateScript.Factory field, ValueSource value, boolean overrideEnabled) { - super(tag); + SetProcessor(String tag, String description, TemplateScript.Factory field, ValueSource value, boolean overrideEnabled) { + super(tag, description); this.overrideEnabled = overrideEnabled; this.field = field; this.value = value; @@ -87,7 +87,7 @@ public Factory(ScriptService scriptService) { @Override public SetProcessor create(Map registry, String processorTag, - Map config) throws Exception { + String description, Map config) throws Exception { String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field"); Object value = ConfigurationUtils.readObject(TYPE, processorTag, config, "value"); boolean overrideEnabled = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "override", true); @@ -95,6 +95,7 @@ public SetProcessor create(Map registry, String proce "field", field, scriptService); return new SetProcessor( processorTag, + description, compiledTemplate, ValueSource.wrap(value, scriptService), overrideEnabled); diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SortProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SortProcessor.java index a29cc34652479..eb9b54aea2562 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SortProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SortProcessor.java @@ -73,8 +73,8 @@ public static SortOrder fromString(String value) { private final SortOrder order; private final String targetField; - SortProcessor(String tag, String field, SortOrder order, String targetField) { - super(tag); + SortProcessor(String tag, String description, String field, SortOrder order, String targetField) { + super(tag, description); this.field = field; this.order = order; this.targetField = targetField; @@ -122,7 +122,7 @@ public static final class Factory implements Processor.Factory { @Override public SortProcessor create(Map registry, String processorTag, - Map config) throws Exception { + String description, Map config) throws Exception { String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, FIELD); String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", field); try { @@ -133,7 +133,7 @@ public SortProcessor create(Map registry, String proc config, ORDER, DEFAULT_ORDER)); - return new SortProcessor(processorTag, field, direction, targetField); + return new SortProcessor(processorTag, description, field, direction, targetField); } catch (IllegalArgumentException e) { throw ConfigurationUtils.newConfigurationException(TYPE, processorTag, ORDER, e.getMessage()); } diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SplitProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SplitProcessor.java index e43c0d4c20fb5..6033d558958d0 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SplitProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SplitProcessor.java @@ -44,8 +44,9 @@ public final class SplitProcessor extends AbstractProcessor { private final boolean preserveTrailing; private final String targetField; - SplitProcessor(String tag, String field, String separator, boolean ignoreMissing, boolean preserveTrailing, String targetField) { - super(tag); + SplitProcessor(String tag, String description, String field, String separator, boolean ignoreMissing, boolean preserveTrailing, + String targetField) { + super(tag, description); this.field = field; this.separator = separator; this.ignoreMissing = ignoreMissing; @@ -96,13 +97,13 @@ public String getType() { public static class Factory implements Processor.Factory { @Override public SplitProcessor create(Map registry, String processorTag, - Map config) throws Exception { + String description, Map config) throws Exception { String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field"); boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false); boolean preserveTrailing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "preserve_trailing", false); String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", field); String separator = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "separator"); - return new SplitProcessor(processorTag, field, separator, ignoreMissing, preserveTrailing, targetField); + return new SplitProcessor(processorTag, description, field, separator, ignoreMissing, preserveTrailing, targetField); } } } diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/TrimProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/TrimProcessor.java index d1b3c87785424..9573585d0602c 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/TrimProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/TrimProcessor.java @@ -29,8 +29,8 @@ public final class TrimProcessor extends AbstractStringProcessor { public static final String TYPE = "trim"; - TrimProcessor(String processorTag, String field, boolean ignoreMissing, String targetField) { - super(processorTag, field, ignoreMissing, targetField); + TrimProcessor(String processorTag, String description, String field, boolean ignoreMissing, String targetField) { + super(processorTag, description, ignoreMissing, targetField, field); } @Override @@ -50,9 +50,9 @@ public Factory() { } @Override - protected TrimProcessor newProcessor(String tag, Map config, String field, + protected TrimProcessor newProcessor(String tag, String description, Map config, String field, boolean ignoreMissing, String targetField) { - return new TrimProcessor(tag, field, ignoreMissing, targetField); + return new TrimProcessor(tag, description, field, ignoreMissing, targetField); } } } diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/URLDecodeProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/URLDecodeProcessor.java index fa9d377714ee9..27b888196cb6c 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/URLDecodeProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/URLDecodeProcessor.java @@ -30,8 +30,8 @@ public final class URLDecodeProcessor extends AbstractStringProcessor { public static final String TYPE = "urldecode"; - URLDecodeProcessor(String processorTag, String field, boolean ignoreMissing, String targetField) { - super(processorTag, field, ignoreMissing, targetField); + URLDecodeProcessor(String processorTag, String description, String field, boolean ignoreMissing, String targetField) { + super(processorTag, description, ignoreMissing, targetField, field); } public static String apply(String value) { @@ -59,9 +59,9 @@ public Factory() { } @Override - protected URLDecodeProcessor newProcessor(String tag, Map config, String field, + protected URLDecodeProcessor newProcessor(String tag, String description, Map config, String field, boolean ignoreMissing, String targetField) { - return new URLDecodeProcessor(tag, field, ignoreMissing, targetField); + return new URLDecodeProcessor(tag, description, field, ignoreMissing, targetField); } } } diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UppercaseProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UppercaseProcessor.java index 4503bfc02f71e..6a8139a685a09 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UppercaseProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UppercaseProcessor.java @@ -30,8 +30,8 @@ public final class UppercaseProcessor extends AbstractStringProcessor { public static final String TYPE = "uppercase"; - UppercaseProcessor(String processorTag, String field, boolean ignoreMissing, String targetField) { - super(processorTag, field, ignoreMissing, targetField); + UppercaseProcessor(String processorTag, String description, String field, boolean ignoreMissing, String targetField) { + super(processorTag, description, ignoreMissing, targetField, field); } public static String apply(String value) { @@ -55,9 +55,9 @@ public Factory() { } @Override - protected UppercaseProcessor newProcessor(String tag, Map config, String field, + protected UppercaseProcessor newProcessor(String tag, String description, Map config, String field, boolean ignoreMissing, String targetField) { - return new UppercaseProcessor(tag, field, ignoreMissing, targetField); + return new UppercaseProcessor(tag, description, field, ignoreMissing, targetField); } } } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AbstractStringProcessorFactoryTestCase.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AbstractStringProcessorFactoryTestCase.java index ba6a2be73465e..1e5b2ae2431eb 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AbstractStringProcessorFactoryTestCase.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AbstractStringProcessorFactoryTestCase.java @@ -47,7 +47,7 @@ public void testCreate() throws Exception { Map config = new HashMap<>(); config.put("field", fieldName); - AbstractStringProcessor processor = factory.create(null, processorTag, modifyConfig(config)); + AbstractStringProcessor processor = factory.create(null, processorTag, null, modifyConfig(config)); assertThat(processor.getTag(), equalTo(processorTag)); assertThat(processor.getField(), equalTo(fieldName)); assertThat(processor.isIgnoreMissing(), is(false)); @@ -64,7 +64,7 @@ public void testCreateWithIgnoreMissing() throws Exception { config.put("field", fieldName); config.put("ignore_missing", true); - AbstractStringProcessor processor = factory.create(null, processorTag, modifyConfig(config)); + AbstractStringProcessor processor = factory.create(null, processorTag, null, modifyConfig(config)); assertThat(processor.getTag(), equalTo(processorTag)); assertThat(processor.getField(), equalTo(fieldName)); assertThat(processor.isIgnoreMissing(), is(true)); @@ -82,7 +82,7 @@ public void testCreateWithTargetField() throws Exception { config.put("field", fieldName); config.put("target_field", targetFieldName); - AbstractStringProcessor processor = factory.create(null, processorTag, modifyConfig(config)); + AbstractStringProcessor processor = factory.create(null, processorTag, null, modifyConfig(config)); assertThat(processor.getTag(), equalTo(processorTag)); assertThat(processor.getField(), equalTo(fieldName)); assertThat(processor.isIgnoreMissing(), is(false)); @@ -94,7 +94,7 @@ public void testCreateMissingField() throws Exception { AbstractStringProcessor.Factory factory = newFactory(); Map config = new HashMap<>(); try { - factory.create(null, null, config); + factory.create(null, null, null, config); fail("factory create should have failed"); } catch(ElasticsearchParseException e) { assertThat(e.getMessage(), equalTo("[field] required property is missing")); diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorFactoryTests.java index 4544140737612..99398316c1ef6 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorFactoryTests.java @@ -52,7 +52,7 @@ public void testCreate() throws Exception { } config.put("value", value); String processorTag = randomAlphaOfLength(10); - AppendProcessor appendProcessor = factory.create(null, processorTag, config); + AppendProcessor appendProcessor = factory.create(null, processorTag, null, config); assertThat(appendProcessor.getTag(), equalTo(processorTag)); assertThat(appendProcessor.getField().newInstance(Collections.emptyMap()).execute(), equalTo("field1")); assertThat(appendProcessor.getValue().copyAndResolve(Collections.emptyMap()), equalTo(value)); @@ -62,7 +62,7 @@ public void testCreateNoFieldPresent() throws Exception { Map config = new HashMap<>(); config.put("value", "value1"); try { - factory.create(null, null, config); + factory.create(null, null, null, config); fail("factory create should have failed"); } catch(ElasticsearchParseException e) { assertThat(e.getMessage(), equalTo("[field] required property is missing")); @@ -73,7 +73,7 @@ public void testCreateNoValuePresent() throws Exception { Map config = new HashMap<>(); config.put("field", "field1"); try { - factory.create(null, null, config); + factory.create(null, null, null, config); fail("factory create should have failed"); } catch(ElasticsearchParseException e) { assertThat(e.getMessage(), equalTo("[value] required property is missing")); @@ -85,7 +85,7 @@ public void testCreateNullValue() throws Exception { config.put("field", "field1"); config.put("value", null); try { - factory.create(null, null, config); + factory.create(null, null, null, config); fail("factory create should have failed"); } catch(ElasticsearchParseException e) { assertThat(e.getMessage(), equalTo("[value] required property is missing")); @@ -98,7 +98,7 @@ public void testInvalidMustacheTemplate() throws Exception { config.put("field", "{{field1}}"); config.put("value", "value1"); String processorTag = randomAlphaOfLength(10); - ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, config)); + ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, null, config)); assertThat(exception.getMessage(), equalTo("java.lang.RuntimeException: could not compile script")); assertThat(exception.getMetadata("es.processor_tag").get(0), equalTo(processorTag)); } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorFactoryTests.java index f3396da64eb5f..862349daaff1d 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorFactoryTests.java @@ -39,7 +39,7 @@ public void testCreate() throws Exception { config.put("field", "field1"); config.put("type", type.toString()); String processorTag = randomAlphaOfLength(10); - ConvertProcessor convertProcessor = factory.create(null, processorTag, config); + ConvertProcessor convertProcessor = factory.create(null, processorTag, null, config); assertThat(convertProcessor.getTag(), equalTo(processorTag)); assertThat(convertProcessor.getField(), equalTo("field1")); assertThat(convertProcessor.getTargetField(), equalTo("field1")); @@ -54,7 +54,7 @@ public void testCreateUnsupportedType() throws Exception { config.put("field", "field1"); config.put("type", type); try { - factory.create(null, null, config); + factory.create(null, null, null, config); fail("factory create should have failed"); } catch (ElasticsearchParseException e) { assertThat(e.getMessage(), Matchers.equalTo("[type] type [" + type + "] not supported, cannot convert field.")); @@ -70,7 +70,7 @@ public void testCreateNoFieldPresent() throws Exception { String type = "type-" + randomAlphaOfLengthBetween(1, 10); config.put("type", type); try { - factory.create(null, null, config); + factory.create(null, null, null, config); fail("factory create should have failed"); } catch (ElasticsearchParseException e) { assertThat(e.getMessage(), Matchers.equalTo("[field] required property is missing")); @@ -82,7 +82,7 @@ public void testCreateNoTypePresent() throws Exception { Map config = new HashMap<>(); config.put("field", "field1"); try { - factory.create(null, null, config); + factory.create(null, null, null, config); fail("factory create should have failed"); } catch (ElasticsearchParseException e) { assertThat(e.getMessage(), Matchers.equalTo("[type] required property is missing")); @@ -97,7 +97,7 @@ public void testCreateWithExplicitTargetField() throws Exception { config.put("target_field", "field2"); config.put("type", type.toString()); String processorTag = randomAlphaOfLength(10); - ConvertProcessor convertProcessor = factory.create(null, processorTag, config); + ConvertProcessor convertProcessor = factory.create(null, processorTag, null, config); assertThat(convertProcessor.getTag(), equalTo(processorTag)); assertThat(convertProcessor.getField(), equalTo("field1")); assertThat(convertProcessor.getTargetField(), equalTo("field2")); @@ -113,7 +113,7 @@ public void testCreateWithIgnoreMissing() throws Exception { config.put("type", type.toString()); config.put("ignore_missing", true); String processorTag = randomAlphaOfLength(10); - ConvertProcessor convertProcessor = factory.create(null, processorTag, config); + ConvertProcessor convertProcessor = factory.create(null, processorTag, null, config); assertThat(convertProcessor.getTag(), equalTo(processorTag)); assertThat(convertProcessor.getField(), equalTo("field1")); assertThat(convertProcessor.getTargetField(), equalTo("field1")); diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CsvProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CsvProcessorFactoryTests.java index ede5d75876104..17f7308b7cb05 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CsvProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CsvProcessorFactoryTests.java @@ -41,7 +41,7 @@ public void testProcessorIsCreated() { properties.put("empty_value", "empty"); properties.put("trim", true); properties.put("ignore_missing", true); - CsvProcessor csv = factory.create(null, "csv", properties); + CsvProcessor csv = factory.create(null, "csv", null, properties); assertThat(csv, notNullValue()); assertThat(csv.field, equalTo("field")); assertThat(csv.headers, equalTo(new String[]{"target"})); diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameFactoryTests.java index 67027f6ab6784..cd3e597f21a7f 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameFactoryTests.java @@ -38,7 +38,7 @@ public void testDefaults() throws Exception { config.put("field", "_field"); config.put("date_rounding", "y"); - DateIndexNameProcessor processor = factory.create(null, null, config); + DateIndexNameProcessor processor = factory.create(null, null, null, config); assertThat(processor.getDateFormats().size(), Matchers.equalTo(1)); assertThat(processor.getField(), Matchers.equalTo("_field")); assertThat(processor.getIndexNamePrefixTemplate().newInstance(Collections.emptyMap()).execute(), Matchers.equalTo("")); @@ -55,7 +55,7 @@ public void testSpecifyOptionalSettings() throws Exception { config.put("date_rounding", "y"); config.put("date_formats", Arrays.asList("UNIX", "UNIX_MS")); - DateIndexNameProcessor processor = factory.create(null, null, config); + DateIndexNameProcessor processor = factory.create(null, null, null, config); assertThat(processor.getDateFormats().size(), Matchers.equalTo(2)); config = new HashMap<>(); @@ -64,7 +64,7 @@ public void testSpecifyOptionalSettings() throws Exception { config.put("date_rounding", "y"); config.put("index_name_format", "yyyyMMdd"); - processor = factory.create(null, null, config); + processor = factory.create(null, null, null, config); assertThat(processor.getIndexNameFormatTemplate().newInstance(Collections.emptyMap()).execute(), Matchers.equalTo("yyyyMMdd")); config = new HashMap<>(); @@ -73,7 +73,7 @@ public void testSpecifyOptionalSettings() throws Exception { config.put("date_rounding", "y"); config.put("timezone", "+02:00"); - processor = factory.create(null, null, config); + processor = factory.create(null, null, null, config); assertThat(processor.getTimezone(), Matchers.equalTo(ZoneOffset.ofHours(2))); config = new HashMap<>(); @@ -81,7 +81,7 @@ public void testSpecifyOptionalSettings() throws Exception { config.put("index_name_prefix", "_prefix"); config.put("date_rounding", "y"); - processor = factory.create(null, null, config); + processor = factory.create(null, null, null, config); assertThat(processor.getIndexNamePrefixTemplate().newInstance(Collections.emptyMap()).execute(), Matchers.equalTo("_prefix")); } @@ -89,12 +89,12 @@ public void testRequiredFields() throws Exception { DateIndexNameProcessor.Factory factory = new DateIndexNameProcessor.Factory(TestTemplateService.instance()); Map config = new HashMap<>(); config.put("date_rounding", "y"); - ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, config)); + ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config)); assertThat(e.getMessage(), Matchers.equalTo("[field] required property is missing")); config.clear(); config.put("field", "_field"); - e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, config)); + e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config)); assertThat(e.getMessage(), Matchers.equalTo("[date_rounding] required property is missing")); } } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorFactoryTests.java index 7d227b222696f..58336d8911ed5 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorFactoryTests.java @@ -49,7 +49,7 @@ public void testBuildDefaults() throws Exception { config.put("field", sourceField); config.put("formats", Collections.singletonList("dd/MM/yyyyy")); String processorTag = randomAlphaOfLength(10); - DateProcessor processor = factory.create(null, processorTag, config); + DateProcessor processor = factory.create(null, processorTag, null, config); assertThat(processor.getTag(), equalTo(processorTag)); assertThat(processor.getField(), equalTo(sourceField)); assertThat(processor.getTargetField(), equalTo(DateProcessor.DEFAULT_TARGET_FIELD)); @@ -65,7 +65,7 @@ public void testMatchFieldIsMandatory() throws Exception { config.put("formats", Collections.singletonList("dd/MM/yyyyy")); try { - factory.create(null, null, config); + factory.create(null, null, null, config); fail("processor creation should have failed"); } catch(ElasticsearchParseException e) { assertThat(e.getMessage(), containsString("[field] required property is missing")); @@ -80,7 +80,7 @@ public void testMatchFormatsIsMandatory() throws Exception { config.put("target_field", targetField); try { - factory.create(null, null, config); + factory.create(null, null, null, config); fail("processor creation should have failed"); } catch(ElasticsearchParseException e) { assertThat(e.getMessage(), containsString("[formats] required property is missing")); @@ -95,7 +95,7 @@ public void testParseLocale() throws Exception { Locale locale = randomFrom(Locale.GERMANY, Locale.FRENCH, Locale.ROOT); config.put("locale", locale.toLanguageTag()); - DateProcessor processor = factory.create(null, null, config); + DateProcessor processor = factory.create(null, null, null, config); assertThat(processor.getLocale().newInstance(Collections.emptyMap()).execute(), equalTo(locale.toLanguageTag())); } @@ -107,7 +107,7 @@ public void testParseTimezone() throws Exception { ZoneId timezone = randomZone(); config.put("timezone", timezone.getId()); - DateProcessor processor = factory.create(null, null, config); + DateProcessor processor = factory.create(null, null, null, config); assertThat(processor.getTimezone().newInstance(Collections.emptyMap()).execute(), equalTo(timezone.getId())); } @@ -117,7 +117,7 @@ public void testParseMatchFormats() throws Exception { config.put("field", sourceField); config.put("formats", Arrays.asList("dd/MM/yyyy", "dd-MM-yyyy")); - DateProcessor processor = factory.create(null, null, config); + DateProcessor processor = factory.create(null, null, null, config); assertThat(processor.getFormats(), equalTo(Arrays.asList("dd/MM/yyyy", "dd-MM-yyyy"))); } @@ -128,7 +128,7 @@ public void testParseMatchFormatsFailure() throws Exception { config.put("formats", "dd/MM/yyyy"); try { - factory.create(null, null, config); + factory.create(null, null, null, config); fail("processor creation should have failed"); } catch(ElasticsearchParseException e) { assertThat(e.getMessage(), containsString("[formats] property isn't a list, but of type [java.lang.String]")); @@ -143,7 +143,7 @@ public void testParseTargetField() throws Exception { config.put("target_field", targetField); config.put("formats", Arrays.asList("dd/MM/yyyy", "dd-MM-yyyy")); - DateProcessor processor = factory.create(null, null, config); + DateProcessor processor = factory.create(null, null, null, config); assertThat(processor.getTargetField(), equalTo(targetField)); } } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DissectProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DissectProcessorFactoryTests.java index ba1b2bd1eb576..f8377acd26839 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DissectProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DissectProcessorFactoryTests.java @@ -47,7 +47,7 @@ public void testCreate() { config.put("append_separator", appendSeparator); config.put("ignore_missing", true); - DissectProcessor processor = factory.create(null, processorTag, config); + DissectProcessor processor = factory.create(null, processorTag, null, config); assertThat(processor.getTag(), equalTo(processorTag)); assertThat(processor.field, equalTo(fieldName)); assertThat(processor.pattern, equalTo(pattern)); @@ -60,7 +60,7 @@ public void testCreateMissingField() { DissectProcessor.Factory factory = new DissectProcessor.Factory(); Map config = new HashMap<>(); config.put("pattern", "%{a},%{b},%{c}"); - Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", config)); + Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", null, config)); assertThat(e.getMessage(), Matchers.equalTo("[field] required property is missing")); } @@ -68,7 +68,7 @@ public void testCreateMissingPattern() { DissectProcessor.Factory factory = new DissectProcessor.Factory(); Map config = new HashMap<>(); config.put("field", randomAlphaOfLength(10)); - Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", config)); + Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", null, config)); assertThat(e.getMessage(), Matchers.equalTo("[pattern] required property is missing")); } @@ -77,7 +77,7 @@ public void testCreateMissingOptionals() { Map config = new HashMap<>(); config.put("pattern", "%{a},%{b},%{c}"); config.put("field", randomAlphaOfLength(10)); - DissectProcessor processor = factory.create(null, "_tag", config); + DissectProcessor processor = factory.create(null, "_tag", null, config); assertThat(processor.appendSeparator, equalTo("")); assertThat(processor.ignoreMissing, is(false)); } @@ -87,6 +87,6 @@ public void testCreateBadPattern() { Map config = new HashMap<>(); config.put("pattern", "no keys defined"); config.put("field", randomAlphaOfLength(10)); - expectThrows(DissectException.class, () -> factory.create(null, "_tag", config)); + expectThrows(DissectException.class, () -> factory.create(null, "_tag", null, config)); } } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorFactoryTests.java index be0695924ef6c..dce33e84c7ddb 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorFactoryTests.java @@ -36,13 +36,13 @@ public void testCreate() throws Exception { Map config = new HashMap<>(); config.put("field", "_field.field"); config.put("path", "_path"); - DotExpanderProcessor processor = (DotExpanderProcessor) factory.create(null, "_tag", config); + DotExpanderProcessor processor = (DotExpanderProcessor) factory.create(null, "_tag", null, config); assertThat(processor.getField(), equalTo("_field.field")); assertThat(processor.getPath(), equalTo("_path")); config = new HashMap<>(); config.put("field", "_field.field"); - processor = (DotExpanderProcessor) factory.create(null, "_tag", config); + processor = (DotExpanderProcessor) factory.create(null, "_tag", null, config); assertThat(processor.getField(), equalTo("_field.field")); assertThat(processor.getPath(), nullValue()); } @@ -55,7 +55,7 @@ public void testValidFields() throws Exception { Map config = new HashMap<>(); config.put("field", field); config.put("path", "_path"); - DotExpanderProcessor processor = (DotExpanderProcessor) factory.create(null, "_tag", config); + DotExpanderProcessor processor = (DotExpanderProcessor) factory.create(null, "_tag", null, config); assertThat(processor.getField(), equalTo(field)); assertThat(processor.getPath(), equalTo("_path")); } @@ -66,7 +66,7 @@ public void testCreate_fieldMissing() throws Exception { Map config = new HashMap<>(); config.put("path", "_path"); - Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", config)); + Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", null, config)); assertThat(e.getMessage(), equalTo("[field] required property is missing")); } @@ -76,7 +76,7 @@ public void testCreate_invalidFields() throws Exception { for (String field : fields) { Map config = new HashMap<>(); config.put("field", field); - Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", config)); + Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", null, config)); assertThat(e.getMessage(), equalTo("[field] field does not contain a dot")); } @@ -84,7 +84,7 @@ public void testCreate_invalidFields() throws Exception { for (String field : fields) { Map config = new HashMap<>(); config.put("field", field); - Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", config)); + Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", null, config)); assertThat(e.getMessage(), equalTo("[field] Field can't start or end with a dot")); } @@ -92,7 +92,7 @@ public void testCreate_invalidFields() throws Exception { for (String field : fields) { Map config = new HashMap<>(); config.put("field", field); - Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", config)); + Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", null, config)); assertThat(e.getMessage(), equalTo("[field] No space between dots")); } } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorFactoryTests.java index 78891e0a02a64..9e8821ca1bdd4 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorFactoryTests.java @@ -44,7 +44,7 @@ public void testCreate() throws Exception { Map config = new HashMap<>(); config.put("message", "error"); String processorTag = randomAlphaOfLength(10); - FailProcessor failProcessor = factory.create(null, processorTag, config); + FailProcessor failProcessor = factory.create(null, processorTag, null, config); assertThat(failProcessor.getTag(), equalTo(processorTag)); assertThat(failProcessor.getMessage().newInstance(Collections.emptyMap()).execute(), equalTo("error")); } @@ -52,7 +52,7 @@ public void testCreate() throws Exception { public void testCreateMissingMessageField() throws Exception { Map config = new HashMap<>(); try { - factory.create(null, null, config); + factory.create(null, null, null, config); fail("factory create should have failed"); } catch(ElasticsearchParseException e) { assertThat(e.getMessage(), equalTo("[message] required property is missing")); @@ -64,7 +64,7 @@ public void testInvalidMustacheTemplate() throws Exception { Map config = new HashMap<>(); config.put("message", "{{error}}"); String processorTag = randomAlphaOfLength(10); - ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, config)); + ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, null, config)); assertThat(exception.getMessage(), equalTo("java.lang.RuntimeException: could not compile script")); assertThat(exception.getMetadata("es.processor_tag").get(0), equalTo(processorTag)); } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorFactoryTests.java index bc522caf9b919..ff43a2f81cb71 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorFactoryTests.java @@ -42,13 +42,13 @@ public class ForEachProcessorFactoryTests extends ESTestCase { public void testCreate() throws Exception { Processor processor = new TestProcessor(ingestDocument -> { }); Map registry = new HashMap<>(); - registry.put("_name", (r, t, c) -> processor); + registry.put("_name", (r, t, description, c) -> processor); ForEachProcessor.Factory forEachFactory = new ForEachProcessor.Factory(scriptService); Map config = new HashMap<>(); config.put("field", "_field"); config.put("processor", Collections.singletonMap("_name", Collections.emptyMap())); - ForEachProcessor forEachProcessor = forEachFactory.create(registry, null, config); + ForEachProcessor forEachProcessor = forEachFactory.create(registry, null, null, config); assertThat(forEachProcessor, Matchers.notNullValue()); assertThat(forEachProcessor.getField(), equalTo("_field")); assertThat(forEachProcessor.getInnerProcessor(), Matchers.sameInstance(processor)); @@ -58,14 +58,14 @@ public void testCreate() throws Exception { public void testSetIgnoreMissing() throws Exception { Processor processor = new TestProcessor(ingestDocument -> { }); Map registry = new HashMap<>(); - registry.put("_name", (r, t, c) -> processor); + registry.put("_name", (r, t, description, c) -> processor); ForEachProcessor.Factory forEachFactory = new ForEachProcessor.Factory(scriptService); Map config = new HashMap<>(); config.put("field", "_field"); config.put("processor", Collections.singletonMap("_name", Collections.emptyMap())); config.put("ignore_missing", true); - ForEachProcessor forEachProcessor = forEachFactory.create(registry, null, config); + ForEachProcessor forEachProcessor = forEachFactory.create(registry, null, null, config); assertThat(forEachProcessor, Matchers.notNullValue()); assertThat(forEachProcessor.getField(), equalTo("_field")); assertThat(forEachProcessor.getInnerProcessor(), Matchers.sameInstance(processor)); @@ -75,8 +75,8 @@ public void testSetIgnoreMissing() throws Exception { public void testCreateWithTooManyProcessorTypes() throws Exception { Processor processor = new TestProcessor(ingestDocument -> { }); Map registry = new HashMap<>(); - registry.put("_first", (r, t, c) -> processor); - registry.put("_second", (r, t, c) -> processor); + registry.put("_first", (r, t, description, c) -> processor); + registry.put("_second", (r, t, description, c) -> processor); ForEachProcessor.Factory forEachFactory = new ForEachProcessor.Factory(scriptService); Map config = new HashMap<>(); @@ -85,7 +85,7 @@ public void testCreateWithTooManyProcessorTypes() throws Exception { processorTypes.put("_first", Collections.emptyMap()); processorTypes.put("_second", Collections.emptyMap()); config.put("processor", processorTypes); - Exception exception = expectThrows(ElasticsearchParseException.class, () -> forEachFactory.create(registry, null, config)); + Exception exception = expectThrows(ElasticsearchParseException.class, () -> forEachFactory.create(registry, null, null, config)); assertThat(exception.getMessage(), equalTo("[processor] Must specify exactly one processor type")); } @@ -95,18 +95,18 @@ public void testCreateWithNonExistingProcessorType() throws Exception { config.put("field", "_field"); config.put("processor", Collections.singletonMap("_name", Collections.emptyMap())); Exception expectedException = expectThrows(ElasticsearchParseException.class, - () -> forEachFactory.create(Collections.emptyMap(), null, config)); + () -> forEachFactory.create(Collections.emptyMap(), null, null, config)); assertThat(expectedException.getMessage(), equalTo("No processor type exists with name [_name]")); } public void testCreateWithMissingField() throws Exception { Processor processor = new TestProcessor(ingestDocument -> { }); Map registry = new HashMap<>(); - registry.put("_name", (r, t, c) -> processor); + registry.put("_name", (r, t, description, c) -> processor); ForEachProcessor.Factory forEachFactory = new ForEachProcessor.Factory(scriptService); Map config = new HashMap<>(); config.put("processor", Collections.singletonList(Collections.singletonMap("_name", Collections.emptyMap()))); - Exception exception = expectThrows(Exception.class, () -> forEachFactory.create(registry, null, config)); + Exception exception = expectThrows(Exception.class, () -> forEachFactory.create(registry, null, null, config)); assertThat(exception.getMessage(), equalTo("[field] required property is missing")); } @@ -114,7 +114,7 @@ public void testCreateWithMissingProcessor() { ForEachProcessor.Factory forEachFactory = new ForEachProcessor.Factory(scriptService); Map config = new HashMap<>(); config.put("field", "_field"); - Exception exception = expectThrows(Exception.class, () -> forEachFactory.create(Collections.emptyMap(), null, config)); + Exception exception = expectThrows(Exception.class, () -> forEachFactory.create(Collections.emptyMap(), null, null, config)); assertThat(exception.getMessage(), equalTo("[processor] required property is missing")); } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorFactoryTests.java index c7029012da7d4..fcc51e10b84f7 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorFactoryTests.java @@ -40,7 +40,7 @@ public void testBuild() throws Exception { config.put("field", "_field"); config.put("patterns", Collections.singletonList("(?\\w+)")); String processorTag = randomAlphaOfLength(10); - GrokProcessor processor = factory.create(null, processorTag, config); + GrokProcessor processor = factory.create(null, processorTag, null, config); assertThat(processor.getTag(), equalTo(processorTag)); assertThat(processor.getMatchField(), equalTo("_field")); assertThat(processor.getGrok(), notNullValue()); @@ -55,7 +55,7 @@ public void testBuildWithIgnoreMissing() throws Exception { config.put("patterns", Collections.singletonList("(?\\w+)")); config.put("ignore_missing", true); String processorTag = randomAlphaOfLength(10); - GrokProcessor processor = factory.create(null, processorTag, config); + GrokProcessor processor = factory.create(null, processorTag, null, config); assertThat(processor.getTag(), equalTo(processorTag)); assertThat(processor.getMatchField(), equalTo("_field")); assertThat(processor.getGrok(), notNullValue()); @@ -66,7 +66,7 @@ public void testBuildMissingField() throws Exception { GrokProcessor.Factory factory = new GrokProcessor.Factory(Collections.emptyMap(), MatcherWatchdog.noop()); Map config = new HashMap<>(); config.put("patterns", Collections.singletonList("(?\\w+)")); - ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, config)); + ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config)); assertThat(e.getMessage(), equalTo("[field] required property is missing")); } @@ -74,7 +74,7 @@ public void testBuildMissingPatterns() throws Exception { GrokProcessor.Factory factory = new GrokProcessor.Factory(Collections.emptyMap(), MatcherWatchdog.noop()); Map config = new HashMap<>(); config.put("field", "foo"); - ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, config)); + ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config)); assertThat(e.getMessage(), equalTo("[patterns] required property is missing")); } @@ -83,7 +83,7 @@ public void testBuildEmptyPatternsList() throws Exception { Map config = new HashMap<>(); config.put("field", "foo"); config.put("patterns", Collections.emptyList()); - ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, config)); + ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config)); assertThat(e.getMessage(), equalTo("[patterns] List of patterns must not be empty")); } @@ -94,7 +94,7 @@ public void testCreateWithCustomPatterns() throws Exception { config.put("field", "_field"); config.put("patterns", Collections.singletonList("%{MY_PATTERN:name}!")); config.put("pattern_definitions", Collections.singletonMap("MY_PATTERN", "foo")); - GrokProcessor processor = factory.create(null, null, config); + GrokProcessor processor = factory.create(null, null, null, config); assertThat(processor.getMatchField(), equalTo("_field")); assertThat(processor.getGrok(), notNullValue()); assertThat(processor.getGrok().match("foo!"), equalTo(true)); @@ -105,7 +105,7 @@ public void testCreateWithInvalidPattern() throws Exception { Map config = new HashMap<>(); config.put("field", "_field"); config.put("patterns", Collections.singletonList("[")); - ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, config)); + ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config)); assertThat(e.getMessage(), equalTo("[patterns] Invalid regex pattern found in: [[]. premature end of char-class")); } @@ -115,7 +115,7 @@ public void testCreateWithInvalidPatternDefinition() throws Exception { config.put("field", "_field"); config.put("patterns", Collections.singletonList("%{MY_PATTERN:name}!")); config.put("pattern_definitions", Collections.singletonMap("MY_PATTERN", "[")); - ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, config)); + ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config)); assertThat(e.getMessage(), equalTo("[patterns] Invalid regex pattern found in: [%{MY_PATTERN:name}!]. premature end of char-class")); } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GsubProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GsubProcessorFactoryTests.java index 0dadefbb4ee64..d5c9a5a9d011d 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GsubProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GsubProcessorFactoryTests.java @@ -54,7 +54,7 @@ public void testCreateNoPatternPresent() throws Exception { config.put("field", "field1"); config.put("replacement", "-"); try { - factory.create(null, null, config); + factory.create(null, null, null, config); fail("factory create should have failed"); } catch(ElasticsearchParseException e) { assertThat(e.getMessage(), equalTo("[pattern] required property is missing")); @@ -67,7 +67,7 @@ public void testCreateNoReplacementPresent() throws Exception { config.put("field", "field1"); config.put("pattern", "\\."); try { - factory.create(null, null, config); + factory.create(null, null, null, config); fail("factory create should have failed"); } catch(ElasticsearchParseException e) { assertThat(e.getMessage(), equalTo("[replacement] required property is missing")); @@ -81,7 +81,7 @@ public void testCreateInvalidPattern() throws Exception { config.put("pattern", "["); config.put("replacement", "-"); try { - factory.create(null, null, config); + factory.create(null, null, null, config); fail("factory create should have failed"); } catch(ElasticsearchParseException e) { assertThat(e.getMessage(), containsString("[pattern] Invalid regex pattern. Unclosed character class")); diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JoinProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JoinProcessorFactoryTests.java index 9078dc484a90f..8da58c8d523ea 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JoinProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JoinProcessorFactoryTests.java @@ -35,7 +35,7 @@ public void testCreate() throws Exception { config.put("field", "field1"); config.put("separator", "-"); String processorTag = randomAlphaOfLength(10); - JoinProcessor joinProcessor = factory.create(null, processorTag, config); + JoinProcessor joinProcessor = factory.create(null, processorTag, null, config); assertThat(joinProcessor.getTag(), equalTo(processorTag)); assertThat(joinProcessor.getField(), equalTo("field1")); assertThat(joinProcessor.getSeparator(), equalTo("-")); @@ -47,7 +47,7 @@ public void testCreateNoFieldPresent() throws Exception { Map config = new HashMap<>(); config.put("separator", "-"); try { - factory.create(null, null, config); + factory.create(null, null, null, config); fail("factory create should have failed"); } catch (ElasticsearchParseException e) { assertThat(e.getMessage(), equalTo("[field] required property is missing")); @@ -59,7 +59,7 @@ public void testCreateNoSeparatorPresent() throws Exception { Map config = new HashMap<>(); config.put("field", "field1"); try { - factory.create(null, null, config); + factory.create(null, null, null, config); fail("factory create should have failed"); } catch (ElasticsearchParseException e) { assertThat(e.getMessage(), equalTo("[separator] required property is missing")); @@ -73,7 +73,7 @@ public void testCreateWithTargetField() throws Exception { config.put("separator", "-"); config.put("target_field", "target"); String processorTag = randomAlphaOfLength(10); - JoinProcessor joinProcessor = factory.create(null, processorTag, config); + JoinProcessor joinProcessor = factory.create(null, processorTag, null, config); assertThat(joinProcessor.getTag(), equalTo(processorTag)); assertThat(joinProcessor.getField(), equalTo("field1")); assertThat(joinProcessor.getSeparator(), equalTo("-")); diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorFactoryTests.java index 66f163a31eb8f..67728ead19b7e 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorFactoryTests.java @@ -39,7 +39,7 @@ public void testCreate() throws Exception { Map config = new HashMap<>(); config.put("field", randomField); config.put("target_field", randomTargetField); - JsonProcessor jsonProcessor = FACTORY.create(null, processorTag, config); + JsonProcessor jsonProcessor = FACTORY.create(null, processorTag, null, config); assertThat(jsonProcessor.getTag(), equalTo(processorTag)); assertThat(jsonProcessor.getField(), equalTo(randomField)); assertThat(jsonProcessor.getTargetField(), equalTo(randomTargetField)); @@ -51,7 +51,7 @@ public void testCreateWithAddToRoot() throws Exception { Map config = new HashMap<>(); config.put("field", randomField); config.put("add_to_root", true); - JsonProcessor jsonProcessor = FACTORY.create(null, processorTag, config); + JsonProcessor jsonProcessor = FACTORY.create(null, processorTag, null, config); assertThat(jsonProcessor.getTag(), equalTo(processorTag)); assertThat(jsonProcessor.getField(), equalTo(randomField)); assertThat(jsonProcessor.getTargetField(), equalTo(randomField)); @@ -63,7 +63,7 @@ public void testCreateWithDefaultTarget() throws Exception { String randomField = randomAlphaOfLength(10); Map config = new HashMap<>(); config.put("field", randomField); - JsonProcessor jsonProcessor = FACTORY.create(null, processorTag, config); + JsonProcessor jsonProcessor = FACTORY.create(null, processorTag, null, config); assertThat(jsonProcessor.getTag(), equalTo(processorTag)); assertThat(jsonProcessor.getField(), equalTo(randomField)); assertThat(jsonProcessor.getTargetField(), equalTo(randomField)); @@ -73,7 +73,7 @@ public void testCreateWithMissingField() throws Exception { Map config = new HashMap<>(); String processorTag = randomAlphaOfLength(10); ElasticsearchException exception = expectThrows(ElasticsearchParseException.class, - () -> FACTORY.create(null, processorTag, config)); + () -> FACTORY.create(null, processorTag, null, config)); assertThat(exception.getMessage(), equalTo("[field] required property is missing")); } @@ -85,7 +85,7 @@ public void testCreateWithBothTargetFieldAndAddToRoot() throws Exception { config.put("target_field", randomTargetField); config.put("add_to_root", true); ElasticsearchException exception = expectThrows(ElasticsearchParseException.class, - () -> FACTORY.create(null, randomAlphaOfLength(10), config)); + () -> FACTORY.create(null, randomAlphaOfLength(10), null, config)); assertThat(exception.getMessage(), equalTo("[target_field] Cannot set a target field while also setting `add_to_root` to true")); } } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/KeyValueProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/KeyValueProcessorFactoryTests.java index cce8cb5b24b1b..7e6aa741cea7c 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/KeyValueProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/KeyValueProcessorFactoryTests.java @@ -42,7 +42,7 @@ public void testCreateWithDefaults() throws Exception { config.put("field_split", "&"); config.put("value_split", "="); String processorTag = randomAlphaOfLength(10); - KeyValueProcessor processor = factory.create(null, processorTag, config); + KeyValueProcessor processor = factory.create(null, processorTag, null, config); assertThat(processor.getTag(), equalTo(processorTag)); assertThat(processor.getField(), equalTo("field1")); assertThat(processor.getFieldSplit(), equalTo("&")); @@ -63,7 +63,7 @@ public void testCreateWithAllFieldsSet() throws Exception { config.put("exclude_keys", Collections.emptyList()); config.put("ignore_missing", true); String processorTag = randomAlphaOfLength(10); - KeyValueProcessor processor = factory.create(null, processorTag, config); + KeyValueProcessor processor = factory.create(null, processorTag, null, config); assertThat(processor.getTag(), equalTo(processorTag)); assertThat(processor.getField(), equalTo("field1")); assertThat(processor.getFieldSplit(), equalTo("&")); @@ -79,7 +79,7 @@ public void testCreateWithMissingField() { Map config = new HashMap<>(); String processorTag = randomAlphaOfLength(10); ElasticsearchException exception = expectThrows(ElasticsearchParseException.class, - () -> factory.create(null, processorTag, config)); + () -> factory.create(null, processorTag, null, config)); assertThat(exception.getMessage(), equalTo("[field] required property is missing")); } @@ -89,7 +89,7 @@ public void testCreateWithMissingFieldSplit() { config.put("field", "field1"); String processorTag = randomAlphaOfLength(10); ElasticsearchException exception = expectThrows(ElasticsearchParseException.class, - () -> factory.create(null, processorTag, config)); + () -> factory.create(null, processorTag, null, config)); assertThat(exception.getMessage(), equalTo("[field_split] required property is missing")); } @@ -100,7 +100,7 @@ public void testCreateWithMissingValueSplit() { config.put("field_split", "&"); String processorTag = randomAlphaOfLength(10); ElasticsearchException exception = expectThrows(ElasticsearchParseException.class, - () -> factory.create(null, processorTag, config)); + () -> factory.create(null, processorTag, null, config)); assertThat(exception.getMessage(), equalTo("[value_split] required property is missing")); } } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/KeyValueProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/KeyValueProcessorTests.java index 591f9994c60b1..34bc6780caee6 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/KeyValueProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/KeyValueProcessorTests.java @@ -223,6 +223,6 @@ private static KeyValueProcessor createKvProcessor(String field, String fieldSpl if (prefix != null) { config.put("prefix", prefix); } - return FACTORY.create(null, randomAlphaOfLength(10), config); + return FACTORY.create(null, randomAlphaOfLength(10), null, config); } } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorFactoryTests.java index 61c1f73142798..b2e9641a240a6 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorFactoryTests.java @@ -46,7 +46,7 @@ public void testCreate() throws Exception { Map config = new HashMap<>(); config.put("field", "field1"); String processorTag = randomAlphaOfLength(10); - RemoveProcessor removeProcessor = factory.create(null, processorTag, config); + RemoveProcessor removeProcessor = factory.create(null, processorTag, null, config); assertThat(removeProcessor.getTag(), equalTo(processorTag)); assertThat(removeProcessor.getFields().get(0).newInstance(Collections.emptyMap()).execute(), equalTo("field1")); } @@ -55,7 +55,7 @@ public void testCreateMultipleFields() throws Exception { Map config = new HashMap<>(); config.put("field", Arrays.asList("field1", "field2")); String processorTag = randomAlphaOfLength(10); - RemoveProcessor removeProcessor = factory.create(null, processorTag, config); + RemoveProcessor removeProcessor = factory.create(null, processorTag, null, config); assertThat(removeProcessor.getTag(), equalTo(processorTag)); assertThat(removeProcessor.getFields().stream() .map(template -> template.newInstance(Collections.emptyMap()).execute()) @@ -65,7 +65,7 @@ public void testCreateMultipleFields() throws Exception { public void testCreateMissingField() throws Exception { Map config = new HashMap<>(); try { - factory.create(null, null, config); + factory.create(null, null, null, config); fail("factory create should have failed"); } catch(ElasticsearchParseException e) { assertThat(e.getMessage(), equalTo("[field] required property is missing")); @@ -77,7 +77,7 @@ public void testInvalidMustacheTemplate() throws Exception { Map config = new HashMap<>(); config.put("field", "{{field1}}"); String processorTag = randomAlphaOfLength(10); - ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, config)); + ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, null, config)); assertThat(exception.getMessage(), equalTo("java.lang.RuntimeException: could not compile script")); assertThat(exception.getMetadata("es.processor_tag").get(0), equalTo(processorTag)); } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorTests.java index b420b2fe3b00b..685b873d507ec 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorTests.java @@ -49,7 +49,7 @@ public void testRemoveNonExistingField() throws Exception { Map config = new HashMap<>(); config.put("field", fieldName); String processorTag = randomAlphaOfLength(10); - Processor processor = new RemoveProcessor.Factory(TestTemplateService.instance()).create(null, processorTag, config); + Processor processor = new RemoveProcessor.Factory(TestTemplateService.instance()).create(null, processorTag, null, config); try { processor.execute(ingestDocument); fail("remove field should have failed"); @@ -65,7 +65,7 @@ public void testIgnoreMissing() throws Exception { config.put("field", fieldName); config.put("ignore_missing", true); String processorTag = randomAlphaOfLength(10); - Processor processor = new RemoveProcessor.Factory(TestTemplateService.instance()).create(null, processorTag, config); + Processor processor = new RemoveProcessor.Factory(TestTemplateService.instance()).create(null, processorTag, null, config); processor.execute(ingestDocument); } } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorFactoryTests.java index 9dd799ff33971..15e07b4deb9b1 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorFactoryTests.java @@ -44,7 +44,7 @@ public void testCreate() throws Exception { config.put("field", "old_field"); config.put("target_field", "new_field"); String processorTag = randomAlphaOfLength(10); - RenameProcessor renameProcessor = factory.create(null, processorTag, config); + RenameProcessor renameProcessor = factory.create(null, processorTag, null, config); assertThat(renameProcessor.getTag(), equalTo(processorTag)); assertThat(renameProcessor.getField().newInstance(Collections.emptyMap()).execute(), equalTo("old_field")); assertThat(renameProcessor.getTargetField().newInstance(Collections.emptyMap()).execute(), equalTo("new_field")); @@ -57,7 +57,7 @@ public void testCreateWithIgnoreMissing() throws Exception { config.put("target_field", "new_field"); config.put("ignore_missing", true); String processorTag = randomAlphaOfLength(10); - RenameProcessor renameProcessor = factory.create(null, processorTag, config); + RenameProcessor renameProcessor = factory.create(null, processorTag, null, config); assertThat(renameProcessor.getTag(), equalTo(processorTag)); assertThat(renameProcessor.getField().newInstance(Collections.emptyMap()).execute(), equalTo("old_field")); assertThat(renameProcessor.getTargetField().newInstance(Collections.emptyMap()).execute(), equalTo("new_field")); @@ -68,7 +68,7 @@ public void testCreateNoFieldPresent() throws Exception { Map config = new HashMap<>(); config.put("target_field", "new_field"); try { - factory.create(null, null, config); + factory.create(null, null, null, config); fail("factory create should have failed"); } catch(ElasticsearchParseException e) { assertThat(e.getMessage(), equalTo("[field] required property is missing")); @@ -79,7 +79,7 @@ public void testCreateNoToPresent() throws Exception { Map config = new HashMap<>(); config.put("field", "old_field"); try { - factory.create(null, null, config); + factory.create(null, null, null, config); fail("factory create should have failed"); } catch(ElasticsearchParseException e) { assertThat(e.getMessage(), equalTo("[target_field] required property is missing")); diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java index 9559c150df68a..8dde090379de0 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java @@ -54,7 +54,7 @@ public void testFactoryValidationWithDefaultLang() throws Exception { Map configMap = new HashMap<>(); String randomType = randomFrom("id", "source"); configMap.put(randomType, "foo"); - ScriptProcessor processor = factory.create(null, randomAlphaOfLength(10), configMap); + ScriptProcessor processor = factory.create(null, randomAlphaOfLength(10), null, configMap); assertThat(processor.getScript().getLang(), equalTo(randomType.equals("id") ? null : Script.DEFAULT_SCRIPT_LANG)); assertThat(processor.getScript().getType().toString(), equalTo(INGEST_SCRIPT_PARAM_TO_TYPE.get(randomType))); assertThat(processor.getScript().getParams(), equalTo(Collections.emptyMap())); @@ -66,7 +66,7 @@ public void testFactoryValidationWithParams() throws Exception { Map randomParams = Collections.singletonMap(randomAlphaOfLength(10), randomAlphaOfLength(10)); configMap.put(randomType, "foo"); configMap.put("params", randomParams); - ScriptProcessor processor = factory.create(null, randomAlphaOfLength(10), configMap); + ScriptProcessor processor = factory.create(null, randomAlphaOfLength(10), null, configMap); assertThat(processor.getScript().getLang(), equalTo(randomType.equals("id") ? null : Script.DEFAULT_SCRIPT_LANG)); assertThat(processor.getScript().getType().toString(), equalTo(INGEST_SCRIPT_PARAM_TO_TYPE.get(randomType))); assertThat(processor.getScript().getParams(), equalTo(randomParams)); @@ -79,7 +79,7 @@ public void testFactoryValidationForMultipleScriptingTypes() throws Exception { configMap.put("lang", "mockscript"); XContentParseException exception = expectThrows(XContentParseException.class, - () -> factory.create(null, randomAlphaOfLength(10), configMap)); + () -> factory.create(null, randomAlphaOfLength(10), null, configMap)); assertThat(exception.getMessage(), containsString("[script] failed to parse field [source]")); } @@ -88,7 +88,7 @@ public void testFactoryValidationAtLeastOneScriptingType() throws Exception { configMap.put("lang", "mockscript"); IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, - () -> factory.create(null, randomAlphaOfLength(10), configMap)); + () -> factory.create(null, randomAlphaOfLength(10), null, configMap)); assertThat(exception.getMessage(), is("must specify either [source] for an inline script or [id] for a stored script")); } @@ -97,7 +97,7 @@ public void testInlineBackcompat() throws Exception { Map configMap = new HashMap<>(); configMap.put("inline", "code"); - factory.create(null, randomAlphaOfLength(10), configMap); + factory.create(null, randomAlphaOfLength(10), null, configMap); assertWarnings("Deprecated field [inline] used, expected [source] instead"); } @@ -113,7 +113,7 @@ public void testFactoryInvalidateWithInvalidCompiledScript() throws Exception { configMap.put(randomType, "my_script"); ElasticsearchException exception = expectThrows(ElasticsearchException.class, - () -> factory.create(null, randomAlphaOfLength(10), configMap)); + () -> factory.create(null, randomAlphaOfLength(10), null, configMap)); assertThat(exception.getMessage(), is("compile-time exception")); } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorFactoryTests.java index b3e183a8ab9ce..145d1ea9f72bc 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorFactoryTests.java @@ -45,7 +45,7 @@ public void testCreate() throws Exception { config.put("field", "field1"); config.put("value", "value1"); String processorTag = randomAlphaOfLength(10); - SetProcessor setProcessor = factory.create(null, processorTag, config); + SetProcessor setProcessor = factory.create(null, processorTag, null, config); assertThat(setProcessor.getTag(), equalTo(processorTag)); assertThat(setProcessor.getField().newInstance(Collections.emptyMap()).execute(), equalTo("field1")); assertThat(setProcessor.getValue().copyAndResolve(Collections.emptyMap()), equalTo("value1")); @@ -59,7 +59,7 @@ public void testCreateWithOverride() throws Exception { config.put("value", "value1"); config.put("override", overrideEnabled); String processorTag = randomAlphaOfLength(10); - SetProcessor setProcessor = factory.create(null, processorTag, config); + SetProcessor setProcessor = factory.create(null, processorTag, null, config); assertThat(setProcessor.getTag(), equalTo(processorTag)); assertThat(setProcessor.getField().newInstance(Collections.emptyMap()).execute(), equalTo("field1")); assertThat(setProcessor.getValue().copyAndResolve(Collections.emptyMap()), equalTo("value1")); @@ -70,7 +70,7 @@ public void testCreateNoFieldPresent() throws Exception { Map config = new HashMap<>(); config.put("value", "value1"); try { - factory.create(null, null, config); + factory.create(null, null, null, config); fail("factory create should have failed"); } catch(ElasticsearchParseException e) { assertThat(e.getMessage(), equalTo("[field] required property is missing")); @@ -81,7 +81,7 @@ public void testCreateNoValuePresent() throws Exception { Map config = new HashMap<>(); config.put("field", "field1"); try { - factory.create(null, null, config); + factory.create(null, null, null, config); fail("factory create should have failed"); } catch(ElasticsearchParseException e) { assertThat(e.getMessage(), equalTo("[value] required property is missing")); @@ -93,7 +93,7 @@ public void testCreateNullValue() throws Exception { config.put("field", "field1"); config.put("value", null); try { - factory.create(null, null, config); + factory.create(null, null, null, config); fail("factory create should have failed"); } catch(ElasticsearchParseException e) { assertThat(e.getMessage(), equalTo("[value] required property is missing")); @@ -106,7 +106,7 @@ public void testInvalidMustacheTemplate() throws Exception { config.put("field", "{{field1}}"); config.put("value", "value1"); String processorTag = randomAlphaOfLength(10); - ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, config)); + ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, null, config)); assertThat(exception.getMessage(), equalTo("java.lang.RuntimeException: could not compile script")); assertThat(exception.getMetadata("es.processor_tag").get(0), equalTo(processorTag)); } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SortProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SortProcessorFactoryTests.java index 452f02e96880a..2e1665fa71124 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SortProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SortProcessorFactoryTests.java @@ -38,7 +38,7 @@ public void testCreate() throws Exception { config.put("field", fieldName); SortProcessor.Factory factory = new SortProcessor.Factory(); - SortProcessor processor = factory.create(null, processorTag, config); + SortProcessor processor = factory.create(null, processorTag, null, config); assertThat(processor.getTag(), equalTo(processorTag)); assertThat(processor.getField(), equalTo(fieldName)); assertThat(processor.getOrder(), equalTo(SortProcessor.SortOrder.ASCENDING)); @@ -54,7 +54,7 @@ public void testCreateWithOrder() throws Exception { config.put("order", "desc"); SortProcessor.Factory factory = new SortProcessor.Factory(); - SortProcessor processor = factory.create(null, processorTag, config); + SortProcessor processor = factory.create(null, processorTag, null, config); assertThat(processor.getTag(), equalTo(processorTag)); assertThat(processor.getField(), equalTo(fieldName)); assertThat(processor.getOrder(), equalTo(SortProcessor.SortOrder.DESCENDING)); @@ -71,7 +71,7 @@ public void testCreateWithTargetField() throws Exception { config.put("target_field", targetFieldName); SortProcessor.Factory factory = new SortProcessor.Factory(); - SortProcessor processor = factory.create(null, processorTag, config); + SortProcessor processor = factory.create(null, processorTag, null, config); assertThat(processor.getTag(), equalTo(processorTag)); assertThat(processor.getField(), equalTo(fieldName)); assertThat(processor.getOrder(), equalTo(SortProcessor.SortOrder.ASCENDING)); @@ -88,7 +88,7 @@ public void testCreateWithInvalidOrder() throws Exception { SortProcessor.Factory factory = new SortProcessor.Factory(); try { - factory.create(null, processorTag, config); + factory.create(null, processorTag, null, config); fail("factory create should have failed"); } catch (ElasticsearchParseException e) { assertThat(e.getMessage(), equalTo("[order] Sort direction [invalid] not recognized. Valid values are: [asc, desc]")); @@ -99,7 +99,7 @@ public void testCreateMissingField() throws Exception { SortProcessor.Factory factory = new SortProcessor.Factory(); Map config = new HashMap<>(); try { - factory.create(null, null, config); + factory.create(null, null, null, config); fail("factory create should have failed"); } catch(ElasticsearchParseException e) { assertThat(e.getMessage(), equalTo("[field] required property is missing")); diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorFactoryTests.java index 8850943c1367f..b70a577e51408 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorFactoryTests.java @@ -35,7 +35,7 @@ public void testCreate() throws Exception { config.put("field", "field1"); config.put("separator", "\\."); String processorTag = randomAlphaOfLength(10); - SplitProcessor splitProcessor = factory.create(null, processorTag, config); + SplitProcessor splitProcessor = factory.create(null, processorTag, null, config); assertThat(splitProcessor.getTag(), equalTo(processorTag)); assertThat(splitProcessor.getField(), equalTo("field1")); assertThat(splitProcessor.getSeparator(), equalTo("\\.")); @@ -48,7 +48,7 @@ public void testCreateNoFieldPresent() throws Exception { Map config = new HashMap<>(); config.put("separator", "\\."); try { - factory.create(null, null, config); + factory.create(null, null, null, config); fail("factory create should have failed"); } catch(ElasticsearchParseException e) { assertThat(e.getMessage(), equalTo("[field] required property is missing")); @@ -60,7 +60,7 @@ public void testCreateNoSeparatorPresent() throws Exception { Map config = new HashMap<>(); config.put("field", "field1"); try { - factory.create(null, null, config); + factory.create(null, null, null, config); fail("factory create should have failed"); } catch(ElasticsearchParseException e) { assertThat(e.getMessage(), equalTo("[separator] required property is missing")); @@ -74,7 +74,7 @@ public void testCreateWithTargetField() throws Exception { config.put("separator", "\\."); config.put("target_field", "target"); String processorTag = randomAlphaOfLength(10); - SplitProcessor splitProcessor = factory.create(null, processorTag, config); + SplitProcessor splitProcessor = factory.create(null, processorTag, null, config); assertThat(splitProcessor.getTag(), equalTo(processorTag)); assertThat(splitProcessor.getField(), equalTo("field1")); assertThat(splitProcessor.getSeparator(), equalTo("\\.")); @@ -91,7 +91,7 @@ public void testCreateWithPreserveTrailing() throws Exception { config.put("target_field", "target"); config.put("preserve_trailing", true); String processorTag = randomAlphaOfLength(10); - SplitProcessor splitProcessor = factory.create(null, processorTag, config); + SplitProcessor splitProcessor = factory.create(null, processorTag, null, config); assertThat(splitProcessor.getTag(), equalTo(processorTag)); assertThat(splitProcessor.getField(), equalTo("field1")); assertThat(splitProcessor.getSeparator(), equalTo("\\.")); diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorTests.java index a979c171ce8cf..5155831c4c7a6 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorTests.java @@ -104,7 +104,7 @@ public void testSplitAppendable() throws Exception { Map splitConfig = new HashMap<>(); splitConfig.put("field", "flags"); splitConfig.put("separator", "\\|"); - Processor splitProcessor = (new SplitProcessor.Factory()).create(null, null, splitConfig); + Processor splitProcessor = (new SplitProcessor.Factory()).create(null, null, null, splitConfig); Map source = new HashMap<>(); source.put("flags", "new|hot|super|fun|interesting"); IngestDocument ingestDocument = new IngestDocument(source, new HashMap<>()); diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java index 41300f71093a0..d66f3feb350fa 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java @@ -75,6 +75,7 @@ public final class GeoIpProcessor extends AbstractProcessor { * Construct a geo-IP processor. * * @param tag the processor tag + * @param description the processor description * @param field the source field to geo-IP map * @param lazyLoader a supplier of a geo-IP database reader; ideally this is lazily-loaded once on first use * @param targetField the target field @@ -85,6 +86,7 @@ public final class GeoIpProcessor extends AbstractProcessor { */ GeoIpProcessor( final String tag, + final String description, final String field, final DatabaseReaderLazyLoader lazyLoader, final String targetField, @@ -92,7 +94,7 @@ public final class GeoIpProcessor extends AbstractProcessor { final boolean ignoreMissing, final GeoIpCache cache, boolean firstOnly) { - super(tag); + super(tag, description); this.field = field; this.targetField = targetField; this.lazyLoader = lazyLoader; @@ -394,9 +396,9 @@ public Factory(Map databaseReaders, GeoIpCache @Override public GeoIpProcessor create( - final Map registry, - final String processorTag, - final Map config) throws IOException { + final Map registry, + final String processorTag, + final String description, final Map config) throws IOException { String ipField = readStringProperty(TYPE, processorTag, config, "field"); String targetField = readStringProperty(TYPE, processorTag, config, "target_field", "geoip"); String databaseFile = readStringProperty(TYPE, processorTag, config, "database_file", "GeoLite2-City.mmdb"); @@ -436,7 +438,8 @@ public GeoIpProcessor create( } } - return new GeoIpProcessor(processorTag, ipField, lazyLoader, targetField, properties, ignoreMissing, cache, firstOnly); + return new GeoIpProcessor(processorTag, description, ipField, lazyLoader, targetField, properties, ignoreMissing, cache, + firstOnly); } } diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java index 001248497f0f9..86f14c5940313 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java @@ -78,7 +78,7 @@ public void testBuildDefaults() throws Exception { config.put("field", "_field"); String processorTag = randomAlphaOfLength(10); - GeoIpProcessor processor = factory.create(null, processorTag, config); + GeoIpProcessor processor = factory.create(null, processorTag, null, config); assertThat(processor.getTag(), equalTo(processorTag)); assertThat(processor.getField(), equalTo("_field")); assertThat(processor.getTargetField(), equalTo("geoip")); @@ -95,7 +95,7 @@ public void testSetIgnoreMissing() throws Exception { config.put("ignore_missing", true); String processorTag = randomAlphaOfLength(10); - GeoIpProcessor processor = factory.create(null, processorTag, config); + GeoIpProcessor processor = factory.create(null, processorTag, null, config); assertThat(processor.getTag(), equalTo(processorTag)); assertThat(processor.getField(), equalTo("_field")); assertThat(processor.getTargetField(), equalTo("geoip")); @@ -112,7 +112,7 @@ public void testCountryBuildDefaults() throws Exception { config.put("database_file", "GeoLite2-Country.mmdb"); String processorTag = randomAlphaOfLength(10); - GeoIpProcessor processor = factory.create(null, processorTag, config); + GeoIpProcessor processor = factory.create(null, processorTag, null, config); assertThat(processor.getTag(), equalTo(processorTag)); assertThat(processor.getField(), equalTo("_field")); @@ -130,7 +130,7 @@ public void testAsnBuildDefaults() throws Exception { config.put("database_file", "GeoLite2-ASN.mmdb"); String processorTag = randomAlphaOfLength(10); - GeoIpProcessor processor = factory.create(null, processorTag, config); + GeoIpProcessor processor = factory.create(null, processorTag, null, config); assertThat(processor.getTag(), equalTo(processorTag)); assertThat(processor.getField(), equalTo("_field")); @@ -145,7 +145,7 @@ public void testBuildTargetField() throws Exception { Map config = new HashMap<>(); config.put("field", "_field"); config.put("target_field", "_field"); - GeoIpProcessor processor = factory.create(null, null, config); + GeoIpProcessor processor = factory.create(null, null, null, config); assertThat(processor.getField(), equalTo("_field")); assertThat(processor.getTargetField(), equalTo("_field")); assertFalse(processor.isIgnoreMissing()); @@ -156,7 +156,7 @@ public void testBuildDbFile() throws Exception { Map config = new HashMap<>(); config.put("field", "_field"); config.put("database_file", "GeoLite2-Country.mmdb"); - GeoIpProcessor processor = factory.create(null, null, config); + GeoIpProcessor processor = factory.create(null, null, null, config); assertThat(processor.getField(), equalTo("_field")); assertThat(processor.getTargetField(), equalTo("geoip")); assertThat(processor.getDatabaseType(), equalTo("GeoLite2-Country")); @@ -173,7 +173,7 @@ public void testBuildWithCountryDbAndAsnFields() throws Exception { asnOnlyProperties.remove(GeoIpProcessor.Property.IP); String asnProperty = RandomPicks.randomFrom(Randomness.get(), asnOnlyProperties).toString(); config.put("properties", Collections.singletonList(asnProperty)); - Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, config)); + Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config)); assertThat(e.getMessage(), equalTo("[properties] illegal property value [" + asnProperty + "]. valid values are [IP, COUNTRY_ISO_CODE, COUNTRY_NAME, CONTINENT_NAME]")); } @@ -187,7 +187,7 @@ public void testBuildWithAsnDbAndCityFields() throws Exception { cityOnlyProperties.remove(GeoIpProcessor.Property.IP); String cityProperty = RandomPicks.randomFrom(Randomness.get(), cityOnlyProperties).toString(); config.put("properties", Collections.singletonList(cityProperty)); - Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, config)); + Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config)); assertThat(e.getMessage(), equalTo("[properties] illegal property value [" + cityProperty + "]. valid values are [IP, ASN, ORGANIZATION_NAME]")); } @@ -198,7 +198,7 @@ public void testBuildNonExistingDbFile() throws Exception { Map config = new HashMap<>(); config.put("field", "_field"); config.put("database_file", "does-not-exist.mmdb"); - Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, config)); + Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config)); assertThat(e.getMessage(), equalTo("[database_file] database file [does-not-exist.mmdb] doesn't exist")); } @@ -220,7 +220,7 @@ public void testBuildFields() throws Exception { Map config = new HashMap<>(); config.put("field", "_field"); config.put("properties", fieldNames); - GeoIpProcessor processor = factory.create(null, null, config); + GeoIpProcessor processor = factory.create(null, null, null, config); assertThat(processor.getField(), equalTo("_field")); assertThat(processor.getProperties(), equalTo(properties)); assertFalse(processor.isIgnoreMissing()); @@ -232,14 +232,14 @@ public void testBuildIllegalFieldOption() throws Exception { Map config1 = new HashMap<>(); config1.put("field", "_field"); config1.put("properties", Collections.singletonList("invalid")); - Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, config1)); + Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config1)); assertThat(e.getMessage(), equalTo("[properties] illegal property value [invalid]. valid values are [IP, COUNTRY_ISO_CODE, " + "COUNTRY_NAME, CONTINENT_NAME, REGION_ISO_CODE, REGION_NAME, CITY_NAME, TIMEZONE, LOCATION]")); Map config2 = new HashMap<>(); config2.put("field", "_field"); config2.put("properties", "invalid"); - e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, config2)); + e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config2)); assertThat(e.getMessage(), equalTo("[properties] property isn't a list, but of type [java.lang.String]")); } @@ -265,7 +265,7 @@ public void testLazyLoading() throws Exception { Map config = new HashMap<>(); config.put("field", "_field"); config.put("database_file", "GeoLite2-City.mmdb"); - final GeoIpProcessor city = factory.create(null, "_tag", config); + final GeoIpProcessor city = factory.create(null, "_tag", null, config); // these are lazy loaded until first use so we expect null here assertNull(databaseReaders.get("GeoLite2-City.mmdb").databaseReader.get()); @@ -276,7 +276,7 @@ public void testLazyLoading() throws Exception { config = new HashMap<>(); config.put("field", "_field"); config.put("database_file", "GeoLite2-Country.mmdb"); - final GeoIpProcessor country = factory.create(null, "_tag", config); + final GeoIpProcessor country = factory.create(null, "_tag", null, config); // these are lazy loaded until first use so we expect null here assertNull(databaseReaders.get("GeoLite2-Country.mmdb").databaseReader.get()); @@ -287,7 +287,7 @@ public void testLazyLoading() throws Exception { config = new HashMap<>(); config.put("field", "_field"); config.put("database_file", "GeoLite2-ASN.mmdb"); - final GeoIpProcessor asn = factory.create(null, "_tag", config); + final GeoIpProcessor asn = factory.create(null, "_tag", null, config); // these are lazy loaded until first use so we expect null here assertNull(databaseReaders.get("GeoLite2-ASN.mmdb").databaseReader.get()); @@ -322,7 +322,7 @@ public void testLoadingCustomDatabase() throws IOException { Map config = new HashMap<>(); config.put("field", "_field"); config.put("database_file", "GeoIP2-City.mmdb"); - final GeoIpProcessor city = factory.create(null, "_tag", config); + final GeoIpProcessor city = factory.create(null, "_tag", null, config); // these are lazy loaded until first use so we expect null here assertNull(databaseReaders.get("GeoIP2-City.mmdb").databaseReader.get()); diff --git a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java index 91a0563b1ca1f..4523b5aff69bb 100644 --- a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java +++ b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java @@ -51,9 +51,9 @@ public class UserAgentProcessor extends AbstractProcessor { private final UserAgentParser parser; private final boolean ignoreMissing; - public UserAgentProcessor(String tag, String field, String targetField, UserAgentParser parser, Set properties, - boolean ignoreMissing) { - super(tag); + public UserAgentProcessor(String tag, String description, String field, String targetField, UserAgentParser parser, + Set properties, boolean ignoreMissing) { + super(tag, description); this.field = field; this.targetField = targetField; this.parser = parser; @@ -179,7 +179,7 @@ public Factory(Map userAgentParsers) { @Override public UserAgentProcessor create(Map factories, String processorTag, - Map config) throws Exception { + String description, Map config) throws Exception { String field = readStringProperty(TYPE, processorTag, config, "field"); String targetField = readStringProperty(TYPE, processorTag, config, "target_field", "user_agent"); String regexFilename = readStringProperty(TYPE, processorTag, config, "regex_file", IngestUserAgentPlugin.DEFAULT_PARSER_NAME); @@ -211,7 +211,7 @@ public UserAgentProcessor create(Map factories, Strin properties = EnumSet.allOf(Property.class); } - return new UserAgentProcessor(processorTag, field, targetField, parser, properties, ignoreMissing); + return new UserAgentProcessor(processorTag, description, field, targetField, parser, properties, ignoreMissing); } } diff --git a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java index f723c13f23022..5b2be7b51eaaf 100644 --- a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java +++ b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java @@ -81,7 +81,7 @@ public void testBuildDefaults() throws Exception { String processorTag = randomAlphaOfLength(10); - UserAgentProcessor processor = factory.create(null, processorTag, config); + UserAgentProcessor processor = factory.create(null, processorTag, null, config); assertThat(processor.getTag(), equalTo(processorTag)); assertThat(processor.getField(), equalTo("_field")); assertThat(processor.getTargetField(), equalTo("user_agent")); @@ -101,7 +101,7 @@ public void testBuildWithIgnoreMissing() throws Exception { String processorTag = randomAlphaOfLength(10); - UserAgentProcessor processor = factory.create(null, processorTag, config); + UserAgentProcessor processor = factory.create(null, processorTag, null, config); assertThat(processor.getTag(), equalTo(processorTag)); assertThat(processor.getField(), equalTo("_field")); assertThat(processor.getTargetField(), equalTo("user_agent")); @@ -119,7 +119,7 @@ public void testBuildTargetField() throws Exception { config.put("field", "_field"); config.put("target_field", "_target_field"); - UserAgentProcessor processor = factory.create(null, null, config); + UserAgentProcessor processor = factory.create(null, null, null, config); assertThat(processor.getField(), equalTo("_field")); assertThat(processor.getTargetField(), equalTo("_target_field")); } @@ -131,7 +131,7 @@ public void testBuildRegexFile() throws Exception { config.put("field", "_field"); config.put("regex_file", regexWithoutDevicesFilename); - UserAgentProcessor processor = factory.create(null, null, config); + UserAgentProcessor processor = factory.create(null, null, null, config); assertThat(processor.getField(), equalTo("_field")); assertThat(processor.getUaParser().getUaPatterns().size(), greaterThan(0)); assertThat(processor.getUaParser().getOsPatterns().size(), greaterThan(0)); @@ -145,7 +145,7 @@ public void testBuildNonExistingRegexFile() throws Exception { config.put("field", "_field"); config.put("regex_file", "does-not-exist.yml"); - ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, config)); + ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config)); assertThat(e.getMessage(), equalTo("[regex_file] regex file [does-not-exist.yml] doesn't exist (has to exist at node startup)")); } @@ -165,7 +165,7 @@ public void testBuildFields() throws Exception { config.put("field", "_field"); config.put("properties", fieldNames); - UserAgentProcessor processor = factory.create(null, null, config); + UserAgentProcessor processor = factory.create(null, null, null, config); assertThat(processor.getField(), equalTo("_field")); assertThat(processor.getProperties(), equalTo(properties)); } @@ -177,7 +177,7 @@ public void testInvalidProperty() throws Exception { config.put("field", "_field"); config.put("properties", Collections.singletonList("invalid")); - ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, config)); + ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config)); assertThat(e.getMessage(), equalTo("[properties] illegal property value [invalid]. valid values are [NAME, OS, DEVICE, " + "ORIGINAL, VERSION]")); } @@ -189,7 +189,7 @@ public void testInvalidPropertiesType() throws Exception { config.put("field", "_field"); config.put("properties", "invalid"); - ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, config)); + ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config)); assertThat(e.getMessage(), equalTo("[properties] property isn't a list, but of type [java.lang.String]")); } } diff --git a/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java b/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java index 90261de7fbe24..e20b8e6025ba6 100644 --- a/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java +++ b/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java @@ -57,9 +57,9 @@ public final class AttachmentProcessor extends AbstractProcessor { private final boolean ignoreMissing; private final String indexedCharsField; - AttachmentProcessor(String tag, String field, String targetField, Set properties, + AttachmentProcessor(String tag, String description, String field, String targetField, Set properties, int indexedChars, boolean ignoreMissing, String indexedCharsField) { - super(tag); + super(tag, description); this.field = field; this.targetField = targetField; this.properties = properties; @@ -195,7 +195,7 @@ public static final class Factory implements Processor.Factory { @Override public AttachmentProcessor create(Map registry, String processorTag, - Map config) throws Exception { + String description, Map config) throws Exception { String field = readStringProperty(TYPE, processorTag, config, "field"); String targetField = readStringProperty(TYPE, processorTag, config, "target_field", "attachment"); List propertyNames = readOptionalList(TYPE, processorTag, config, "properties"); @@ -218,7 +218,8 @@ public AttachmentProcessor create(Map registry, Strin properties = DEFAULT_PROPERTIES; } - return new AttachmentProcessor(processorTag, field, targetField, properties, indexedChars, ignoreMissing, indexedCharsField); + return new AttachmentProcessor(processorTag, description, field, targetField, properties, indexedChars, ignoreMissing, + indexedCharsField); } } diff --git a/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorFactoryTests.java b/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorFactoryTests.java index c5f2b0d7fbd35..8bb5ce3d35fc5 100644 --- a/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorFactoryTests.java +++ b/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorFactoryTests.java @@ -46,7 +46,7 @@ public void testBuildDefaults() throws Exception { String processorTag = randomAlphaOfLength(10); - AttachmentProcessor processor = factory.create(null, processorTag, config); + AttachmentProcessor processor = factory.create(null, processorTag, null, config); assertThat(processor.getTag(), equalTo(processorTag)); assertThat(processor.getField(), equalTo("_field")); assertThat(processor.getTargetField(), equalTo("attachment")); @@ -61,7 +61,7 @@ public void testConfigureIndexedChars() throws Exception { config.put("indexed_chars", indexedChars); String processorTag = randomAlphaOfLength(10); - AttachmentProcessor processor = factory.create(null, processorTag, config); + AttachmentProcessor processor = factory.create(null, processorTag, null, config); assertThat(processor.getTag(), equalTo(processorTag)); assertThat(processor.getIndexedChars(), is(indexedChars)); assertFalse(processor.isIgnoreMissing()); @@ -71,7 +71,7 @@ public void testBuildTargetField() throws Exception { Map config = new HashMap<>(); config.put("field", "_field"); config.put("target_field", "_field"); - AttachmentProcessor processor = factory.create(null, null, config); + AttachmentProcessor processor = factory.create(null, null, null, config); assertThat(processor.getField(), equalTo("_field")); assertThat(processor.getTargetField(), equalTo("_field")); assertFalse(processor.isIgnoreMissing()); @@ -89,7 +89,7 @@ public void testBuildFields() throws Exception { Map config = new HashMap<>(); config.put("field", "_field"); config.put("properties", fieldNames); - AttachmentProcessor processor = factory.create(null, null, config); + AttachmentProcessor processor = factory.create(null, null, null, config); assertThat(processor.getField(), equalTo("_field")); assertThat(processor.getProperties(), equalTo(properties)); assertFalse(processor.isIgnoreMissing()); @@ -100,7 +100,7 @@ public void testBuildIllegalFieldOption() throws Exception { config.put("field", "_field"); config.put("properties", Collections.singletonList("invalid")); try { - factory.create(null, null, config); + factory.create(null, null, null, config); fail("exception expected"); } catch (ElasticsearchParseException e) { assertThat(e.getMessage(), containsString("[properties] illegal field option [invalid]")); @@ -114,7 +114,7 @@ public void testBuildIllegalFieldOption() throws Exception { config.put("field", "_field"); config.put("properties", "invalid"); try { - factory.create(null, null, config); + factory.create(null, null, null, config); fail("exception expected"); } catch (ElasticsearchParseException e) { assertThat(e.getMessage(), equalTo("[properties] property isn't a list, but of type [java.lang.String]")); @@ -128,7 +128,7 @@ public void testIgnoreMissing() throws Exception { String processorTag = randomAlphaOfLength(10); - AttachmentProcessor processor = factory.create(null, processorTag, config); + AttachmentProcessor processor = factory.create(null, processorTag, null, config); assertThat(processor.getTag(), equalTo(processorTag)); assertThat(processor.getField(), equalTo("_field")); assertThat(processor.getTargetField(), equalTo("attachment")); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/ingest/AsyncIngestProcessorIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/ingest/AsyncIngestProcessorIT.java index 3908d3cb149fc..9cb1248f0b11d 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/ingest/AsyncIngestProcessorIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/ingest/AsyncIngestProcessorIT.java @@ -112,8 +112,8 @@ public Collection createComponents(Client client, ClusterService cluster @Override public Map getProcessors(Processor.Parameters parameters) { return Map.of( - "test-async", (factories, tag, config) -> { - return new AbstractProcessor(tag) { + "test-async", (factories, tag, description, config) -> { + return new AbstractProcessor(tag, description) { @Override public void execute(IngestDocument ingestDocument, BiConsumer handler) { @@ -142,8 +142,8 @@ public String getType() { } }; }, - "test", (processorFactories, tag, config) -> { - return new AbstractProcessor(tag) { + "test", (processorFactories, tag, description, config) -> { + return new AbstractProcessor(tag, description) { @Override public IngestDocument execute(IngestDocument ingestDocument) throws Exception { String id = (String) ingestDocument.getSourceAndMetadata().get("_id"); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/FinalPipelineIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/FinalPipelineIT.java index fbf97ff851b17..dcf29e60571d7 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/FinalPipelineIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/FinalPipelineIT.java @@ -255,8 +255,8 @@ public Collection createComponents( public Map getProcessors(Processor.Parameters parameters) { return Map.of( "default", - (factories, tag, config) -> - new AbstractProcessor(tag) { + (factories, tag, description, config) -> + new AbstractProcessor(tag, description) { @Override public IngestDocument execute(final IngestDocument ingestDocument) throws Exception { @@ -270,9 +270,9 @@ public String getType() { } }, "final", - (processorFactories, tag, config) -> { + (processorFactories, tag, description, config) -> { final String exists = (String) config.remove("exists"); - return new AbstractProcessor(tag) { + return new AbstractProcessor(tag, description) { @Override public IngestDocument execute(final IngestDocument ingestDocument) throws Exception { // this asserts that this pipeline is the final pipeline executed @@ -293,8 +293,8 @@ public String getType() { }; }, "request", - (processorFactories, tag, config) -> - new AbstractProcessor(tag) { + (processorFactories, tag, description, config) -> + new AbstractProcessor(tag, description) { @Override public IngestDocument execute(final IngestDocument ingestDocument) throws Exception { ingestDocument.setFieldValue("request", true); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestClientIT.java b/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestClientIT.java index 512962325a27c..9436dc8ecf538 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestClientIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestClientIT.java @@ -438,8 +438,10 @@ public static class ExtendedIngestTestPlugin extends IngestTestPlugin { public Map getProcessors(Processor.Parameters parameters) { Map factories = new HashMap<>(super.getProcessors(parameters)); factories.put(PipelineProcessor.TYPE, new PipelineProcessor.Factory(parameters.ingestService)); - factories.put("fail", (processorFactories, tag, config) -> new TestProcessor(tag, "fail", new RuntimeException())); - factories.put("onfailure_processor", (processorFactories, tag, config) -> new TestProcessor(tag, "fail", document -> { + factories.put("fail", (processorFactories, tag, description, config) -> + new TestProcessor(tag, "fail", description, new RuntimeException())); + factories.put("onfailure_processor", (processorFactories, tag, description, config) -> new TestProcessor(tag, "fail", + description, document -> { String onFailurePipeline = document.getFieldValue("_ingest.on_failure_pipeline", String.class); document.setFieldValue("readme", "pipeline with id [" + onFailurePipeline + "] is a bad pipeline"); })); diff --git a/server/src/main/java/org/elasticsearch/ingest/AbstractProcessor.java b/server/src/main/java/org/elasticsearch/ingest/AbstractProcessor.java index c7204ee3e6969..ddcf1e23e3c71 100644 --- a/server/src/main/java/org/elasticsearch/ingest/AbstractProcessor.java +++ b/server/src/main/java/org/elasticsearch/ingest/AbstractProcessor.java @@ -20,18 +20,25 @@ package org.elasticsearch.ingest; /** - * An Abstract Processor that holds a processorTag field to be used - * by other processors. + * An Abstract Processor that holds tag and description information + * about the processor. */ public abstract class AbstractProcessor implements Processor { protected final String tag; + protected final String description; - protected AbstractProcessor(String tag) { + protected AbstractProcessor(String tag, String description) { this.tag = tag; + this.description = description; } @Override public String getTag() { return tag; } + + @Override + public String getDescription() { + return description; + } } diff --git a/server/src/main/java/org/elasticsearch/ingest/CompoundProcessor.java b/server/src/main/java/org/elasticsearch/ingest/CompoundProcessor.java index 9cc414c5a15d6..3bd90f67c0e61 100644 --- a/server/src/main/java/org/elasticsearch/ingest/CompoundProcessor.java +++ b/server/src/main/java/org/elasticsearch/ingest/CompoundProcessor.java @@ -114,6 +114,11 @@ public String getTag() { return "CompoundProcessor-" + flattenProcessors().stream().map(Processor::getTag).collect(Collectors.joining("-")); } + @Override + public String getDescription() { + return null; + } + @Override public IngestDocument execute(IngestDocument ingestDocument) throws Exception { throw new UnsupportedOperationException("this method should not get executed"); diff --git a/server/src/main/java/org/elasticsearch/ingest/ConditionalProcessor.java b/server/src/main/java/org/elasticsearch/ingest/ConditionalProcessor.java index cf82f23f3aa61..c2ad72bfd3e23 100644 --- a/server/src/main/java/org/elasticsearch/ingest/ConditionalProcessor.java +++ b/server/src/main/java/org/elasticsearch/ingest/ConditionalProcessor.java @@ -62,12 +62,13 @@ public class ConditionalProcessor extends AbstractProcessor implements WrappingP private final IngestMetric metric; private final LongSupplier relativeTimeProvider; - ConditionalProcessor(String tag, Script script, ScriptService scriptService, Processor processor) { - this(tag, script, scriptService, processor, System::nanoTime); + ConditionalProcessor(String tag, String description, Script script, ScriptService scriptService, Processor processor) { + this(tag, description, script, scriptService, processor, System::nanoTime); } - ConditionalProcessor(String tag, Script script, ScriptService scriptService, Processor processor, LongSupplier relativeTimeProvider) { - super(tag); + ConditionalProcessor(String tag, String description, Script script, ScriptService scriptService, Processor processor, + LongSupplier relativeTimeProvider) { + super(tag, description); this.condition = script; this.scriptService = scriptService; this.processor = processor; diff --git a/server/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java b/server/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java index c2a5b09cd5352..cee797e94a313 100644 --- a/server/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java +++ b/server/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java @@ -48,6 +48,7 @@ public final class ConfigurationUtils { public static final String TAG_KEY = "tag"; + public static final String DESCRIPTION_KEY = "description"; private ConfigurationUtils() { } @@ -410,6 +411,7 @@ public static Processor readProcessor(Map processorFa ScriptService scriptService, String type, Map config) throws Exception { String tag = ConfigurationUtils.readOptionalStringProperty(null, null, config, TAG_KEY); + String description = ConfigurationUtils.readOptionalStringProperty(null, tag, config, DESCRIPTION_KEY); Script conditionalScript = extractConditional(config); Processor.Factory factory = processorFactories.get(type); if (factory != null) { @@ -425,7 +427,7 @@ public static Processor readProcessor(Map processorFa } try { - Processor processor = factory.create(processorFactories, tag, config); + Processor processor = factory.create(processorFactories, tag, description, config); if (config.isEmpty() == false) { throw new ElasticsearchParseException("processor [{}] doesn't support one or more provided configuration parameters {}", type, Arrays.toString(config.keySet().toArray())); @@ -434,7 +436,7 @@ public static Processor readProcessor(Map processorFa processor = new CompoundProcessor(ignoreFailure, Collections.singletonList(processor), onFailureProcessors); } if (conditionalScript != null) { - processor = new ConditionalProcessor(tag, conditionalScript, scriptService, processor); + processor = new ConditionalProcessor(tag, description, conditionalScript, scriptService, processor); } return processor; } catch (Exception e) { diff --git a/server/src/main/java/org/elasticsearch/ingest/DropProcessor.java b/server/src/main/java/org/elasticsearch/ingest/DropProcessor.java index d4ccf7ad6358d..5c9999fb795b1 100644 --- a/server/src/main/java/org/elasticsearch/ingest/DropProcessor.java +++ b/server/src/main/java/org/elasticsearch/ingest/DropProcessor.java @@ -29,8 +29,8 @@ public final class DropProcessor extends AbstractProcessor { public static final String TYPE = "drop"; - private DropProcessor(final String tag) { - super(tag); + private DropProcessor(final String tag, final String description) { + super(tag, description); } @Override @@ -47,8 +47,8 @@ public static final class Factory implements Processor.Factory { @Override public Processor create(final Map processorFactories, final String tag, - final Map config) { - return new DropProcessor(tag); + final String description, final Map config) { + return new DropProcessor(tag, description); } } } diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestService.java b/server/src/main/java/org/elasticsearch/ingest/IngestService.java index 2364e3187fb65..7cfde71a86406 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestService.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestService.java @@ -691,7 +691,7 @@ private static Pipeline substitutePipeline(String id, ElasticsearchParseExceptio String tag = e.getHeaderKeys().contains("processor_tag") ? e.getHeader("processor_tag").get(0) : null; String type = e.getHeaderKeys().contains("processor_type") ? e.getHeader("processor_type").get(0) : "unknown"; String errorMessage = "pipeline with id [" + id + "] could not be loaded, caused by [" + e.getDetailedMessage() + "]"; - Processor failureProcessor = new AbstractProcessor(tag) { + Processor failureProcessor = new AbstractProcessor(tag, "this is a placeholder processor") { @Override public IngestDocument execute(IngestDocument ingestDocument) { throw new IllegalStateException(errorMessage); diff --git a/server/src/main/java/org/elasticsearch/ingest/PipelineProcessor.java b/server/src/main/java/org/elasticsearch/ingest/PipelineProcessor.java index 2ee99d6e5d48f..b362259ea03db 100644 --- a/server/src/main/java/org/elasticsearch/ingest/PipelineProcessor.java +++ b/server/src/main/java/org/elasticsearch/ingest/PipelineProcessor.java @@ -31,8 +31,8 @@ public class PipelineProcessor extends AbstractProcessor { private final TemplateScript.Factory pipelineTemplate; private final IngestService ingestService; - PipelineProcessor(String tag, TemplateScript.Factory pipelineTemplate, IngestService ingestService) { - super(tag); + PipelineProcessor(String tag, String description, TemplateScript.Factory pipelineTemplate, IngestService ingestService) { + super(tag, description); this.pipelineTemplate = pipelineTemplate; this.ingestService = ingestService; } @@ -78,10 +78,10 @@ public Factory(IngestService ingestService) { @Override public PipelineProcessor create(Map registry, String processorTag, - Map config) throws Exception { + String description, Map config) throws Exception { TemplateScript.Factory pipelineTemplate = ConfigurationUtils.readTemplateProperty(TYPE, processorTag, config, "name", ingestService.getScriptService()); - return new PipelineProcessor(processorTag, pipelineTemplate, ingestService); + return new PipelineProcessor(processorTag, description, pipelineTemplate, ingestService); } } } diff --git a/server/src/main/java/org/elasticsearch/ingest/Processor.java b/server/src/main/java/org/elasticsearch/ingest/Processor.java index d0b96c99747a6..fa84ffa0504f7 100644 --- a/server/src/main/java/org/elasticsearch/ingest/Processor.java +++ b/server/src/main/java/org/elasticsearch/ingest/Processor.java @@ -75,6 +75,11 @@ default void execute(IngestDocument ingestDocument, BiConsumerNote: Implementations are responsible for removing the used configuration keys, so that after - * creating a pipeline ingest can verify if all configurations settings have been used. */ - Processor create(Map processorFactories, String tag, - Map config) throws Exception; + Processor create(Map processorFactories, String tag, + String description, Map config) throws Exception; } /** diff --git a/server/src/main/java/org/elasticsearch/ingest/TrackingResultProcessor.java b/server/src/main/java/org/elasticsearch/ingest/TrackingResultProcessor.java index 8a84c40f088ca..5582275b8e7bd 100644 --- a/server/src/main/java/org/elasticsearch/ingest/TrackingResultProcessor.java +++ b/server/src/main/java/org/elasticsearch/ingest/TrackingResultProcessor.java @@ -118,6 +118,11 @@ public String getTag() { return actualProcessor.getTag(); } + @Override + public String getDescription() { + return actualProcessor.getDescription(); + } + public static CompoundProcessor decorate(CompoundProcessor compoundProcessor, ConditionalProcessor parentCondition, List processorResultList) { List processors = new ArrayList<>(); diff --git a/server/src/test/java/org/elasticsearch/action/ingest/SimulateExecutionServiceTests.java b/server/src/test/java/org/elasticsearch/action/ingest/SimulateExecutionServiceTests.java index 1af375ced8337..83264ff1cb6b6 100644 --- a/server/src/test/java/org/elasticsearch/action/ingest/SimulateExecutionServiceTests.java +++ b/server/src/test/java/org/elasticsearch/action/ingest/SimulateExecutionServiceTests.java @@ -76,7 +76,7 @@ public void destroy() { } public void testExecuteVerboseItem() throws Exception { - TestProcessor processor = new TestProcessor("test-id", "mock", ingestDocument -> {}); + TestProcessor processor = new TestProcessor("test-id", "mock", null, ingestDocument -> {}); Pipeline pipeline = new Pipeline("_id", "_description", version, new CompoundProcessor(processor, processor)); CountDownLatch latch = new CountDownLatch(1); @@ -103,7 +103,7 @@ public void testExecuteVerboseItem() throws Exception { assertThat(simulateDocumentVerboseResult.getProcessorResults().get(1).getFailure(), nullValue()); } public void testExecuteItem() throws Exception { - TestProcessor processor = new TestProcessor("processor_0", "mock", ingestDocument -> {}); + TestProcessor processor = new TestProcessor("processor_0", "mock", null, ingestDocument -> {}); Pipeline pipeline = new Pipeline("_id", "_description", version, new CompoundProcessor(processor, processor)); CountDownLatch latch = new CountDownLatch(1); AtomicReference holder = new AtomicReference<>(); @@ -121,9 +121,9 @@ public void testExecuteItem() throws Exception { } public void testExecuteVerboseItemExceptionWithoutOnFailure() throws Exception { - TestProcessor processor1 = new TestProcessor("processor_0", "mock", ingestDocument -> {}); - TestProcessor processor2 = new TestProcessor("processor_1", "mock", new RuntimeException("processor failed")); - TestProcessor processor3 = new TestProcessor("processor_2", "mock", ingestDocument -> {}); + TestProcessor processor1 = new TestProcessor("processor_0", "mock", null, ingestDocument -> {}); + TestProcessor processor2 = new TestProcessor("processor_1", "mock", null, new RuntimeException("processor failed")); + TestProcessor processor3 = new TestProcessor("processor_2", "mock", null, ingestDocument -> {}); Pipeline pipeline = new Pipeline("_id", "_description", version, new CompoundProcessor(processor1, processor2, processor3)); CountDownLatch latch = new CountDownLatch(1); AtomicReference holder = new AtomicReference<>(); @@ -150,9 +150,9 @@ public void testExecuteVerboseItemExceptionWithoutOnFailure() throws Exception { } public void testExecuteVerboseItemWithOnFailure() throws Exception { - TestProcessor processor1 = new TestProcessor("processor_0", "mock", new RuntimeException("processor failed")); - TestProcessor processor2 = new TestProcessor("processor_1", "mock", ingestDocument -> {}); - TestProcessor processor3 = new TestProcessor("processor_2", "mock", ingestDocument -> {}); + TestProcessor processor1 = new TestProcessor("processor_0", "mock", null, new RuntimeException("processor failed")); + TestProcessor processor2 = new TestProcessor("processor_1", "mock", null, ingestDocument -> {}); + TestProcessor processor3 = new TestProcessor("processor_2", "mock", null, ingestDocument -> {}); Pipeline pipeline = new Pipeline("_id", "_description", version, new CompoundProcessor(new CompoundProcessor(false, Collections.singletonList(processor1), Collections.singletonList(processor2)), processor3)); @@ -193,7 +193,7 @@ public void testExecuteVerboseItemWithOnFailure() throws Exception { public void testExecuteVerboseItemExceptionWithIgnoreFailure() throws Exception { RuntimeException exception = new RuntimeException("processor failed"); - TestProcessor testProcessor = new TestProcessor("processor_0", "mock", exception); + TestProcessor testProcessor = new TestProcessor("processor_0", "mock", null, exception); CompoundProcessor processor = new CompoundProcessor(true, Collections.singletonList(testProcessor), Collections.emptyList()); Pipeline pipeline = new Pipeline("_id", "_description", version, new CompoundProcessor(processor)); CountDownLatch latch = new CountDownLatch(1); @@ -214,7 +214,7 @@ public void testExecuteVerboseItemExceptionWithIgnoreFailure() throws Exception } public void testExecuteVerboseItemWithoutExceptionAndWithIgnoreFailure() throws Exception { - TestProcessor testProcessor = new TestProcessor("processor_0", "mock", ingestDocument -> { }); + TestProcessor testProcessor = new TestProcessor("processor_0", "mock", null, ingestDocument -> { }); CompoundProcessor processor = new CompoundProcessor(true, Collections.singletonList(testProcessor), Collections.emptyList()); Pipeline pipeline = new Pipeline("_id", "_description", version, new CompoundProcessor(processor)); CountDownLatch latch = new CountDownLatch(1); @@ -257,7 +257,7 @@ public void testExecuteItemWithFailure() throws Exception { public void testDropDocument() throws Exception { TestProcessor processor1 = new TestProcessor(ingestDocument -> ingestDocument.setFieldValue("field", "value")); - Processor processor2 = new DropProcessor.Factory().create(Map.of(), null, Map.of()); + Processor processor2 = new DropProcessor.Factory().create(Map.of(), null, null, Map.of()); Pipeline pipeline = new Pipeline("_id", "_description", version, new CompoundProcessor(processor1, processor2)); CountDownLatch latch = new CountDownLatch(1); @@ -277,7 +277,7 @@ public void testDropDocument() throws Exception { public void testDropDocumentVerbose() throws Exception { TestProcessor processor1 = new TestProcessor(ingestDocument -> ingestDocument.setFieldValue("field", "value")); - Processor processor2 = new DropProcessor.Factory().create(Map.of(), null, Map.of()); + Processor processor2 = new DropProcessor.Factory().create(Map.of(), null, null, Map.of()); Pipeline pipeline = new Pipeline("_id", "_description", version, new CompoundProcessor(processor1, processor2)); CountDownLatch latch = new CountDownLatch(1); @@ -300,7 +300,7 @@ public void testDropDocumentVerbose() throws Exception { public void testDropDocumentVerboseExtraProcessor() throws Exception { TestProcessor processor1 = new TestProcessor(ingestDocument -> ingestDocument.setFieldValue("field1", "value")); - Processor processor2 = new DropProcessor.Factory().create(Map.of(), null, Map.of()); + Processor processor2 = new DropProcessor.Factory().create(Map.of(), null, null, Map.of()); TestProcessor processor3 = new TestProcessor(ingestDocument -> ingestDocument.setFieldValue("field2", "value")); Pipeline pipeline = new Pipeline("_id", "_description", version, new CompoundProcessor(processor1, processor2, processor3)); @@ -329,7 +329,7 @@ public void testAsyncSimulation() throws Exception { for (int id = 0; id < numDocs; id++) { documents.add(new IngestDocument("_index", Integer.toString(id), null, 0L, VersionType.INTERNAL, new HashMap<>())); } - Processor processor1 = new AbstractProcessor(null) { + Processor processor1 = new AbstractProcessor(null, null) { @Override public void execute(IngestDocument ingestDocument, BiConsumer handler) { diff --git a/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java b/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java index 424451700284a..d02db6be0cd2f 100644 --- a/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java +++ b/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java @@ -64,7 +64,7 @@ public void init() throws IOException { CompoundProcessor pipelineCompoundProcessor = new CompoundProcessor(processor); Pipeline pipeline = new Pipeline(SIMULATED_PIPELINE_ID, null, null, pipelineCompoundProcessor); Map registry = - Collections.singletonMap("mock_processor", (factories, tag, config) -> processor); + Collections.singletonMap("mock_processor", (factories, tag, description, config) -> processor); ingestService = mock(IngestService.class); when(ingestService.getPipeline(SIMULATED_PIPELINE_ID)).thenReturn(pipeline); when(ingestService.getProcessorFactories()).thenReturn(registry); diff --git a/server/src/test/java/org/elasticsearch/ingest/CompoundProcessorTests.java b/server/src/test/java/org/elasticsearch/ingest/CompoundProcessorTests.java index 75f1d4d7afb6b..90a7162d65193 100644 --- a/server/src/test/java/org/elasticsearch/ingest/CompoundProcessorTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/CompoundProcessorTests.java @@ -110,7 +110,7 @@ public void testIgnoreFailure() throws Exception { } public void testSingleProcessorWithOnFailureProcessor() throws Exception { - TestProcessor processor1 = new TestProcessor("id", "first", new RuntimeException("error")); + TestProcessor processor1 = new TestProcessor("id", "first", null, new RuntimeException("error")); TestProcessor processor2 = new TestProcessor(ingestDocument -> { Map ingestMetadata = ingestDocument.getIngestMetadata(); assertThat(ingestMetadata.size(), equalTo(3)); @@ -132,7 +132,7 @@ public void testSingleProcessorWithOnFailureProcessor() throws Exception { } public void testSingleProcessorWithOnFailureDropProcessor() throws Exception { - TestProcessor processor1 = new TestProcessor("id", "first", new RuntimeException("error")); + TestProcessor processor1 = new TestProcessor("id", "first", null, new RuntimeException("error")); Processor processor2 = new Processor() { @Override public IngestDocument execute(IngestDocument ingestDocument) throws Exception { @@ -149,6 +149,11 @@ public String getType() { public String getTag() { return null; } + + @Override + public String getDescription() { + return null; + } }; LongSupplier relativeTimeProvider = mock(LongSupplier.class); @@ -163,8 +168,8 @@ public String getTag() { } public void testSingleProcessorWithNestedFailures() throws Exception { - TestProcessor processor = new TestProcessor("id", "first", new RuntimeException("error")); - TestProcessor processorToFail = new TestProcessor("id2", "second", (Consumer) ingestDocument -> { + TestProcessor processor = new TestProcessor("id", "first", null, new RuntimeException("error")); + TestProcessor processorToFail = new TestProcessor("id2", "second", null, (Consumer) ingestDocument -> { Map ingestMetadata = ingestDocument.getIngestMetadata(); assertThat(ingestMetadata.size(), equalTo(3)); assertThat(ingestMetadata.get(CompoundProcessor.ON_FAILURE_MESSAGE_FIELD), equalTo("error")); @@ -193,8 +198,8 @@ public void testSingleProcessorWithNestedFailures() throws Exception { } public void testCompoundProcessorExceptionFailWithoutOnFailure() throws Exception { - TestProcessor firstProcessor = new TestProcessor("id1", "first", new RuntimeException("error")); - TestProcessor secondProcessor = new TestProcessor("id3", "second", ingestDocument -> { + TestProcessor firstProcessor = new TestProcessor("id1", "first", null, new RuntimeException("error")); + TestProcessor secondProcessor = new TestProcessor("id3", "second", null, ingestDocument -> { Map ingestMetadata = ingestDocument.getIngestMetadata(); assertThat(ingestMetadata.entrySet(), hasSize(3)); assertThat(ingestMetadata.get(CompoundProcessor.ON_FAILURE_MESSAGE_FIELD), equalTo("error")); @@ -216,10 +221,10 @@ public void testCompoundProcessorExceptionFailWithoutOnFailure() throws Exceptio } public void testCompoundProcessorExceptionFail() throws Exception { - TestProcessor firstProcessor = new TestProcessor("id1", "first", new RuntimeException("error")); + TestProcessor firstProcessor = new TestProcessor("id1", "first", null, new RuntimeException("error")); TestProcessor failProcessor = - new TestProcessor("tag_fail", "fail", new RuntimeException("custom error message")); - TestProcessor secondProcessor = new TestProcessor("id3", "second", ingestDocument -> { + new TestProcessor("tag_fail", "fail", null, new RuntimeException("custom error message")); + TestProcessor secondProcessor = new TestProcessor("id3", "second", null, ingestDocument -> { Map ingestMetadata = ingestDocument.getIngestMetadata(); assertThat(ingestMetadata.entrySet(), hasSize(3)); assertThat(ingestMetadata.get(CompoundProcessor.ON_FAILURE_MESSAGE_FIELD), equalTo("custom error message")); @@ -242,10 +247,10 @@ public void testCompoundProcessorExceptionFail() throws Exception { } public void testCompoundProcessorExceptionFailInOnFailure() throws Exception { - TestProcessor firstProcessor = new TestProcessor("id1", "first", new RuntimeException("error")); + TestProcessor firstProcessor = new TestProcessor("id1", "first", null, new RuntimeException("error")); TestProcessor failProcessor = - new TestProcessor("tag_fail", "fail", new RuntimeException("custom error message")); - TestProcessor secondProcessor = new TestProcessor("id3", "second", ingestDocument -> { + new TestProcessor("tag_fail", "fail", null, new RuntimeException("custom error message")); + TestProcessor secondProcessor = new TestProcessor("id3", "second", null, ingestDocument -> { Map ingestMetadata = ingestDocument.getIngestMetadata(); assertThat(ingestMetadata.entrySet(), hasSize(3)); assertThat(ingestMetadata.get(CompoundProcessor.ON_FAILURE_MESSAGE_FIELD), equalTo("custom error message")); @@ -268,9 +273,9 @@ public void testCompoundProcessorExceptionFailInOnFailure() throws Exception { } public void testBreakOnFailure() throws Exception { - TestProcessor firstProcessor = new TestProcessor("id1", "first", new RuntimeException("error1")); - TestProcessor secondProcessor = new TestProcessor("id2", "second", new RuntimeException("error2")); - TestProcessor onFailureProcessor = new TestProcessor("id2", "on_failure", ingestDocument -> {}); + TestProcessor firstProcessor = new TestProcessor("id1", "first", null, new RuntimeException("error1")); + TestProcessor secondProcessor = new TestProcessor("id2", "second", null, new RuntimeException("error2")); + TestProcessor onFailureProcessor = new TestProcessor("id2", "on_failure", null, ingestDocument -> {}); LongSupplier relativeTimeProvider = mock(LongSupplier.class); when(relativeTimeProvider.getAsLong()).thenReturn(0L); CompoundProcessor pipeline = new CompoundProcessor(false, Arrays.asList(firstProcessor, secondProcessor), @@ -283,7 +288,7 @@ public void testBreakOnFailure() throws Exception { } public void testFailureProcessorIsInvokedOnFailure() { - TestProcessor onFailureProcessor = new TestProcessor(null, "on_failure", ingestDocument -> { + TestProcessor onFailureProcessor = new TestProcessor(null, "on_failure", null, ingestDocument -> { Map ingestMetadata = ingestDocument.getIngestMetadata(); assertThat(ingestMetadata.entrySet(), hasSize(5)); assertThat(ingestMetadata.get(CompoundProcessor.ON_FAILURE_MESSAGE_FIELD), equalTo("failure!")); @@ -294,7 +299,7 @@ public void testFailureProcessorIsInvokedOnFailure() { }); Pipeline pipeline2 = new Pipeline("2", null, null, new CompoundProcessor(new TestProcessor(new RuntimeException("failure!")))); - Pipeline pipeline1 = new Pipeline("1", null, null, new CompoundProcessor(false, List.of(new AbstractProcessor(null) { + Pipeline pipeline1 = new Pipeline("1", null, null, new CompoundProcessor(false,List.of(new AbstractProcessor(null, null) { @Override public void execute(IngestDocument ingestDocument, BiConsumer handler) { ingestDocument.executePipeline(pipeline2, handler); @@ -319,7 +324,7 @@ public String getType() { } public void testNewCompoundProcessorException() { - TestProcessor processor = new TestProcessor("my_tag", "my_type", new RuntimeException()); + TestProcessor processor = new TestProcessor("my_tag", "my_type", null, new RuntimeException()); IngestProcessorException ingestProcessorException1 = CompoundProcessor.newCompoundProcessorException(new RuntimeException(), processor, ingestDocument); assertThat(ingestProcessorException1.getHeader("processor_tag"), equalTo(List.of("my_tag"))); @@ -333,8 +338,8 @@ public void testNewCompoundProcessorException() { public void testNewCompoundProcessorExceptionPipelineOrigin() { Pipeline pipeline2 = new Pipeline("2", null, null, - new CompoundProcessor(new TestProcessor("my_tag", "my_type", new RuntimeException()))); - Pipeline pipeline1 = new Pipeline("1", null, null, new CompoundProcessor(new AbstractProcessor(null) { + new CompoundProcessor(new TestProcessor("my_tag", "my_type", null, new RuntimeException()))); + Pipeline pipeline1 = new Pipeline("1", null, null, new CompoundProcessor(new AbstractProcessor(null, null) { @Override public IngestDocument execute(IngestDocument ingestDocument) throws Exception { throw new UnsupportedOperationException(); diff --git a/server/src/test/java/org/elasticsearch/ingest/ConditionalProcessorTests.java b/server/src/test/java/org/elasticsearch/ingest/ConditionalProcessorTests.java index 204a0bddbc339..6bcbba09ab044 100644 --- a/server/src/test/java/org/elasticsearch/ingest/ConditionalProcessorTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/ConditionalProcessorTests.java @@ -70,6 +70,7 @@ public void testChecksCondition() throws Exception { when(relativeTimeProvider.getAsLong()).thenReturn(0L, TimeUnit.MILLISECONDS.toNanos(1), 0L, TimeUnit.MILLISECONDS.toNanos(2)); ConditionalProcessor processor = new ConditionalProcessor( randomAlphaOfLength(10), + "description", new Script( ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptName, Collections.emptyMap()), scriptService, @@ -92,6 +93,11 @@ public String getType() { public String getTag() { return null; } + + @Override + public String getDescription() { + return null; + } }, relativeTimeProvider); //false, never call processor never increments metrics @@ -158,6 +164,7 @@ public void testTypeDeprecation() throws Exception { when(relativeTimeProvider.getAsLong()).thenReturn(0L, TimeUnit.MILLISECONDS.toNanos(1), 0L, TimeUnit.MILLISECONDS.toNanos(2)); ConditionalProcessor processor = new ConditionalProcessor( randomAlphaOfLength(10), + "description", new Script( ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptName, Collections.emptyMap()), scriptService, @@ -176,6 +183,11 @@ public String getType() { public String getTag() { return null; } + + @Override + public String getDescription() { + return null; + } }, relativeTimeProvider); IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap()); @@ -209,6 +221,7 @@ private static void assertMutatingCtxThrows(Consumer> mutati Map document = new HashMap<>(); ConditionalProcessor processor = new ConditionalProcessor( randomAlphaOfLength(10), + "desription", new Script( ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptName, Collections.emptyMap()), scriptService, null diff --git a/server/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java b/server/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java index 20f67fd10a36d..3eb6ada4e725d 100644 --- a/server/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java @@ -117,7 +117,7 @@ public void testReadStringOrIntPropertyInvalidType() { public void testReadProcessors() throws Exception { Processor processor = mock(Processor.class); Map registry = - Collections.singletonMap("test_processor", (factories, tag, config) -> processor); + Collections.singletonMap("test_processor", (factories, tag, description, config) -> processor); List> config = new ArrayList<>(); Map emptyConfig = Collections.emptyMap(); @@ -131,6 +131,9 @@ public void testReadProcessors() throws Exception { Map unknownTaggedConfig = new HashMap<>(); unknownTaggedConfig.put("tag", "my_unknown"); + if (randomBoolean()) { + unknownTaggedConfig.put("description", "my_description"); + } config.add(Collections.singletonMap("unknown_processor", unknownTaggedConfig)); ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> ConfigurationUtils.readProcessorConfigs(config, scriptService, registry)); @@ -138,6 +141,7 @@ public void testReadProcessors() throws Exception { assertThat(e.getMetadata("es.processor_tag"), equalTo(Collections.singletonList("my_unknown"))); assertThat(e.getMetadata("es.processor_type"), equalTo(Collections.singletonList("unknown_processor"))); assertThat(e.getMetadata("es.property_name"), is(nullValue())); + assertThat(e.getMetadata("es.processor_description"), is(nullValue())); List> config2 = new ArrayList<>(); unknownTaggedConfig = new HashMap<>(); @@ -164,10 +168,44 @@ public void testReadProcessors() throws Exception { assertThat(e2.getMetadata("es.property_name"), is(nullValue())); } + public void testReadProcessorNullDescription() throws Exception { + Processor processor = new TestProcessor("tag", "type", null, (ingestDocument) -> {}); + Map registry = + Collections.singletonMap("test_processor", (factories, tag, description, config) -> { + assertNull(description); + return processor; + }); + + List> config = new ArrayList<>(); + Map emptyConfig = Collections.emptyMap(); + config.add(Collections.singletonMap("test_processor", emptyConfig)); + List result = ConfigurationUtils.readProcessorConfigs(config, scriptService, registry); + assertThat(result.size(), equalTo(1)); + assertThat(result.get(0), sameInstance(processor)); + } + + public void testReadProcessorDescription() throws Exception { + String testDescription = randomAlphaOfLengthBetween(10, 20); + Processor processor = new TestProcessor("tag", "type", testDescription, (ingestDocument) -> {}); + Map registry = + Collections.singletonMap("test_processor", (factories, tag, description, config) -> { + assertThat(description, equalTo(processor.getDescription())); + return processor; + }); + + List> config = new ArrayList<>(); + Map processorConfig = new HashMap<>(); + processorConfig.put(ConfigurationUtils.DESCRIPTION_KEY, testDescription); + config.add(Collections.singletonMap("test_processor", processorConfig)); + List result = ConfigurationUtils.readProcessorConfigs(config, scriptService, registry); + assertThat(result.size(), equalTo(1)); + assertThat(result.get(0), sameInstance(processor)); + } + public void testReadProcessorFromObjectOrMap() throws Exception { Processor processor = mock(Processor.class); Map registry = - Collections.singletonMap("script", (processorFactories, tag, config) -> { + Collections.singletonMap("script", (processorFactories, tag, description, config) -> { config.clear(); return processor; }); diff --git a/server/src/test/java/org/elasticsearch/ingest/FakeProcessor.java b/server/src/test/java/org/elasticsearch/ingest/FakeProcessor.java index 90c14a20a8f00..cd340880eff6e 100644 --- a/server/src/test/java/org/elasticsearch/ingest/FakeProcessor.java +++ b/server/src/test/java/org/elasticsearch/ingest/FakeProcessor.java @@ -24,11 +24,13 @@ class FakeProcessor implements Processor { private String type; private String tag; + private String description; private Consumer executor; - FakeProcessor(String type, String tag, Consumer executor) { + FakeProcessor(String type, String tag, String description, Consumer executor) { this.type = type; this.tag = tag; + this.description = description; this.executor = executor; } @@ -47,4 +49,9 @@ public String getType() { public String getTag() { return tag; } + + @Override + public String getDescription() { + return description; + } } diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java index 47161ce8055d6..bf92faac3deec 100644 --- a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java @@ -108,7 +108,7 @@ public class IngestServiceTests extends ESTestCase { private static final IngestPlugin DUMMY_PLUGIN = new IngestPlugin() { @Override public Map getProcessors(Processor.Parameters parameters) { - return Collections.singletonMap("foo", (factories, tag, config) -> null); + return Collections.singletonMap("foo", (factories, tag, description, config) -> null); } }; @@ -338,19 +338,19 @@ public void testGetProcessorsInPipelineComplexConditional() throws Exception { ); Map processors = new HashMap<>(); - processors.put("complexSet", (factories, tag, config) -> { + processors.put("complexSet", (factories, tag, description, config) -> { String field = (String) config.remove("field"); String value = (String) config.remove("value"); - return new ConditionalProcessor(randomAlphaOfLength(10), + return new ConditionalProcessor(randomAlphaOfLength(10), null, new Script( ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptName, Collections.emptyMap()), scriptService, - new ConditionalProcessor(randomAlphaOfLength(10) + "-nested", + new ConditionalProcessor(randomAlphaOfLength(10) + "-nested", null, new Script( ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptName, Collections.emptyMap()), scriptService, - new FakeProcessor("complexSet", tag, (ingestDocument) -> ingestDocument.setFieldValue(field, value)))); + new FakeProcessor("complexSet", tag, description, (ingestDocument) -> ingestDocument.setFieldValue(field, value)))); }); IngestService ingestService = createWithProcessors(processors); @@ -612,7 +612,7 @@ public void testValidate() throws Exception { public void testExecuteIndexPipelineExistsButFailedParsing() { IngestService ingestService = createWithProcessors(Collections.singletonMap( - "mock", (factories, tag, config) -> new AbstractProcessor("mock") { + "mock", (factories, tag, description, config) -> new AbstractProcessor("mock", "description") { @Override public IngestDocument execute(IngestDocument ingestDocument) { throw new IllegalStateException("error"); @@ -650,7 +650,7 @@ public String getType() { public void testExecuteBulkPipelineDoesNotExist() { IngestService ingestService = createWithProcessors(Collections.singletonMap( - "mock", (factories, tag, config) -> mockCompoundProcessor())); + "mock", (factories, tag, description, config) -> mockCompoundProcessor())); PutPipelineRequest putRequest = new PutPipelineRequest("_id", new BytesArray("{\"processors\": [{\"mock\" : {}}]}"), XContentType.JSON); @@ -692,7 +692,7 @@ protected boolean matchesSafely(IllegalArgumentException iae) { public void testExecuteSuccess() { IngestService ingestService = createWithProcessors(Collections.singletonMap( - "mock", (factories, tag, config) -> mockCompoundProcessor())); + "mock", (factories, tag, description, config) -> mockCompoundProcessor())); PutPipelineRequest putRequest = new PutPipelineRequest("_id", new BytesArray("{\"processors\": [{\"mock\" : {}}]}"), XContentType.JSON); ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build(); // Start empty @@ -732,7 +732,7 @@ public void testExecuteEmptyPipeline() throws Exception { public void testExecutePropagateAllMetadataUpdates() throws Exception { final CompoundProcessor processor = mockCompoundProcessor(); IngestService ingestService = createWithProcessors(Collections.singletonMap( - "mock", (factories, tag, config) -> processor)); + "mock", (factories, tag, description, config) -> processor)); PutPipelineRequest putRequest = new PutPipelineRequest("_id", new BytesArray("{\"processors\": [{\"mock\" : {}}]}"), XContentType.JSON); ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build(); // Start empty @@ -786,7 +786,7 @@ public void testExecutePropagateAllMetadataUpdates() throws Exception { public void testExecuteFailure() throws Exception { final CompoundProcessor processor = mockCompoundProcessor(); IngestService ingestService = createWithProcessors(Collections.singletonMap( - "mock", (factories, tag, config) -> processor)); + "mock", (factories, tag, description, config) -> processor)); PutPipelineRequest putRequest = new PutPipelineRequest("_id", new BytesArray("{\"processors\": [{\"mock\" : {}}]}"), XContentType.JSON); ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build(); // Start empty @@ -831,7 +831,7 @@ public void testExecuteSuccessWithOnFailure() throws Exception { final CompoundProcessor compoundProcessor = new CompoundProcessor( false, Collections.singletonList(processor), Collections.singletonList(new CompoundProcessor(onFailureProcessor))); IngestService ingestService = createWithProcessors(Collections.singletonMap( - "mock", (factories, tag, config) -> compoundProcessor)); + "mock", (factories, tag, description, config) -> compoundProcessor)); PutPipelineRequest putRequest = new PutPipelineRequest("_id", new BytesArray("{\"processors\": [{\"mock\" : {}}]}"), XContentType.JSON); ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build(); // Start empty @@ -860,7 +860,7 @@ public void testExecuteFailureWithNestedOnFailure() throws Exception { Collections.singletonList(processor), Collections.singletonList(new CompoundProcessor(false, processors, onFailureProcessors))); IngestService ingestService = createWithProcessors(Collections.singletonMap( - "mock", (factories, tag, config) -> compoundProcessor)); + "mock", (factories, tag, description, config) -> compoundProcessor)); PutPipelineRequest putRequest = new PutPipelineRequest("_id", new BytesArray("{\"processors\": [{\"mock\" : {}}]}"), XContentType.JSON); ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build(); // Start empty @@ -921,7 +921,7 @@ public void testBulkRequestExecutionWithFailures() throws Exception { return null; }).when(processor).execute(any(), any()); IngestService ingestService = createWithProcessors(Collections.singletonMap( - "mock", (factories, tag, config) -> processor)); + "mock", (factories, tag, description, config) -> processor)); PutPipelineRequest putRequest = new PutPipelineRequest("_id", new BytesArray("{\"processors\": [{\"mock\" : {}}]}"), XContentType.JSON); ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build(); // Start empty @@ -971,7 +971,7 @@ public void testBulkRequestExecution() throws Exception { return null; }).when(processor).execute(any(), any()); Map map = new HashMap<>(2); - map.put("mock", (factories, tag, config) -> processor); + map.put("mock", (factories, tag, description, config) -> processor); IngestService ingestService = createWithProcessors(map); PutPipelineRequest putRequest = new PutPipelineRequest("_id", @@ -1016,8 +1016,8 @@ public void testStats() throws Exception { return null; }).when(processorFailure).execute(any(IngestDocument.class), any()); Map map = new HashMap<>(2); - map.put("mock", (factories, tag, config) -> processor); - map.put("failure-mock", (factories, tag, config) -> processorFailure); + map.put("mock", (factories, tag, description, config) -> processor); + map.put("failure-mock", (factories, tag, description, config) -> processorFailure); IngestService ingestService = createWithProcessors(map); final IngestStats initialStats = ingestService.stats(); @@ -1144,7 +1144,7 @@ public void testStatName(){ public void testExecuteWithDrop() { Map factories = new HashMap<>(); factories.put("drop", new DropProcessor.Factory()); - factories.put("mock", (processorFactories, tag, config) -> new Processor() { + factories.put("mock", (processorFactories, tag, description, config) -> new Processor() { @Override public IngestDocument execute(final IngestDocument ingestDocument) { throw new AssertionError("Document should have been dropped but reached this processor"); @@ -1159,6 +1159,11 @@ public String getType() { public String getTag() { return null; } + + @Override + public String getDescription() { + return null; + } }); IngestService ingestService = createWithProcessors(factories); PutPipelineRequest putRequest = new PutPipelineRequest("_id", @@ -1193,9 +1198,9 @@ public void testIngestClusterStateListeners_orderOfExecution() { IngestPlugin testPlugin = new IngestPlugin() { @Override public Map getProcessors(Processor.Parameters parameters) { - return Collections.singletonMap("test", (factories, tag, config) -> { + return Collections.singletonMap("test", (factories, tag, description, config) -> { assertThat(counter.compareAndSet(1, 2), is(true)); - return new FakeProcessor("test", tag, ingestDocument -> {}); + return new FakeProcessor("test", tag, description, ingestDocument -> {}); }); } }; @@ -1222,7 +1227,7 @@ public void testCBORParsing() throws Exception { AtomicReference reference = new AtomicReference<>(); Consumer executor = doc -> reference.set(doc.getFieldValueAsBytes("data")); final IngestService ingestService = createWithProcessors(Collections.singletonMap("foo", - (factories, tag, config) -> new FakeProcessor("foo", tag, executor))); + (factories, tag, description, config) -> new FakeProcessor("foo", tag, description, executor))); ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build(); ClusterState previousClusterState = clusterState; @@ -1257,14 +1262,14 @@ private IngestDocument eqIndexTypeId(final Long version, final VersionType versi private static IngestService createWithProcessors() { Map processors = new HashMap<>(); - processors.put("set", (factories, tag, config) -> { + processors.put("set", (factories, tag, description, config) -> { String field = (String) config.remove("field"); String value = (String) config.remove("value"); - return new FakeProcessor("set", tag, (ingestDocument) ->ingestDocument.setFieldValue(field, value)); + return new FakeProcessor("set", tag, description, (ingestDocument) ->ingestDocument.setFieldValue(field, value)); }); - processors.put("remove", (factories, tag, config) -> { + processors.put("remove", (factories, tag, description, config) -> { String field = (String) config.remove("field"); - return new WrappingProcessorImpl("remove", tag, (ingestDocument -> ingestDocument.removeField(field))) { + return new WrappingProcessorImpl("remove", tag, description, (ingestDocument -> ingestDocument.removeField(field))) { }; }); return createWithProcessors(processors); diff --git a/server/src/test/java/org/elasticsearch/ingest/PipelineProcessorTests.java b/server/src/test/java/org/elasticsearch/ingest/PipelineProcessorTests.java index 74dba097db792..ffe6f8e9a5b64 100644 --- a/server/src/test/java/org/elasticsearch/ingest/PipelineProcessorTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/PipelineProcessorTests.java @@ -64,13 +64,18 @@ public String getType() { public String getTag() { return null; } + + @Override + public String getDescription() { + return null; + } }) ); when(ingestService.getPipeline(pipelineId)).thenReturn(pipeline); PipelineProcessor.Factory factory = new PipelineProcessor.Factory(ingestService); Map config = new HashMap<>(); config.put("name", pipelineId); - factory.create(Collections.emptyMap(), null, config).execute(testIngestDocument, (result, e) -> {}); + factory.create(Collections.emptyMap(), null, null, config).execute(testIngestDocument, (result, e) -> {}); assertEquals(testIngestDocument, invoked.get()); } @@ -81,7 +86,7 @@ public void testThrowsOnMissingPipeline() throws Exception { Map config = new HashMap<>(); config.put("name", "missingPipelineId"); IllegalStateException[] e = new IllegalStateException[1]; - factory.create(Collections.emptyMap(), null, config) + factory.create(Collections.emptyMap(), null, null, config) .execute(testIngestDocument, (result, e1) -> e[0] = (IllegalStateException) e1); assertEquals( "Pipeline processor configured for non-existent pipeline [missingPipelineId]", e[0].getMessage() @@ -98,19 +103,19 @@ public void testThrowsOnRecursivePipelineInvocations() throws Exception { PipelineProcessor.Factory factory = new PipelineProcessor.Factory(ingestService); Pipeline outer = new Pipeline( outerPipelineId, null, null, - new CompoundProcessor(factory.create(Collections.emptyMap(), null, outerConfig)) + new CompoundProcessor(factory.create(Collections.emptyMap(), null, null, outerConfig)) ); Map innerConfig = new HashMap<>(); innerConfig.put("name", outerPipelineId); Pipeline inner = new Pipeline( innerPipelineId, null, null, - new CompoundProcessor(factory.create(Collections.emptyMap(), null, innerConfig)) + new CompoundProcessor(factory.create(Collections.emptyMap(), null, null, innerConfig)) ); when(ingestService.getPipeline(outerPipelineId)).thenReturn(outer); when(ingestService.getPipeline(innerPipelineId)).thenReturn(inner); outerConfig.put("name", innerPipelineId); ElasticsearchException[] e = new ElasticsearchException[1]; - factory.create(Collections.emptyMap(), null, outerConfig) + factory.create(Collections.emptyMap(), null, null, outerConfig) .execute(testIngestDocument, (result, e1) -> e[0] = (ElasticsearchException) e1); assertEquals( "Cycle detected for pipeline: inner", e[0].getRootCause().getMessage() @@ -128,7 +133,7 @@ public void testAllowsRepeatedPipelineInvocations() throws Exception { innerPipelineId, null, null, new CompoundProcessor() ); when(ingestService.getPipeline(innerPipelineId)).thenReturn(inner); - Processor outerProc = factory.create(Collections.emptyMap(), null, outerConfig); + Processor outerProc = factory.create(Collections.emptyMap(), null, null, outerConfig); outerProc.execute(testIngestDocument, (result, e) -> {}); outerProc.execute(testIngestDocument, (result, e) -> {}); } @@ -142,11 +147,11 @@ public void testPipelineProcessorWithPipelineChain() throws Exception { Map pipeline1ProcessorConfig = new HashMap<>(); pipeline1ProcessorConfig.put("name", pipeline2Id); - PipelineProcessor pipeline1Processor = factory.create(Collections.emptyMap(), null, pipeline1ProcessorConfig); + PipelineProcessor pipeline1Processor = factory.create(Collections.emptyMap(), null, null, pipeline1ProcessorConfig); Map pipeline2ProcessorConfig = new HashMap<>(); pipeline2ProcessorConfig.put("name", pipeline3Id); - PipelineProcessor pipeline2Processor = factory.create(Collections.emptyMap(), null, pipeline2ProcessorConfig); + PipelineProcessor pipeline2Processor = factory.create(Collections.emptyMap(), null, null, pipeline2ProcessorConfig); LongSupplier relativeTimeProvider = mock(LongSupplier.class); when(relativeTimeProvider.getAsLong()).thenReturn(0L); @@ -218,7 +223,7 @@ public void testIngestPipelineMetadata() { for (int i = 0; i < numPipelines; i++) { String pipelineId = Integer.toString(i); List processors = new ArrayList<>(); - processors.add(new AbstractProcessor(null) { + processors.add(new AbstractProcessor(null, null) { @Override public IngestDocument execute(final IngestDocument ingestDocument) throws Exception { ingestDocument.appendFieldValue("pipelines", ingestDocument.getIngestMetadata().get("pipeline")); @@ -233,7 +238,7 @@ public String getType() { }); if (i < (numPipelines - 1)) { TemplateScript.Factory pipelineName = new TestTemplateService.MockTemplateScript.Factory(Integer.toString(i + 1)); - processors.add(new PipelineProcessor(null, pipelineName, ingestService)); + processors.add(new PipelineProcessor(null, null, pipelineName, ingestService)); } diff --git a/server/src/test/java/org/elasticsearch/ingest/TrackingResultProcessorTests.java b/server/src/test/java/org/elasticsearch/ingest/TrackingResultProcessorTests.java index 5aec8a5fcde00..dce1e6d32559b 100644 --- a/server/src/test/java/org/elasticsearch/ingest/TrackingResultProcessorTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/TrackingResultProcessorTests.java @@ -97,8 +97,8 @@ public void testActualCompoundProcessorWithoutOnFailure() throws Exception { public void testActualCompoundProcessorWithOnFailure() throws Exception { RuntimeException exception = new RuntimeException("fail"); - TestProcessor failProcessor = new TestProcessor("fail", "test", exception); - TestProcessor onFailureProcessor = new TestProcessor("success", "test", ingestDocument -> {}); + TestProcessor failProcessor = new TestProcessor("fail", "test", null, exception); + TestProcessor onFailureProcessor = new TestProcessor("success", "test", null, ingestDocument -> {}); CompoundProcessor actualProcessor = new CompoundProcessor(false, Arrays.asList(new CompoundProcessor(false, Arrays.asList(failProcessor, onFailureProcessor), @@ -144,12 +144,13 @@ public void testActualCompoundProcessorWithOnFailureAndTrueCondition() throws Ex new HashMap<>(ScriptModule.CORE_CONTEXTS) ); RuntimeException exception = new RuntimeException("fail"); - TestProcessor failProcessor = new TestProcessor("fail", "test", exception); + TestProcessor failProcessor = new TestProcessor("fail", "test", null, exception); ConditionalProcessor conditionalProcessor = new ConditionalProcessor( randomAlphaOfLength(10), + null, new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptName, Collections.emptyMap()), scriptService, failProcessor); - TestProcessor onFailureProcessor = new TestProcessor("success", "test", ingestDocument -> {}); + TestProcessor onFailureProcessor = new TestProcessor("success", "test", null, ingestDocument -> {}); CompoundProcessor actualProcessor = new CompoundProcessor(false, Arrays.asList(conditionalProcessor), @@ -210,6 +211,7 @@ public void testActualCompoundProcessorWithFalseConditional() throws Exception { new TestProcessor(ingestDocument -> {ingestDocument.setFieldValue(key1, randomInt()); }), new ConditionalProcessor( randomAlphaOfLength(10), + null, new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptName, Collections.emptyMap()), scriptService, new TestProcessor(ingestDocument -> {ingestDocument.setFieldValue(key2, randomInt()); })), new TestProcessor(ingestDocument -> { ingestDocument.setFieldValue(key3, randomInt()); })); @@ -253,7 +255,7 @@ pipelineId, null, null, new CompoundProcessor( ); when(ingestService.getPipeline(pipelineId)).thenReturn(pipeline); - PipelineProcessor pipelineProcessor = factory.create(Collections.emptyMap(), null, pipelineConfig); + PipelineProcessor pipelineProcessor = factory.create(Collections.emptyMap(), null, null, pipelineConfig); CompoundProcessor actualProcessor = new CompoundProcessor(pipelineProcessor); CompoundProcessor trackingProcessor = decorate(actualProcessor, null, resultList); @@ -307,8 +309,9 @@ pipelineId1, null, null, new CompoundProcessor( new TestProcessor(ingestDocument -> {ingestDocument.setFieldValue(key1, randomInt()); }), new ConditionalProcessor( randomAlphaOfLength(10), + null, new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptName, Collections.emptyMap()), scriptService, - factory.create(Collections.emptyMap(), null, pipelineConfig2)), + factory.create(Collections.emptyMap(), null, null, pipelineConfig2)), new TestProcessor(ingestDocument -> {ingestDocument.setFieldValue(key3, randomInt()); }) ) ); @@ -320,7 +323,7 @@ pipelineId2, null, null, new CompoundProcessor( when(ingestService.getPipeline(pipelineId1)).thenReturn(pipeline1); when(ingestService.getPipeline(pipelineId2)).thenReturn(pipeline2); - PipelineProcessor pipelineProcessor = factory.create(Collections.emptyMap(), null, pipelineConfig0); + PipelineProcessor pipelineProcessor = factory.create(Collections.emptyMap(), null, null, pipelineConfig0); CompoundProcessor actualProcessor = new CompoundProcessor(pipelineProcessor); CompoundProcessor trackingProcessor = decorate(actualProcessor, null, resultList); @@ -376,8 +379,9 @@ pipelineId1, null, null, new CompoundProcessor( new TestProcessor(ingestDocument -> {ingestDocument.setFieldValue(key1, randomInt()); }), new ConditionalProcessor( randomAlphaOfLength(10), + null, new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptName, Collections.emptyMap()), scriptService, - factory.create(Collections.emptyMap(), null, pipelineConfig2)), + factory.create(Collections.emptyMap(), null, null, pipelineConfig2)), new TestProcessor(ingestDocument -> {ingestDocument.setFieldValue(key3, randomInt()); }) ) ); @@ -389,7 +393,7 @@ pipelineId2, null, null, new CompoundProcessor( when(ingestService.getPipeline(pipelineId1)).thenReturn(pipeline1); when(ingestService.getPipeline(pipelineId2)).thenReturn(pipeline2); - PipelineProcessor pipelineProcessor = factory.create(Collections.emptyMap(), null, pipelineConfig0); + PipelineProcessor pipelineProcessor = factory.create(Collections.emptyMap(), null, null, pipelineConfig0); CompoundProcessor actualProcessor = new CompoundProcessor(pipelineProcessor); CompoundProcessor trackingProcessor = decorate(actualProcessor, null, resultList); @@ -442,7 +446,7 @@ pipelineId, null, null, new CompoundProcessor( ); when(ingestService.getPipeline(pipelineId)).thenReturn(pipeline); - PipelineProcessor pipelineProcessor = factory.create(Collections.emptyMap(), null, pipelineConfig); + PipelineProcessor pipelineProcessor = factory.create(Collections.emptyMap(), null, null, pipelineConfig); CompoundProcessor actualProcessor = new CompoundProcessor(pipelineProcessor); CompoundProcessor trackingProcessor = decorate(actualProcessor, null, resultList); @@ -485,15 +489,15 @@ public void testActualPipelineProcessorWithCycle() throws Exception { PipelineProcessor.Factory factory = new PipelineProcessor.Factory(ingestService); Pipeline pipeline1 = new Pipeline( - pipelineId1, null, null, new CompoundProcessor(factory.create(Collections.emptyMap(), null, pipelineConfig2))); + pipelineId1, null, null, new CompoundProcessor(factory.create(Collections.emptyMap(), null, null, pipelineConfig2))); Pipeline pipeline2 = new Pipeline( - pipelineId2, null, null, new CompoundProcessor(factory.create(Collections.emptyMap(), null, pipelineConfig1))); + pipelineId2, null, null, new CompoundProcessor(factory.create(Collections.emptyMap(), null, null, pipelineConfig1))); when(ingestService.getPipeline(pipelineId1)).thenReturn(pipeline1); when(ingestService.getPipeline(pipelineId2)).thenReturn(pipeline2); - PipelineProcessor pipelineProcessor = factory.create(Collections.emptyMap(), null, pipelineConfig0); + PipelineProcessor pipelineProcessor = factory.create(Collections.emptyMap(), null, null, pipelineConfig0); CompoundProcessor actualProcessor = new CompoundProcessor(pipelineProcessor); CompoundProcessor trackingProcessor = decorate(actualProcessor, null, resultList); @@ -513,7 +517,7 @@ public void testActualPipelineProcessorRepeatedInvocation() throws Exception { PipelineProcessor.Factory factory = new PipelineProcessor.Factory(ingestService); String key1 = randomAlphaOfLength(10); - PipelineProcessor pipelineProcessor = factory.create(Collections.emptyMap(), null, pipelineConfig); + PipelineProcessor pipelineProcessor = factory.create(Collections.emptyMap(), null, null, pipelineConfig); Pipeline pipeline = new Pipeline( pipelineId, null, null, new CompoundProcessor( new TestProcessor(ingestDocument -> { ingestDocument.setFieldValue(key1, randomInt()); })) diff --git a/server/src/test/java/org/elasticsearch/ingest/WrappingProcessorImpl.java b/server/src/test/java/org/elasticsearch/ingest/WrappingProcessorImpl.java index db2b827c0144b..d600c73586b87 100644 --- a/server/src/test/java/org/elasticsearch/ingest/WrappingProcessorImpl.java +++ b/server/src/test/java/org/elasticsearch/ingest/WrappingProcessorImpl.java @@ -23,14 +23,15 @@ class WrappingProcessorImpl extends FakeProcessor implements WrappingProcessor { - WrappingProcessorImpl(String type, String tag, Consumer executor) { - super(type, tag, executor); + WrappingProcessorImpl(String type, String tag, String description, Consumer executor) { + super(type, tag, description, executor); } @Override public Processor getInnerProcessor() { String theType = getType(); String theTag = getTag(); + String theDescription = getDescription(); return new Processor() { @Override public IngestDocument execute(IngestDocument ingestDocument) throws Exception { @@ -46,6 +47,11 @@ public String getType() { public String getTag() { return theTag; } + + @Override + public String getDescription() { + return theDescription; + } }; } } diff --git a/test/framework/src/main/java/org/elasticsearch/ingest/IngestTestPlugin.java b/test/framework/src/main/java/org/elasticsearch/ingest/IngestTestPlugin.java index 1fe7edc745242..c96f0e88a5b72 100644 --- a/test/framework/src/main/java/org/elasticsearch/ingest/IngestTestPlugin.java +++ b/test/framework/src/main/java/org/elasticsearch/ingest/IngestTestPlugin.java @@ -31,8 +31,8 @@ public class IngestTestPlugin extends Plugin implements IngestPlugin { @Override public Map getProcessors(Processor.Parameters parameters) { - return Collections.singletonMap("test", (factories, tag, config) -> - new TestProcessor("id", "test", doc -> { + return Collections.singletonMap("test", (factories, tag, description, config) -> + new TestProcessor("id", "test", "description", doc -> { doc.setFieldValue("processed", true); if (doc.hasField("fail") && doc.getFieldValue("fail", Boolean.class)) { throw new IllegalArgumentException("test processor failed"); diff --git a/test/framework/src/main/java/org/elasticsearch/ingest/TestProcessor.java b/test/framework/src/main/java/org/elasticsearch/ingest/TestProcessor.java index 80579831475e0..b0ab3ebc0699f 100644 --- a/test/framework/src/main/java/org/elasticsearch/ingest/TestProcessor.java +++ b/test/framework/src/main/java/org/elasticsearch/ingest/TestProcessor.java @@ -32,32 +32,34 @@ public class TestProcessor implements Processor { private final String type; private final String tag; + private final String description; private final Function ingestDocumentMapper; private final AtomicInteger invokedCounter = new AtomicInteger(); public TestProcessor(Consumer ingestDocumentConsumer) { - this(null, "test-processor", ingestDocumentConsumer); + this(null, "test-processor", null, ingestDocumentConsumer); } public TestProcessor(RuntimeException e) { - this(null, "test-processor", e); + this(null, "test-processor", null, e); } - public TestProcessor(String tag, String type, RuntimeException e) { - this(tag, type, (Consumer) i -> { throw e; }); + public TestProcessor(String tag, String type, String description, RuntimeException e) { + this(tag, type, description, (Consumer) i -> { throw e; }); } - public TestProcessor(String tag, String type, Consumer ingestDocumentConsumer) { - this(tag, type, id -> { + public TestProcessor(String tag, String type, String description, Consumer ingestDocumentConsumer) { + this(tag, type, description, id -> { ingestDocumentConsumer.accept(id); return id; }); } - public TestProcessor(String tag, String type, Function ingestDocumentMapper) { + public TestProcessor(String tag, String type, String description, Function ingestDocumentMapper) { this.ingestDocumentMapper = ingestDocumentMapper; this.type = type; this.tag = tag; + this.description = description; } @Override @@ -76,6 +78,11 @@ public String getTag() { return tag; } + @Override + public String getDescription() { + return description; + } + public int getInvokedCounter() { return invokedCounter.get(); } @@ -83,8 +90,8 @@ public int getInvokedCounter() { public static final class Factory implements Processor.Factory { @Override public TestProcessor create(Map registry, String processorTag, - Map config) throws Exception { - return new TestProcessor(processorTag, "test-processor", ingestDocument -> {}); + String description, Map config) throws Exception { + return new TestProcessor(processorTag, "test-processor", description, ingestDocument -> {}); } } } diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/AbstractEnrichProcessor.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/AbstractEnrichProcessor.java index 07be2f59cbda7..4c219c0779092 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/AbstractEnrichProcessor.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/AbstractEnrichProcessor.java @@ -38,6 +38,7 @@ public abstract class AbstractEnrichProcessor extends AbstractProcessor { protected AbstractEnrichProcessor( String tag, + String description, Client client, String policyName, TemplateScript.Factory field, @@ -47,11 +48,13 @@ protected AbstractEnrichProcessor( String matchField, int maxMatches ) { - this(tag, createSearchRunner(client), policyName, field, targetField, ignoreMissing, overrideEnabled, matchField, maxMatches); + this(tag, description, createSearchRunner(client), policyName, field, targetField, ignoreMissing, overrideEnabled, matchField, + maxMatches); } protected AbstractEnrichProcessor( String tag, + String description, BiConsumer> searchRunner, String policyName, TemplateScript.Factory field, @@ -61,7 +64,7 @@ protected AbstractEnrichProcessor( String matchField, int maxMatches ) { - super(tag); + super(tag, description); this.policyName = policyName; this.searchRunner = searchRunner; this.field = field; diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactory.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactory.java index 9df68629b6b58..fa2d5ecadfece 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactory.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactory.java @@ -35,7 +35,7 @@ final class EnrichProcessorFactory implements Processor.Factory, Consumer processorFactories, String tag, Map config) throws Exception { + public Processor create(Map processorFactories, String tag, String description, Map config) throws Exception { String policyName = ConfigurationUtils.readStringProperty(TYPE, tag, config, "policy_name"); String policyAlias = EnrichPolicy.getBaseName(policyName); if (metadata == null) { diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/GeoMatchProcessor.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/GeoMatchProcessor.java index 62f8c16dd0d0b..e882b05af4d09 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/GeoMatchProcessor.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/GeoMatchProcessor.java @@ -28,6 +28,7 @@ public final class GeoMatchProcessor extends AbstractEnrichProcessor { GeoMatchProcessor( String tag, + String description, Client client, String policyName, TemplateScript.Factory field, @@ -38,13 +39,14 @@ public final class GeoMatchProcessor extends AbstractEnrichProcessor { int maxMatches, ShapeRelation shapeRelation ) { - super(tag, client, policyName, field, targetField, ignoreMissing, overrideEnabled, matchField, maxMatches); + super(tag, description, client, policyName, field, targetField, ignoreMissing, overrideEnabled, matchField, maxMatches); this.shapeRelation = shapeRelation; } /** used in tests **/ GeoMatchProcessor( String tag, + String description, BiConsumer> searchRunner, String policyName, TemplateScript.Factory field, @@ -55,7 +57,7 @@ public final class GeoMatchProcessor extends AbstractEnrichProcessor { int maxMatches, ShapeRelation shapeRelation ) { - super(tag, searchRunner, policyName, field, targetField, ignoreMissing, overrideEnabled, matchField, maxMatches); + super(tag, description, searchRunner, policyName, field, targetField, ignoreMissing, overrideEnabled, matchField, maxMatches); this.shapeRelation = shapeRelation; } diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/MatchProcessor.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/MatchProcessor.java index 317e470b764d9..01b031382e85c 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/MatchProcessor.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/MatchProcessor.java @@ -20,6 +20,7 @@ public final class MatchProcessor extends AbstractEnrichProcessor { MatchProcessor( String tag, + String description, Client client, String policyName, TemplateScript.Factory field, @@ -29,12 +30,13 @@ public final class MatchProcessor extends AbstractEnrichProcessor { String matchField, int maxMatches ) { - super(tag, client, policyName, field, targetField, ignoreMissing, overrideEnabled, matchField, maxMatches); + super(tag, description, client, policyName, field, targetField, ignoreMissing, overrideEnabled, matchField, maxMatches); } /** used in tests **/ MatchProcessor( String tag, + String description, BiConsumer> searchRunner, String policyName, TemplateScript.Factory field, @@ -44,7 +46,7 @@ public final class MatchProcessor extends AbstractEnrichProcessor { String matchField, int maxMatches ) { - super(tag, searchRunner, policyName, field, targetField, ignoreMissing, overrideEnabled, matchField, maxMatches); + super(tag, description, searchRunner, policyName, field, targetField, ignoreMissing, overrideEnabled, matchField, maxMatches); } @Override diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactoryTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactoryTests.java index 37ed55ed16329..c2f31b063dee7 100644 --- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactoryTests.java +++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactoryTests.java @@ -70,7 +70,7 @@ public void testCreateProcessorInstance() throws Exception { randomValues.add(new Tuple<>(randomFrom(enrichValues), randomAlphaOfLength(4))); } - MatchProcessor result = (MatchProcessor) factory.create(Collections.emptyMap(), "_tag", config); + MatchProcessor result = (MatchProcessor) factory.create(Collections.emptyMap(), "_tag", null, config); assertThat(result, notNullValue()); assertThat(result.getPolicyName(), equalTo("majestic")); assertThat(result.getField(), equalTo("host")); @@ -114,7 +114,7 @@ public void testPolicyDoesNotExist() { } config.put("set_from", valuesConfig); - Exception e = expectThrows(IllegalArgumentException.class, () -> factory.create(Collections.emptyMap(), "_tag", config)); + Exception e = expectThrows(IllegalArgumentException.class, () -> factory.create(Collections.emptyMap(), "_tag", null, config)); assertThat(e.getMessage(), equalTo("no enrich index exists for policy with name [majestic]")); } @@ -141,7 +141,7 @@ public void testPolicyNameMissing() { } config.put("set_from", valuesConfig); - Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(Collections.emptyMap(), "_tag", config)); + Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(Collections.emptyMap(), "_tag", null, config)); assertThat(e.getMessage(), equalTo("[policy_name] required property is missing")); } @@ -160,7 +160,7 @@ public void testUnsupportedPolicy() throws Exception { config.put("ignore_missing", keyIgnoreMissing); } - Exception e = expectThrows(IllegalArgumentException.class, () -> factory.create(Collections.emptyMap(), "_tag", config)); + Exception e = expectThrows(IllegalArgumentException.class, () -> factory.create(Collections.emptyMap(), "_tag", null, config)); assertThat(e.getMessage(), equalTo("unsupported policy type [unsupported]")); } @@ -175,7 +175,7 @@ public void testCompactEnrichValuesFormat() throws Exception { config.put("field", "host"); config.put("target_field", "entry"); - MatchProcessor result = (MatchProcessor) factory.create(Collections.emptyMap(), "_tag", config); + MatchProcessor result = (MatchProcessor) factory.create(Collections.emptyMap(), "_tag", null, config); assertThat(result, notNullValue()); assertThat(result.getPolicyName(), equalTo("majestic")); assertThat(result.getField(), equalTo("host")); @@ -192,7 +192,7 @@ public void testNoTargetField() throws Exception { config1.put("policy_name", "majestic"); config1.put("field", "host"); - Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(Collections.emptyMap(), "_tag", config1)); + Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(Collections.emptyMap(), "_tag", null, config1)); assertThat(e.getMessage(), equalTo("[target_field] required property is missing")); } @@ -208,7 +208,7 @@ public void testIllegalMaxMatches() throws Exception { config.put("target_field", "entry"); config.put("max_matches", randomBoolean() ? between(-2048, 0) : between(129, 2048)); - Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(Collections.emptyMap(), "_tag", config)); + Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(Collections.emptyMap(), "_tag", null, config)); assertThat(e.getMessage(), equalTo("[max_matches] should be between 1 and 128")); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java index 2be07dc64b3f0..55538a211773e 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java @@ -84,11 +84,12 @@ public class InferenceProcessor extends AbstractProcessor { public InferenceProcessor(Client client, InferenceAuditor auditor, String tag, + String description, String targetField, String modelId, InferenceConfigUpdate inferenceConfig, Map fieldMap) { - super(tag); + super(tag, description); this.client = ExceptionsHelper.requireNonNull(client, "client"); this.targetField = ExceptionsHelper.requireNonNull(targetField, TARGET_FIELD); this.auditor = ExceptionsHelper.requireNonNull(auditor, "auditor"); @@ -270,7 +271,7 @@ int numInferenceProcessors() { } @Override - public InferenceProcessor create(Map processorFactories, String tag, Map config) { + public InferenceProcessor create(Map processorFactories, String tag, String description, Map config) { if (this.maxIngestProcessors <= currentInferenceProcessors) { throw new ElasticsearchStatusException("Max number of inference processors reached, total inference processors [{}]. " + @@ -300,6 +301,7 @@ public InferenceProcessor create(Map processorFactori return new InferenceProcessor(client, auditor, tag, + description, targetField, modelId, inferenceConfig, diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportGetTrainedModelsStatsActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportGetTrainedModelsStatsActionTests.java index e061c18912d70..d2e68f00f5e08 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportGetTrainedModelsStatsActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportGetTrainedModelsStatsActionTests.java @@ -76,7 +76,7 @@ public String getTag() { static class Factory implements Processor.Factory { @Override - public Processor create(Map processorFactories, String tag, Map config) { + public Processor create(Map processorFactories, String tag, String description, Map config) { return new NotInferenceProcessor(); } } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessorFactoryTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessorFactoryTests.java index 3d4496a8dfdc5..1375234734f3d 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessorFactoryTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessorFactoryTests.java @@ -149,7 +149,7 @@ public void testCreateProcessorWithTooManyExisting() throws Exception { processorFactory.accept(buildClusterStateWithModelReferences("model1")); ElasticsearchStatusException ex = expectThrows(ElasticsearchStatusException.class, - () -> processorFactory.create(Collections.emptyMap(), "my_inference_processor", Collections.emptyMap())); + () -> processorFactory.create(Collections.emptyMap(), "my_inference_processor", null, Collections.emptyMap())); assertThat(ex.getMessage(), equalTo("Max number of inference processors reached, total inference processors [1]. " + "Adjust the setting [xpack.ml.max_inference_processors]: [1] if a greater number is desired.")); @@ -168,7 +168,7 @@ public void testCreateProcessorWithInvalidInferenceConfig() { }}; ElasticsearchStatusException ex = expectThrows(ElasticsearchStatusException.class, - () -> processorFactory.create(Collections.emptyMap(), "my_inference_processor", config)); + () -> processorFactory.create(Collections.emptyMap(), "my_inference_processor", null, config)); assertThat(ex.getMessage(), equalTo("unrecognized inference configuration type [unknown_type]. Supported types [classification, regression]")); @@ -179,7 +179,7 @@ public void testCreateProcessorWithInvalidInferenceConfig() { put(InferenceProcessor.INFERENCE_CONFIG, Collections.singletonMap("regression", "boom")); }}; ex = expectThrows(ElasticsearchStatusException.class, - () -> processorFactory.create(Collections.emptyMap(), "my_inference_processor", config2)); + () -> processorFactory.create(Collections.emptyMap(), "my_inference_processor", null, config2)); assertThat(ex.getMessage(), equalTo("inference_config must be an object with one inference type mapped to an object.")); @@ -190,7 +190,7 @@ public void testCreateProcessorWithInvalidInferenceConfig() { put(InferenceProcessor.INFERENCE_CONFIG, Collections.emptyMap()); }}; ex = expectThrows(ElasticsearchStatusException.class, - () -> processorFactory.create(Collections.emptyMap(), "my_inference_processor", config3)); + () -> processorFactory.create(Collections.emptyMap(), "my_inference_processor", null, config3)); assertThat(ex.getMessage(), equalTo("inference_config must be an object with one inference type mapped to an object.")); } @@ -210,7 +210,7 @@ public void testCreateProcessorWithTooOldMinNodeVersion() throws IOException { }}; try { - processorFactory.create(Collections.emptyMap(), "my_inference_processor", regression); + processorFactory.create(Collections.emptyMap(), "my_inference_processor", null, regression); fail("Should not have successfully created"); } catch (ElasticsearchException ex) { assertThat(ex.getMessage(), @@ -228,7 +228,7 @@ public void testCreateProcessorWithTooOldMinNodeVersion() throws IOException { }}; try { - processorFactory.create(Collections.emptyMap(), "my_inference_processor", classification); + processorFactory.create(Collections.emptyMap(), "my_inference_processor", null, classification); fail("Should not have successfully created"); } catch (ElasticsearchException ex) { assertThat(ex.getMessage(), @@ -252,7 +252,7 @@ public void testCreateProcessor() { }}; try { - processorFactory.create(Collections.emptyMap(), "my_inference_processor", regression); + processorFactory.create(Collections.emptyMap(), "my_inference_processor", null, regression); } catch (Exception ex) { fail(ex.getMessage()); } @@ -266,7 +266,7 @@ public void testCreateProcessor() { }}; try { - processorFactory.create(Collections.emptyMap(), "my_inference_processor", classification); + processorFactory.create(Collections.emptyMap(), "my_inference_processor", null, classification); } catch (Exception ex) { fail(ex.getMessage()); } @@ -286,7 +286,7 @@ public void testCreateProcessorWithDuplicateFields() { }}; try { - processorFactory.create(Collections.emptyMap(), "my_inference_processor", regression); + processorFactory.create(Collections.emptyMap(), "my_inference_processor", null, regression); fail("should not have succeeded creating with duplicate fields"); } catch (Exception ex) { assertThat(ex.getMessage(), equalTo("Cannot create processor as configured. " + diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/test/MockIngestPlugin.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/test/MockIngestPlugin.java index b4521ad58b222..650df8243bb2d 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/test/MockIngestPlugin.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/test/MockIngestPlugin.java @@ -52,7 +52,7 @@ static class MockProcessorFactory implements Processor.Factory { @Override public Processor create(Map processorFactories, String tag, - Map config) throws Exception { + String description, Map config) throws Exception { // read fields so the processor succeeds for (final String field : fields) { ConfigurationUtils.readObject(type, tag, config, field); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessor.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessor.java index 0647ab4588eb1..048879c4dc30f 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessor.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessor.java @@ -44,9 +44,9 @@ public final class SetSecurityUserProcessor extends AbstractProcessor { private final String field; private final Set properties; - public SetSecurityUserProcessor(String tag, SecurityContext securityContext, XPackLicenseState licenseState, String field, - Set properties) { - super(tag); + public SetSecurityUserProcessor(String tag, String description, SecurityContext securityContext, XPackLicenseState licenseState, + String field, Set properties) { + super(tag, description); this.securityContext = securityContext; this.licenseState = Objects.requireNonNull(licenseState, "license state cannot be null"); if (licenseState.isSecurityEnabled() == false) { @@ -191,7 +191,7 @@ public Factory(Supplier securityContext, Supplier processorFactories, String tag, - Map config) throws Exception { + String description, Map config) throws Exception { String field = readStringProperty(TYPE, tag, config, "field"); List propertyNames = readOptionalList(TYPE, tag, config, "properties"); Set properties; @@ -203,7 +203,7 @@ public SetSecurityUserProcessor create(Map processorF } else { properties = EnumSet.allOf(Property.class); } - return new SetSecurityUserProcessor(tag, securityContext.get(), licenseState.get(), field, properties); + return new SetSecurityUserProcessor(tag, description, securityContext.get(), licenseState.get(), field, properties); } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessorFactoryTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessorFactoryTests.java index aa7339576b0ec..2214fa62dc31f 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessorFactoryTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessorFactoryTests.java @@ -40,7 +40,7 @@ public void testProcessor() throws Exception { SetSecurityUserProcessor.Factory factory = new SetSecurityUserProcessor.Factory(() -> securityContext, () -> licenseState); Map config = new HashMap<>(); config.put("field", "_field"); - SetSecurityUserProcessor processor = factory.create(null, "_tag", config); + SetSecurityUserProcessor processor = factory.create(null, "_tag", null, config); assertThat(processor.getField(), equalTo("_field")); assertThat(processor.getProperties(), equalTo(EnumSet.allOf(Property.class))); } @@ -48,7 +48,7 @@ public void testProcessor() throws Exception { public void testProcessor_noField() throws Exception { SetSecurityUserProcessor.Factory factory = new SetSecurityUserProcessor.Factory(() -> securityContext, () -> licenseState); Map config = new HashMap<>(); - ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", config)); + ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", null, config)); assertThat(e.getMetadata("es.property_name").get(0), equalTo("field")); assertThat(e.getMetadata("es.processor_type").get(0), equalTo(SetSecurityUserProcessor.TYPE)); assertThat(e.getMetadata("es.processor_tag").get(0), equalTo("_tag")); @@ -59,7 +59,7 @@ public void testProcessor_validProperties() throws Exception { Map config = new HashMap<>(); config.put("field", "_field"); config.put("properties", Arrays.asList(Property.USERNAME.name(), Property.ROLES.name())); - SetSecurityUserProcessor processor = factory.create(null, "_tag", config); + SetSecurityUserProcessor processor = factory.create(null, "_tag", null, config); assertThat(processor.getField(), equalTo("_field")); assertThat(processor.getProperties(), equalTo(EnumSet.of(Property.USERNAME, Property.ROLES))); } @@ -69,7 +69,7 @@ public void testProcessor_invalidProperties() throws Exception { Map config = new HashMap<>(); config.put("field", "_field"); config.put("properties", Arrays.asList("invalid")); - ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", config)); + ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", null, config)); assertThat(e.getMetadata("es.property_name").get(0), equalTo("properties")); assertThat(e.getMetadata("es.processor_type").get(0), equalTo(SetSecurityUserProcessor.TYPE)); assertThat(e.getMetadata("es.processor_tag").get(0), equalTo("_tag")); @@ -80,7 +80,7 @@ public void testCanConstructorProcessorWithoutSecurityEnabled() throws Exception SetSecurityUserProcessor.Factory factory = new SetSecurityUserProcessor.Factory(() -> null, () -> licenseState); Map config = new HashMap<>(); config.put("field", "_field"); - final SetSecurityUserProcessor processor = factory.create(null, "_tag", config); + final SetSecurityUserProcessor processor = factory.create(null, "_tag", null, config); assertThat(processor, notNullValue()); } diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/ingest/CircleProcessor.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/ingest/CircleProcessor.java index 34f589b7cf3e2..2b7dd4ff9fe81 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/ingest/CircleProcessor.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/ingest/CircleProcessor.java @@ -45,9 +45,9 @@ public final class CircleProcessor extends AbstractProcessor { private final double errorDistance; private final CircleShapeFieldType circleShapeFieldType; - CircleProcessor(String tag, String field, String targetField, boolean ignoreMissing, double errorDistance, + CircleProcessor(String tag, String description, String field, String targetField, boolean ignoreMissing, double errorDistance, CircleShapeFieldType circleShapeFieldType) { - super(tag); + super(tag, description); this.field = field; this.targetField = targetField; this.ignoreMissing = ignoreMissing; @@ -141,14 +141,14 @@ int numSides(double radiusMeters) { public static final class Factory implements Processor.Factory { - public CircleProcessor create(Map registry, String processorTag, Map config) { + public CircleProcessor create(Map registry, String processorTag, String description, Map config) { String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field"); String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", field); boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false); double radiusDistance = Math.abs(ConfigurationUtils.readDoubleProperty(TYPE, processorTag, config, "error_distance")); CircleShapeFieldType circleFieldType = CircleShapeFieldType.parse( ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "shape_type")); - return new CircleProcessor(processorTag, field, targetField, ignoreMissing, radiusDistance, circleFieldType); + return new CircleProcessor(processorTag, description, field, targetField, ignoreMissing, radiusDistance, circleFieldType); } } diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/ingest/CircleProcessorFactoryTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/ingest/CircleProcessorFactoryTests.java index be4544c10305e..4b6a997f656d0 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/ingest/CircleProcessorFactoryTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/ingest/CircleProcessorFactoryTests.java @@ -30,7 +30,7 @@ public void testCreateGeoShape() { config.put("error_distance", 0.002); config.put("shape_type", "geo_shape"); String processorTag = randomAlphaOfLength(10); - CircleProcessor processor = factory.create(null, processorTag, config); + CircleProcessor processor = factory.create(null, processorTag, null, config); assertThat(processor.getTag(), equalTo(processorTag)); assertThat(processor.field(), equalTo("field1")); assertThat(processor.targetField(), equalTo("field1")); @@ -44,7 +44,7 @@ public void testCreateShape() { config.put("error_distance", 0.002); config.put("shape_type", "shape"); String processorTag = randomAlphaOfLength(10); - CircleProcessor processor = factory.create(null, processorTag, config); + CircleProcessor processor = factory.create(null, processorTag, null, config); assertThat(processor.getTag(), equalTo(processorTag)); assertThat(processor.field(), equalTo("field1")); assertThat(processor.targetField(), equalTo("field1")); @@ -58,14 +58,14 @@ public void testCreateInvalidShapeType() { config.put("error_distance", 0.002); config.put("shape_type", "invalid"); String processorTag = randomAlphaOfLength(10); - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> factory.create(null, processorTag, config)); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> factory.create(null, processorTag, null, config)); assertThat(e.getMessage(), equalTo("illegal [shape_type] value [invalid]. valid values are [SHAPE, GEO_SHAPE]")); } public void testCreateMissingField() { Map config = new HashMap<>(); String processorTag = randomAlphaOfLength(10); - ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, processorTag, config)); + ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, processorTag, null, config)); assertThat(e.getMessage(), equalTo("[field] required property is missing")); } @@ -76,7 +76,7 @@ public void testCreateWithTargetField() { config.put("error_distance", 0.002); config.put("shape_type", "geo_shape"); String processorTag = randomAlphaOfLength(10); - CircleProcessor processor = factory.create(null, processorTag, config); + CircleProcessor processor = factory.create(null, processorTag, null, config); assertThat(processor.getTag(), equalTo(processorTag)); assertThat(processor.field(), equalTo("field1")); assertThat(processor.targetField(), equalTo("other")); @@ -88,7 +88,7 @@ public void testCreateWithNoErrorDistanceDefined() { Map config = new HashMap<>(); config.put("field", "field1"); String processorTag = randomAlphaOfLength(10); - ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, processorTag, config)); + ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, processorTag, null, config)); assertThat(e.getMessage(), equalTo("[error_distance] required property is missing")); } } From bf0305f6f458ada023f2e90d8dec99cd5f433611 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Wed, 10 Jun 2020 11:07:56 -0700 Subject: [PATCH 2/6] fix ingest-common test refactoring --- .../common/AbstractStringProcessor.java | 3 +- .../ingest/common/DissectProcessor.java | 3 +- .../ingest/common/DotExpanderProcessor.java | 2 +- .../ingest/common/ScriptProcessor.java | 3 +- .../common/AppendProcessorFactoryTests.java | 3 +- .../ingest/common/AppendProcessorTests.java | 2 +- .../ingest/common/BytesProcessorTests.java | 2 +- .../ingest/common/ConvertProcessorTests.java | 70 +++++++++---------- .../ingest/common/CsvProcessorTests.java | 9 +-- .../common/DateIndexNameProcessorTests.java | 2 +- .../ingest/common/DateProcessorTests.java | 24 +++---- .../ingest/common/DissectProcessorTests.java | 14 ++-- .../common/DotExpanderProcessorTests.java | 28 ++++---- .../common/FailProcessorFactoryTests.java | 3 +- .../ingest/common/FailProcessorTests.java | 2 +- .../ingest/common/ForEachProcessorTests.java | 36 ++++++---- .../ingest/common/GrokProcessorTests.java | 34 ++++----- .../ingest/common/GsubProcessorTests.java | 2 +- .../common/HtmlStripProcessorTests.java | 2 +- .../ingest/common/JoinProcessorTests.java | 12 ++-- .../ingest/common/JsonProcessorTests.java | 24 +++---- .../common/LowercaseProcessorTests.java | 2 +- .../common/RemoveProcessorFactoryTests.java | 3 +- .../ingest/common/RemoveProcessorTests.java | 2 +- .../ingest/common/RenameProcessorTests.java | 2 +- .../ingest/common/ScriptProcessorTests.java | 2 +- .../common/SetProcessorFactoryTests.java | 3 +- .../ingest/common/SetProcessorTests.java | 2 +- .../ingest/common/SortProcessorTests.java | 30 ++++---- .../ingest/common/SplitProcessorTests.java | 16 ++--- .../ingest/common/TrimProcessorTests.java | 2 +- .../common/URLDecodeProcessorTests.java | 2 +- .../common/UppercaseProcessorTests.java | 2 +- 33 files changed, 182 insertions(+), 166 deletions(-) diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java index ad3a53c7ef7ef..8e0ca574676a8 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java @@ -114,7 +114,8 @@ public AbstractStringProcessor create(Map registry return newProcessor(tag, description, config, field, ignoreMissing, targetField); } - protected abstract AbstractStringProcessor newProcessor(String processorTag, String description, Map config, String field, + protected abstract AbstractStringProcessor newProcessor(String processorTag, String description, + Map config, String field, boolean ignoreMissing, String targetField); } } diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DissectProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DissectProcessor.java index 2ed4349a69730..c9d5a3059d1fe 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DissectProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DissectProcessor.java @@ -66,7 +66,8 @@ public String getType() { public static final class Factory implements Processor.Factory { @Override - public DissectProcessor create(Map registry, String processorTag, String description, Map config) { + public DissectProcessor create(Map registry, String processorTag, String description, + Map config) { String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field"); String pattern = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "pattern"); String appendSeparator = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "append_separator", ""); diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DotExpanderProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DotExpanderProcessor.java index ea198b7b4f842..6c22343fefd0e 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DotExpanderProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DotExpanderProcessor.java @@ -33,7 +33,7 @@ public final class DotExpanderProcessor extends AbstractProcessor { private final String path; private final String field; - DotExpanderProcessor(String tag, String description, String path, String field) { + DotExpanderProcessor(String tag, String description, String field, String path) { super(tag, description); this.path = path; this.field = field; diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java index ddc5d78be629f..e882eac9ef8b8 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java @@ -53,8 +53,7 @@ public final class ScriptProcessor extends AbstractProcessor { /** * Processor that evaluates a script with an ingest document in its context - * - * @param tag The processor's tag. + * @param tag The processor's tag. * @param description The processor's description. * @param script The {@link Script} to execute. * @param scriptService The {@link ScriptService} used to execute the script. diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorFactoryTests.java index 99398316c1ef6..bed9e91cb4c25 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorFactoryTests.java @@ -98,7 +98,8 @@ public void testInvalidMustacheTemplate() throws Exception { config.put("field", "{{field1}}"); config.put("value", "value1"); String processorTag = randomAlphaOfLength(10); - ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, null, config)); + ElasticsearchException exception = expectThrows(ElasticsearchException.class, + () -> factory.create(null, processorTag, null, config)); assertThat(exception.getMessage(), equalTo("java.lang.RuntimeException: could not compile script")); assertThat(exception.getMetadata("es.processor_tag").get(0), equalTo(processorTag)); } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorTests.java index 7f1aa11bd4485..6e54e359ec520 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorTests.java @@ -158,7 +158,7 @@ public void testAppendMetadataExceptVersion() throws Exception { private static Processor createAppendProcessor(String fieldName, Object fieldValue) { return new AppendProcessor(randomAlphaOfLength(10), - new TestTemplateService.MockTemplateScript.Factory(fieldName), + null, new TestTemplateService.MockTemplateScript.Factory(fieldName), ValueSource.wrap(fieldValue, TestTemplateService.instance())); } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/BytesProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/BytesProcessorTests.java index 2520f3e5ad17f..b7382f04538dd 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/BytesProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/BytesProcessorTests.java @@ -35,7 +35,7 @@ public class BytesProcessorTests extends AbstractStringProcessorTestCase { @Override protected AbstractStringProcessor newProcessor(String field, boolean ignoreMissing, String targetField) { - return new BytesProcessor(randomAlphaOfLength(10), field, ignoreMissing, targetField); + return new BytesProcessor(randomAlphaOfLength(10), null, field, ignoreMissing, targetField); } @Override diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java index 34fd4ec27a3b3..02598f4134858 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java @@ -44,7 +44,7 @@ public void testConvertInt() throws Exception { IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random()); int randomInt = randomInt(); String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, randomInt); - Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.INTEGER, false); + Processor processor = new ConvertProcessor(randomAlphaOfLength(10), null, fieldName, fieldName, Type.INTEGER, false); processor.execute(ingestDocument); assertThat(ingestDocument.getFieldValue(fieldName, Integer.class), equalTo(randomInt)); } @@ -54,7 +54,7 @@ public void testConvertIntHex() throws Exception { int randomInt = randomInt(); String intString = randomInt < 0 ? "-0x" + Integer.toHexString(-randomInt) : "0x" + Integer.toHexString(randomInt); String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, intString); - Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.INTEGER, false); + Processor processor = new ConvertProcessor(randomAlphaOfLength(10), null, fieldName, fieldName, Type.INTEGER, false); processor.execute(ingestDocument); assertThat(ingestDocument.getFieldValue(fieldName, Integer.class), equalTo(randomInt)); } @@ -62,7 +62,7 @@ public void testConvertIntHex() throws Exception { public void testConvertIntLeadingZero() throws Exception { IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random()); String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, "010"); - Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.INTEGER, false); + Processor processor = new ConvertProcessor(randomAlphaOfLength(10), null, fieldName, fieldName, Type.INTEGER, false); processor.execute(ingestDocument); assertThat(ingestDocument.getFieldValue(fieldName, Integer.class), equalTo(10)); } @@ -71,7 +71,7 @@ public void testConvertIntHexError() { IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random()); String value = "0xnotanumber"; String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, value); - Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.INTEGER, false); + Processor processor = new ConvertProcessor(randomAlphaOfLength(10), null, fieldName, fieldName, Type.INTEGER, false); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> processor.execute(ingestDocument)); assertThat(e.getMessage(), equalTo("unable to convert [" + value + "] to integer")); } @@ -87,7 +87,7 @@ public void testConvertIntList() throws Exception { expectedList.add(randomInt); } String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue); - Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.INTEGER, false); + Processor processor = new ConvertProcessor(randomAlphaOfLength(10), null, fieldName, fieldName, Type.INTEGER, false); processor.execute(ingestDocument); assertThat(ingestDocument.getFieldValue(fieldName, List.class), equalTo(expectedList)); } @@ -98,7 +98,7 @@ public void testConvertIntError() throws Exception { String value = "string-" + randomAlphaOfLengthBetween(1, 10); ingestDocument.setFieldValue(fieldName, value); - Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.INTEGER, false); + Processor processor = new ConvertProcessor(randomAlphaOfLength(10), null, fieldName, fieldName, Type.INTEGER, false); try { processor.execute(ingestDocument); fail("processor execute should have failed"); @@ -114,7 +114,7 @@ public void testConvertLong() throws Exception { String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, randomLong); expectedResult.put(fieldName, randomLong); - Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.LONG, false); + Processor processor = new ConvertProcessor(randomAlphaOfLength(10), null, fieldName, fieldName, Type.LONG, false); processor.execute(ingestDocument); assertThat(ingestDocument.getFieldValue(fieldName, Long.class), equalTo(randomLong)); } @@ -124,7 +124,7 @@ public void testConvertLongHex() throws Exception { long randomLong = randomLong(); String longString = randomLong < 0 ? "-0x" + Long.toHexString(-randomLong) : "0x" + Long.toHexString(randomLong); String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, longString); - Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.LONG, false); + Processor processor = new ConvertProcessor(randomAlphaOfLength(10), null, fieldName, fieldName, Type.LONG, false); processor.execute(ingestDocument); assertThat(ingestDocument.getFieldValue(fieldName, Long.class), equalTo(randomLong)); } @@ -132,7 +132,7 @@ public void testConvertLongHex() throws Exception { public void testConvertLongLeadingZero() throws Exception { IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random()); String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, "010"); - Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.LONG, false); + Processor processor = new ConvertProcessor(randomAlphaOfLength(10), null, fieldName, fieldName, Type.LONG, false); processor.execute(ingestDocument); assertThat(ingestDocument.getFieldValue(fieldName, Long.class), equalTo(10L)); } @@ -141,7 +141,7 @@ public void testConvertLongHexError() { IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random()); String value = "0xnotanumber"; String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, value); - Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.LONG, false); + Processor processor = new ConvertProcessor(randomAlphaOfLength(10), null, fieldName, fieldName, Type.LONG, false); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> processor.execute(ingestDocument)); assertThat(e.getMessage(), equalTo("unable to convert [" + value + "] to long")); } @@ -157,7 +157,7 @@ public void testConvertLongList() throws Exception { expectedList.add(randomLong); } String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue); - Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.LONG, false); + Processor processor = new ConvertProcessor(randomAlphaOfLength(10), null, fieldName, fieldName, Type.LONG, false); processor.execute(ingestDocument); assertThat(ingestDocument.getFieldValue(fieldName, List.class), equalTo(expectedList)); } @@ -168,7 +168,7 @@ public void testConvertLongError() throws Exception { String value = "string-" + randomAlphaOfLengthBetween(1, 10); ingestDocument.setFieldValue(fieldName, value); - Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.LONG, false); + Processor processor = new ConvertProcessor(randomAlphaOfLength(10), null, fieldName, fieldName, Type.LONG, false); try { processor.execute(ingestDocument); fail("processor execute should have failed"); @@ -184,7 +184,7 @@ public void testConvertDouble() throws Exception { String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, randomDouble); expectedResult.put(fieldName, randomDouble); - Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.DOUBLE, false); + Processor processor = new ConvertProcessor(randomAlphaOfLength(10), null, fieldName, fieldName, Type.DOUBLE, false); processor.execute(ingestDocument); assertThat(ingestDocument.getFieldValue(fieldName, Double.class), equalTo(randomDouble)); } @@ -200,7 +200,7 @@ public void testConvertDoubleList() throws Exception { expectedList.add(randomDouble); } String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue); - Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.DOUBLE, false); + Processor processor = new ConvertProcessor(randomAlphaOfLength(10), null, fieldName, fieldName, Type.DOUBLE, false); processor.execute(ingestDocument); assertThat(ingestDocument.getFieldValue(fieldName, List.class), equalTo(expectedList)); } @@ -211,7 +211,7 @@ public void testConvertDoubleError() throws Exception { String value = "string-" + randomAlphaOfLengthBetween(1, 10); ingestDocument.setFieldValue(fieldName, value); - Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.DOUBLE, false); + Processor processor = new ConvertProcessor(randomAlphaOfLength(10), null, fieldName, fieldName, Type.DOUBLE, false); try { processor.execute(ingestDocument); fail("processor execute should have failed"); @@ -227,7 +227,7 @@ public void testConvertFloat() throws Exception { String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, randomFloat); expectedResult.put(fieldName, randomFloat); - Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.FLOAT, false); + Processor processor = new ConvertProcessor(randomAlphaOfLength(10), null, fieldName, fieldName, Type.FLOAT, false); processor.execute(ingestDocument); assertThat(ingestDocument.getFieldValue(fieldName, Float.class), equalTo(randomFloat)); } @@ -243,7 +243,7 @@ public void testConvertFloatList() throws Exception { expectedList.add(randomFloat); } String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue); - Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.FLOAT, false); + Processor processor = new ConvertProcessor(randomAlphaOfLength(10), null, fieldName, fieldName, Type.FLOAT, false); processor.execute(ingestDocument); assertThat(ingestDocument.getFieldValue(fieldName, List.class), equalTo(expectedList)); } @@ -254,7 +254,7 @@ public void testConvertFloatError() throws Exception { String value = "string-" + randomAlphaOfLengthBetween(1, 10); ingestDocument.setFieldValue(fieldName, value); - Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.FLOAT, false); + Processor processor = new ConvertProcessor(randomAlphaOfLength(10), null, fieldName, fieldName, Type.FLOAT, false); try { processor.execute(ingestDocument); fail("processor execute should have failed"); @@ -272,7 +272,7 @@ public void testConvertBoolean() throws Exception { } String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, booleanString); - Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.BOOLEAN, false); + Processor processor = new ConvertProcessor(randomAlphaOfLength(10), null, fieldName, fieldName, Type.BOOLEAN, false); processor.execute(ingestDocument); assertThat(ingestDocument.getFieldValue(fieldName, Boolean.class), equalTo(randomBoolean)); } @@ -292,7 +292,7 @@ public void testConvertBooleanList() throws Exception { expectedList.add(randomBoolean); } String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue); - Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.BOOLEAN, false); + Processor processor = new ConvertProcessor(randomAlphaOfLength(10), null, fieldName, fieldName, Type.BOOLEAN, false); processor.execute(ingestDocument); assertThat(ingestDocument.getFieldValue(fieldName, List.class), equalTo(expectedList)); } @@ -309,7 +309,7 @@ public void testConvertBooleanError() throws Exception { } ingestDocument.setFieldValue(fieldName, fieldValue); - Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.BOOLEAN, false); + Processor processor = new ConvertProcessor(randomAlphaOfLength(10), null, fieldName, fieldName, Type.BOOLEAN, false); try { processor.execute(ingestDocument); fail("processor execute should have failed"); @@ -343,7 +343,7 @@ public void testConvertString() throws Exception { } String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue); - Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.STRING, false); + Processor processor = new ConvertProcessor(randomAlphaOfLength(10), null, fieldName, fieldName, Type.STRING, false); processor.execute(ingestDocument); assertThat(ingestDocument.getFieldValue(fieldName, String.class), equalTo(expectedFieldValue)); } @@ -389,7 +389,7 @@ public void testConvertStringList() throws Exception { expectedList.add(randomValueString); } String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue); - Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.STRING, false); + Processor processor = new ConvertProcessor(randomAlphaOfLength(10), null, fieldName, fieldName, Type.STRING, false); processor.execute(ingestDocument); assertThat(ingestDocument.getFieldValue(fieldName, List.class), equalTo(expectedList)); } @@ -398,7 +398,7 @@ public void testConvertNonExistingField() throws Exception { IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>()); String fieldName = RandomDocumentPicks.randomFieldName(random()); Type type = randomFrom(Type.values()); - Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, type, false); + Processor processor = new ConvertProcessor(randomAlphaOfLength(10), null, fieldName, fieldName, type, false); try { processor.execute(ingestDocument); fail("processor execute should have failed"); @@ -410,7 +410,7 @@ public void testConvertNonExistingField() throws Exception { public void testConvertNullField() throws Exception { IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", null)); Type type = randomFrom(Type.values()); - Processor processor = new ConvertProcessor(randomAlphaOfLength(10), "field", "field", type, false); + Processor processor = new ConvertProcessor(randomAlphaOfLength(10), null, "field", "field", type, false); try { processor.execute(ingestDocument); fail("processor execute should have failed"); @@ -424,7 +424,7 @@ public void testConvertNonExistingFieldWithIgnoreMissing() throws Exception { IngestDocument ingestDocument = new IngestDocument(originalIngestDocument); String fieldName = RandomDocumentPicks.randomFieldName(random()); Type type = randomFrom(Type.values()); - Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, type, true); + Processor processor = new ConvertProcessor(randomAlphaOfLength(10), null, fieldName, fieldName, type, true); processor.execute(ingestDocument); assertIngestDocument(originalIngestDocument, ingestDocument); } @@ -433,7 +433,7 @@ public void testConvertNullFieldWithIgnoreMissing() throws Exception { IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", null)); IngestDocument ingestDocument = new IngestDocument(originalIngestDocument); Type type = randomFrom(Type.values()); - Processor processor = new ConvertProcessor(randomAlphaOfLength(10), "field", "field", type, true); + Processor processor = new ConvertProcessor(randomAlphaOfLength(10), null, "field", "field", type, true); processor.execute(ingestDocument); assertIngestDocument(originalIngestDocument, ingestDocument); } @@ -457,7 +457,7 @@ public void testAutoConvertNotString() throws Exception { throw new UnsupportedOperationException(); } IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", randomValue)); - Processor processor = new ConvertProcessor(randomAlphaOfLength(10), "field", "field", Type.AUTO, false); + Processor processor = new ConvertProcessor(randomAlphaOfLength(10), null, "field", "field", Type.AUTO, false); processor.execute(ingestDocument); Object convertedValue = ingestDocument.getFieldValue("field", Object.class); assertThat(convertedValue, sameInstance(randomValue)); @@ -466,7 +466,7 @@ public void testAutoConvertNotString() throws Exception { public void testAutoConvertStringNotMatched() throws Exception { String value = "notAnIntFloatOrBool"; IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", value)); - Processor processor = new ConvertProcessor(randomAlphaOfLength(10), "field", "field", Type.AUTO, false); + Processor processor = new ConvertProcessor(randomAlphaOfLength(10), null, "field", "field", Type.AUTO, false); processor.execute(ingestDocument); Object convertedValue = ingestDocument.getFieldValue("field", Object.class); assertThat(convertedValue, sameInstance(value)); @@ -477,7 +477,7 @@ public void testAutoConvertMatchBoolean() throws Exception { String booleanString = Boolean.toString(randomBoolean); IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", booleanString)); - Processor processor = new ConvertProcessor(randomAlphaOfLength(10), "field", "field", Type.AUTO, false); + Processor processor = new ConvertProcessor(randomAlphaOfLength(10), null, "field", "field", Type.AUTO, false); processor.execute(ingestDocument); Object convertedValue = ingestDocument.getFieldValue("field", Object.class); assertThat(convertedValue, equalTo(randomBoolean)); @@ -487,7 +487,7 @@ public void testAutoConvertMatchInteger() throws Exception { int randomInt = randomInt(); String randomString = Integer.toString(randomInt); IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", randomString)); - Processor processor = new ConvertProcessor(randomAlphaOfLength(10), "field", "field", Type.AUTO, false); + Processor processor = new ConvertProcessor(randomAlphaOfLength(10), null, "field", "field", Type.AUTO, false); processor.execute(ingestDocument); Object convertedValue = ingestDocument.getFieldValue("field", Object.class); assertThat(convertedValue, equalTo(randomInt)); @@ -497,7 +497,7 @@ public void testAutoConvertMatchLong() throws Exception { long randomLong = randomLong(); String randomString = Long.toString(randomLong); IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", randomString)); - Processor processor = new ConvertProcessor(randomAlphaOfLength(10), "field", "field", Type.AUTO, false); + Processor processor = new ConvertProcessor(randomAlphaOfLength(10), null, "field", "field", Type.AUTO, false); processor.execute(ingestDocument); Object convertedValue = ingestDocument.getFieldValue("field", Object.class); assertThat(convertedValue, equalTo(randomLong)); @@ -508,7 +508,7 @@ public void testAutoConvertDoubleNotMatched() throws Exception { String randomString = Double.toString(randomDouble); float randomFloat = Float.parseFloat(randomString); IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", randomString)); - Processor processor = new ConvertProcessor(randomAlphaOfLength(10), "field", "field", Type.AUTO, false); + Processor processor = new ConvertProcessor(randomAlphaOfLength(10), null, "field", "field", Type.AUTO, false); processor.execute(ingestDocument); Object convertedValue = ingestDocument.getFieldValue("field", Object.class); assertThat(convertedValue, not(randomDouble)); @@ -519,7 +519,7 @@ public void testAutoConvertMatchFloat() throws Exception { float randomFloat = randomFloat(); String randomString = Float.toString(randomFloat); IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", randomString)); - Processor processor = new ConvertProcessor(randomAlphaOfLength(10), "field", "field", Type.AUTO, false); + Processor processor = new ConvertProcessor(randomAlphaOfLength(10), null, "field", "field", Type.AUTO, false); processor.execute(ingestDocument); Object convertedValue = ingestDocument.getFieldValue("field", Object.class); assertThat(convertedValue, equalTo(randomFloat)); @@ -530,7 +530,7 @@ public void testTargetField() throws Exception { int randomInt = randomInt(); String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, String.valueOf(randomInt)); String targetField = fieldName + randomAlphaOfLength(5); - Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, targetField, Type.INTEGER, false); + Processor processor = new ConvertProcessor(randomAlphaOfLength(10), null, fieldName, targetField, Type.INTEGER, false); processor.execute(ingestDocument); assertThat(ingestDocument.getFieldValue(fieldName, String.class), equalTo(String.valueOf(randomInt))); assertThat(ingestDocument.getFieldValue(targetField, Integer.class), equalTo(randomInt)); diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CsvProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CsvProcessorTests.java index 639e345599ba3..aeb64dc0ffdf6 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CsvProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CsvProcessorTests.java @@ -257,9 +257,10 @@ public void testIgnoreMissing() { if (ingestDocument.hasField(fieldName)) { ingestDocument.removeField(fieldName); } - CsvProcessor processor = new CsvProcessor(randomAlphaOfLength(5), fieldName, new String[]{"a"}, false, ',', '"', true, null); + CsvProcessor processor = new CsvProcessor(randomAlphaOfLength(5), null, fieldName, new String[]{"a"}, false, ',', '"', true, null); processor.execute(ingestDocument); - CsvProcessor processor2 = new CsvProcessor(randomAlphaOfLength(5), fieldName, new String[]{"a"}, false, ',', '"', false, null); + CsvProcessor processor2 = new CsvProcessor(randomAlphaOfLength(5), null, fieldName, new String[]{"a"}, false, + ',', '"', false, null); expectThrows(IllegalArgumentException.class, () -> processor2.execute(ingestDocument)); } @@ -269,7 +270,7 @@ public void testEmptyHeaders() throws Exception { String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, "abc,abc"); HashMap metadata = new HashMap<>(ingestDocument.getSourceAndMetadata()); - CsvProcessor processor = new CsvProcessor(randomAlphaOfLength(5), fieldName, new String[0], false, ',', '"', false, null); + CsvProcessor processor = new CsvProcessor(randomAlphaOfLength(5), null, fieldName, new String[0], false, ',', '"', false, null); processor.execute(ingestDocument); @@ -291,7 +292,7 @@ private IngestDocument processDocument(String[] headers, String csv, boolean tri ingestDocument.setFieldValue(fieldName, csv); char quoteChar = quote.isEmpty() ? '"' : quote.charAt(0); - CsvProcessor processor = new CsvProcessor(randomAlphaOfLength(5), fieldName, headers, trim, separator, quoteChar, false, + CsvProcessor processor = new CsvProcessor(randomAlphaOfLength(5), null, fieldName, headers, trim, separator, quoteChar, false, emptyValue); processor.execute(ingestDocument); diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java index 9d79a14568583..25474dcb0e13f 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java @@ -103,7 +103,7 @@ public void testTemplatedFields() throws Exception { private DateIndexNameProcessor createProcessor(String field, List> dateFormats, ZoneId timezone, String indexNamePrefix, String dateRounding, String indexNameFormat) { - return new DateIndexNameProcessor(randomAlphaOfLength(10), field, dateFormats, timezone, + return new DateIndexNameProcessor(randomAlphaOfLength(10), null, field, dateFormats, timezone, new TestTemplateService.MockTemplateScript.Factory(indexNamePrefix), new TestTemplateService.MockTemplateScript.Factory(dateRounding), new TestTemplateService.MockTemplateScript.Factory(indexNameFormat) diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java index c390f14029c39..8f3fd31ab4ac8 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java @@ -51,7 +51,7 @@ private TemplateScript.Factory templatize(ZoneId timezone) { public void testJavaPattern() { DateProcessor dateProcessor = new DateProcessor(randomAlphaOfLength(10), - templatize(ZoneId.of("Europe/Amsterdam")), templatize(Locale.ENGLISH), + null, templatize(ZoneId.of("Europe/Amsterdam")), templatize(Locale.ENGLISH), "date_as_string", Collections.singletonList("yyyy dd MM HH:mm:ss"), "date_as_date"); Map document = new HashMap<>(); document.put("date_as_string", "2010 12 06 11:05:15"); @@ -66,7 +66,7 @@ public void testJavaPatternMultipleFormats() { matchFormats.add("dd/MM/yyyy"); matchFormats.add("dd-MM-yyyy"); DateProcessor dateProcessor = new DateProcessor(randomAlphaOfLength(10), - templatize(ZoneId.of("Europe/Amsterdam")), templatize(Locale.ENGLISH), + null, templatize(ZoneId.of("Europe/Amsterdam")), templatize(Locale.ENGLISH), "date_as_string", matchFormats, "date_as_date"); Map document = new HashMap<>(); @@ -100,7 +100,7 @@ public void testJavaPatternMultipleFormats() { public void testJavaPatternNoTimezone() { DateProcessor dateProcessor = new DateProcessor(randomAlphaOfLength(10), - null, null, + null, null, null, "date_as_string", Arrays.asList("yyyy dd MM HH:mm:ss XXX"), "date_as_date"); Map document = new HashMap<>(); @@ -113,7 +113,7 @@ public void testJavaPatternNoTimezone() { public void testInvalidJavaPattern() { try { DateProcessor processor = new DateProcessor(randomAlphaOfLength(10), - templatize(ZoneOffset.UTC), templatize(randomLocale(random())), + null, templatize(ZoneOffset.UTC), templatize(randomLocale(random())), "date_as_string", Collections.singletonList("invalid pattern"), "date_as_date"); Map document = new HashMap<>(); document.put("date_as_string", "2010"); @@ -128,7 +128,7 @@ public void testInvalidJavaPattern() { public void testJavaPatternLocale() { assumeFalse("Can't run in a FIPS JVM, Joda parse date error", inFipsJvm()); DateProcessor dateProcessor = new DateProcessor(randomAlphaOfLength(10), - templatize(ZoneId.of("Europe/Amsterdam")), templatize(Locale.ITALIAN), + null, templatize(ZoneId.of("Europe/Amsterdam")), templatize(Locale.ITALIAN), "date_as_string", Collections.singletonList("yyyy dd MMMM"), "date_as_date"); Map document = new HashMap<>(); document.put("date_as_string", "2010 12 giugno"); @@ -140,7 +140,7 @@ public void testJavaPatternLocale() { public void testJavaPatternEnglishLocale() { // Since testJavaPatternLocale is muted in FIPS mode, test that we can correctly parse dates in english DateProcessor dateProcessor = new DateProcessor(randomAlphaOfLength(10), - templatize(ZoneId.of("Europe/Amsterdam")), templatize(Locale.ENGLISH), + null, templatize(ZoneId.of("Europe/Amsterdam")), templatize(Locale.ENGLISH), "date_as_string", Collections.singletonList("yyyy dd MMMM"), "date_as_date"); Map document = new HashMap<>(); document.put("date_as_string", "2010 12 June"); @@ -152,7 +152,7 @@ public void testJavaPatternEnglishLocale() { public void testJavaPatternDefaultYear() { String format = randomFrom("dd/MM", "8dd/MM"); DateProcessor dateProcessor = new DateProcessor(randomAlphaOfLength(10), - templatize(ZoneId.of("Europe/Amsterdam")), templatize(Locale.ENGLISH), + null, templatize(ZoneId.of("Europe/Amsterdam")), templatize(Locale.ENGLISH), "date_as_string", Collections.singletonList(format), "date_as_date"); Map document = new HashMap<>(); document.put("date_as_string", "12/06"); @@ -163,7 +163,7 @@ public void testJavaPatternDefaultYear() { } public void testTAI64N() { - DateProcessor dateProcessor = new DateProcessor(randomAlphaOfLength(10), templatize(ZoneOffset.ofHours(2)), + DateProcessor dateProcessor = new DateProcessor(randomAlphaOfLength(10), null, templatize(ZoneOffset.ofHours(2)), templatize(randomLocale(random())), "date_as_string", Collections.singletonList("TAI64N"), "date_as_date"); Map document = new HashMap<>(); @@ -175,7 +175,7 @@ public void testTAI64N() { } public void testUnixMs() { - DateProcessor dateProcessor = new DateProcessor(randomAlphaOfLength(10), templatize(ZoneOffset.UTC), + DateProcessor dateProcessor = new DateProcessor(randomAlphaOfLength(10), null, templatize(ZoneOffset.UTC), templatize(randomLocale(random())), "date_as_string", Collections.singletonList("UNIX_MS"), "date_as_date"); Map document = new HashMap<>(); document.put("date_as_string", "1000500"); @@ -191,7 +191,7 @@ public void testUnixMs() { } public void testUnix() { - DateProcessor dateProcessor = new DateProcessor(randomAlphaOfLength(10), templatize(ZoneOffset.UTC), + DateProcessor dateProcessor = new DateProcessor(randomAlphaOfLength(10), null, templatize(ZoneOffset.UTC), templatize(randomLocale(random())), "date_as_string", Collections.singletonList("UNIX"), "date_as_date"); Map document = new HashMap<>(); @@ -203,7 +203,7 @@ public void testUnix() { public void testInvalidTimezone() { DateProcessor processor = new DateProcessor(randomAlphaOfLength(10), - new TestTemplateService.MockTemplateScript.Factory("invalid_timezone"), templatize(randomLocale(random())), + null, new TestTemplateService.MockTemplateScript.Factory("invalid_timezone"), templatize(randomLocale(random())), "date_as_string", Collections.singletonList("yyyy"), "date_as_date"); Map document = new HashMap<>(); document.put("date_as_string", "2010"); @@ -215,7 +215,7 @@ public void testInvalidTimezone() { public void testInvalidLocale() { DateProcessor processor = new DateProcessor(randomAlphaOfLength(10), - templatize(ZoneOffset.UTC), new TestTemplateService.MockTemplateScript.Factory("invalid_locale"), + null, templatize(ZoneOffset.UTC), new TestTemplateService.MockTemplateScript.Factory("invalid_locale"), "date_as_string", Collections.singletonList("yyyy"), "date_as_date"); Map document = new HashMap<>(); document.put("date_as_string", "2010"); diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DissectProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DissectProcessorTests.java index 1910c6cd1d37a..c49fd6451edf3 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DissectProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DissectProcessorTests.java @@ -42,7 +42,7 @@ public class DissectProcessorTests extends ESTestCase { public void testMatch() { IngestDocument ingestDocument = new IngestDocument("_index", "_id", null, null, null, Collections.singletonMap("message", "foo,bar,baz")); - DissectProcessor dissectProcessor = new DissectProcessor("", "message", "%{a},%{b},%{c}", "", true); + DissectProcessor dissectProcessor = new DissectProcessor("", null, "message", "%{a},%{b},%{c}", "", true); dissectProcessor.execute(ingestDocument); assertThat(ingestDocument.getFieldValue("a", String.class), equalTo("foo")); assertThat(ingestDocument.getFieldValue("b", String.class), equalTo("bar")); @@ -56,7 +56,7 @@ public void testMatchOverwrite() { .put("a", "willgetstompped") .map()); assertThat(ingestDocument.getFieldValue("a", String.class), equalTo("willgetstompped")); - DissectProcessor dissectProcessor = new DissectProcessor("", "message", "%{a},%{b},%{c}", "", true); + DissectProcessor dissectProcessor = new DissectProcessor("", null, "message", "%{a},%{b},%{c}", "", true); dissectProcessor.execute(ingestDocument); assertThat(ingestDocument.getFieldValue("a", String.class), equalTo("foo")); assertThat(ingestDocument.getFieldValue("b", String.class), equalTo("bar")); @@ -67,7 +67,7 @@ public void testAdvancedMatch() { IngestDocument ingestDocument = new IngestDocument("_index", "_id", null, null, null, Collections.singletonMap("message", "foo bar,,,,,,,baz nope:notagain 😊 🐇 🙃")); DissectProcessor dissectProcessor = - new DissectProcessor("", "message", "%{a->} %{*b->},%{&b} %{}:%{?skipme} %{+smile/2} 🐇 %{+smile/1}", "::::", true); + new DissectProcessor("", null, "message", "%{a->} %{*b->},%{&b} %{}:%{?skipme} %{+smile/2} 🐇 %{+smile/1}", "::::", true); dissectProcessor.execute(ingestDocument); assertThat(ingestDocument.getFieldValue("a", String.class), equalTo("foo")); assertThat(ingestDocument.getFieldValue("bar", String.class), equalTo("baz")); @@ -79,14 +79,14 @@ public void testAdvancedMatch() { public void testMiss() { IngestDocument ingestDocument = new IngestDocument("_index", "_id", null, null, null, Collections.singletonMap("message", "foo:bar,baz")); - DissectProcessor dissectProcessor = new DissectProcessor("", "message", "%{a},%{b},%{c}", "", true); + DissectProcessor dissectProcessor = new DissectProcessor("", null, "message", "%{a},%{b},%{c}", "", true); DissectException e = expectThrows(DissectException.class, () -> dissectProcessor.execute(ingestDocument)); assertThat(e.getMessage(), CoreMatchers.containsString("Unable to find match for dissect pattern")); } public void testNonStringValueWithIgnoreMissing() { String fieldName = RandomDocumentPicks.randomFieldName(random()); - Processor processor = new DissectProcessor("", fieldName, "%{a},%{b},%{c}", "", true); + Processor processor = new DissectProcessor("", null, fieldName, "%{a},%{b},%{c}", "", true); IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>()); ingestDocument.setFieldValue(fieldName, randomInt()); Exception e = expectThrows(IllegalArgumentException.class, () -> processor.execute(ingestDocument)); @@ -95,7 +95,7 @@ public void testNonStringValueWithIgnoreMissing() { public void testNullValueWithIgnoreMissing() throws Exception { String fieldName = RandomDocumentPicks.randomFieldName(random()); - Processor processor = new DissectProcessor("", fieldName, "%{a},%{b},%{c}", "", true); + Processor processor = new DissectProcessor("", null, fieldName, "%{a},%{b},%{c}", "", true); IngestDocument originalIngestDocument = RandomDocumentPicks .randomIngestDocument(random(), Collections.singletonMap(fieldName, null)); IngestDocument ingestDocument = new IngestDocument(originalIngestDocument); @@ -105,7 +105,7 @@ public void testNullValueWithIgnoreMissing() throws Exception { public void testNullValueWithOutIgnoreMissing() { String fieldName = RandomDocumentPicks.randomFieldName(random()); - Processor processor = new DissectProcessor("", fieldName, "%{a},%{b},%{c}", "", false); + Processor processor = new DissectProcessor("", null, fieldName, "%{a},%{b},%{c}", "", false); IngestDocument originalIngestDocument = RandomDocumentPicks .randomIngestDocument(random(), Collections.singletonMap(fieldName, null)); IngestDocument ingestDocument = new IngestDocument(originalIngestDocument); diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorTests.java index d6a207b859eb0..ed1799c05ef67 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorTests.java @@ -37,7 +37,7 @@ public void testEscapeFields() throws Exception { Map source = new HashMap<>(); source.put("foo.bar", "baz1"); IngestDocument document = new IngestDocument(source, Collections.emptyMap()); - DotExpanderProcessor processor = new DotExpanderProcessor("_tag", null, "foo.bar"); + DotExpanderProcessor processor = new DotExpanderProcessor("_tag", null, "foo.bar", null); processor.execute(document); assertThat(document.getFieldValue("foo", Map.class).size(), equalTo(1)); assertThat(document.getFieldValue("foo.bar", String.class), equalTo("baz1")); @@ -45,7 +45,7 @@ public void testEscapeFields() throws Exception { source = new HashMap<>(); source.put("foo.bar.baz", "value"); document = new IngestDocument(source, Collections.emptyMap()); - processor = new DotExpanderProcessor("_tag", null, "foo.bar.baz"); + processor = new DotExpanderProcessor("_tag", null, "foo.bar.baz", null); processor.execute(document); assertThat(document.getFieldValue("foo", Map.class).size(), equalTo(1)); assertThat(document.getFieldValue("foo.bar", Map.class).size(), equalTo(1)); @@ -55,7 +55,7 @@ public void testEscapeFields() throws Exception { source.put("foo.bar", "baz1"); source.put("foo", new HashMap<>(Collections.singletonMap("bar", "baz2"))); document = new IngestDocument(source, Collections.emptyMap()); - processor = new DotExpanderProcessor("_tag", null, "foo.bar"); + processor = new DotExpanderProcessor("_tag", null, "foo.bar", null); processor.execute(document); assertThat(document.getSourceAndMetadata().size(), equalTo(1)); assertThat(document.getFieldValue("foo.bar", List.class).size(), equalTo(2)); @@ -66,7 +66,7 @@ public void testEscapeFields() throws Exception { source.put("foo.bar", "2"); source.put("foo", new HashMap<>(Collections.singletonMap("bar", 1))); document = new IngestDocument(source, Collections.emptyMap()); - processor = new DotExpanderProcessor("_tag", null, "foo.bar"); + processor = new DotExpanderProcessor("_tag", null, "foo.bar", null); processor.execute(document); assertThat(document.getSourceAndMetadata().size(), equalTo(1)); assertThat(document.getFieldValue("foo.bar", List.class).size(), equalTo(2)); @@ -79,7 +79,7 @@ public void testEscapeFields_valueField() throws Exception { source.put("foo.bar", "baz1"); source.put("foo", "baz2"); IngestDocument document1 = new IngestDocument(source, Collections.emptyMap()); - Processor processor1 = new DotExpanderProcessor("_tag", null, "foo.bar"); + Processor processor1 = new DotExpanderProcessor("_tag", null, "foo.bar", null); // foo already exists and if a leaf field and therefor can't be replaced by a map field: Exception e = expectThrows(IllegalArgumentException.class, () -> processor1.execute(document1)); assertThat(e.getMessage(), equalTo("cannot expend [foo.bar], because [foo] is not an object field, but a value field")); @@ -87,10 +87,10 @@ public void testEscapeFields_valueField() throws Exception { // so because foo is no branch field but a value field the `foo.bar` field can't be expanded // into [foo].[bar], so foo should be renamed first into `[foo].[bar]: IngestDocument document = new IngestDocument(source, Collections.emptyMap()); - Processor processor = new RenameProcessor("_tag", new TestTemplateService.MockTemplateScript.Factory("foo"), + Processor processor = new RenameProcessor("_tag", null, new TestTemplateService.MockTemplateScript.Factory("foo"), new TestTemplateService.MockTemplateScript.Factory("foo.bar"), false); processor.execute(document); - processor = new DotExpanderProcessor("_tag", null, "foo.bar"); + processor = new DotExpanderProcessor("_tag", null, "foo.bar", null); processor.execute(document); assertThat(document.getFieldValue("foo", Map.class).size(), equalTo(1)); assertThat(document.getFieldValue("foo.bar.0", String.class), equalTo("baz2")); @@ -99,7 +99,7 @@ public void testEscapeFields_valueField() throws Exception { source = new HashMap<>(); source.put("foo.bar", "baz1"); document = new IngestDocument(source, Collections.emptyMap()); - processor = new DotExpanderProcessor("_tag", null, "foo.bar"); + processor = new DotExpanderProcessor("_tag", null, "foo.bar", null); processor.execute(document); assertThat(document.getFieldValue("foo", Map.class).size(), equalTo(1)); assertThat(document.getFieldValue("foo.bar", String.class), equalTo("baz1")); @@ -108,7 +108,7 @@ public void testEscapeFields_valueField() throws Exception { source.put("foo.bar.baz", "baz1"); source.put("foo", new HashMap<>(Collections.singletonMap("bar", new HashMap<>()))); document = new IngestDocument(source, Collections.emptyMap()); - processor = new DotExpanderProcessor("_tag", null, "foo.bar.baz"); + processor = new DotExpanderProcessor("_tag", null, "foo.bar.baz", null); processor.execute(document); assertThat(document.getFieldValue("foo", Map.class).size(), equalTo(1)); assertThat(document.getFieldValue("foo.bar", Map.class).size(), equalTo(1)); @@ -118,7 +118,7 @@ public void testEscapeFields_valueField() throws Exception { source.put("foo.bar.baz", "baz1"); source.put("foo", new HashMap<>(Collections.singletonMap("bar", "baz2"))); IngestDocument document2 = new IngestDocument(source, Collections.emptyMap()); - Processor processor2 = new DotExpanderProcessor("_tag", null, "foo.bar.baz"); + Processor processor2 = new DotExpanderProcessor("_tag", null, "foo.bar.baz", null); e = expectThrows(IllegalArgumentException.class, () -> processor2.execute(document2)); assertThat(e.getMessage(), equalTo("cannot expend [foo.bar.baz], because [foo.bar] is not an object field, but a value field")); } @@ -127,7 +127,7 @@ public void testEscapeFields_path() throws Exception { Map source = new HashMap<>(); source.put("foo", new HashMap<>(Collections.singletonMap("bar.baz", "value"))); IngestDocument document = new IngestDocument(source, Collections.emptyMap()); - DotExpanderProcessor processor = new DotExpanderProcessor("_tag", "foo", "bar.baz"); + DotExpanderProcessor processor = new DotExpanderProcessor("_tag", null, "bar.baz", "foo"); processor.execute(document); assertThat(document.getFieldValue("foo", Map.class).size(), equalTo(1)); assertThat(document.getFieldValue("foo.bar", Map.class).size(), equalTo(1)); @@ -136,7 +136,7 @@ public void testEscapeFields_path() throws Exception { source = new HashMap<>(); source.put("field", new HashMap<>(Collections.singletonMap("foo.bar.baz", "value"))); document = new IngestDocument(source, Collections.emptyMap()); - processor = new DotExpanderProcessor("_tag", "field", "foo.bar.baz"); + processor = new DotExpanderProcessor("_tag", null, "foo.bar.baz", "field"); processor.execute(document); assertThat(document.getFieldValue("field.foo", Map.class).size(), equalTo(1)); assertThat(document.getFieldValue("field.foo.bar", Map.class).size(), equalTo(1)); @@ -150,7 +150,7 @@ public void testEscapeFields_doNothingIfFieldNotInSourceDoc() throws Exception { source.put("foo.bar", "baz1"); IngestDocument document = new IngestDocument(source, Collections.emptyMap()); //abc.def does not exist in source, so don't mutate document - DotExpanderProcessor processor = new DotExpanderProcessor("_tag", null, "abc.def"); + DotExpanderProcessor processor = new DotExpanderProcessor("_tag", null, "abc.def", null); processor.execute(document); //hasField returns false since it requires the expanded form, which is not expanded since we did not ask for it to be assertFalse(document.hasField("foo.bar")); @@ -168,7 +168,7 @@ public void testEscapeFields_doNothingIfFieldNotInSourceDoc() throws Exception { source.put("foo", inner); document = new IngestDocument(source, Collections.emptyMap()); //foo.bar, the literal value (as opposed to nested value) does not exist in source, so don't mutate document - processor = new DotExpanderProcessor("_tag", null, "foo.bar"); + processor = new DotExpanderProcessor("_tag", null, "foo.bar", null); processor.execute(document); //hasField returns true because the nested/expanded form exists in the source document assertTrue(document.hasField("foo.bar")); diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorFactoryTests.java index 9e8821ca1bdd4..b8c761f182721 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorFactoryTests.java @@ -64,7 +64,8 @@ public void testInvalidMustacheTemplate() throws Exception { Map config = new HashMap<>(); config.put("message", "{{error}}"); String processorTag = randomAlphaOfLength(10); - ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, null, config)); + ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, + null, config)); assertThat(exception.getMessage(), equalTo("java.lang.RuntimeException: could not compile script")); assertThat(exception.getMetadata("es.processor_tag").get(0), equalTo(processorTag)); } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorTests.java index 31cf830369eae..96342ed50b86d 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorTests.java @@ -33,7 +33,7 @@ public void test() throws Exception { IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random()); String message = randomAlphaOfLength(10); Processor processor = new FailProcessor(randomAlphaOfLength(10), - new TestTemplateService.MockTemplateScript.Factory(message)); + null, new TestTemplateService.MockTemplateScript.Factory(message)); try { processor.execute(ingestDocument); fail("fail processor should throw an exception"); diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorTests.java index a1aa7be272143..318afab4667cf 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorTests.java @@ -52,7 +52,7 @@ public void testExecuteWithAsyncProcessor() throws Exception { "_index", "_id", null, null, null, Collections.singletonMap("values", values) ); - ForEachProcessor processor = new ForEachProcessor("_tag", "values", new AsyncUpperCaseProcessor("_ingest._value"), + ForEachProcessor processor = new ForEachProcessor("_tag", null, "values", new AsyncUpperCaseProcessor("_ingest._value"), false); processor.execute(ingestDocument, (result, e) -> { }); @@ -77,7 +77,7 @@ public void testExecuteWithFailure() throws Exception { throw new RuntimeException("failure"); } }); - ForEachProcessor processor = new ForEachProcessor("_tag", "values", testProcessor, false); + ForEachProcessor processor = new ForEachProcessor("_tag", null, "values", testProcessor, false); Exception[] exceptions = new Exception[1]; processor.execute(ingestDocument, (result, e) -> {exceptions[0] = e;}); assertThat(exceptions[0].getMessage(), equalTo("failure")); @@ -94,7 +94,7 @@ public void testExecuteWithFailure() throws Exception { }); Processor onFailureProcessor = new TestProcessor(ingestDocument1 -> {}); processor = new ForEachProcessor( - "_tag", "values", new CompoundProcessor(false, Arrays.asList(testProcessor), Arrays.asList(onFailureProcessor)), + "_tag", null, "values", new CompoundProcessor(false, Arrays.asList(testProcessor), Arrays.asList(onFailureProcessor)), false ); processor.execute(ingestDocument, (result, e) -> {}); @@ -114,7 +114,7 @@ public void testMetadataAvailable() throws Exception { id.setFieldValue("_ingest._value.index", id.getSourceAndMetadata().get("_index")); id.setFieldValue("_ingest._value.id", id.getSourceAndMetadata().get("_id")); }); - ForEachProcessor processor = new ForEachProcessor("_tag", "values", innerProcessor, false); + ForEachProcessor processor = new ForEachProcessor("_tag", null, "values", innerProcessor, false); processor.execute(ingestDocument, (result, e) -> {}); assertThat(innerProcessor.getInvokedCounter(), equalTo(2)); @@ -138,8 +138,8 @@ public void testRestOfTheDocumentIsAvailable() throws Exception { IngestDocument ingestDocument = new IngestDocument("_index", "_id", null, null, null, document); ForEachProcessor processor = new ForEachProcessor( - "_tag", "values", new SetProcessor("_tag", - new TestTemplateService.MockTemplateScript.Factory("_ingest._value.new_field"), + "_tag", null, "values", new SetProcessor("_tag", + null, new TestTemplateService.MockTemplateScript.Factory("_ingest._value.new_field"), (model) -> model.get("other")), false); processor.execute(ingestDocument, (result, e) -> {}); @@ -168,6 +168,11 @@ public String getType() { public String getTag() { return null; } + + @Override + public String getDescription() { + return null; + } }; int numValues = randomIntBetween(1, 10000); List values = IntStream.range(0, numValues).mapToObj(i->"").collect(Collectors.toList()); @@ -176,7 +181,7 @@ public String getTag() { "_index", "_id", null, null, null, Collections.singletonMap("values", values) ); - ForEachProcessor processor = new ForEachProcessor("_tag", "values", innerProcessor, false); + ForEachProcessor processor = new ForEachProcessor("_tag", null, "values", innerProcessor, false); processor.execute(ingestDocument, (result, e) -> {}); @SuppressWarnings("unchecked") @@ -197,9 +202,9 @@ public void testModifyFieldsOutsideArray() throws Exception { TemplateScript.Factory template = new TestTemplateService.MockTemplateScript.Factory("errors"); ForEachProcessor processor = new ForEachProcessor( - "_tag", "values", new CompoundProcessor(false, - Collections.singletonList(new UppercaseProcessor("_tag_upper", "_ingest._value", false, "_ingest._value")), - Collections.singletonList(new AppendProcessor("_tag", template, (model) -> (Collections.singletonList("added")))) + "_tag", null, "values", new CompoundProcessor(false, + Collections.singletonList(new UppercaseProcessor("_tag_upper", null, "_ingest._value", false, "_ingest._value")), + Collections.singletonList(new AppendProcessor("_tag", null, template, (model) -> (Collections.singletonList("added")))) ), false); processor.execute(ingestDocument, (result, e) -> {}); @@ -226,7 +231,7 @@ public void testScalarValueAllowsUnderscoreValueFieldToRemainAccessible() throws TestProcessor processor = new TestProcessor(doc -> doc.setFieldValue("_ingest._value", doc.getFieldValue("_source._value", String.class))); - ForEachProcessor forEachProcessor = new ForEachProcessor("_tag", "values", processor, false); + ForEachProcessor forEachProcessor = new ForEachProcessor("_tag", null, "values", processor, false); forEachProcessor.execute(ingestDocument, (result, e) -> {}); List result = ingestDocument.getFieldValue("values", List.class); @@ -259,7 +264,7 @@ public void testNestedForEach() throws Exception { doc -> doc.setFieldValue("_ingest._value", doc.getFieldValue("_ingest._value", String.class).toUpperCase(Locale.ENGLISH)) ); ForEachProcessor processor = new ForEachProcessor( - "_tag", "values1", new ForEachProcessor("_tag", "_ingest._value.values2", testProcessor, false), + "_tag", null, "values1", new ForEachProcessor("_tag", null, "_ingest._value.values2", testProcessor, false), false); processor.execute(ingestDocument, (result, e) -> {}); @@ -278,7 +283,7 @@ public void testIgnoreMissing() throws Exception { ); IngestDocument ingestDocument = new IngestDocument(originalIngestDocument); TestProcessor testProcessor = new TestProcessor(doc -> {}); - ForEachProcessor processor = new ForEachProcessor("_tag", "_ingest._value", testProcessor, true); + ForEachProcessor processor = new ForEachProcessor("_tag", null, "_ingest._value", testProcessor, true); processor.execute(ingestDocument, (result, e) -> {}); assertIngestDocument(originalIngestDocument, ingestDocument); assertThat(testProcessor.getInvokedCounter(), equalTo(0)); @@ -319,6 +324,11 @@ public String getType() { public String getTag() { return getType(); } + + @Override + public String getDescription() { + return "async uppercase processor description"; + } } } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorTests.java index faabeacd94058..eac07bf5430e8 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorTests.java @@ -39,7 +39,7 @@ public void testMatch() throws Exception { String fieldName = RandomDocumentPicks.randomFieldName(random()); IngestDocument doc = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>()); doc.setFieldValue(fieldName, "1"); - GrokProcessor processor = new GrokProcessor(randomAlphaOfLength(10), Collections.singletonMap("ONE", "1"), + GrokProcessor processor = new GrokProcessor(randomAlphaOfLength(10), null, Collections.singletonMap("ONE", "1"), Collections.singletonList("%{ONE:one}"), fieldName, false, false, MatcherWatchdog.noop()); processor.execute(doc); assertThat(doc.getFieldValue("one", String.class), equalTo("1")); @@ -49,7 +49,7 @@ public void testIgnoreCase() throws Exception { String fieldName = RandomDocumentPicks.randomFieldName(random()); IngestDocument doc = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>()); doc.setFieldValue(fieldName, "A"); - GrokProcessor processor = new GrokProcessor(randomAlphaOfLength(10), Collections.emptyMap(), + GrokProcessor processor = new GrokProcessor(randomAlphaOfLength(10), null, Collections.emptyMap(), Collections.singletonList("(?(?i)A)"), fieldName, false, false, MatcherWatchdog.noop()); processor.execute(doc); assertThat(doc.getFieldValue("a", String.class), equalTo("A")); @@ -59,7 +59,7 @@ public void testNoMatch() { String fieldName = RandomDocumentPicks.randomFieldName(random()); IngestDocument doc = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>()); doc.setFieldValue(fieldName, "23"); - GrokProcessor processor = new GrokProcessor(randomAlphaOfLength(10), Collections.singletonMap("ONE", "1"), + GrokProcessor processor = new GrokProcessor(randomAlphaOfLength(10), null, Collections.singletonMap("ONE", "1"), Collections.singletonList("%{ONE:one}"), fieldName, false, false, MatcherWatchdog.noop()); Exception e = expectThrows(Exception.class, () -> processor.execute(doc)); assertThat(e.getMessage(), equalTo("Provided Grok expressions do not match field value: [23]")); @@ -70,7 +70,7 @@ public void testNoMatchingPatternName() { IngestDocument doc = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>()); doc.setFieldValue(fieldName, "23"); Exception e = expectThrows(IllegalArgumentException.class, () -> new GrokProcessor(randomAlphaOfLength(10), - Collections.singletonMap("ONE", "1"), Collections.singletonList("%{NOTONE:not_one}"), fieldName, + null, Collections.singletonMap("ONE", "1"), Collections.singletonList("%{NOTONE:not_one}"), fieldName, false, false, MatcherWatchdog.noop())); assertThat(e.getMessage(), equalTo("Unable to find pattern [NOTONE] in Grok's pattern dictionary")); } @@ -80,7 +80,7 @@ public void testMatchWithoutCaptures() throws Exception { IngestDocument originalDoc = new IngestDocument(new HashMap<>(), new HashMap<>()); originalDoc.setFieldValue(fieldName, fieldName); IngestDocument doc = new IngestDocument(originalDoc); - GrokProcessor processor = new GrokProcessor(randomAlphaOfLength(10), Collections.emptyMap(), + GrokProcessor processor = new GrokProcessor(randomAlphaOfLength(10), null, Collections.emptyMap(), Collections.singletonList(fieldName), fieldName, false, false, MatcherWatchdog.noop()); processor.execute(doc); assertThat(doc, equalTo(originalDoc)); @@ -90,7 +90,7 @@ public void testNullField() { String fieldName = RandomDocumentPicks.randomFieldName(random()); IngestDocument doc = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>()); doc.setFieldValue(fieldName, null); - GrokProcessor processor = new GrokProcessor(randomAlphaOfLength(10), Collections.singletonMap("ONE", "1"), + GrokProcessor processor = new GrokProcessor(randomAlphaOfLength(10), null, Collections.singletonMap("ONE", "1"), Collections.singletonList("%{ONE:one}"), fieldName, false, false, MatcherWatchdog.noop()); Exception e = expectThrows(Exception.class, () -> processor.execute(doc)); assertThat(e.getMessage(), equalTo("field [" + fieldName + "] is null, cannot process it.")); @@ -101,7 +101,7 @@ public void testNullFieldWithIgnoreMissing() throws Exception { IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>()); originalIngestDocument.setFieldValue(fieldName, null); IngestDocument ingestDocument = new IngestDocument(originalIngestDocument); - GrokProcessor processor = new GrokProcessor(randomAlphaOfLength(10), Collections.singletonMap("ONE", "1"), + GrokProcessor processor = new GrokProcessor(randomAlphaOfLength(10), null, Collections.singletonMap("ONE", "1"), Collections.singletonList("%{ONE:one}"), fieldName, false, true, MatcherWatchdog.noop()); processor.execute(ingestDocument); assertIngestDocument(originalIngestDocument, ingestDocument); @@ -111,7 +111,7 @@ public void testNotStringField() { String fieldName = RandomDocumentPicks.randomFieldName(random()); IngestDocument doc = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>()); doc.setFieldValue(fieldName, 1); - GrokProcessor processor = new GrokProcessor(randomAlphaOfLength(10), Collections.singletonMap("ONE", "1"), + GrokProcessor processor = new GrokProcessor(randomAlphaOfLength(10), null, Collections.singletonMap("ONE", "1"), Collections.singletonList("%{ONE:one}"), fieldName, false, false, MatcherWatchdog.noop()); Exception e = expectThrows(Exception.class, () -> processor.execute(doc)); assertThat(e.getMessage(), equalTo("field [" + fieldName + "] of type [java.lang.Integer] cannot be cast to [java.lang.String]")); @@ -121,7 +121,7 @@ public void testNotStringFieldWithIgnoreMissing() { String fieldName = RandomDocumentPicks.randomFieldName(random()); IngestDocument doc = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>()); doc.setFieldValue(fieldName, 1); - GrokProcessor processor = new GrokProcessor(randomAlphaOfLength(10), Collections.singletonMap("ONE", "1"), + GrokProcessor processor = new GrokProcessor(randomAlphaOfLength(10), null, Collections.singletonMap("ONE", "1"), Collections.singletonList("%{ONE:one}"), fieldName, false, true, MatcherWatchdog.noop()); Exception e = expectThrows(Exception.class, () -> processor.execute(doc)); assertThat(e.getMessage(), equalTo("field [" + fieldName + "] of type [java.lang.Integer] cannot be cast to [java.lang.String]")); @@ -130,7 +130,7 @@ public void testNotStringFieldWithIgnoreMissing() { public void testMissingField() { String fieldName = "foo.bar"; IngestDocument doc = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>()); - GrokProcessor processor = new GrokProcessor(randomAlphaOfLength(10), Collections.singletonMap("ONE", "1"), + GrokProcessor processor = new GrokProcessor(randomAlphaOfLength(10), null, Collections.singletonMap("ONE", "1"), Collections.singletonList("%{ONE:one}"), fieldName, false, false, MatcherWatchdog.noop()); Exception e = expectThrows(Exception.class, () -> processor.execute(doc)); assertThat(e.getMessage(), equalTo("field [foo] not present as part of path [foo.bar]")); @@ -140,7 +140,7 @@ public void testMissingFieldWithIgnoreMissing() throws Exception { String fieldName = "foo.bar"; IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>()); IngestDocument ingestDocument = new IngestDocument(originalIngestDocument); - GrokProcessor processor = new GrokProcessor(randomAlphaOfLength(10), Collections.singletonMap("ONE", "1"), + GrokProcessor processor = new GrokProcessor(randomAlphaOfLength(10), null, Collections.singletonMap("ONE", "1"), Collections.singletonList("%{ONE:one}"), fieldName, false, true, MatcherWatchdog.noop()); processor.execute(ingestDocument); assertIngestDocument(originalIngestDocument, ingestDocument); @@ -154,7 +154,7 @@ public void testMultiplePatternsWithMatchReturn() throws Exception { patternBank.put("ONE", "1"); patternBank.put("TWO", "2"); patternBank.put("THREE", "3"); - GrokProcessor processor = new GrokProcessor(randomAlphaOfLength(10), patternBank, + GrokProcessor processor = new GrokProcessor(randomAlphaOfLength(10), null, patternBank, Arrays.asList("%{ONE:one}", "%{TWO:two}", "%{THREE:three}"), fieldName, false, false, MatcherWatchdog.noop()); processor.execute(doc); assertThat(doc.hasField("one"), equalTo(false)); @@ -170,7 +170,7 @@ public void testSetMetadata() throws Exception { patternBank.put("ONE", "1"); patternBank.put("TWO", "2"); patternBank.put("THREE", "3"); - GrokProcessor processor = new GrokProcessor(randomAlphaOfLength(10), patternBank, + GrokProcessor processor = new GrokProcessor(randomAlphaOfLength(10), null, patternBank, Arrays.asList("%{ONE:one}", "%{TWO:two}", "%{THREE:three}"), fieldName, true, false, MatcherWatchdog.noop()); processor.execute(doc); assertThat(doc.hasField("one"), equalTo(false)); @@ -185,7 +185,7 @@ public void testTraceWithOnePattern() throws Exception { doc.setFieldValue(fieldName, "first1"); Map patternBank = new HashMap<>(); patternBank.put("ONE", "1"); - GrokProcessor processor = new GrokProcessor(randomAlphaOfLength(10), patternBank, + GrokProcessor processor = new GrokProcessor(randomAlphaOfLength(10), null, patternBank, Arrays.asList("%{ONE:one}"), fieldName, true, false, MatcherWatchdog.noop()); processor.execute(doc); assertThat(doc.hasField("one"), equalTo(true)); @@ -216,7 +216,7 @@ public void testCombineSamePatternNameAcrossPatterns() throws Exception { patternBank.put("ONE", "1"); patternBank.put("TWO", "2"); patternBank.put("THREE", "3"); - GrokProcessor processor = new GrokProcessor(randomAlphaOfLength(10), patternBank, Arrays.asList("%{ONE:first}-%{TWO:second}", + GrokProcessor processor = new GrokProcessor(randomAlphaOfLength(10), null, patternBank, Arrays.asList("%{ONE:first}-%{TWO:second}", "%{ONE:first}-%{THREE:second}"), fieldName, randomBoolean(), randomBoolean(), MatcherWatchdog.noop()); processor.execute(doc); assertThat(doc.getFieldValue("first", String.class), equalTo("1")); @@ -229,7 +229,7 @@ public void testFirstWinNamedCapture() throws Exception { doc.setFieldValue(fieldName, "12"); Map patternBank = new HashMap<>(); patternBank.put("ONETWO", "1|2"); - GrokProcessor processor = new GrokProcessor(randomAlphaOfLength(10), patternBank, + GrokProcessor processor = new GrokProcessor(randomAlphaOfLength(10), null, patternBank, Collections.singletonList("%{ONETWO:first}%{ONETWO:first}"), fieldName, randomBoolean(), randomBoolean(), MatcherWatchdog.noop()); processor.execute(doc); @@ -243,7 +243,7 @@ public void testUnmatchedNamesNotIncludedInDocument() throws Exception { Map patternBank = new HashMap<>(); patternBank.put("ONETWO", "1|2"); patternBank.put("THREE", "3"); - GrokProcessor processor = new GrokProcessor(randomAlphaOfLength(10), patternBank, + GrokProcessor processor = new GrokProcessor(randomAlphaOfLength(10), null, patternBank, Collections.singletonList("%{ONETWO:first}|%{THREE:second}"), fieldName, randomBoolean(), randomBoolean(), MatcherWatchdog.noop()); processor.execute(doc); diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GsubProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GsubProcessorTests.java index 9c003356c3dff..75c2d46e97b7b 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GsubProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GsubProcessorTests.java @@ -25,7 +25,7 @@ public class GsubProcessorTests extends AbstractStringProcessorTestCase @Override protected AbstractStringProcessor newProcessor(String field, boolean ignoreMissing, String targetField) { - return new GsubProcessor(randomAlphaOfLength(10), field, Pattern.compile("\\."), "-", ignoreMissing, targetField); + return new GsubProcessor(randomAlphaOfLength(10), null, field, Pattern.compile("\\."), "-", ignoreMissing, targetField); } @Override diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/HtmlStripProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/HtmlStripProcessorTests.java index 79ccff84a726e..fe88a93e3a92e 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/HtmlStripProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/HtmlStripProcessorTests.java @@ -23,7 +23,7 @@ public class HtmlStripProcessorTests extends AbstractStringProcessorTestCase newProcessor(String field, boolean ignoreMissing, String targetField) { - return new HtmlStripProcessor(randomAlphaOfLength(10), field, ignoreMissing, targetField); + return new HtmlStripProcessor(randomAlphaOfLength(10), null, field, ignoreMissing, targetField); } @Override diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JoinProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JoinProcessorTests.java index a866b4164bf11..dc8a20dd4da1c 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JoinProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JoinProcessorTests.java @@ -51,7 +51,7 @@ public void testJoinStrings() throws Exception { } } String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue); - Processor processor = new JoinProcessor(randomAlphaOfLength(10), fieldName, separator, fieldName); + Processor processor = new JoinProcessor(randomAlphaOfLength(10), null, fieldName, separator, fieldName); processor.execute(ingestDocument); assertThat(ingestDocument.getFieldValue(fieldName, String.class), equalTo(expectedResult)); } @@ -71,7 +71,7 @@ public void testJoinIntegers() throws Exception { } } String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue); - Processor processor = new JoinProcessor(randomAlphaOfLength(10), fieldName, separator, fieldName); + Processor processor = new JoinProcessor(randomAlphaOfLength(10), null, fieldName, separator, fieldName); processor.execute(ingestDocument); assertThat(ingestDocument.getFieldValue(fieldName, String.class), equalTo(expectedResult)); } @@ -80,7 +80,7 @@ public void testJoinNonListField() throws Exception { IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>()); String fieldName = RandomDocumentPicks.randomFieldName(random()); ingestDocument.setFieldValue(fieldName, randomAlphaOfLengthBetween(1, 10)); - Processor processor = new JoinProcessor(randomAlphaOfLength(10), fieldName, "-", fieldName); + Processor processor = new JoinProcessor(randomAlphaOfLength(10), null, fieldName, "-", fieldName); try { processor.execute(ingestDocument); } catch(IllegalArgumentException e) { @@ -91,7 +91,7 @@ public void testJoinNonListField() throws Exception { public void testJoinNonExistingField() throws Exception { IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>()); String fieldName = RandomDocumentPicks.randomFieldName(random()); - Processor processor = new JoinProcessor(randomAlphaOfLength(10), fieldName, "-", fieldName); + Processor processor = new JoinProcessor(randomAlphaOfLength(10), null, fieldName, "-", fieldName); try { processor.execute(ingestDocument); } catch(IllegalArgumentException e) { @@ -101,7 +101,7 @@ public void testJoinNonExistingField() throws Exception { public void testJoinNullValue() throws Exception { IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", null)); - Processor processor = new JoinProcessor(randomAlphaOfLength(10), "field", "-", "field"); + Processor processor = new JoinProcessor(randomAlphaOfLength(10), null, "field", "-", "field"); try { processor.execute(ingestDocument); } catch(IllegalArgumentException e) { @@ -125,7 +125,7 @@ public void testJoinWithTargetField() throws Exception { } String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue); String targetFieldName = fieldName + randomAlphaOfLength(5); - Processor processor = new JoinProcessor(randomAlphaOfLength(10), fieldName, separator, targetFieldName); + Processor processor = new JoinProcessor(randomAlphaOfLength(10), null, fieldName, separator, targetFieldName); processor.execute(ingestDocument); assertThat(ingestDocument.getFieldValue(targetFieldName, String.class), equalTo(expectedResult)); } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorTests.java index c9e81ccf4c811..35fdfc10ad67f 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorTests.java @@ -43,7 +43,7 @@ public void testExecute() throws Exception { String processorTag = randomAlphaOfLength(3); String randomField = randomAlphaOfLength(3); String randomTargetField = randomAlphaOfLength(2); - JsonProcessor jsonProcessor = new JsonProcessor(processorTag, randomField, randomTargetField, false); + JsonProcessor jsonProcessor = new JsonProcessor(processorTag, null, randomField, randomTargetField, false); Map document = new HashMap<>(); Map randomJsonMap = RandomDocumentPicks.randomSource(random()); @@ -58,7 +58,7 @@ public void testExecute() throws Exception { } public void testInvalidValue() { - JsonProcessor jsonProcessor = new JsonProcessor("tag", "field", "target_field", false); + JsonProcessor jsonProcessor = new JsonProcessor("tag", null, "field", "target_field", false); Map document = new HashMap<>(); document.put("field", "blah blah"); IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); @@ -69,7 +69,7 @@ public void testInvalidValue() { } public void testByteArray() { - JsonProcessor jsonProcessor = new JsonProcessor("tag", "field", "target_field", false); + JsonProcessor jsonProcessor = new JsonProcessor("tag", null, "field", "target_field", false); Map document = new HashMap<>(); document.put("field", new byte[] { 0, 1 }); IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); @@ -84,7 +84,7 @@ public void testByteArray() { } public void testNull() throws Exception { - JsonProcessor jsonProcessor = new JsonProcessor("tag", "field", "target_field", false); + JsonProcessor jsonProcessor = new JsonProcessor("tag", null, "field", "target_field", false); Map document = new HashMap<>(); document.put("field", null); IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); @@ -93,7 +93,7 @@ public void testNull() throws Exception { } public void testBoolean() throws Exception { - JsonProcessor jsonProcessor = new JsonProcessor("tag", "field", "target_field", false); + JsonProcessor jsonProcessor = new JsonProcessor("tag", null, "field", "target_field", false); Map document = new HashMap<>(); boolean value = true; document.put("field", value); @@ -103,7 +103,7 @@ public void testBoolean() throws Exception { } public void testInteger() throws Exception { - JsonProcessor jsonProcessor = new JsonProcessor("tag", "field", "target_field", false); + JsonProcessor jsonProcessor = new JsonProcessor("tag", null, "field", "target_field", false); Map document = new HashMap<>(); int value = 3; document.put("field", value); @@ -113,7 +113,7 @@ public void testInteger() throws Exception { } public void testDouble() throws Exception { - JsonProcessor jsonProcessor = new JsonProcessor("tag", "field", "target_field", false); + JsonProcessor jsonProcessor = new JsonProcessor("tag", null, "field", "target_field", false); Map document = new HashMap<>(); double value = 3.0; document.put("field", value); @@ -123,7 +123,7 @@ public void testDouble() throws Exception { } public void testString() throws Exception { - JsonProcessor jsonProcessor = new JsonProcessor("tag", "field", "target_field", false); + JsonProcessor jsonProcessor = new JsonProcessor("tag", null, "field", "target_field", false); Map document = new HashMap<>(); String value = "hello world"; document.put("field", "\"" + value + "\""); @@ -133,7 +133,7 @@ public void testString() throws Exception { } public void testArray() throws Exception { - JsonProcessor jsonProcessor = new JsonProcessor("tag", "field", "target_field", false); + JsonProcessor jsonProcessor = new JsonProcessor("tag", null, "field", "target_field", false); Map document = new HashMap<>(); List value = Arrays.asList(true, true, false); document.put("field", value.toString()); @@ -143,7 +143,7 @@ public void testArray() throws Exception { } public void testFieldMissing() { - JsonProcessor jsonProcessor = new JsonProcessor("tag", "field", "target_field", false); + JsonProcessor jsonProcessor = new JsonProcessor("tag", null, "field", "target_field", false); Map document = new HashMap<>(); IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); @@ -154,7 +154,7 @@ public void testFieldMissing() { public void testAddToRoot() throws Exception { String processorTag = randomAlphaOfLength(3); String randomTargetField = randomAlphaOfLength(2); - JsonProcessor jsonProcessor = new JsonProcessor(processorTag, "a", randomTargetField, true); + JsonProcessor jsonProcessor = new JsonProcessor(processorTag, null, "a", randomTargetField, true); Map document = new HashMap<>(); String json = "{\"a\": 1, \"b\": 2}"; @@ -171,7 +171,7 @@ public void testAddToRoot() throws Exception { } public void testAddBoolToRoot() { - JsonProcessor jsonProcessor = new JsonProcessor("tag", "field", "target_field", true); + JsonProcessor jsonProcessor = new JsonProcessor("tag", null, "field", "target_field", true); Map document = new HashMap<>(); document.put("field", true); IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/LowercaseProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/LowercaseProcessorTests.java index b804d3a0221c2..db5af63aa0934 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/LowercaseProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/LowercaseProcessorTests.java @@ -24,7 +24,7 @@ public class LowercaseProcessorTests extends AbstractStringProcessorTestCase { @Override protected AbstractStringProcessor newProcessor(String field, boolean ignoreMissing, String targetField) { - return new LowercaseProcessor(randomAlphaOfLength(10), field, ignoreMissing, targetField); + return new LowercaseProcessor(randomAlphaOfLength(10), null, field, ignoreMissing, targetField); } @Override diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorFactoryTests.java index b2e9641a240a6..2a137f48b117f 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorFactoryTests.java @@ -77,7 +77,8 @@ public void testInvalidMustacheTemplate() throws Exception { Map config = new HashMap<>(); config.put("field", "{{field1}}"); String processorTag = randomAlphaOfLength(10); - ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, null, config)); + ElasticsearchException exception = expectThrows(ElasticsearchException.class, + () -> factory.create(null, processorTag, null, config)); assertThat(exception.getMessage(), equalTo("java.lang.RuntimeException: could not compile script")); assertThat(exception.getMetadata("es.processor_tag").get(0), equalTo(processorTag)); } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorTests.java index 685b873d507ec..aeed73d428279 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorTests.java @@ -38,7 +38,7 @@ public void testRemoveFields() throws Exception { IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random()); String field = RandomDocumentPicks.randomExistingFieldName(random(), ingestDocument); Processor processor = new RemoveProcessor(randomAlphaOfLength(10), - Collections.singletonList(new TestTemplateService.MockTemplateScript.Factory(field)), false); + null, Collections.singletonList(new TestTemplateService.MockTemplateScript.Factory(field)), false); processor.execute(ingestDocument); assertThat(ingestDocument.hasField(field), equalTo(false)); } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorTests.java index 5c3b0b0e7f3b5..962611ec6c3d6 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorTests.java @@ -212,7 +212,7 @@ public void testRenameLeafIntoBranch() throws Exception { } private RenameProcessor createRenameProcessor(String field, String targetField, boolean ignoreMissing) { - return new RenameProcessor(randomAlphaOfLength(10), new TestTemplateService.MockTemplateScript.Factory(field), + return new RenameProcessor(randomAlphaOfLength(10), null, new TestTemplateService.MockTemplateScript.Factory(field), new TestTemplateService.MockTemplateScript.Factory(targetField), ignoreMissing); } } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java index 10fcf5fe602ad..b7a35fcb42870 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java @@ -65,7 +65,7 @@ Script.DEFAULT_SCRIPT_LANG, new MockScriptEngine( document.put("bytes_out", randomInt()); IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); - ScriptProcessor processor = new ScriptProcessor(randomAlphaOfLength(10), script, scriptService); + ScriptProcessor processor = new ScriptProcessor(randomAlphaOfLength(10), null, script, scriptService); processor.execute(ingestDocument); diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorFactoryTests.java index 145d1ea9f72bc..f051eaf45a34b 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorFactoryTests.java @@ -106,7 +106,8 @@ public void testInvalidMustacheTemplate() throws Exception { config.put("field", "{{field1}}"); config.put("value", "value1"); String processorTag = randomAlphaOfLength(10); - ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, null, config)); + ElasticsearchException exception = expectThrows(ElasticsearchException.class, + () -> factory.create(null, processorTag, null, config)); assertThat(exception.getMessage(), equalTo("java.lang.RuntimeException: could not compile script")); assertThat(exception.getMetadata("es.processor_tag").get(0), equalTo(processorTag)); } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorTests.java index ccf4c90340dea..b0a67800f6089 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorTests.java @@ -141,7 +141,7 @@ public void testSetMetadataIfPrimaryTerm() throws Exception { } private static Processor createSetProcessor(String fieldName, Object fieldValue, boolean overrideEnabled) { - return new SetProcessor(randomAlphaOfLength(10), new TestTemplateService.MockTemplateScript.Factory(fieldName), + return new SetProcessor(randomAlphaOfLength(10), null, new TestTemplateService.MockTemplateScript.Factory(fieldName), ValueSource.wrap(fieldValue, TestTemplateService.instance()), overrideEnabled); } } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SortProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SortProcessorTests.java index 5eca68f35de21..bff052b0d7c91 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SortProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SortProcessorTests.java @@ -54,7 +54,7 @@ public void testSortStrings() throws Exception { } String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue); - Processor processor = new SortProcessor(randomAlphaOfLength(10), fieldName, order, fieldName); + Processor processor = new SortProcessor(randomAlphaOfLength(10), null, fieldName, order, fieldName); processor.execute(ingestDocument); assertEquals(ingestDocument.getFieldValue(fieldName, List.class), expectedResult); } @@ -68,7 +68,7 @@ public void testSortIntegersNonRandom() throws Exception { Collections.shuffle(fieldValue, random()); String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue); - Processor processor = new SortProcessor(randomAlphaOfLength(10), fieldName, SortOrder.ASCENDING, fieldName); + Processor processor = new SortProcessor(randomAlphaOfLength(10), null, fieldName, SortOrder.ASCENDING, fieldName); processor.execute(ingestDocument); assertThat(ingestDocument.getFieldValue(fieldName, List.class).toArray(), equalTo(expectedResult)); } @@ -91,7 +91,7 @@ public void testSortIntegers() throws Exception { } String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue); - Processor processor = new SortProcessor(randomAlphaOfLength(10), fieldName, order, fieldName); + Processor processor = new SortProcessor(randomAlphaOfLength(10), null, fieldName, order, fieldName); processor.execute(ingestDocument); assertEquals(ingestDocument.getFieldValue(fieldName, List.class), expectedResult); } @@ -114,7 +114,7 @@ public void testSortShorts() throws Exception { } String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue); - Processor processor = new SortProcessor(randomAlphaOfLength(10), fieldName, order, fieldName); + Processor processor = new SortProcessor(randomAlphaOfLength(10), null, fieldName, order, fieldName); processor.execute(ingestDocument); assertEquals(ingestDocument.getFieldValue(fieldName, List.class), expectedResult); } @@ -137,7 +137,7 @@ public void testSortDoubles() throws Exception { } String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue); - Processor processor = new SortProcessor(randomAlphaOfLength(10), fieldName, order, fieldName); + Processor processor = new SortProcessor(randomAlphaOfLength(10), null, fieldName, order, fieldName); processor.execute(ingestDocument); assertEquals(ingestDocument.getFieldValue(fieldName, List.class), expectedResult); } @@ -160,7 +160,7 @@ public void testSortFloats() throws Exception { } String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue); - Processor processor = new SortProcessor(randomAlphaOfLength(10), fieldName, order, fieldName); + Processor processor = new SortProcessor(randomAlphaOfLength(10), null, fieldName, order, fieldName); processor.execute(ingestDocument); assertEquals(ingestDocument.getFieldValue(fieldName, List.class), expectedResult); } @@ -183,7 +183,7 @@ public void testSortBytes() throws Exception { } String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue); - Processor processor = new SortProcessor(randomAlphaOfLength(10), fieldName, order, fieldName); + Processor processor = new SortProcessor(randomAlphaOfLength(10), null, fieldName, order, fieldName); processor.execute(ingestDocument); assertEquals(ingestDocument.getFieldValue(fieldName, List.class), expectedResult); } @@ -206,7 +206,7 @@ public void testSortBooleans() throws Exception { } String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue); - Processor processor = new SortProcessor(randomAlphaOfLength(10), fieldName, order, fieldName); + Processor processor = new SortProcessor(randomAlphaOfLength(10), null, fieldName, order, fieldName); processor.execute(ingestDocument); assertEquals(ingestDocument.getFieldValue(fieldName, List.class), expectedResult); } @@ -234,7 +234,7 @@ public void testSortMixedStrings() throws Exception { } String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue); - Processor processor = new SortProcessor(randomAlphaOfLength(10), fieldName, order, fieldName); + Processor processor = new SortProcessor(randomAlphaOfLength(10), null, fieldName, order, fieldName); processor.execute(ingestDocument); assertEquals(ingestDocument.getFieldValue(fieldName, List.class), expectedResult); } @@ -244,7 +244,7 @@ public void testSortNonListField() throws Exception { String fieldName = RandomDocumentPicks.randomFieldName(random()); ingestDocument.setFieldValue(fieldName, randomAlphaOfLengthBetween(1, 10)); SortOrder order = randomBoolean() ? SortOrder.ASCENDING : SortOrder.DESCENDING; - Processor processor = new SortProcessor(randomAlphaOfLength(10), fieldName, order, fieldName); + Processor processor = new SortProcessor(randomAlphaOfLength(10), null, fieldName, order, fieldName); try { processor.execute(ingestDocument); } catch(IllegalArgumentException e) { @@ -256,7 +256,7 @@ public void testSortNonExistingField() throws Exception { IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>()); String fieldName = RandomDocumentPicks.randomFieldName(random()); SortOrder order = randomBoolean() ? SortOrder.ASCENDING : SortOrder.DESCENDING; - Processor processor = new SortProcessor(randomAlphaOfLength(10), fieldName, order, fieldName); + Processor processor = new SortProcessor(randomAlphaOfLength(10), null, fieldName, order, fieldName); try { processor.execute(ingestDocument); } catch(IllegalArgumentException e) { @@ -267,7 +267,7 @@ public void testSortNonExistingField() throws Exception { public void testSortNullValue() throws Exception { IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", null)); SortOrder order = randomBoolean() ? SortOrder.ASCENDING : SortOrder.DESCENDING; - Processor processor = new SortProcessor(randomAlphaOfLength(10), "field", order, "field"); + Processor processor = new SortProcessor(randomAlphaOfLength(10), null, "field", order, "field"); try { processor.execute(ingestDocument); } catch(IllegalArgumentException e) { @@ -290,7 +290,7 @@ public void testDescendingSortWithTargetField() throws Exception { String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue); String targetFieldName = fieldName + "foo"; - Processor processor = new SortProcessor(randomAlphaOfLength(10), fieldName, + Processor processor = new SortProcessor(randomAlphaOfLength(10), null, fieldName, SortOrder.DESCENDING, targetFieldName); processor.execute(ingestDocument); assertEquals(ingestDocument.getFieldValue(targetFieldName, List.class), expectedResult); @@ -311,7 +311,7 @@ public void testAscendingSortWithTargetField() throws Exception { String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue); String targetFieldName = fieldName + "foo"; - Processor processor = new SortProcessor(randomAlphaOfLength(10), fieldName, + Processor processor = new SortProcessor(randomAlphaOfLength(10), null, fieldName, SortOrder.ASCENDING, targetFieldName); processor.execute(ingestDocument); assertEquals(ingestDocument.getFieldValue(targetFieldName, List.class), expectedResult); @@ -330,7 +330,7 @@ public void testSortWithTargetFieldLeavesOriginalUntouched() throws Exception { String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, new ArrayList<>(fieldValue)); String targetFieldName = fieldName + "foo"; - Processor processor = new SortProcessor(randomAlphaOfLength(10), fieldName, order, targetFieldName); + Processor processor = new SortProcessor(randomAlphaOfLength(10), null, fieldName, order, targetFieldName); processor.execute(ingestDocument); assertEquals(ingestDocument.getFieldValue(targetFieldName, List.class), expectedResult); assertEquals(ingestDocument.getFieldValue(fieldName, List.class), fieldValue); diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorTests.java index 5155831c4c7a6..fe6de358e5afd 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorTests.java @@ -39,7 +39,7 @@ public class SplitProcessorTests extends ESTestCase { public void testSplit() throws Exception { IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random()); String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, "127.0.0.1"); - Processor processor = new SplitProcessor(randomAlphaOfLength(10), fieldName, "\\.", false, false, fieldName); + Processor processor = new SplitProcessor(randomAlphaOfLength(10), null, fieldName, "\\.", false, false, fieldName); processor.execute(ingestDocument); assertThat(ingestDocument.getFieldValue(fieldName, List.class), equalTo(Arrays.asList("127", "0", "0", "1"))); } @@ -47,7 +47,7 @@ public void testSplit() throws Exception { public void testSplitFieldNotFound() throws Exception { IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>()); String fieldName = RandomDocumentPicks.randomFieldName(random()); - Processor processor = new SplitProcessor(randomAlphaOfLength(10), fieldName, "\\.", false, false, fieldName); + Processor processor = new SplitProcessor(randomAlphaOfLength(10), null, fieldName, "\\.", false, false, fieldName); try { processor.execute(ingestDocument); fail("split processor should have failed"); @@ -59,7 +59,7 @@ public void testSplitFieldNotFound() throws Exception { public void testSplitNullValue() throws Exception { IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", null)); - Processor processor = new SplitProcessor(randomAlphaOfLength(10), "field", "\\.", false, false, "field"); + Processor processor = new SplitProcessor(randomAlphaOfLength(10), null, "field", "\\.", false, false, "field"); try { processor.execute(ingestDocument); fail("split processor should have failed"); @@ -73,7 +73,7 @@ public void testSplitNullValueWithIgnoreMissing() throws Exception { IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap(fieldName, null)); IngestDocument ingestDocument = new IngestDocument(originalIngestDocument); - Processor processor = new SplitProcessor(randomAlphaOfLength(10), fieldName, "\\.", true, false, fieldName); + Processor processor = new SplitProcessor(randomAlphaOfLength(10), null, fieldName, "\\.", true, false, fieldName); processor.execute(ingestDocument); assertIngestDocument(originalIngestDocument, ingestDocument); } @@ -81,7 +81,7 @@ public void testSplitNullValueWithIgnoreMissing() throws Exception { public void testSplitNonExistentWithIgnoreMissing() throws Exception { IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap()); IngestDocument ingestDocument = new IngestDocument(originalIngestDocument); - Processor processor = new SplitProcessor(randomAlphaOfLength(10), "field", "\\.", true, false, "field"); + Processor processor = new SplitProcessor(randomAlphaOfLength(10), null, "field", "\\.", true, false, "field"); processor.execute(ingestDocument); assertIngestDocument(originalIngestDocument, ingestDocument); } @@ -90,7 +90,7 @@ public void testSplitNonStringValue() throws Exception { IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>()); String fieldName = RandomDocumentPicks.randomFieldName(random()); ingestDocument.setFieldValue(fieldName, randomInt()); - Processor processor = new SplitProcessor(randomAlphaOfLength(10), fieldName, "\\.", false, false, fieldName); + Processor processor = new SplitProcessor(randomAlphaOfLength(10), null, fieldName, "\\.", false, false, fieldName); try { processor.execute(ingestDocument); fail("split processor should have failed"); @@ -121,7 +121,7 @@ public void testSplitWithTargetField() throws Exception { IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random()); String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, "127.0.0.1"); String targetFieldName = fieldName + randomAlphaOfLength(5); - Processor processor = new SplitProcessor(randomAlphaOfLength(10), fieldName, "\\.", false, false, targetFieldName); + Processor processor = new SplitProcessor(randomAlphaOfLength(10), null, fieldName, "\\.", false, false, targetFieldName); processor.execute(ingestDocument); assertThat(ingestDocument.getFieldValue(targetFieldName, List.class), equalTo(Arrays.asList("127", "0", "0", "1"))); } @@ -137,7 +137,7 @@ public void testSplitWithoutPreserveTrailing() throws Exception { private void doTestSplitWithPreserveTrailing(boolean preserveTrailing, String fieldValue, List expected) throws Exception { IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random()); String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue); - Processor processor = new SplitProcessor(randomAlphaOfLength(10), fieldName, "\\|", false, preserveTrailing, fieldName); + Processor processor = new SplitProcessor(randomAlphaOfLength(10), null, fieldName, "\\|", false, preserveTrailing, fieldName); processor.execute(ingestDocument); assertThat(ingestDocument.getFieldValue(fieldName, List.class), equalTo(expected)); } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/TrimProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/TrimProcessorTests.java index abd7cae12fe91..77fc5b501a984 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/TrimProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/TrimProcessorTests.java @@ -23,7 +23,7 @@ public class TrimProcessorTests extends AbstractStringProcessorTestCase @Override protected AbstractStringProcessor newProcessor(String field, boolean ignoreMissing, String targetField) { - return new TrimProcessor(randomAlphaOfLength(10), field, ignoreMissing, targetField); + return new TrimProcessor(randomAlphaOfLength(10), null, field, ignoreMissing, targetField); } @Override diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/URLDecodeProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/URLDecodeProcessorTests.java index 150d594afd9af..497314d7ce13a 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/URLDecodeProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/URLDecodeProcessorTests.java @@ -30,7 +30,7 @@ protected String modifyInput(String input) { @Override protected AbstractStringProcessor newProcessor(String field, boolean ignoreMissing, String targetField) { - return new URLDecodeProcessor(randomAlphaOfLength(10), field, ignoreMissing, targetField); + return new URLDecodeProcessor(randomAlphaOfLength(10), null, field, ignoreMissing, targetField); } @Override diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UppercaseProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UppercaseProcessorTests.java index 1b027c4380837..9090ba9cd3e28 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UppercaseProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UppercaseProcessorTests.java @@ -25,7 +25,7 @@ public class UppercaseProcessorTests extends AbstractStringProcessorTestCase newProcessor(String field, boolean ignoreMissing, String targetField) { - return new UppercaseProcessor(randomAlphaOfLength(10), field, ignoreMissing, targetField); + return new UppercaseProcessor(randomAlphaOfLength(10), null, field, ignoreMissing, targetField); } @Override From 882d17c78f8cea4dae14c2abb0508dc83c1962c6 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Wed, 10 Jun 2020 11:35:50 -0700 Subject: [PATCH 3/6] more precommit fixes --- .../ingest/geoip/GeoIpProcessor.java | 6 +- .../ingest/geoip/GeoIpProcessorTests.java | 34 ++++---- .../useragent/UserAgentProcessorTests.java | 10 +-- .../attachment/AttachmentProcessorTests.java | 16 ++-- .../xpack/enrich/AbstractEnrichProcessor.java | 14 +++- .../xpack/enrich/EnrichProcessorFactory.java | 5 +- .../xpack/enrich/GeoMatchProcessorTests.java | 1 + .../xpack/enrich/MatchProcessorTests.java | 81 +++++++++++++++++-- .../inference/ingest/InferenceProcessor.java | 3 +- ...sportGetTrainedModelsStatsActionTests.java | 8 +- .../ingest/InferenceProcessorTests.java | 22 ++--- .../monitoring/test/MockIngestPlugin.java | 11 ++- .../ingest/SetSecurityUserProcessorTests.java | 26 +++--- .../xpack/spatial/ingest/CircleProcessor.java | 3 +- .../ingest/CircleProcessorFactoryTests.java | 6 +- .../spatial/ingest/CircleProcessorTests.java | 24 +++--- 16 files changed, 184 insertions(+), 86 deletions(-) diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java index d66f3feb350fa..2fe9c76451bc0 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java @@ -73,8 +73,7 @@ public final class GeoIpProcessor extends AbstractProcessor { /** * Construct a geo-IP processor. - * - * @param tag the processor tag + * @param tag the processor tag * @param description the processor description * @param field the source field to geo-IP map * @param lazyLoader a supplier of a geo-IP database reader; ideally this is lazily-loaded once on first use @@ -86,8 +85,7 @@ public final class GeoIpProcessor extends AbstractProcessor { */ GeoIpProcessor( final String tag, - final String description, - final String field, + String description, final String field, final DatabaseReaderLazyLoader lazyLoader, final String targetField, final Set properties, diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java index f0e9b30ae69ca..c33080891644f 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java @@ -46,7 +46,7 @@ public class GeoIpProcessorTests extends ESTestCase { public void testCity() throws Exception { - GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), "source_field", + GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), null, "source_field", loader("/GeoLite2-City.mmdb"), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), false, new GeoIpCache(1000), false); @@ -71,7 +71,7 @@ public void testCity() throws Exception { } public void testNullValueWithIgnoreMissing() throws Exception { - GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), "source_field", + GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), null, "source_field", loader("/GeoLite2-City.mmdb"), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), true, new GeoIpCache(1000), false); IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), @@ -82,7 +82,7 @@ public void testNullValueWithIgnoreMissing() throws Exception { } public void testNonExistentWithIgnoreMissing() throws Exception { - GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), "source_field", + GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), null, "source_field", loader("/GeoLite2-City.mmdb"), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), true, new GeoIpCache(1000), false); IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap()); @@ -92,7 +92,7 @@ public void testNonExistentWithIgnoreMissing() throws Exception { } public void testNullWithoutIgnoreMissing() throws Exception { - GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), "source_field", + GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), null, "source_field", loader("/GeoLite2-City.mmdb"), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), false, new GeoIpCache(1000), false); IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), @@ -103,7 +103,7 @@ public void testNullWithoutIgnoreMissing() throws Exception { } public void testNonExistentWithoutIgnoreMissing() throws Exception { - GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), "source_field", + GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), null, "source_field", loader("/GeoLite2-City.mmdb"), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), false, new GeoIpCache(1000), false); IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap()); @@ -113,7 +113,7 @@ public void testNonExistentWithoutIgnoreMissing() throws Exception { } public void testCity_withIpV6() throws Exception { - GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), "source_field", + GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), null, "source_field", loader("/GeoLite2-City.mmdb"), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), false, new GeoIpCache(1000), false); @@ -142,7 +142,7 @@ public void testCity_withIpV6() throws Exception { } public void testCityWithMissingLocation() throws Exception { - GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), "source_field", + GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), null, "source_field", loader("/GeoLite2-City.mmdb"), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), false, new GeoIpCache(1000), false); @@ -159,7 +159,7 @@ public void testCityWithMissingLocation() throws Exception { } public void testCountry() throws Exception { - GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), "source_field", + GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), null, "source_field", loader("/GeoLite2-Country.mmdb"), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), false, new GeoIpCache(1000), false); @@ -179,7 +179,7 @@ public void testCountry() throws Exception { } public void testCountryWithMissingLocation() throws Exception { - GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), "source_field", + GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), null, "source_field", loader("/GeoLite2-Country.mmdb"), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), false, new GeoIpCache(1000), false); @@ -197,7 +197,7 @@ public void testCountryWithMissingLocation() throws Exception { public void testAsn() throws Exception { String ip = "82.171.64.0"; - GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), "source_field", + GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), null, "source_field", loader("/GeoLite2-ASN.mmdb"), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), false, new GeoIpCache(1000), false); @@ -216,7 +216,7 @@ public void testAsn() throws Exception { } public void testAddressIsNotInTheDatabase() throws Exception { - GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), "source_field", + GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), null, "source_field", loader("/GeoLite2-City.mmdb"), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), false, new GeoIpCache(1000), false); @@ -229,7 +229,7 @@ public void testAddressIsNotInTheDatabase() throws Exception { /** Don't silently do DNS lookups or anything trappy on bogus data */ public void testInvalid() throws Exception { - GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), "source_field", + GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), null, "source_field", loader("/GeoLite2-City.mmdb"), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), false, new GeoIpCache(1000), false); @@ -241,7 +241,7 @@ public void testInvalid() throws Exception { } public void testListAllValid() throws Exception { - GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), "source_field", + GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), null, "source_field", loader("/GeoLite2-City.mmdb"), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), false, new GeoIpCache(1000), false); @@ -262,7 +262,7 @@ public void testListAllValid() throws Exception { } public void testListPartiallyValid() throws Exception { - GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), "source_field", + GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), null, "source_field", loader("/GeoLite2-City.mmdb"), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), false, new GeoIpCache(1000), false); @@ -283,7 +283,7 @@ public void testListPartiallyValid() throws Exception { } public void testListNoMatches() throws Exception { - GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), "source_field", + GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), null, "source_field", loader("/GeoLite2-City.mmdb"), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), false, new GeoIpCache(1000), false); @@ -296,7 +296,7 @@ public void testListNoMatches() throws Exception { } public void testListFirstOnly() throws Exception { - GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), "source_field", + GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), null, "source_field", loader("/GeoLite2-City.mmdb"), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), false, new GeoIpCache(1000), true); @@ -315,7 +315,7 @@ public void testListFirstOnly() throws Exception { } public void testListFirstOnlyNoMatches() throws Exception { - GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), "source_field", + GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), null, "source_field", loader("/GeoLite2-City.mmdb"), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), false, new GeoIpCache(1000), true); diff --git a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java index e5e8a2dd8b70a..5738495b98424 100644 --- a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java +++ b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java @@ -47,12 +47,12 @@ public static void setupProcessor() throws IOException { UserAgentParser parser = new UserAgentParser(randomAlphaOfLength(10), regexStream, new UserAgentCache(1000)); - processor = new UserAgentProcessor(randomAlphaOfLength(10), "source_field", "target_field", parser, + processor = new UserAgentProcessor(randomAlphaOfLength(10), null, "source_field", "target_field", parser, EnumSet.allOf(UserAgentProcessor.Property.class), false); } public void testNullValueWithIgnoreMissing() throws Exception { - UserAgentProcessor processor = new UserAgentProcessor(randomAlphaOfLength(10), "source_field", "target_field", null, + UserAgentProcessor processor = new UserAgentProcessor(randomAlphaOfLength(10), null, "source_field", "target_field", null, EnumSet.allOf(UserAgentProcessor.Property.class), true); IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("source_field", null)); @@ -62,7 +62,7 @@ public void testNullValueWithIgnoreMissing() throws Exception { } public void testNonExistentWithIgnoreMissing() throws Exception { - UserAgentProcessor processor = new UserAgentProcessor(randomAlphaOfLength(10), "source_field", "target_field", null, + UserAgentProcessor processor = new UserAgentProcessor(randomAlphaOfLength(10), null, "source_field", "target_field", null, EnumSet.allOf(UserAgentProcessor.Property.class), true); IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap()); IngestDocument ingestDocument = new IngestDocument(originalIngestDocument); @@ -71,7 +71,7 @@ public void testNonExistentWithIgnoreMissing() throws Exception { } public void testNullWithoutIgnoreMissing() throws Exception { - UserAgentProcessor processor = new UserAgentProcessor(randomAlphaOfLength(10), "source_field", "target_field", null, + UserAgentProcessor processor = new UserAgentProcessor(randomAlphaOfLength(10), null, "source_field", "target_field", null, EnumSet.allOf(UserAgentProcessor.Property.class), false); IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("source_field", null)); @@ -81,7 +81,7 @@ public void testNullWithoutIgnoreMissing() throws Exception { } public void testNonExistentWithoutIgnoreMissing() throws Exception { - UserAgentProcessor processor = new UserAgentProcessor(randomAlphaOfLength(10), "source_field", "target_field", null, + UserAgentProcessor processor = new UserAgentProcessor(randomAlphaOfLength(10), null, "source_field", "target_field", null, EnumSet.allOf(UserAgentProcessor.Property.class), false); IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap()); IngestDocument ingestDocument = new IngestDocument(originalIngestDocument); diff --git a/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorTests.java b/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorTests.java index f08e147129ce2..38b03c1879e67 100644 --- a/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorTests.java +++ b/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorTests.java @@ -54,7 +54,7 @@ public class AttachmentProcessorTests extends ESTestCase { @Before public void createStandardProcessor() { - processor = new AttachmentProcessor(randomAlphaOfLength(10), "source_field", + processor = new AttachmentProcessor(randomAlphaOfLength(10), null, "source_field", "target_field", EnumSet.allOf(AttachmentProcessor.Property.class), 10000, false, null); } @@ -87,7 +87,7 @@ public void testHtmlDocumentWithRandomFields() throws Exception { if (randomBoolean()) { selectedProperties.add(AttachmentProcessor.Property.DATE); } - processor = new AttachmentProcessor(randomAlphaOfLength(10), "source_field", + processor = new AttachmentProcessor(randomAlphaOfLength(10), null, "source_field", "target_field", selectedProperties, 10000, false, null); Map attachmentData = parseDocument("htmlWithEmptyDateMeta.html", processor); @@ -247,7 +247,7 @@ public void testNullValueWithIgnoreMissing() throws Exception { IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("source_field", null)); IngestDocument ingestDocument = new IngestDocument(originalIngestDocument); - Processor processor = new AttachmentProcessor(randomAlphaOfLength(10), "source_field", "randomTarget", null, 10, true, null); + Processor processor = new AttachmentProcessor(randomAlphaOfLength(10), null, "source_field", "randomTarget", null, 10, true, null); processor.execute(ingestDocument); assertIngestDocument(originalIngestDocument, ingestDocument); } @@ -255,7 +255,7 @@ public void testNullValueWithIgnoreMissing() throws Exception { public void testNonExistentWithIgnoreMissing() throws Exception { IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap()); IngestDocument ingestDocument = new IngestDocument(originalIngestDocument); - Processor processor = new AttachmentProcessor(randomAlphaOfLength(10), "source_field", "randomTarget", null, 10, true, null); + Processor processor = new AttachmentProcessor(randomAlphaOfLength(10), null, "source_field", "randomTarget", null, 10, true, null); processor.execute(ingestDocument); assertIngestDocument(originalIngestDocument, ingestDocument); } @@ -264,7 +264,7 @@ public void testNullWithoutIgnoreMissing() throws Exception { IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("source_field", null)); IngestDocument ingestDocument = new IngestDocument(originalIngestDocument); - Processor processor = new AttachmentProcessor(randomAlphaOfLength(10), "source_field", "randomTarget", null, 10, false, null); + Processor processor = new AttachmentProcessor(randomAlphaOfLength(10), null, "source_field", "randomTarget", null, 10, false, null); Exception exception = expectThrows(Exception.class, () -> processor.execute(ingestDocument)); assertThat(exception.getMessage(), equalTo("field [source_field] is null, cannot parse.")); } @@ -272,7 +272,7 @@ public void testNullWithoutIgnoreMissing() throws Exception { public void testNonExistentWithoutIgnoreMissing() throws Exception { IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap()); IngestDocument ingestDocument = new IngestDocument(originalIngestDocument); - Processor processor = new AttachmentProcessor(randomAlphaOfLength(10), "source_field", "randomTarget", null, 10, false, null); + Processor processor = new AttachmentProcessor(randomAlphaOfLength(10), null, "source_field", "randomTarget", null, 10, false, null); Exception exception = expectThrows(Exception.class, () -> processor.execute(ingestDocument)); assertThat(exception.getMessage(), equalTo("field [source_field] not present as part of path [source_field]")); } @@ -296,7 +296,7 @@ private Map parseDocument(String file, AttachmentProcessor proce } public void testIndexedChars() throws Exception { - processor = new AttachmentProcessor(randomAlphaOfLength(10), "source_field", + processor = new AttachmentProcessor(randomAlphaOfLength(10), null, "source_field", "target_field", EnumSet.allOf(AttachmentProcessor.Property.class), 19, false, null); Map attachmentData = parseDocument("text-in-english.txt", processor); @@ -307,7 +307,7 @@ public void testIndexedChars() throws Exception { assertThat(attachmentData.get("content_type").toString(), containsString("text/plain")); assertThat(attachmentData.get("content_length"), is(19L)); - processor = new AttachmentProcessor(randomAlphaOfLength(10), "source_field", + processor = new AttachmentProcessor(randomAlphaOfLength(10), null, "source_field", "target_field", EnumSet.allOf(AttachmentProcessor.Property.class), 19, false, "max_length"); attachmentData = parseDocument("text-in-english.txt", processor); diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/AbstractEnrichProcessor.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/AbstractEnrichProcessor.java index 4c219c0779092..fc4d0691b39a3 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/AbstractEnrichProcessor.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/AbstractEnrichProcessor.java @@ -48,8 +48,18 @@ protected AbstractEnrichProcessor( String matchField, int maxMatches ) { - this(tag, description, createSearchRunner(client), policyName, field, targetField, ignoreMissing, overrideEnabled, matchField, - maxMatches); + this( + tag, + description, + createSearchRunner(client), + policyName, + field, + targetField, + ignoreMissing, + overrideEnabled, + matchField, + maxMatches + ); } protected AbstractEnrichProcessor( diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactory.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactory.java index fa2d5ecadfece..b89e1485b3b5c 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactory.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactory.java @@ -35,7 +35,8 @@ final class EnrichProcessorFactory implements Processor.Factory, Consumer processorFactories, String tag, String description, Map config) throws Exception { + public Processor create(Map processorFactories, String tag, String description, Map config) + throws Exception { String policyName = ConfigurationUtils.readStringProperty(TYPE, tag, config, "policy_name"); String policyAlias = EnrichPolicy.getBaseName(policyName); if (metadata == null) { @@ -69,6 +70,7 @@ public Processor create(Map processorFactories, Strin case EnrichPolicy.MATCH_TYPE: return new MatchProcessor( tag, + description, client, policyName, field, @@ -83,6 +85,7 @@ public Processor create(Map processorFactories, Strin ShapeRelation shapeRelation = ShapeRelation.getRelationByName(relationStr); return new GeoMatchProcessor( tag, + description, client, policyName, field, diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/GeoMatchProcessorTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/GeoMatchProcessorTests.java index 7228690a871b9..1e87364c74068 100644 --- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/GeoMatchProcessorTests.java +++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/GeoMatchProcessorTests.java @@ -65,6 +65,7 @@ private void testBasicsForFieldValue(Object fieldValue, Geometry expectedGeometr MockSearchFunction mockSearch = mockedSearchFunction(Map.of("key", Map.of("shape", "object", "zipcode", 94040))); GeoMatchProcessor processor = new GeoMatchProcessor( "_tag", + null, mockSearch, "_name", str("location"), diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/MatchProcessorTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/MatchProcessorTests.java index 89861cd230e20..500165e56fbd3 100644 --- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/MatchProcessorTests.java +++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/MatchProcessorTests.java @@ -51,6 +51,7 @@ public void testBasics() throws Exception { MockSearchFunction mockSearch = mockedSearchFunction(Map.of("elastic.co", Map.of("globalRank", 451, "tldRank", 23, "tld", "co"))); MatchProcessor processor = new MatchProcessor( "_tag", + null, mockSearch, "_name", str("domain"), @@ -103,7 +104,18 @@ public void testBasics() throws Exception { public void testNoMatch() throws Exception { MockSearchFunction mockSearch = mockedSearchFunction(); - MatchProcessor processor = new MatchProcessor("_tag", mockSearch, "_name", str("domain"), str("entry"), true, false, "domain", 1); + MatchProcessor processor = new MatchProcessor( + "_tag", + null, + mockSearch, + "_name", + str("domain"), + str("entry"), + true, + false, + "domain", + 1 + ); IngestDocument ingestDocument = new IngestDocument( "_index", "_id", @@ -139,7 +151,18 @@ public void testNoMatch() throws Exception { public void testSearchFailure() throws Exception { String indexName = ".enrich-_name"; MockSearchFunction mockSearch = mockedSearchFunction(new IndexNotFoundException(indexName)); - MatchProcessor processor = new MatchProcessor("_tag", mockSearch, "_name", str("domain"), str("entry"), true, false, "domain", 1); + MatchProcessor processor = new MatchProcessor( + "_tag", + null, + mockSearch, + "_name", + str("domain"), + str("entry"), + true, + false, + "domain", + 1 + ); IngestDocument ingestDocument = new IngestDocument( "_index", "_id", @@ -181,6 +204,7 @@ public void testIgnoreKeyMissing() throws Exception { { MatchProcessor processor = new MatchProcessor( "_tag", + null, mockedSearchFunction(), "_name", str("domain"), @@ -201,6 +225,7 @@ public void testIgnoreKeyMissing() throws Exception { { MatchProcessor processor = new MatchProcessor( "_tag", + null, mockedSearchFunction(), "_name", str("domain"), @@ -225,7 +250,18 @@ public void testIgnoreKeyMissing() throws Exception { public void testExistingFieldWithOverrideDisabled() throws Exception { MockSearchFunction mockSearch = mockedSearchFunction(Map.of("elastic.co", Map.of("globalRank", 451, "tldRank", 23, "tld", "co"))); - MatchProcessor processor = new MatchProcessor("_tag", mockSearch, "_name", str("domain"), str("entry"), false, false, "domain", 1); + MatchProcessor processor = new MatchProcessor( + "_tag", + null, + mockSearch, + "_name", + str("domain"), + str("entry"), + false, + false, + "domain", + 1 + ); IngestDocument ingestDocument = new IngestDocument(new HashMap<>(Map.of("domain", "elastic.co", "tld", "tld")), Map.of()); IngestDocument[] resultHolder = new IngestDocument[1]; @@ -241,7 +277,18 @@ public void testExistingFieldWithOverrideDisabled() throws Exception { public void testExistingNullFieldWithOverrideDisabled() throws Exception { MockSearchFunction mockSearch = mockedSearchFunction(Map.of("elastic.co", Map.of("globalRank", 451, "tldRank", 23, "tld", "co"))); - MatchProcessor processor = new MatchProcessor("_tag", mockSearch, "_name", str("domain"), str("entry"), false, false, "domain", 1); + MatchProcessor processor = new MatchProcessor( + "_tag", + null, + mockSearch, + "_name", + str("domain"), + str("entry"), + false, + false, + "domain", + 1 + ); Map source = new HashMap<>(); source.put("domain", "elastic.co"); @@ -260,7 +307,18 @@ public void testExistingNullFieldWithOverrideDisabled() throws Exception { public void testNumericValue() { MockSearchFunction mockSearch = mockedSearchFunction(Map.of(2, Map.of("globalRank", 451, "tldRank", 23, "tld", "co"))); - MatchProcessor processor = new MatchProcessor("_tag", mockSearch, "_name", str("domain"), str("entry"), false, true, "domain", 1); + MatchProcessor processor = new MatchProcessor( + "_tag", + null, + mockSearch, + "_name", + str("domain"), + str("entry"), + false, + true, + "domain", + 1 + ); IngestDocument ingestDocument = new IngestDocument("_index", "_id", "_routing", 1L, VersionType.INTERNAL, Map.of("domain", 2)); // Execute @@ -288,7 +346,18 @@ public void testArray() { MockSearchFunction mockSearch = mockedSearchFunction( Map.of(List.of("1", "2"), Map.of("globalRank", 451, "tldRank", 23, "tld", "co")) ); - MatchProcessor processor = new MatchProcessor("_tag", mockSearch, "_name", str("domain"), str("entry"), false, true, "domain", 1); + MatchProcessor processor = new MatchProcessor( + "_tag", + null, + mockSearch, + "_name", + str("domain"), + str("entry"), + false, + true, + "domain", + 1 + ); IngestDocument ingestDocument = new IngestDocument( "_index", "_id", diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java index 55538a211773e..35d1a608fcec2 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java @@ -271,7 +271,8 @@ int numInferenceProcessors() { } @Override - public InferenceProcessor create(Map processorFactories, String tag, String description, Map config) { + public InferenceProcessor create(Map processorFactories, String tag, String description, + Map config) { if (this.maxIngestProcessors <= currentInferenceProcessors) { throw new ElasticsearchStatusException("Max number of inference processors reached, total inference processors [{}]. " + diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportGetTrainedModelsStatsActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportGetTrainedModelsStatsActionTests.java index d2e68f00f5e08..c136cb0a0cc4c 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportGetTrainedModelsStatsActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportGetTrainedModelsStatsActionTests.java @@ -73,10 +73,16 @@ public String getTag() { return null; } + @Override + public String getDescription() { + return null; + } + static class Factory implements Processor.Factory { @Override - public Processor create(Map processorFactories, String tag, String description, Map config) { + public Processor create(Map processorFactories, String tag, String description, + Map config) { return new NotInferenceProcessor(); } } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessorTests.java index 6313ba2546e8c..0ce10b8d17a47 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessorTests.java @@ -54,7 +54,7 @@ public void testMutateDocumentWithClassification() { InferenceProcessor inferenceProcessor = new InferenceProcessor(client, auditor, "my_processor", - targetField, + null, targetField, "classification_model", ClassificationConfigUpdate.EMPTY_PARAMS, Collections.emptyMap()); @@ -83,7 +83,7 @@ public void testMutateDocumentClassificationTopNClasses() { InferenceProcessor inferenceProcessor = new InferenceProcessor(client, auditor, "my_processor", - "ml.my_processor", + null, "ml.my_processor", "classification_model", classificationConfigUpdate, Collections.emptyMap()); @@ -113,7 +113,7 @@ public void testMutateDocumentClassificationFeatureInfluence() { InferenceProcessor inferenceProcessor = new InferenceProcessor(client, auditor, "my_processor", - "ml.my_processor", + null, "ml.my_processor", "classification_model", classificationConfigUpdate, Collections.emptyMap()); @@ -154,7 +154,7 @@ public void testMutateDocumentClassificationTopNClassesWithSpecificField() { InferenceProcessor inferenceProcessor = new InferenceProcessor(client, auditor, "my_processor", - "ml.my_processor", + null, "ml.my_processor", "classification_model", classificationConfigUpdate, Collections.emptyMap()); @@ -184,7 +184,7 @@ public void testMutateDocumentRegression() { InferenceProcessor inferenceProcessor = new InferenceProcessor(client, auditor, "my_processor", - "ml.my_processor", + null, "ml.my_processor", "regression_model", regressionConfigUpdate, Collections.emptyMap()); @@ -207,7 +207,7 @@ public void testMutateDocumentRegressionWithTopFetures() { InferenceProcessor inferenceProcessor = new InferenceProcessor(client, auditor, "my_processor", - "ml.my_processor", + null, "ml.my_processor", "regression_model", regressionConfigUpdate, Collections.emptyMap()); @@ -239,7 +239,7 @@ public void testGenerateRequestWithEmptyMapping() { InferenceProcessor processor = new InferenceProcessor(client, auditor, "my_processor", - "my_field", + null, "my_field", modelId, new ClassificationConfigUpdate(topNClasses, null, null, null, null), Collections.emptyMap()); @@ -268,7 +268,7 @@ public void testGenerateWithMapping() { InferenceProcessor processor = new InferenceProcessor(client, auditor, "my_processor", - "my_field", + null, "my_field", modelId, new ClassificationConfigUpdate(topNClasses, null, null, null, null), fieldMapping); @@ -304,7 +304,7 @@ public void testGenerateWithMappingNestedFields() { InferenceProcessor processor = new InferenceProcessor(client, auditor, "my_processor", - "my_field", + null, "my_field", modelId, new ClassificationConfigUpdate(topNClasses, null, null, null, null), fieldMapping); @@ -332,7 +332,7 @@ public void testHandleResponseLicenseChanged() { InferenceProcessor inferenceProcessor = new InferenceProcessor(client, auditor, "my_processor", - targetField, + null, targetField, "regression_model", RegressionConfigUpdate.EMPTY_PARAMS, Collections.emptyMap()); @@ -375,7 +375,7 @@ public void testMutateDocumentWithWarningResult() { InferenceProcessor inferenceProcessor = new InferenceProcessor(client, auditor, "my_processor", - "ml", + null, "ml", "regression_model", RegressionConfigUpdate.EMPTY_PARAMS, Collections.emptyMap()); diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/test/MockIngestPlugin.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/test/MockIngestPlugin.java index 650df8243bb2d..7db2358a3ce9d 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/test/MockIngestPlugin.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/test/MockIngestPlugin.java @@ -58,7 +58,7 @@ public Processor create(Map processorFactories, ConfigurationUtils.readObject(type, tag, config, field); } - return new MockProcessor(type, tag); + return new MockProcessor(type, tag, description); } } @@ -67,10 +67,12 @@ static class MockProcessor implements Processor { private final String type; private final String tag; + private final String description; - MockProcessor(final String type, final String tag) { + MockProcessor(final String type, final String tag, final String description) { this.type = type; this.tag = tag; + this.description = description; } @Override @@ -89,6 +91,11 @@ public String getTag() { return tag; } + @Override + public String getDescription() { + return description; + } + } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessorTests.java index 8707c5d427099..e0903feb5e2ba 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessorTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessorTests.java @@ -52,7 +52,7 @@ public void testProcessorWithData() throws Exception { IngestDocument ingestDocument = new IngestDocument(new HashMap<>(), new HashMap<>()); SetSecurityUserProcessor processor = new SetSecurityUserProcessor( - "_tag", securityContext, licenseState, "_field", EnumSet.allOf(Property.class)); + "_tag", null, securityContext, licenseState, "_field", EnumSet.allOf(Property.class)); processor.execute(ingestDocument); Map result = ingestDocument.getFieldValue("_field", Map.class); @@ -82,7 +82,7 @@ public void testProcessorWithEmptyUserData() throws Exception { IngestDocument ingestDocument = new IngestDocument(new HashMap<>(), new HashMap<>()); SetSecurityUserProcessor processor = new SetSecurityUserProcessor( - "_tag", securityContext, licenseState, "_field", EnumSet.allOf(Property.class)); + "_tag", null, securityContext, licenseState, "_field", EnumSet.allOf(Property.class)); processor.execute(ingestDocument); Map result = ingestDocument.getFieldValue("_field", Map.class); // Still holds data for realm and authentication type @@ -95,7 +95,7 @@ public void testProcessorWithEmptyUserData() throws Exception { public void testNoCurrentUser() throws Exception { IngestDocument ingestDocument = new IngestDocument(new HashMap<>(), new HashMap<>()); SetSecurityUserProcessor processor = new SetSecurityUserProcessor( - "_tag", securityContext, licenseState, "_field", EnumSet.allOf(Property.class)); + "_tag", null, securityContext, licenseState, "_field", EnumSet.allOf(Property.class)); IllegalStateException e = expectThrows(IllegalStateException.class, () -> processor.execute(ingestDocument)); assertThat(e.getMessage(), equalTo("There is no authenticated user - the [set_security_user] processor requires an authenticated user")); @@ -105,7 +105,7 @@ public void testSecurityDisabled() throws Exception { when(licenseState.isSecurityEnabled()).thenReturn(false); IngestDocument ingestDocument = new IngestDocument(new HashMap<>(), new HashMap<>()); SetSecurityUserProcessor processor = new SetSecurityUserProcessor( - "_tag", securityContext, licenseState, "_field", EnumSet.allOf(Property.class)); + "_tag", null, securityContext, licenseState, "_field", EnumSet.allOf(Property.class)); IllegalStateException e = expectThrows(IllegalStateException.class, () -> processor.execute(ingestDocument)); assertThat(e.getMessage(), equalTo("Security (authentication) is not enabled on this cluster, so there is no active user" + " - the [set_security_user] processor cannot be used without security")); @@ -118,7 +118,7 @@ public void testUsernameProperties() throws Exception { IngestDocument ingestDocument = new IngestDocument(new HashMap<>(), new HashMap<>()); SetSecurityUserProcessor processor = new SetSecurityUserProcessor( - "_tag", securityContext, licenseState, "_field", EnumSet.of(Property.USERNAME)); + "_tag", null, securityContext, licenseState, "_field", EnumSet.of(Property.USERNAME)); processor.execute(ingestDocument); @SuppressWarnings("unchecked") @@ -134,7 +134,7 @@ public void testRolesProperties() throws Exception { IngestDocument ingestDocument = new IngestDocument(new HashMap<>(), new HashMap<>()); SetSecurityUserProcessor processor = new SetSecurityUserProcessor( - "_tag", securityContext, licenseState, "_field", EnumSet.of(Property.ROLES)); + "_tag", null, securityContext, licenseState, "_field", EnumSet.of(Property.ROLES)); processor.execute(ingestDocument); @SuppressWarnings("unchecked") @@ -150,7 +150,7 @@ public void testFullNameProperties() throws Exception { IngestDocument ingestDocument = new IngestDocument(new HashMap<>(), new HashMap<>()); SetSecurityUserProcessor processor - = new SetSecurityUserProcessor("_tag", securityContext, licenseState, "_field", EnumSet.of(Property.FULL_NAME)); + = new SetSecurityUserProcessor("_tag", null, securityContext, licenseState, "_field", EnumSet.of(Property.FULL_NAME)); processor.execute(ingestDocument); @SuppressWarnings("unchecked") @@ -166,7 +166,7 @@ public void testEmailProperties() throws Exception { IngestDocument ingestDocument = new IngestDocument(new HashMap<>(), new HashMap<>()); SetSecurityUserProcessor processor = new SetSecurityUserProcessor( - "_tag", securityContext, licenseState, "_field", EnumSet.of(Property.EMAIL)); + "_tag", null, securityContext, licenseState, "_field", EnumSet.of(Property.EMAIL)); processor.execute(ingestDocument); @SuppressWarnings("unchecked") @@ -182,7 +182,7 @@ public void testMetadataProperties() throws Exception { IngestDocument ingestDocument = new IngestDocument(new HashMap<>(), new HashMap<>()); SetSecurityUserProcessor processor = new SetSecurityUserProcessor( - "_tag", securityContext, licenseState, "_field", EnumSet.of(Property.METADATA)); + "_tag", null, securityContext, licenseState, "_field", EnumSet.of(Property.METADATA)); processor.execute(ingestDocument); @SuppressWarnings("unchecked") @@ -198,7 +198,7 @@ public void testOverwriteExistingField() throws Exception { new Authentication(user, realmRef, null).writeToContext(threadContext); SetSecurityUserProcessor processor = new SetSecurityUserProcessor( - "_tag", securityContext, licenseState, "_field", EnumSet.of(Property.USERNAME)); + "_tag", null, securityContext, licenseState, "_field", EnumSet.of(Property.USERNAME)); IngestDocument ingestDocument = new IngestDocument(new HashMap<>(), new HashMap<>()); ingestDocument.setFieldValue("_field", "test"); @@ -238,7 +238,7 @@ public void testApiKeyPopulation() throws Exception { IngestDocument ingestDocument = new IngestDocument(new HashMap<>(), new HashMap<>()); SetSecurityUserProcessor processor = new SetSecurityUserProcessor( - "_tag", securityContext, licenseState, "_field", EnumSet.allOf(Property.class)); + "_tag", null, securityContext, licenseState, "_field", EnumSet.allOf(Property.class)); processor.execute(ingestDocument); Map result = ingestDocument.getFieldValue("_field", Map.class); @@ -269,7 +269,7 @@ public void testWillNotOverwriteExistingApiKeyAndRealm() throws Exception { "_field", Map.of("api_key", Map.of("version", 42), "realm", Map.of("id", 7)) )), new HashMap<>()); SetSecurityUserProcessor processor = new SetSecurityUserProcessor( - "_tag", securityContext, licenseState, "_field", EnumSet.allOf(Property.class)); + "_tag", null, securityContext, licenseState, "_field", EnumSet.allOf(Property.class)); processor.execute(ingestDocument); Map result = ingestDocument.getFieldValue("_field", Map.class); @@ -292,7 +292,7 @@ public void testWillSetRunAsRealmForNonApiAuth() throws Exception { IngestDocument ingestDocument = new IngestDocument(new HashMap<>(), new HashMap<>()); SetSecurityUserProcessor processor = new SetSecurityUserProcessor( - "_tag", securityContext, licenseState, "_field", EnumSet.allOf(Property.class)); + "_tag", null, securityContext, licenseState, "_field", EnumSet.allOf(Property.class)); processor.execute(ingestDocument); Map result = ingestDocument.getFieldValue("_field", Map.class); diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/ingest/CircleProcessor.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/ingest/CircleProcessor.java index 2b7dd4ff9fe81..e8b6345208d8a 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/ingest/CircleProcessor.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/ingest/CircleProcessor.java @@ -141,7 +141,8 @@ int numSides(double radiusMeters) { public static final class Factory implements Processor.Factory { - public CircleProcessor create(Map registry, String processorTag, String description, Map config) { + public CircleProcessor create(Map registry, String processorTag, String description, + Map config) { String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field"); String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", field); boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false); diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/ingest/CircleProcessorFactoryTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/ingest/CircleProcessorFactoryTests.java index 4b6a997f656d0..a4ccea8399765 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/ingest/CircleProcessorFactoryTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/ingest/CircleProcessorFactoryTests.java @@ -65,7 +65,8 @@ public void testCreateInvalidShapeType() { public void testCreateMissingField() { Map config = new HashMap<>(); String processorTag = randomAlphaOfLength(10); - ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, processorTag, null, config)); + ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, + () -> factory.create(null, processorTag, null, config)); assertThat(e.getMessage(), equalTo("[field] required property is missing")); } @@ -88,7 +89,8 @@ public void testCreateWithNoErrorDistanceDefined() { Map config = new HashMap<>(); config.put("field", "field1"); String processorTag = randomAlphaOfLength(10); - ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, processorTag, null, config)); + ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, + () -> factory.create(null, processorTag, null, config)); assertThat(e.getMessage(), equalTo("[error_distance] required property is missing")); } } diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/ingest/CircleProcessorTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/ingest/CircleProcessorTests.java index 65d71f2b7f6ce..7caafaefafac4 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/ingest/CircleProcessorTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/ingest/CircleProcessorTests.java @@ -63,7 +63,7 @@ public class CircleProcessorTests extends ESTestCase { public void testNumSides() { double radiusDistanceMeters = randomDoubleBetween(0.01, 6371000, true); CircleShapeFieldType shapeType = randomFrom(SHAPE, GEO_SHAPE); - CircleProcessor processor = new CircleProcessor("tag", "field", "field", false, radiusDistanceMeters, shapeType); + CircleProcessor processor = new CircleProcessor("tag", null, "field", "field", false, radiusDistanceMeters, shapeType); // radius is same as error distance assertThat(processor.numSides(radiusDistanceMeters), equalTo(4)); @@ -77,14 +77,14 @@ public void testNumSides() { } public void testFieldNotFound() throws Exception { - CircleProcessor processor = new CircleProcessor("tag", "field", "field", false, 10, GEO_SHAPE); + CircleProcessor processor = new CircleProcessor("tag", null, "field", "field", false, 10, GEO_SHAPE); IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>()); Exception e = expectThrows(Exception.class, () -> processor.execute(ingestDocument)); assertThat(e.getMessage(), containsString("not present as part of path [field]")); } public void testFieldNotFoundWithIgnoreMissing() throws Exception { - CircleProcessor processor = new CircleProcessor("tag", "field", "field", true, 10, GEO_SHAPE); + CircleProcessor processor = new CircleProcessor("tag", null, "field", "field", true, 10, GEO_SHAPE); IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>()); IngestDocument ingestDocument = new IngestDocument(originalIngestDocument); processor.execute(ingestDocument); @@ -92,14 +92,14 @@ public void testFieldNotFoundWithIgnoreMissing() throws Exception { } public void testNullValue() throws Exception { - CircleProcessor processor = new CircleProcessor("tag", "field", "field", false, 10, GEO_SHAPE); + CircleProcessor processor = new CircleProcessor("tag", null, "field", "field", false, 10, GEO_SHAPE); IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", null)); Exception e = expectThrows(Exception.class, () -> processor.execute(ingestDocument)); assertThat(e.getMessage(), equalTo("field [field] is null, cannot process it.")); } public void testNullValueWithIgnoreMissing() throws Exception { - CircleProcessor processor = new CircleProcessor("tag", "field", "field", true, 10, GEO_SHAPE); + CircleProcessor processor = new CircleProcessor("tag", null, "field", "field", true, 10, GEO_SHAPE); IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", null)); IngestDocument ingestDocument = new IngestDocument(originalIngestDocument); processor.execute(ingestDocument); @@ -118,7 +118,7 @@ public void testJson() throws IOException { Geometry expectedPoly = SpatialUtils.createRegularGeoShapePolygon(circle, 4); assertThat(expectedPoly, instanceOf(Polygon.class)); IngestDocument ingestDocument = new IngestDocument(map, Collections.emptyMap()); - CircleProcessor processor = new CircleProcessor("tag", "field", "field", false, 10, GEO_SHAPE); + CircleProcessor processor = new CircleProcessor("tag", null, "field", "field", false, 10, GEO_SHAPE); processor.execute(ingestDocument); Map polyMap = ingestDocument.getFieldValue("field", Map.class); XContentBuilder builder = XContentFactory.jsonBuilder(); @@ -134,7 +134,7 @@ public void testWKT() { map.put("field", WKT.toWKT(circle)); Geometry expectedPoly = SpatialUtils.createRegularGeoShapePolygon(circle, 4); IngestDocument ingestDocument = new IngestDocument(map, Collections.emptyMap()); - CircleProcessor processor = new CircleProcessor("tag", "field", "field",false, 2, GEO_SHAPE); + CircleProcessor processor = new CircleProcessor("tag", null, "field", "field",false, 2, GEO_SHAPE); processor.execute(ingestDocument); String polyString = ingestDocument.getFieldValue("field", String.class); assertThat(polyString, equalTo(WKT.toWKT(expectedPoly))); @@ -144,7 +144,7 @@ public void testInvalidWKT() { HashMap map = new HashMap<>(); map.put("field", "invalid"); IngestDocument ingestDocument = new IngestDocument(map, Collections.emptyMap()); - CircleProcessor processor = new CircleProcessor("tag", "field", "field", false, 10, GEO_SHAPE); + CircleProcessor processor = new CircleProcessor("tag", null, "field", "field", false, 10, GEO_SHAPE); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> processor.execute(ingestDocument)); assertThat(e.getMessage(), equalTo("invalid circle definition")); map.put("field", "POINT (30 10)"); @@ -154,7 +154,7 @@ public void testInvalidWKT() { public void testMissingField() { IngestDocument ingestDocument = new IngestDocument(new HashMap<>(), Collections.emptyMap()); - CircleProcessor processor = new CircleProcessor("tag", "field", "field", false, 10, GEO_SHAPE); + CircleProcessor processor = new CircleProcessor("tag", null, "field", "field", false, 10, GEO_SHAPE); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> processor.execute(ingestDocument)); assertThat(e.getMessage(), equalTo("field [field] not present as part of path [field]")); } @@ -166,7 +166,7 @@ public void testInvalidType() { Map map = new HashMap<>(); map.put("field", field); IngestDocument ingestDocument = new IngestDocument(map, Collections.emptyMap()); - CircleProcessor processor = new CircleProcessor("tag", "field", "field", false, 10, GEO_SHAPE); + CircleProcessor processor = new CircleProcessor("tag", null, "field", "field", false, 10, GEO_SHAPE); for (Object value : new Object[] { null, 4.0, "not_circle"}) { field.put("type", value); @@ -182,7 +182,7 @@ public void testInvalidCoordinates() { Map map = new HashMap<>(); map.put("field", field); IngestDocument ingestDocument = new IngestDocument(map, Collections.emptyMap()); - CircleProcessor processor = new CircleProcessor("tag", "field", "field", false, 10, GEO_SHAPE); + CircleProcessor processor = new CircleProcessor("tag", null, "field", "field", false, 10, GEO_SHAPE); for (Object value : new Object[] { null, "not_circle"}) { field.put("coordinates", value); @@ -198,7 +198,7 @@ public void testInvalidRadius() { Map map = new HashMap<>(); map.put("field", field); IngestDocument ingestDocument = new IngestDocument(map, Collections.emptyMap()); - CircleProcessor processor = new CircleProcessor("tag", "field", "field", false, 10, GEO_SHAPE); + CircleProcessor processor = new CircleProcessor("tag", null, "field", "field", false, 10, GEO_SHAPE); for (Object value : new Object[] { null, "NotNumber", "10.0fs"}) { field.put("radius", value); From 590e2cd5b36cdbfce3a7c52e296d4c9efdca6f5b Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Wed, 10 Jun 2020 12:58:05 -0700 Subject: [PATCH 4/6] misordering in dotexpander --- .../org/elasticsearch/ingest/common/DotExpanderProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DotExpanderProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DotExpanderProcessor.java index 6c22343fefd0e..ea198b7b4f842 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DotExpanderProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DotExpanderProcessor.java @@ -33,7 +33,7 @@ public final class DotExpanderProcessor extends AbstractProcessor { private final String path; private final String field; - DotExpanderProcessor(String tag, String description, String field, String path) { + DotExpanderProcessor(String tag, String description, String path, String field) { super(tag, description); this.path = path; this.field = field; From 6316c2b9c3e128474c5d8a72b3df2eaa0ae01d11 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Wed, 10 Jun 2020 13:55:12 -0700 Subject: [PATCH 5/6] more fixes --- .../ingest/common/DotExpanderProcessor.java | 6 ++--- .../common/DotExpanderProcessorTests.java | 26 +++++++++---------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DotExpanderProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DotExpanderProcessor.java index ea198b7b4f842..d6b3441013718 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DotExpanderProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DotExpanderProcessor.java @@ -96,8 +96,8 @@ String getField() { public static final class Factory implements Processor.Factory { @Override - public Processor create(Map processorFactories, String tag, - String description, Map config) throws Exception { + public Processor create(Map processorFactories, String tag, String description, + Map config) throws Exception { String field = ConfigurationUtils.readStringProperty(TYPE, tag, config, "field"); if (field.contains(".") == false) { throw ConfigurationUtils.newConfigurationException(ConfigurationUtils.TAG_KEY, tag, "field", @@ -117,7 +117,7 @@ public Processor create(Map processorFactories, Strin } String path = ConfigurationUtils.readOptionalStringProperty(TYPE, tag, config, "path"); - return new DotExpanderProcessor(tag, description, path, field); + return new DotExpanderProcessor(tag, null, path, field); } } } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorTests.java index ed1799c05ef67..71d7461440252 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorTests.java @@ -37,7 +37,7 @@ public void testEscapeFields() throws Exception { Map source = new HashMap<>(); source.put("foo.bar", "baz1"); IngestDocument document = new IngestDocument(source, Collections.emptyMap()); - DotExpanderProcessor processor = new DotExpanderProcessor("_tag", null, "foo.bar", null); + DotExpanderProcessor processor = new DotExpanderProcessor("_tag", null, null, "foo.bar"); processor.execute(document); assertThat(document.getFieldValue("foo", Map.class).size(), equalTo(1)); assertThat(document.getFieldValue("foo.bar", String.class), equalTo("baz1")); @@ -45,7 +45,7 @@ public void testEscapeFields() throws Exception { source = new HashMap<>(); source.put("foo.bar.baz", "value"); document = new IngestDocument(source, Collections.emptyMap()); - processor = new DotExpanderProcessor("_tag", null, "foo.bar.baz", null); + processor = new DotExpanderProcessor("_tag", null, null, "foo.bar.baz"); processor.execute(document); assertThat(document.getFieldValue("foo", Map.class).size(), equalTo(1)); assertThat(document.getFieldValue("foo.bar", Map.class).size(), equalTo(1)); @@ -55,7 +55,7 @@ public void testEscapeFields() throws Exception { source.put("foo.bar", "baz1"); source.put("foo", new HashMap<>(Collections.singletonMap("bar", "baz2"))); document = new IngestDocument(source, Collections.emptyMap()); - processor = new DotExpanderProcessor("_tag", null, "foo.bar", null); + processor = new DotExpanderProcessor("_tag", null, null, "foo.bar"); processor.execute(document); assertThat(document.getSourceAndMetadata().size(), equalTo(1)); assertThat(document.getFieldValue("foo.bar", List.class).size(), equalTo(2)); @@ -66,7 +66,7 @@ public void testEscapeFields() throws Exception { source.put("foo.bar", "2"); source.put("foo", new HashMap<>(Collections.singletonMap("bar", 1))); document = new IngestDocument(source, Collections.emptyMap()); - processor = new DotExpanderProcessor("_tag", null, "foo.bar", null); + processor = new DotExpanderProcessor("_tag", null, null, "foo.bar"); processor.execute(document); assertThat(document.getSourceAndMetadata().size(), equalTo(1)); assertThat(document.getFieldValue("foo.bar", List.class).size(), equalTo(2)); @@ -79,7 +79,7 @@ public void testEscapeFields_valueField() throws Exception { source.put("foo.bar", "baz1"); source.put("foo", "baz2"); IngestDocument document1 = new IngestDocument(source, Collections.emptyMap()); - Processor processor1 = new DotExpanderProcessor("_tag", null, "foo.bar", null); + Processor processor1 = new DotExpanderProcessor("_tag", null, null, "foo.bar"); // foo already exists and if a leaf field and therefor can't be replaced by a map field: Exception e = expectThrows(IllegalArgumentException.class, () -> processor1.execute(document1)); assertThat(e.getMessage(), equalTo("cannot expend [foo.bar], because [foo] is not an object field, but a value field")); @@ -90,7 +90,7 @@ public void testEscapeFields_valueField() throws Exception { Processor processor = new RenameProcessor("_tag", null, new TestTemplateService.MockTemplateScript.Factory("foo"), new TestTemplateService.MockTemplateScript.Factory("foo.bar"), false); processor.execute(document); - processor = new DotExpanderProcessor("_tag", null, "foo.bar", null); + processor = new DotExpanderProcessor("_tag", null, null, "foo.bar"); processor.execute(document); assertThat(document.getFieldValue("foo", Map.class).size(), equalTo(1)); assertThat(document.getFieldValue("foo.bar.0", String.class), equalTo("baz2")); @@ -99,7 +99,7 @@ public void testEscapeFields_valueField() throws Exception { source = new HashMap<>(); source.put("foo.bar", "baz1"); document = new IngestDocument(source, Collections.emptyMap()); - processor = new DotExpanderProcessor("_tag", null, "foo.bar", null); + processor = new DotExpanderProcessor("_tag", null, null, "foo.bar"); processor.execute(document); assertThat(document.getFieldValue("foo", Map.class).size(), equalTo(1)); assertThat(document.getFieldValue("foo.bar", String.class), equalTo("baz1")); @@ -108,7 +108,7 @@ public void testEscapeFields_valueField() throws Exception { source.put("foo.bar.baz", "baz1"); source.put("foo", new HashMap<>(Collections.singletonMap("bar", new HashMap<>()))); document = new IngestDocument(source, Collections.emptyMap()); - processor = new DotExpanderProcessor("_tag", null, "foo.bar.baz", null); + processor = new DotExpanderProcessor("_tag", null, null, "foo.bar.baz"); processor.execute(document); assertThat(document.getFieldValue("foo", Map.class).size(), equalTo(1)); assertThat(document.getFieldValue("foo.bar", Map.class).size(), equalTo(1)); @@ -118,7 +118,7 @@ public void testEscapeFields_valueField() throws Exception { source.put("foo.bar.baz", "baz1"); source.put("foo", new HashMap<>(Collections.singletonMap("bar", "baz2"))); IngestDocument document2 = new IngestDocument(source, Collections.emptyMap()); - Processor processor2 = new DotExpanderProcessor("_tag", null, "foo.bar.baz", null); + Processor processor2 = new DotExpanderProcessor("_tag", null, null, "foo.bar.baz"); e = expectThrows(IllegalArgumentException.class, () -> processor2.execute(document2)); assertThat(e.getMessage(), equalTo("cannot expend [foo.bar.baz], because [foo.bar] is not an object field, but a value field")); } @@ -127,7 +127,7 @@ public void testEscapeFields_path() throws Exception { Map source = new HashMap<>(); source.put("foo", new HashMap<>(Collections.singletonMap("bar.baz", "value"))); IngestDocument document = new IngestDocument(source, Collections.emptyMap()); - DotExpanderProcessor processor = new DotExpanderProcessor("_tag", null, "bar.baz", "foo"); + DotExpanderProcessor processor = new DotExpanderProcessor("_tag", null, "foo", "bar.baz"); processor.execute(document); assertThat(document.getFieldValue("foo", Map.class).size(), equalTo(1)); assertThat(document.getFieldValue("foo.bar", Map.class).size(), equalTo(1)); @@ -136,7 +136,7 @@ public void testEscapeFields_path() throws Exception { source = new HashMap<>(); source.put("field", new HashMap<>(Collections.singletonMap("foo.bar.baz", "value"))); document = new IngestDocument(source, Collections.emptyMap()); - processor = new DotExpanderProcessor("_tag", null, "foo.bar.baz", "field"); + processor = new DotExpanderProcessor("_tag", null, "field", "foo.bar.baz"); processor.execute(document); assertThat(document.getFieldValue("field.foo", Map.class).size(), equalTo(1)); assertThat(document.getFieldValue("field.foo.bar", Map.class).size(), equalTo(1)); @@ -150,7 +150,7 @@ public void testEscapeFields_doNothingIfFieldNotInSourceDoc() throws Exception { source.put("foo.bar", "baz1"); IngestDocument document = new IngestDocument(source, Collections.emptyMap()); //abc.def does not exist in source, so don't mutate document - DotExpanderProcessor processor = new DotExpanderProcessor("_tag", null, "abc.def", null); + DotExpanderProcessor processor = new DotExpanderProcessor("_tag", null, null, "abc.def"); processor.execute(document); //hasField returns false since it requires the expanded form, which is not expanded since we did not ask for it to be assertFalse(document.hasField("foo.bar")); @@ -168,7 +168,7 @@ public void testEscapeFields_doNothingIfFieldNotInSourceDoc() throws Exception { source.put("foo", inner); document = new IngestDocument(source, Collections.emptyMap()); //foo.bar, the literal value (as opposed to nested value) does not exist in source, so don't mutate document - processor = new DotExpanderProcessor("_tag", null, "foo.bar", null); + processor = new DotExpanderProcessor("_tag", null, null, "foo.bar"); processor.execute(document); //hasField returns true because the nested/expanded form exists in the source document assertTrue(document.hasField("foo.bar")); From 22e0e2fc2289596af0fc301e5cc8be02b20297b7 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Fri, 12 Jun 2020 16:53:35 -0700 Subject: [PATCH 6/6] add more yml tests --- .../rest-api-spec/test/ingest/20_crud.yml | 25 +++++++++++++++ .../rest-api-spec/test/ingest/90_simulate.yml | 32 +++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/20_crud.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/20_crud.yml index bd6a3e6ca14fd..11b24087d55e3 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/20_crud.yml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/20_crud.yml @@ -203,3 +203,28 @@ teardown: catch: missing ingest.get_pipeline: id: "my_pipeline" + +--- +"Test processor with description": + - do: + ingest.put_pipeline: + id: "my_pipeline" + body: > + { + "description": "_description", + "processors": [ + { + "set" : { + "description": "this processor sets the [field2] field to [_value]", + "field" : "field2", + "value": "_value" + } + } + ] + } + - match: { acknowledged: true } + + - do: + ingest.get_pipeline: + id: "my_pipeline" + - match: { my_pipeline.processors.0.set.description: "this processor sets the [field2] field to [_value]" } diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/90_simulate.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/90_simulate.yml index a33ae7778d46f..ff51f91a75867 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/90_simulate.yml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/90_simulate.yml @@ -812,3 +812,35 @@ teardown: - match: { docs.0.processor_results.3.doc._ingest.on_failure_processor_tag: "gunna_fail_again" } +--- +"Test simulate with provided pipeline definition with description in processors": + - do: + ingest.simulate: + verbose: true + body: > + { + "pipeline": { + "description": "_description", + "processors": [ + { + "set" : { + "description": "processor_description", + "field" : "field2", + "value" : "_value" + } + } + ] + }, + "docs": [ + { + "_index": "index", + "_id": "id", + "_source": { + "foo": "bar" + } + } + ] + } + - length: { docs: 1 } + - length: { docs.0.processor_results: 1 } + - match: { docs.0.processor_results.0.doc._source.field2: "_value" }