diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexUpgradeService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexUpgradeService.java index 44b871b7f1f9d..9c37f8199f09d 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexUpgradeService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexUpgradeService.java @@ -61,14 +61,16 @@ public class MetadataIndexUpgradeService { private final MapperRegistry mapperRegistry; private final IndexScopedSettings indexScopedSettings; private final SystemIndices systemIndices; + private final ScriptService scriptService; public MetadataIndexUpgradeService(Settings settings, NamedXContentRegistry xContentRegistry, MapperRegistry mapperRegistry, - IndexScopedSettings indexScopedSettings, SystemIndices systemIndices) { + IndexScopedSettings indexScopedSettings, SystemIndices systemIndices, ScriptService scriptService) { this.settings = settings; this.xContentRegistry = xContentRegistry; this.mapperRegistry = mapperRegistry; this.indexScopedSettings = indexScopedSettings; this.systemIndices = systemIndices; + this.scriptService = scriptService; } /** @@ -185,7 +187,7 @@ public Set> entrySet() { try (IndexAnalyzers fakeIndexAnalzyers = new IndexAnalyzers(analyzerMap, analyzerMap, analyzerMap)) { MapperService mapperService = new MapperService(indexSettings, fakeIndexAnalzyers, xContentRegistry, similarityService, - mapperRegistry, () -> null, () -> false); + mapperRegistry, () -> null, () -> false, scriptService); mapperService.merge(indexMetadata, MapperService.MergeReason.MAPPING_RECOVERY); } } catch (Exception ex) { diff --git a/server/src/main/java/org/elasticsearch/index/IndexModule.java b/server/src/main/java/org/elasticsearch/index/IndexModule.java index fea123aa820d5..030ec8745d6a1 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexModule.java +++ b/server/src/main/java/org/elasticsearch/index/IndexModule.java @@ -505,7 +505,7 @@ public MapperService newIndexMapperService(NamedXContentRegistry xContentRegistr ScriptService scriptService) throws IOException { return new MapperService(indexSettings, analysisRegistry.build(indexSettings), xContentRegistry, new SimilarityService(indexSettings, scriptService, similarities), mapperRegistry, - () -> { throw new UnsupportedOperationException("no index query shard context available"); }, () -> false); + () -> { throw new UnsupportedOperationException("no index query shard context available"); }, () -> false, scriptService); } /** diff --git a/server/src/main/java/org/elasticsearch/index/IndexService.java b/server/src/main/java/org/elasticsearch/index/IndexService.java index ed26a3651ed89..4d3dad75841c7 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexService.java +++ b/server/src/main/java/org/elasticsearch/index/IndexService.java @@ -192,16 +192,14 @@ public IndexService( assert indexAnalyzers != null; this.mapperService = new MapperService(indexSettings, indexAnalyzers, xContentRegistry, similarityService, mapperRegistry, // we parse all percolator queries as they would be parsed on shard 0 - () -> newQueryShardContext(0, null, System::currentTimeMillis, null), idFieldDataEnabled); + () -> newQueryShardContext(0, null, System::currentTimeMillis, null), idFieldDataEnabled, scriptService); this.indexFieldData = new IndexFieldDataService(indexSettings, indicesFieldDataCache, circuitBreakerService, mapperService); if (indexSettings.getIndexSortConfig().hasIndexSort()) { // we delay the actual creation of the sort order for this index because the mapping has not been merged yet. // The sort order is validated right after the merge of the mapping later in the process. this.indexSortSupplier = () -> indexSettings.getIndexSortConfig().buildIndexSort( mapperService::fieldType, - fieldType -> indexFieldData.getForField(fieldType, indexFieldData.index().getName(), () -> { - throw new UnsupportedOperationException("search lookup not available for index sorting"); - }) + (fieldType, searchLookup) -> indexFieldData.getForField(fieldType, indexFieldData.index().getName(), searchLookup) ); } else { this.indexSortSupplier = () -> null; diff --git a/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java b/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java index eb83e85bb7c0b..8bc69f4f02fb2 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java @@ -28,12 +28,15 @@ import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.search.MultiValueMode; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.sort.SortOrder; import java.util.Collections; import java.util.EnumSet; import java.util.List; +import java.util.function.BiFunction; import java.util.function.Function; +import java.util.function.Supplier; /** * Holds all the information that is used to build the sort order of an index. @@ -171,7 +174,7 @@ public boolean hasPrimarySortOnField(String field) { * or returns null if this index has no sort. */ public Sort buildIndexSort(Function fieldTypeLookup, - Function> fieldDataLookup) { + BiFunction, IndexFieldData> fieldDataLookup) { if (hasIndexSort() == false) { return null; } @@ -190,7 +193,9 @@ public Sort buildIndexSort(Function fieldTypeLookup, } IndexFieldData fieldData; try { - fieldData = fieldDataLookup.apply(ft); + fieldData = fieldDataLookup.apply(ft, () -> { + throw new UnsupportedOperationException("index sorting not supported on runtime field [" + ft.name() + "]"); + }); } catch (Exception e) { throw new IllegalArgumentException("docvalues not found for index sort field:[" + sortSpec.field + "]", e); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java index 080ef97d5f86d..1dfb10098b8b5 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java @@ -481,25 +481,30 @@ protected void resize(int newSize) { public int size() { return count; } - } - public static final class Strings extends BinaryScriptDocValues { - + public static class Strings extends BinaryScriptDocValues { public Strings(SortedBinaryDocValues in) { super(in); } @Override - public String get(int index) { + public final String get(int index) { if (count == 0) { throw new IllegalStateException("A document doesn't have a value for a field! " + "Use doc[].size()==0 to check if a document is missing a field!"); } - return values[index].get().utf8ToString(); + return bytesToString(values[index].get()); + } + + /** + * Convert the stored bytes to a String. + */ + protected String bytesToString(BytesRef bytes) { + return bytes.utf8ToString(); } - public String getValue() { + public final String getValue() { return get(0); } } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafDoubleFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafDoubleFieldData.java index 46cd3a5ccaef2..3ab7847f641d7 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafDoubleFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafDoubleFieldData.java @@ -34,11 +34,11 @@ /** * Specialization of {@link LeafNumericFieldData} for floating-point numerics. */ -abstract class LeafDoubleFieldData implements LeafNumericFieldData { +public abstract class LeafDoubleFieldData implements LeafNumericFieldData { private final long ramBytesUsed; - LeafDoubleFieldData(long ramBytesUsed) { + protected LeafDoubleFieldData(long ramBytesUsed) { this.ramBytesUsed = ramBytesUsed; } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafLongFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafLongFieldData.java index cf46d51340227..38dede8537b94 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafLongFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafLongFieldData.java @@ -29,7 +29,7 @@ /** * Specialization of {@link LeafNumericFieldData} for integers. */ -abstract class LeafLongFieldData implements LeafNumericFieldData { +public abstract class LeafLongFieldData implements LeafNumericFieldData { private final long ramBytesUsed; /** @@ -37,7 +37,7 @@ abstract class LeafLongFieldData implements LeafNumericFieldData { */ private final NumericType numericType; - LeafLongFieldData(long ramBytesUsed, NumericType numericType) { + protected LeafLongFieldData(long ramBytesUsed, NumericType numericType) { this.ramBytesUsed = ramBytesUsed; this.numericType = numericType; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index d29ee3ad3374e..22d796c4169ea 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -31,10 +31,10 @@ import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; -import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.geo.ShapeRelation; +import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.time.DateFormatters; import org.elasticsearch.common.time.DateMathParser; @@ -60,6 +60,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.function.BiFunction; import java.util.function.Function; import java.util.function.LongSupplier; import java.util.function.Supplier; @@ -318,60 +319,83 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower DateMathParser parser = forcedDateParser == null ? dateMathParser : forcedDateParser; + return dateRangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, timeZone, parser, context, resolution, (l, u) -> { + Query query = LongPoint.newRangeQuery(name(), l, u); + if (hasDocValues()) { + Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(name(), l, u); + query = new IndexOrDocValuesQuery(query, dvQuery); + + if (context.indexSortedOnField(name())) { + query = new IndexSortSortedNumericDocValuesRangeQuery(name(), l, u, query); + } + } + return query; + }); + } + + public static Query dateRangeQuery( + Object lowerTerm, + Object upperTerm, + boolean includeLower, + boolean includeUpper, + @Nullable ZoneId timeZone, + DateMathParser parser, + QueryShardContext context, + Resolution resolution, + BiFunction builder + ) { + return handleNow(context, nowSupplier -> { + long l, u; + if (lowerTerm == null) { + l = Long.MIN_VALUE; + } else { + l = parseToLong(lowerTerm, !includeLower, timeZone, parser, nowSupplier, resolution); + if (includeLower == false) { + ++l; + } + } + if (upperTerm == null) { + u = Long.MAX_VALUE; + } else { + u = parseToLong(upperTerm, includeUpper, timeZone, parser, nowSupplier, resolution); + if (includeUpper == false) { + --u; + } + } + return builder.apply(l, u); + }); + } + + /** + * Handle {@code now} in queries. + * @param context context from which to read the current time + * @param builder build the query + * @return the result of the builder, wrapped in {@link DateRangeIncludingNowQuery} if {@code now} was used. + */ + public static Query handleNow(QueryShardContext context, Function builder) { boolean[] nowUsed = new boolean[1]; LongSupplier nowSupplier = () -> { nowUsed[0] = true; return context.nowInMillis(); }; - long l, u; - if (lowerTerm == null) { - l = Long.MIN_VALUE; - } else { - l = parseToLong(lowerTerm, !includeLower, timeZone, parser, nowSupplier); - if (includeLower == false) { - ++l; - } - } - if (upperTerm == null) { - u = Long.MAX_VALUE; - } else { - u = parseToLong(upperTerm, includeUpper, timeZone, parser, nowSupplier); - if (includeUpper == false) { - --u; - } - } - - Query query = LongPoint.newRangeQuery(name(), l, u); - if (hasDocValues()) { - Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(name(), l, u); - query = new IndexOrDocValuesQuery(query, dvQuery); - - if (context.indexSortedOnField(name())) { - query = new IndexSortSortedNumericDocValuesRangeQuery(name(), l, u, query); - } - } - - if (nowUsed[0]) { - query = new DateRangeIncludingNowQuery(query); - } - return query; + Query query = builder.apply(nowSupplier); + return nowUsed[0] ? new DateRangeIncludingNowQuery(query) : query; } - public long parseToLong(Object value, boolean roundUp, - @Nullable ZoneId zone, @Nullable DateMathParser forcedDateParser, LongSupplier now) { - DateMathParser dateParser = dateMathParser(); - if (forcedDateParser != null) { - dateParser = forcedDateParser; - } + public long parseToLong(Object value, boolean roundUp, @Nullable ZoneId zone, DateMathParser dateParser, LongSupplier now) { + dateParser = dateParser == null ? dateMathParser() : dateParser; + return parseToLong(value, roundUp, zone, dateParser, now, resolution); + } - String strValue; - if (value instanceof BytesRef) { - strValue = ((BytesRef) value).utf8ToString(); - } else { - strValue = value.toString(); - } - Instant instant = dateParser.parse(strValue, now, roundUp, zone); - return resolution.convert(instant); + public static long parseToLong( + Object value, + boolean roundUp, + @Nullable ZoneId zone, + DateMathParser dateParser, + LongSupplier now, + Resolution resolution + ) { + return resolution.convert(dateParser.parse(BytesRefs.toString(value), now, roundUp, zone)); } @Override @@ -391,7 +415,7 @@ public Relation isFieldWithinQuery(IndexReader reader, long fromInclusive = Long.MIN_VALUE; if (from != null) { - fromInclusive = parseToLong(from, !includeLower, timeZone, dateParser, context::nowInMillis); + fromInclusive = parseToLong(from, !includeLower, timeZone, dateParser, context::nowInMillis, resolution); if (includeLower == false) { if (fromInclusive == Long.MAX_VALUE) { return Relation.DISJOINT; @@ -402,7 +426,7 @@ public Relation isFieldWithinQuery(IndexReader reader, long toInclusive = Long.MAX_VALUE; if (to != null) { - toInclusive = parseToLong(to, includeUpper, timeZone, dateParser, context::nowInMillis); + toInclusive = parseToLong(to, includeUpper, timeZone, dateParser, context::nowInMillis, resolution); if (includeUpper == false) { if (toInclusive == Long.MIN_VALUE) { return Relation.DISJOINT; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java index 313cf0e0cc872..7c22b309f5288 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java @@ -33,6 +33,7 @@ import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.indices.mapper.MapperRegistry; +import org.elasticsearch.script.ScriptService; import java.util.Collections; import java.util.HashMap; @@ -53,13 +54,16 @@ public class DocumentMapperParser { private final Map typeParsers; private final Map rootTypeParsers; + private final ScriptService scriptService; public DocumentMapperParser(IndexSettings indexSettings, MapperService mapperService, NamedXContentRegistry xContentRegistry, - SimilarityService similarityService, MapperRegistry mapperRegistry, Supplier queryShardContextSupplier) { + SimilarityService similarityService, MapperRegistry mapperRegistry, + Supplier queryShardContextSupplier, ScriptService scriptService) { this.mapperService = mapperService; this.xContentRegistry = xContentRegistry; this.similarityService = similarityService; this.queryShardContextSupplier = queryShardContextSupplier; + this.scriptService = scriptService; this.typeParsers = mapperRegistry.getMapperParsers(); this.indexVersionCreated = indexSettings.getIndexVersionCreated(); this.rootTypeParsers = mapperRegistry.getMetadataMapperParsers(indexVersionCreated); @@ -67,12 +71,12 @@ public DocumentMapperParser(IndexSettings indexSettings, MapperService mapperSer public Mapper.TypeParser.ParserContext parserContext() { return new Mapper.TypeParser.ParserContext(similarityService::getSimilarity, mapperService, - typeParsers::get, indexVersionCreated, queryShardContextSupplier, null); + typeParsers::get, indexVersionCreated, queryShardContextSupplier, null, scriptService); } public Mapper.TypeParser.ParserContext parserContext(DateFormatter dateFormatter) { return new Mapper.TypeParser.ParserContext(similarityService::getSimilarity, mapperService, - typeParsers::get, indexVersionCreated, queryShardContextSupplier, dateFormatter); + typeParsers::get, indexVersionCreated, queryShardContextSupplier, dateFormatter, scriptService); } public DocumentMapper parse(@Nullable String type, CompressedXContent source) throws MapperParsingException { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index aac7fda42cef7..1b75f09c7177a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -49,6 +49,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.function.BiFunction; import java.util.function.Supplier; /** A {@link FieldMapper} for ip addresses. */ @@ -128,7 +129,7 @@ public String typeName() { return CONTENT_TYPE; } - private InetAddress parse(Object value) { + private static InetAddress parse(Object value) { if (value instanceof InetAddress) { return (InetAddress) value; } else { @@ -194,6 +195,26 @@ public Query termsQuery(List values, QueryShardContext context) { @Override public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, QueryShardContext context) { failIfNotIndexed(); + return rangeQuery( + lowerTerm, + upperTerm, + includeLower, + includeUpper, + (lower, upper) -> InetAddressPoint.newRangeQuery(name(), lower, upper) + ); + } + + /** + * Processes query bounds into {@code long}s and delegates the + * provided {@code builder} to build a range query. + */ + public static Query rangeQuery( + Object lowerTerm, + Object upperTerm, + boolean includeLower, + boolean includeUpper, + BiFunction builder + ) { InetAddress lower; if (lowerTerm == null) { lower = InetAddressPoint.MIN_VALUE; @@ -220,7 +241,7 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower } } - return InetAddressPoint.newRangeQuery(name(), lower, upper); + return builder.apply(lower, upper); } public static final class IpScriptDocValues extends ScriptDocValues { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java index 666a02a71f6f3..007b267a30a88 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java @@ -26,6 +26,7 @@ import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.similarity.SimilarityProvider; +import org.elasticsearch.script.ScriptService; import java.util.Map; import java.util.Objects; @@ -91,16 +92,19 @@ class ParserContext { private final DateFormatter dateFormatter; + private final ScriptService scriptService; + public ParserContext(Function similarityLookupService, MapperService mapperService, Function typeParsers, Version indexVersionCreated, Supplier queryShardContextSupplier, - DateFormatter dateFormatter) { + DateFormatter dateFormatter, ScriptService scriptService) { this.similarityLookupService = similarityLookupService; this.mapperService = mapperService; this.typeParsers = typeParsers; this.indexVersionCreated = indexVersionCreated; this.queryShardContextSupplier = queryShardContextSupplier; this.dateFormatter = dateFormatter; + this.scriptService = scriptService; } public IndexAnalyzers getIndexAnalyzers() { @@ -146,6 +150,13 @@ public DateFormatter getDateFormatter() { protected Function similarityLookupService() { return similarityLookupService; } + /** + * The {@linkplain ScriptService} to compile scripts needed by the {@linkplain Mapper}. + */ + public ScriptService scriptService() { + return scriptService; + } + public ParserContext createMultiFieldContext(ParserContext in) { return new MultiFieldParserContext(in); } @@ -153,7 +164,7 @@ public ParserContext createMultiFieldContext(ParserContext in) { static class MultiFieldParserContext extends ParserContext { MultiFieldParserContext(ParserContext in) { super(in.similarityLookupService(), in.mapperService(), in.typeParsers(), - in.indexVersionCreated(), in.queryShardContextSupplier(), in.getDateFormatter()); + in.indexVersionCreated(), in.queryShardContextSupplier(), in.getDateFormatter(), in.scriptService()); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 66caf9d41bdeb..750b217059027 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -51,6 +51,7 @@ import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.indices.mapper.MapperRegistry; +import org.elasticsearch.script.ScriptService; import java.io.Closeable; import java.io.IOException; @@ -125,12 +126,13 @@ public enum MergeReason { public MapperService(IndexSettings indexSettings, IndexAnalyzers indexAnalyzers, NamedXContentRegistry xContentRegistry, SimilarityService similarityService, MapperRegistry mapperRegistry, - Supplier queryShardContextSupplier, BooleanSupplier idFieldDataEnabled) { + Supplier queryShardContextSupplier, BooleanSupplier idFieldDataEnabled, + ScriptService scriptService) { super(indexSettings); this.indexVersionCreated = indexSettings.getIndexVersionCreated(); this.indexAnalyzers = indexAnalyzers; this.documentParser = new DocumentMapperParser(indexSettings, this, xContentRegistry, similarityService, mapperRegistry, - queryShardContextSupplier); + queryShardContextSupplier, scriptService); this.indexAnalyzer = new MapperAnalyzerWrapper(indexAnalyzers.getDefaultIndexAnalyzer(), MappedFieldType::indexAnalyzer); this.searchAnalyzer = new MapperAnalyzerWrapper(indexAnalyzers.getDefaultSearchAnalyzer(), p -> p.getTextSearchInfo().getSearchAnalyzer()); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 16ff074e5bea2..dfa43ba7ea663 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.core.exc.InputCoercionException; + import org.apache.lucene.document.DoublePoint; import org.apache.lucene.document.Field; import org.apache.lucene.document.FloatPoint; @@ -63,6 +64,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.function.BiFunction; import java.util.function.Function; import java.util.function.Supplier; @@ -367,28 +369,16 @@ public Query termsQuery(String field, List values) { public Query rangeQuery(String field, Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, boolean hasDocValues, QueryShardContext context) { - double l = Double.NEGATIVE_INFINITY; - double u = Double.POSITIVE_INFINITY; - if (lowerTerm != null) { - l = parse(lowerTerm, false); - if (includeLower == false) { - l = DoublePoint.nextUp(l); - } - } - if (upperTerm != null) { - u = parse(upperTerm, false); - if (includeUpper == false) { - u = DoublePoint.nextDown(u); + return doubleRangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, (l, u) -> { + Query query = DoublePoint.newRangeQuery(field, l, u); + if (hasDocValues) { + Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(field, + NumericUtils.doubleToSortableLong(l), + NumericUtils.doubleToSortableLong(u)); + query = new IndexOrDocValuesQuery(query, dvQuery); } - } - Query query = DoublePoint.newRangeQuery(field, l, u); - if (hasDocValues) { - Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(field, - NumericUtils.doubleToSortableLong(l), - NumericUtils.doubleToSortableLong(u)); - query = new IndexOrDocValuesQuery(query, dvQuery); - } - return query; + return query; + }); } @Override @@ -654,23 +644,7 @@ public List createFields(String name, Number value, LONG("long", NumericType.LONG) { @Override public Long parse(Object value, boolean coerce) { - if (value instanceof Long) { - return (Long)value; - } - - double doubleValue = objectToDouble(value); - // this check does not guarantee that value is inside MIN_VALUE/MAX_VALUE because values up to 9223372036854776832 will - // be equal to Long.MAX_VALUE after conversion to double. More checks ahead. - if (doubleValue < Long.MIN_VALUE || doubleValue > Long.MAX_VALUE) { - throw new IllegalArgumentException("Value [" + value + "] is out of range for a long"); - } - if (!coerce && doubleValue % 1 != 0) { - throw new IllegalArgumentException("Value [" + value + "] has a decimal part"); - } - - // longs need special handling so we don't lose precision while parsing - String stringValue = (value instanceof BytesRef) ? ((BytesRef) value).utf8ToString() : value.toString(); - return Numbers.toLong(stringValue, coerce); + return objectToLong(value, coerce); } @Override @@ -717,44 +691,17 @@ public Query termsQuery(String field, List values) { public Query rangeQuery(String field, Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, boolean hasDocValues, QueryShardContext context) { - long l = Long.MIN_VALUE; - long u = Long.MAX_VALUE; - if (lowerTerm != null) { - l = parse(lowerTerm, true); - // if the lower bound is decimal: - // - if the bound is positive then we increment it: - // if lowerTerm=1.5 then the (inclusive) bound becomes 2 - // - if the bound is negative then we leave it as is: - // if lowerTerm=-1.5 then the (inclusive) bound becomes -1 due to the call to longValue - boolean lowerTermHasDecimalPart = hasDecimalPart(lowerTerm); - if ((lowerTermHasDecimalPart == false && includeLower == false) || - (lowerTermHasDecimalPart && signum(lowerTerm) > 0)) { - if (l == Long.MAX_VALUE) { - return new MatchNoDocsQuery(); + return longRangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, (l, u) -> { + Query query = LongPoint.newRangeQuery(field, l, u); + if (hasDocValues) { + Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(field, l, u); + query = new IndexOrDocValuesQuery(query, dvQuery); + if (context.indexSortedOnField(field)) { + query = new IndexSortSortedNumericDocValuesRangeQuery(field, l, u, query); } - ++l; } - } - if (upperTerm != null) { - u = parse(upperTerm, true); - boolean upperTermHasDecimalPart = hasDecimalPart(upperTerm); - if ((upperTermHasDecimalPart == false && includeUpper == false) || - (upperTermHasDecimalPart && signum(upperTerm) < 0)) { - if (u == Long.MIN_VALUE) { - return new MatchNoDocsQuery(); - } - --u; - } - } - Query query = LongPoint.newRangeQuery(field, l, u); - if (hasDocValues) { - Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(field, l, u); - query = new IndexOrDocValuesQuery(query, dvQuery); - if (context.indexSortedOnField(field)) { - query = new IndexSortSortedNumericDocValuesRangeQuery(field, l, u, query); - } - } - return query; + return query; + }); } @Override @@ -812,7 +759,7 @@ Number valueForSearch(Number value) { /** * Returns true if the object is a number and has a decimal part */ - boolean hasDecimalPart(Object number) { + public static boolean hasDecimalPart(Object number) { if (number instanceof Number) { double doubleValue = ((Number) number).doubleValue(); return doubleValue % 1 != 0; @@ -829,7 +776,7 @@ boolean hasDecimalPart(Object number) { /** * Returns -1, 0, or 1 if the value is lower than, equal to, or greater than 0 */ - double signum(Object value) { + static double signum(Object value) { if (value instanceof Number) { double doubleValue = ((Number) value).doubleValue(); return Math.signum(doubleValue); @@ -843,7 +790,7 @@ boolean hasDecimalPart(Object number) { /** * Converts an Object to a double by checking it against known types first */ - private static double objectToDouble(Object value) { + public static double objectToDouble(Object value) { double doubleValue; if (value instanceof Number) { @@ -856,6 +803,95 @@ private static double objectToDouble(Object value) { return doubleValue; } + + /** + * Converts and Object to a {@code long} by checking it against known + * types and checking its range. + */ + public static long objectToLong(Object value, boolean coerce) { + if (value instanceof Long) { + return (Long)value; + } + + double doubleValue = objectToDouble(value); + // this check does not guarantee that value is inside MIN_VALUE/MAX_VALUE because values up to 9223372036854776832 will + // be equal to Long.MAX_VALUE after conversion to double. More checks ahead. + if (doubleValue < Long.MIN_VALUE || doubleValue > Long.MAX_VALUE) { + throw new IllegalArgumentException("Value [" + value + "] is out of range for a long"); + } + if (!coerce && doubleValue % 1 != 0) { + throw new IllegalArgumentException("Value [" + value + "] has a decimal part"); + } + + // longs need special handling so we don't lose precision while parsing + String stringValue = (value instanceof BytesRef) ? ((BytesRef) value).utf8ToString() : value.toString(); + return Numbers.toLong(stringValue, coerce); + } + + public static Query doubleRangeQuery( + Object lowerTerm, + Object upperTerm, + boolean includeLower, + boolean includeUpper, + BiFunction builder + ) { + double l = Double.NEGATIVE_INFINITY; + double u = Double.POSITIVE_INFINITY; + if (lowerTerm != null) { + l = objectToDouble(lowerTerm); + if (includeLower == false) { + l = DoublePoint.nextUp(l); + } + } + if (upperTerm != null) { + u = objectToDouble(upperTerm); + if (includeUpper == false) { + u = DoublePoint.nextDown(u); + } + } + return builder.apply(l, u); + } + + /** + * Processes query bounds into {@code long}s and delegates the + * provided {@code builder} to build a range query. + */ + public static Query longRangeQuery( + Object lowerTerm, + Object upperTerm, + boolean includeLower, + boolean includeUpper, + BiFunction builder + ) { + long l = Long.MIN_VALUE; + long u = Long.MAX_VALUE; + if (lowerTerm != null) { + l = objectToLong(lowerTerm, true); + // if the lower bound is decimal: + // - if the bound is positive then we increment it: + // if lowerTerm=1.5 then the (inclusive) bound becomes 2 + // - if the bound is negative then we leave it as is: + // if lowerTerm=-1.5 then the (inclusive) bound becomes -1 due to the call to longValue + boolean lowerTermHasDecimalPart = hasDecimalPart(lowerTerm); + if ((lowerTermHasDecimalPart == false && includeLower == false) || (lowerTermHasDecimalPart && signum(lowerTerm) > 0)) { + if (l == Long.MAX_VALUE) { + return new MatchNoDocsQuery(); + } + ++l; + } + } + if (upperTerm != null) { + u = objectToLong(upperTerm, true); + boolean upperTermHasDecimalPart = hasDecimalPart(upperTerm); + if ((upperTermHasDecimalPart == false && includeUpper == false) || (upperTermHasDecimalPart && signum(upperTerm) < 0)) { + if (u == Long.MIN_VALUE) { + return new MatchNoDocsQuery(); + } + --u; + } + } + return builder.apply(l, u); + } } public static final class NumberFieldType extends SimpleMappedFieldType { diff --git a/server/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java b/server/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java index 0b4177490b17b..47e08b1286ea3 100644 --- a/server/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java @@ -210,6 +210,7 @@ private AbstractDistanceScoreFunction parseVariable(String fieldName, XContentPa // dates and time and geo need special handling parser.nextToken(); + // TODO these ain't gonna work with runtime fields if (fieldType instanceof DateFieldMapper.DateFieldType) { return parseDateVariable(parser, context, fieldType, mode); } else if (fieldType instanceof GeoPointFieldType) { diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index 55df41ac89433..16a2fc9dc03ec 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -533,7 +533,7 @@ protected Node(final Environment initialEnvironment, .collect(Collectors.toList()); final MetadataUpgrader metadataUpgrader = new MetadataUpgrader(indexTemplateMetadataUpgraders); final MetadataIndexUpgradeService metadataIndexUpgradeService = new MetadataIndexUpgradeService(settings, xContentRegistry, - indicesModule.getMapperRegistry(), settingsModule.getIndexScopedSettings(), systemIndices); + indicesModule.getMapperRegistry(), settingsModule.getIndexScopedSettings(), systemIndices, scriptService); clusterService.addListener(new SystemIndexMetadataUpgradeService(systemIndices, clusterService)); new TemplateUpgradeService(client, clusterService, threadPool, indexTemplateMetadataUpgraders); final Transport transport = networkModule.getTransportSupplier().get(); diff --git a/server/src/main/java/org/elasticsearch/script/ScriptCache.java b/server/src/main/java/org/elasticsearch/script/ScriptCache.java index e156cfa125b19..e7f1245d0afd9 100644 --- a/server/src/main/java/org/elasticsearch/script/ScriptCache.java +++ b/server/src/main/java/org/elasticsearch/script/ScriptCache.java @@ -42,7 +42,7 @@ public class ScriptCache { private static final Logger logger = LogManager.getLogger(ScriptService.class); - static final CompilationRate UNLIMITED_COMPILATION_RATE = new CompilationRate(0, TimeValue.ZERO); + public static final CompilationRate UNLIMITED_COMPILATION_RATE = new CompilationRate(0, TimeValue.ZERO); private final Cache cache; private final ScriptMetrics scriptMetrics; diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexUpgradeServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexUpgradeServiceTests.java index dfd7505e67a93..227436f56ec9c 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexUpgradeServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexUpgradeServiceTests.java @@ -161,7 +161,8 @@ private MetadataIndexUpgradeService getMetadataIndexUpgradeService() { xContentRegistry(), new MapperRegistry(Collections.emptyMap(), Collections.emptyMap(), MapperPlugin.NOOP_FIELD_FILTER), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, - new SystemIndices(Map.of("system-plugin", List.of(new SystemIndexDescriptor(".system", "a system index")))) + new SystemIndices(Map.of("system-plugin", List.of(new SystemIndexDescriptor(".system", "a system index")))), + null ); } diff --git a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java index f260cb767c925..6ceee1e13c691 100644 --- a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java @@ -159,7 +159,7 @@ private static class MockMetadataIndexUpgradeService extends MetadataIndexUpgrad private final boolean upgrade; MockMetadataIndexUpgradeService(boolean upgrade) { - super(Settings.EMPTY, null, null, null, null); + super(Settings.EMPTY, null, null, null, null, null); this.upgrade = upgrade; } diff --git a/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java b/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java index 331a2c138f6f2..47cfea54c65a0 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java @@ -94,7 +94,7 @@ private CodecService createCodecService() throws IOException { IndexAnalyzers indexAnalyzers = createTestAnalysis(settings, nodeSettings).indexAnalyzers; MapperRegistry mapperRegistry = new MapperRegistry(Collections.emptyMap(), Collections.emptyMap(), MapperPlugin.NOOP_FIELD_FILTER); MapperService service = new MapperService(settings, indexAnalyzers, xContentRegistry(), similarityService, mapperRegistry, - () -> null, () -> false); + () -> null, () -> false, null); return new CodecService(service, LogManager.getLogger("test")); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ExternalFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ExternalFieldMapperTests.java index 04218959a51ff..fc652effc7dcb 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ExternalFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ExternalFieldMapperTests.java @@ -73,7 +73,7 @@ public void testExternalValues() throws Exception { return indexService.newQueryShardContext(0, null, () -> { throw new UnsupportedOperationException(); }, null); }; DocumentMapperParser parser = new DocumentMapperParser(indexService.getIndexSettings(), indexService.mapperService(), - indexService.xContentRegistry(), indexService.similarityService(), mapperRegistry, queryShardContext); + indexService.xContentRegistry(), indexService.similarityService(), mapperRegistry, queryShardContext, null); DocumentMapper documentMapper = parser.parse("type", new CompressedXContent( Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type") .startObject(ExternalMetadataMapper.CONTENT_TYPE) @@ -120,7 +120,7 @@ public void testExternalValuesWithMultifield() throws Exception { return indexService.newQueryShardContext(0, null, () -> { throw new UnsupportedOperationException(); }, null); }; DocumentMapperParser parser = new DocumentMapperParser(indexService.getIndexSettings(), indexService.mapperService(), - indexService.xContentRegistry(), indexService.similarityService(), mapperRegistry, queryShardContext); + indexService.xContentRegistry(), indexService.similarityService(), mapperRegistry, queryShardContext, null); DocumentMapper documentMapper = parser.parse("type", new CompressedXContent( Strings diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java index 9f14d58128ef7..2820e98385fe0 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java @@ -215,7 +215,7 @@ private static TestMapper fromMapping(String mapping, Version version) { return BinaryFieldMapper.PARSER; } return null; - }, version, () -> null, null); + }, version, () -> null, null, null); return (TestMapper) new TypeParser() .parse("field", XContentHelper.convertToMap(JsonXContent.jsonXContent, mapping, true), pc) .build(new Mapper.BuilderContext(Settings.EMPTY, new ContentPath(0))); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java index fab5b62c09eba..6b54d9759d590 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java @@ -195,7 +195,7 @@ public void testMultiFieldWithinMultiField() throws IOException { when(mapperService.getIndexAnalyzers()).thenReturn(indexAnalyzers); Version olderVersion = VersionUtils.randomPreviousCompatibleVersion(random(), Version.V_8_0_0); Mapper.TypeParser.ParserContext olderContext = new Mapper.TypeParser.ParserContext( - null, mapperService, type -> typeParser, olderVersion, null, null); + null, mapperService, type -> typeParser, olderVersion, null, null, null); TypeParsers.parseField(builder, "some-field", fieldNode, olderContext); assertWarnings("At least one multi-field, [sub-field], " + @@ -210,7 +210,7 @@ public void testMultiFieldWithinMultiField() throws IOException { Version version = VersionUtils.randomVersionBetween(random(), Version.V_8_0_0, Version.CURRENT); Mapper.TypeParser.ParserContext context = new Mapper.TypeParser.ParserContext( - null, mapperService, type -> typeParser, version, null, null); + null, mapperService, type -> typeParser, version, null, null, null); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> TypeParsers.parseField(builder, "some-field", fieldNodeCopy, context)); diff --git a/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java b/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java index 28b322c77fac3..316da9cf2810b 100644 --- a/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java +++ b/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java @@ -193,8 +193,14 @@ public ClusterStateChanges(NamedXContentRegistry xContentRegistry, ThreadPool th TransportService.NOOP_TRANSPORT_INTERCEPTOR, boundAddress -> DiscoveryNode.createLocal(SETTINGS, boundAddress.publishAddress(), UUIDs.randomBase64UUID()), clusterSettings, Collections.emptySet()); - MetadataIndexUpgradeService metadataIndexUpgradeService = - new MetadataIndexUpgradeService(SETTINGS, xContentRegistry, null, null, null) { + MetadataIndexUpgradeService metadataIndexUpgradeService = new MetadataIndexUpgradeService( + SETTINGS, + xContentRegistry, + null, + null, + null, + null + ) { // metadata upgrader should do nothing @Override public IndexMetadata upgradeIndexMetadata(IndexMetadata indexMetadata, Version minimumIndexCompatibilityVersion) { diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java index 5005aaed31f1a..3fc5a05ea4d16 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java @@ -1591,7 +1591,8 @@ clusterService, indicesService, threadPool, shardStateAction, mappingUpdatedActi settings, namedXContentRegistry, mapperRegistry, indexScopedSettings, - new SystemIndices(Map.of())), + new SystemIndices(Map.of()), + null), clusterSettings, shardLimitValidator ); diff --git a/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java b/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java index f8057d7bfa8a2..e9e9b3b043d40 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java @@ -73,7 +73,7 @@ public static MapperService newMapperService(NamedXContentRegistry xContentRegis xContentRegistry, similarityService, mapperRegistry, - () -> null, () -> false); + () -> null, () -> false, null); } public static void assertConflicts(String mapping1, diff --git a/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java b/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java index ee9755af4196b..71aab46fe8da8 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java +++ b/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java @@ -65,7 +65,7 @@ public TranslogHandler(NamedXContentRegistry xContentRegistry, IndexSettings ind SimilarityService similarityService = new SimilarityService(indexSettings, null, emptyMap()); MapperRegistry mapperRegistry = new IndicesModule(emptyList()).getMapperRegistry(); mapperService = new MapperService(indexSettings, indexAnalyzers, xContentRegistry, similarityService, mapperRegistry, - () -> null, () -> false); + () -> null, () -> false, null); } private DocumentMapperForType docMapper(String type) { diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java index bfd8b8948a228..668956385dd65 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java @@ -44,6 +44,8 @@ import org.elasticsearch.indices.mapper.MapperRegistry; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.ScriptPlugin; +import org.elasticsearch.script.ScriptModule; import org.elasticsearch.script.ScriptService; import org.elasticsearch.test.ESTestCase; @@ -52,7 +54,6 @@ import java.util.Map; import static java.util.Collections.emptyList; -import static java.util.Collections.emptyMap; import static java.util.stream.Collectors.toList; import static org.mockito.Matchers.anyObject; import static org.mockito.Matchers.anyString; @@ -100,11 +101,15 @@ protected final MapperService createMapperService(Settings settings, XContentBui .numberOfReplicas(0) .numberOfShards(1) .build(); - IndexSettings indexSettings = new IndexSettings(meta, settings); + IndexSettings indexSettings = new IndexSettings(meta, Settings.EMPTY); MapperRegistry mapperRegistry = new IndicesModule( getPlugins().stream().filter(p -> p instanceof MapperPlugin).map(p -> (MapperPlugin) p).collect(toList()) ).getMapperRegistry(); - ScriptService scriptService = new ScriptService(settings, emptyMap(), emptyMap()); + ScriptModule scriptModule = new ScriptModule( + Settings.EMPTY, + getPlugins().stream().filter(p -> p instanceof ScriptPlugin).map(p -> (ScriptPlugin) p).collect(toList()) + ); + ScriptService scriptService = new ScriptService(Settings.EMPTY, scriptModule.engines, scriptModule.contexts); SimilarityService similarityService = new SimilarityService(indexSettings, scriptService, Map.of()); MapperService mapperService = new MapperService( indexSettings, @@ -113,7 +118,8 @@ protected final MapperService createMapperService(Settings settings, XContentBui similarityService, mapperRegistry, () -> { throw new UnsupportedOperationException(); }, - () -> true + () -> true, + scriptService ); merge(mapperService, mapping); return mapperService; diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index be92d8fe51a6f..afe51f2537aee 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -40,7 +40,6 @@ * Base class for testing {@link Mapper}s. */ public abstract class MapperTestCase extends MapperServiceTestCase { - protected abstract void minimalMapping(XContentBuilder b) throws IOException; public final void testEmptyName() { diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index 8eb5ad44b522a..4ddda5fc9f084 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -830,7 +830,7 @@ private void writeTestDoc(MappedFieldType fieldType, String fieldName, RandomInd private class MockParserContext extends Mapper.TypeParser.ParserContext { MockParserContext() { - super(null, null, null, null, null, null); + super(null, null, null, null, null, null, null); } @Override diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java index aa41bcad47c32..a5efdae46d58f 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java @@ -359,7 +359,7 @@ private static class ServiceHolder implements Closeable { similarityService = new SimilarityService(idxSettings, null, Collections.emptyMap()); MapperRegistry mapperRegistry = indicesModule.getMapperRegistry(); mapperService = new MapperService(idxSettings, indexAnalyzers, xContentRegistry, similarityService, mapperRegistry, - () -> createShardContext(null), () -> false); + () -> createShardContext(null), () -> false, null); IndicesFieldDataCache indicesFieldDataCache = new IndicesFieldDataCache(nodeSettings, new IndexFieldDataCache.Listener() { }); indexFieldDataService = new IndexFieldDataService(idxSettings, indicesFieldDataCache, diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java index 48e7fc031139b..0a1b03279f7bb 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java @@ -60,7 +60,12 @@ public static ClientYamlTestSection parse(XContentParser parser) throws IOExcept private final SkipSection skipSection; private final List executableSections; - ClientYamlTestSection(XContentLocation location, String name, SkipSection skipSection, List executableSections) { + public ClientYamlTestSection( + XContentLocation location, + String name, + SkipSection skipSection, + List executableSections + ) { this.location = location; this.name = name; this.skipSection = Objects.requireNonNull(skipSection, "skip section cannot be null"); diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuite.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuite.java index fbccce730523a..57070ef8d2c8f 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuite.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuite.java @@ -111,7 +111,7 @@ public static ClientYamlTestSuite parse(String api, String suiteName, XContentPa private final TeardownSection teardownSection; private final List testSections; - ClientYamlTestSuite(String api, String name, SetupSection setupSection, TeardownSection teardownSection, + public ClientYamlTestSuite(String api, String name, SetupSection setupSection, TeardownSection teardownSection, List testSections) { this.api = api; this.name = name; diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SetupSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SetupSection.java index 38a034d47e769..dd718ea684b05 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SetupSection.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SetupSection.java @@ -70,7 +70,7 @@ public static SetupSection parse(XContentParser parser) throws IOException { private final SkipSection skipSection; private final List executableSections; - SetupSection(SkipSection skipSection, List executableSections) { + public SetupSection(SkipSection skipSection, List executableSections) { this.skipSection = Objects.requireNonNull(skipSection, "skip section cannot be null"); this.executableSections = Collections.unmodifiableList(executableSections); } diff --git a/x-pack/plugin/runtime-fields/build.gradle b/x-pack/plugin/runtime-fields/build.gradle new file mode 100644 index 0000000000000..1f44b5c019e55 --- /dev/null +++ b/x-pack/plugin/runtime-fields/build.gradle @@ -0,0 +1,26 @@ +evaluationDependsOn(xpackModule('core')) + +apply plugin: 'elasticsearch.esplugin' + +esplugin { + name 'x-pack-runtime-fields' + description 'A module which adds support for runtime fields' + classname 'org.elasticsearch.xpack.runtimefields.RuntimeFields' + extendedPlugins = ['x-pack-core', 'lang-painless'] +} +archivesBaseName = 'x-pack-runtime-fields' + +compileJava.options.compilerArgs << "-Xlint:-rawtypes" +compileTestJava.options.compilerArgs << "-Xlint:-rawtypes" + +dependencies { + compileOnly project(":server") + compileOnly project(':modules:lang-painless:spi') + compileOnly project(path: xpackModule('core'), configuration: 'default') +} + +dependencyLicenses { + ignoreSha 'x-pack-core' +} + +integTest.enabled = false diff --git a/x-pack/plugin/runtime-fields/qa/build.gradle b/x-pack/plugin/runtime-fields/qa/build.gradle new file mode 100644 index 0000000000000..7dc01b73ed9ec --- /dev/null +++ b/x-pack/plugin/runtime-fields/qa/build.gradle @@ -0,0 +1 @@ +// Empty project so we can pick up its subproject diff --git a/x-pack/plugin/runtime-fields/qa/rest/build.gradle b/x-pack/plugin/runtime-fields/qa/rest/build.gradle new file mode 100644 index 0000000000000..9ca0108efb1d0 --- /dev/null +++ b/x-pack/plugin/runtime-fields/qa/rest/build.gradle @@ -0,0 +1,50 @@ +apply plugin: 'elasticsearch.yaml-rest-test' + +restResources { + restApi { + includeXpack 'async_search', 'graph', '*_point_in_time' + } + restTests { + includeCore '*' + includeXpack 'async_search', 'graph' + } +} + +testClusters.yamlRestTest { + testDistribution = 'DEFAULT' + setting 'xpack.license.self_generated.type', 'trial' +} + +yamlRestTest { + systemProperty 'tests.rest.suite', + [ + 'async_search', + 'field_caps', + 'graph', + 'msearch', + 'search', + 'search.aggregation', + 'search.highlight', + 'search.inner_hits', + 'search_shards', + 'suggest', + ].join(',') + systemProperty 'tests.rest.blacklist', + [ + /////// TO FIX /////// + 'search/330_fetch_fields/*', // The whole API is not yet supported + 'search.highlight/40_keyword_ignore/Plain Highligher should skip highlighting ignored keyword values', // The plain highlighter is incompatible with runtime fields. Worth fixing? + 'search/115_multiple_field_collapsing/two levels fields collapsing', // Broken. Gotta fix. + 'field_caps/30_filter/Field caps with index filter', // We don't support filtering field caps on runtime fields. What should we do? + 'search.aggregation/10_histogram/*', // runtime_script doesn't support sub-fields. Maybe it should? + 'search/140_pre_filter_search_shards/pre_filter_shard_size with shards that have no hit', + /////// TO FIX /////// + + /////// NOT SUPPORTED /////// + 'search.aggregation/280_rare_terms/*', // Requires an index and we won't have it + // Runtime fields don't have global ords + 'search.aggregation/20_terms/string profiler via global ordinals', + 'search.aggregation/20_terms/Global ordinals are loaded with the global_ordinals execution hint', + /////// NOT SUPPORTED /////// + ].join(',') +} diff --git a/x-pack/plugin/runtime-fields/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/rest/CoreTestsWithRuntimeFieldsIT.java b/x-pack/plugin/runtime-fields/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/rest/CoreTestsWithRuntimeFieldsIT.java new file mode 100644 index 0000000000000..8d679ded47261 --- /dev/null +++ b/x-pack/plugin/runtime-fields/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/rest/CoreTestsWithRuntimeFieldsIT.java @@ -0,0 +1,265 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.rest; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.common.xcontent.XContentLocation; +import org.elasticsearch.index.mapper.BooleanFieldMapper; +import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.index.mapper.IpFieldMapper; +import org.elasticsearch.index.mapper.KeywordFieldMapper; +import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; +import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; +import org.elasticsearch.test.rest.yaml.ClientYamlTestExecutionContext; +import org.elasticsearch.test.rest.yaml.ClientYamlTestResponse; +import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSection; +import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSuite; +import org.elasticsearch.test.rest.yaml.section.DoSection; +import org.elasticsearch.test.rest.yaml.section.ExecutableSection; +import org.elasticsearch.test.rest.yaml.section.SetupSection; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static org.hamcrest.Matchers.equalTo; + +public class CoreTestsWithRuntimeFieldsIT extends ESClientYamlSuiteTestCase { + public CoreTestsWithRuntimeFieldsIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + /** + * Builds test parameters similarly to {@link ESClientYamlSuiteTestCase#createParameters()}, + * replacing the body of index creation commands so that fields are {@code runtime_script}s + * that load from {@code source} instead of their original type. Test configurations that + * do are not modified to contain runtime fields are not returned as they are tested + * elsewhere. + */ + @ParametersFactory + public static Iterable parameters() throws Exception { + Map suites = new HashMap<>(); + List result = new ArrayList<>(); + for (Object[] orig : ESClientYamlSuiteTestCase.createParameters()) { + assert orig.length == 1; + ClientYamlTestCandidate candidate = (ClientYamlTestCandidate) orig[0]; + ClientYamlTestSuite suite = suites.computeIfAbsent(candidate.getTestPath(), k -> modifiedSuite(candidate)); + if (suite == null) { + // The setup section contains an unsupported option + continue; + } + if (false == modifySection(candidate.getTestPath(), candidate.getTestSection().getExecutableSections())) { + // The test section contains an unsupported option + continue; + } + ClientYamlTestSection modified = new ClientYamlTestSection( + candidate.getTestSection().getLocation(), + candidate.getTestSection().getName(), + candidate.getTestSection().getSkipSection(), + candidate.getTestSection().getExecutableSections() + ); + result.add(new Object[] { new ClientYamlTestCandidate(suite, modified) }); + } + return result; + } + + /** + * Modify the setup section to setup a dynamic template that replaces + * field configurations with scripts that load from source + * and replaces field configurations in {@code incides.create} + * with scripts that load from source. + */ + private static ClientYamlTestSuite modifiedSuite(ClientYamlTestCandidate candidate) { + if (false == modifySection(candidate.getSuitePath() + "/setup", candidate.getSetupSection().getExecutableSections())) { + return null; + } + List setup = new ArrayList<>(candidate.getSetupSection().getExecutableSections().size() + 1); + setup.add(ADD_TEMPLATE); + setup.addAll(candidate.getSetupSection().getExecutableSections()); + return new ClientYamlTestSuite( + candidate.getApi(), + candidate.getName(), + new SetupSection(candidate.getSetupSection().getSkipSection(), setup), + candidate.getTeardownSection(), + List.of() + ); + } + + /** + * Replace field configuration in {@code indices.create} with scripts + * that load from the source. + */ + private static boolean modifySection(String sectionName, List executables) { + for (ExecutableSection section : executables) { + if (false == (section instanceof DoSection)) { + continue; + } + DoSection doSection = (DoSection) section; + if (false == doSection.getApiCallSection().getApi().equals("indices.create")) { + continue; + } + for (Map body : doSection.getApiCallSection().getBodies()) { + Object settings = body.get("settings"); + if (settings instanceof Map && ((Map) settings).containsKey("sort.field")) { + /* + * You can't sort the index on a runtime_keyword and it is + * hard to figure out if the sort was a runtime_keyword so + * let's just skip this test. + */ + continue; + } + Object mappings = body.get("mappings"); + if (false == (mappings instanceof Map)) { + continue; + } + Object properties = ((Map) mappings).get("properties"); + if (false == (properties instanceof Map)) { + continue; + } + for (Map.Entry property : ((Map) properties).entrySet()) { + if (false == property.getValue() instanceof Map) { + continue; + } + @SuppressWarnings("unchecked") + Map propertyMap = (Map) property.getValue(); + String name = property.getKey().toString(); + String type = Objects.toString(propertyMap.get("type")); + if ("nested".equals(type)) { + // Our loading scripts can't be made to manage nested fields so we have to skip those tests. + return false; + } + if ("false".equals(Objects.toString(propertyMap.get("doc_values")))) { + // If doc_values is false we can't emulate with scripts. `null` and `true` are fine. + continue; + } + if ("false".equals(Objects.toString(propertyMap.get("index")))) { + // If index is false we can't emulate with scripts + continue; + } + if ("true".equals(Objects.toString(propertyMap.get("store")))) { + // If store is true we can't emulate with scripts + continue; + } + if (propertyMap.containsKey("ignore_above")) { + // Scripts don't support ignore_above so we skip those fields + continue; + } + if (propertyMap.containsKey("ignore_malformed")) { + // Our source reading script doesn't emulate ignore_malformed + continue; + } + String toLoad = painlessToLoadFromSource(name, type); + if (toLoad == null) { + continue; + } + propertyMap.put("type", "runtime_script"); + propertyMap.put("runtime_type", type); + propertyMap.put("script", toLoad); + propertyMap.remove("store"); + propertyMap.remove("index"); + propertyMap.remove("doc_values"); + } + } + } + return true; + } + + private static String painlessToLoadFromSource(String name, String type) { + String emit = PAINLESS_TO_EMIT.get(type); + if (emit == null) { + return null; + } + StringBuilder b = new StringBuilder(); + b.append("def v = params._source['").append(name).append("'];\n"); + b.append("if (v instanceof Iterable) {\n"); + b.append(" for (def vv : ((Iterable) v)) {\n"); + b.append(" if (vv != null) {\n"); + b.append(" def value = vv;\n"); + b.append(" ").append(emit).append("\n"); + b.append(" }\n"); + b.append(" }\n"); + b.append("} else {\n"); + b.append(" if (v != null) {\n"); + b.append(" def value = v;\n"); + b.append(" ").append(emit).append("\n"); + b.append(" }\n"); + b.append("}\n"); + return b.toString(); + } + + private static final Map PAINLESS_TO_EMIT = Map.ofEntries( + Map.entry(BooleanFieldMapper.CONTENT_TYPE, "emitValue(parse(value));"), + Map.entry(DateFieldMapper.CONTENT_TYPE, "emitValue(parse(value.toString()));"), + Map.entry( + NumberType.DOUBLE.typeName(), + "emitValue(value instanceof Number ? ((Number) value).doubleValue() : Double.parseDouble(value.toString()));" + ), + Map.entry(KeywordFieldMapper.CONTENT_TYPE, "emitValue(value.toString());"), + Map.entry(IpFieldMapper.CONTENT_TYPE, "emitValue(value.toString());"), + Map.entry( + NumberType.LONG.typeName(), + "emitValue(value instanceof Number ? ((Number) value).longValue() : Long.parseLong(value.toString()));" + ) + ); + + private static final ExecutableSection ADD_TEMPLATE = new ExecutableSection() { + @Override + public XContentLocation getLocation() { + return new XContentLocation(-1, -1); + } + + @Override + public void execute(ClientYamlTestExecutionContext executionContext) throws IOException { + Map params = Map.of("name", "convert_to_source_only", "create", "true"); + List> dynamicTemplates = new ArrayList<>(); + for (String type : PAINLESS_TO_EMIT.keySet()) { + if (type.equals("ip")) { + // There isn't a dynamic template to pick up ips. They'll just look like strings. + continue; + } + Map mapping = Map.ofEntries( + Map.entry("type", "runtime_script"), + Map.entry("runtime_type", type), + Map.entry("script", painlessToLoadFromSource("{name}", type)) + ); + Map body = Map.ofEntries( + Map.entry("match_mapping_type", type.equals("keyword") ? "string" : type), + Map.entry("mapping", mapping) + ); + if (type.contentEquals("keyword")) { + /* + * For "string"-type dynamic mappings emulate our default + * behavior with a top level text field and a `.keyword` + * multi-field. But instead of the default, use a runtime + * field for the multi-field. + */ + mapping = Map.of("type", "text", "fields", Map.of("keyword", mapping)); + dynamicTemplates.add(Map.of(type, Map.of("match_mapping_type", "string", "mapping", mapping))); + } else { + dynamicTemplates.add(Map.of(type, Map.of("match_mapping_type", type, "mapping", mapping))); + } + dynamicTemplates.add(Map.of(type, body)); + } + List> bodies = List.of( + Map.ofEntries( + Map.entry("index_patterns", "*"), + Map.entry("priority", Integer.MAX_VALUE - 1), + Map.entry("template", Map.of("settings", Map.of(), "mappings", Map.of("dynamic_templates", dynamicTemplates))) + ) + ); + ClientYamlTestResponse response = executionContext.callApi("indices.put_index_template", params, bodies, Map.of()); + assertThat(response.getStatusCode(), equalTo(200)); + // There are probably some warning about overlapping templates. Ignore them. + } + }; +} diff --git a/x-pack/plugin/runtime-fields/qa/with-security/build.gradle b/x-pack/plugin/runtime-fields/qa/with-security/build.gradle new file mode 100644 index 0000000000000..8442e1aa7b0c2 --- /dev/null +++ b/x-pack/plugin/runtime-fields/qa/with-security/build.gradle @@ -0,0 +1,26 @@ +apply plugin: 'elasticsearch.testclusters' +apply plugin: 'elasticsearch.standalone-rest-test' +apply plugin: 'elasticsearch.rest-test' + +dependencies { + testImplementation project(path: xpackProject('plugin').path, configuration: 'testArtifacts') +} + +def clusterCredentials = [username: System.getProperty('tests.rest.cluster.username', 'test_admin'), + password: System.getProperty('tests.rest.cluster.password', 'x-pack-test-password')] + +integTest { + systemProperty 'tests.rest.cluster.username', clusterCredentials.username + systemProperty 'tests.rest.cluster.password', clusterCredentials.password +} + +testClusters.integTest { + testDistribution = 'DEFAULT' + setting 'xpack.security.enabled', 'true' + setting 'xpack.watcher.enabled', 'false' + setting 'xpack.ml.enabled', 'false' + setting 'xpack.license.self_generated.type', 'trial' + extraConfigFile 'roles.yml', file('roles.yml') + user clusterCredentials + user username: "test", password: "x-pack-test-password", role: "test" +} diff --git a/x-pack/plugin/runtime-fields/qa/with-security/roles.yml b/x-pack/plugin/runtime-fields/qa/with-security/roles.yml new file mode 100644 index 0000000000000..b38ad1b8d0ecf --- /dev/null +++ b/x-pack/plugin/runtime-fields/qa/with-security/roles.yml @@ -0,0 +1,13 @@ +test: + indices: + - names: [ 'dls' ] + privileges: + - read + query: "{\"match\": {\"year\": 2016}}" + - names: [ 'fls' ] + privileges: + - read + field_security: + grant: [ '*' ] + except: [ 'year', 'hidden' ] + diff --git a/x-pack/plugin/runtime-fields/qa/with-security/src/test/java/org/elasticsearch/xpack/security/PermissionsIT.java b/x-pack/plugin/runtime-fields/qa/with-security/src/test/java/org/elasticsearch/xpack/security/PermissionsIT.java new file mode 100644 index 0000000000000..fe6732519a782 --- /dev/null +++ b/x-pack/plugin/runtime-fields/qa/with-security/src/test/java/org/elasticsearch/xpack/security/PermissionsIT.java @@ -0,0 +1,228 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.security; + +import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest; +import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.common.document.DocumentField; +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.junit.AfterClass; +import org.junit.Before; + +import java.io.IOException; +import java.util.Collections; +import java.util.Map; + +import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; + +public class PermissionsIT extends ESRestTestCase { + + private static HighLevelClient highLevelClient; + private static HighLevelClient adminHighLevelClient; + + @Override + protected Settings restClientSettings() { + String token = basicAuthHeaderValue("test", new SecureString("x-pack-test-password".toCharArray())); + return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build(); + } + + @Override + protected Settings restAdminSettings() { + String token = basicAuthHeaderValue("test_admin", new SecureString("x-pack-test-password".toCharArray())); + return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build(); + } + + @Before + public void initHighLevelClient() { + if (highLevelClient == null) { + highLevelClient = new HighLevelClient(client()); + adminHighLevelClient = new HighLevelClient(adminClient()); + } + } + + @AfterClass + public static void closeHighLevelClients() throws IOException { + highLevelClient.close(); + adminHighLevelClient.close(); + highLevelClient = null; + adminHighLevelClient = null; + } + + public void testDLS() throws IOException { + Request createIndex = new Request("PUT", "/dls"); + createIndex.setJsonEntity( + "{\n" + + " \"mappings\" : {\n" + + " \"properties\" : {\n" + + " \"date\" : {\"type\" : \"keyword\"},\n" + + " \"year\" : {\n" + + " \"type\" : \"runtime_script\", \n" + + " \"runtime_type\" : \"keyword\",\n" + + " \"script\" : \"emitValue(doc['date'].value.substring(0,4))\"\n" + + " }\n" + + " }\n" + + " }\n" + + "}\n" + ); + assertOK(adminClient().performRequest(createIndex)); + + Request indexDoc1 = new Request("PUT", "/dls/_doc/1"); + indexDoc1.setJsonEntity("{\n" + " \"date\" : \"2009-11-15T14:12:12\"\n" + "}\n"); + assertOK(adminClient().performRequest(indexDoc1)); + + Request indexDoc2 = new Request("PUT", "/dls/_doc/2"); + indexDoc2.setJsonEntity("{\n" + " \"date\" : \"2016-11-15T14:12:12\"\n" + "}\n"); + assertOK(adminClient().performRequest(indexDoc2)); + + Request indexDoc3 = new Request("PUT", "/dls/_doc/3"); + indexDoc3.addParameter("refresh", "true"); + indexDoc3.setJsonEntity("{\n" + " \"date\" : \"2018-11-15T14:12:12\"\n" + "}\n"); + assertOK(adminClient().performRequest(indexDoc3)); + + SearchRequest searchRequest = new SearchRequest("dls"); + { + SearchResponse searchResponse = adminHighLevelClient.search(searchRequest, RequestOptions.DEFAULT); + assertEquals(3, searchResponse.getHits().getTotalHits().value); + } + { + SearchResponse searchResponse = highLevelClient.search(searchRequest, RequestOptions.DEFAULT); + assertEquals(1, searchResponse.getHits().getTotalHits().value); + } + } + + public void testFLSProtectsData() throws IOException { + Request createIndex = new Request("PUT", "/fls"); + createIndex.setJsonEntity( + "{\n" + + " \"mappings\" : {\n" + + " \"properties\" : {\n" + + " \"hidden\" : {\"type\" : \"keyword\"},\n" + + " \"hidden_values_count\" : {\n" + + " \"type\" : \"runtime_script\", \n" + + " \"runtime_type\" : \"long\",\n" + + " \"script\" : \"emitValue(doc['hidden'].size())\"\n" + + " }\n" + + " }\n" + + " }\n" + + "}\n" + ); + assertOK(adminClient().performRequest(createIndex)); + + Request indexDoc1 = new Request("PUT", "/fls/_doc/1"); + indexDoc1.setJsonEntity("{\n" + " \"hidden\" : \"should not be read\"\n" + "}\n"); + assertOK(adminClient().performRequest(indexDoc1)); + + Request indexDoc2 = new Request("PUT", "/fls/_doc/2"); + indexDoc2.setJsonEntity("{\n" + " \"hidden\" : \"should not be read\"\n" + "}\n"); + assertOK(adminClient().performRequest(indexDoc2)); + + Request indexDoc3 = new Request("PUT", "/fls/_doc/3"); + indexDoc3.addParameter("refresh", "true"); + indexDoc3.setJsonEntity("{\n" + " \"hidden\" : \"should not be read\"\n" + "}\n"); + assertOK(adminClient().performRequest(indexDoc3)); + + SearchRequest searchRequest = new SearchRequest("fls").source(new SearchSourceBuilder().docValueField("hidden_values_count")); + { + SearchResponse searchResponse = adminHighLevelClient.search(searchRequest, RequestOptions.DEFAULT); + assertEquals(3, searchResponse.getHits().getTotalHits().value); + for (SearchHit hit : searchResponse.getHits().getHits()) { + assertEquals(1, hit.getFields().size()); + assertEquals(1, (int) hit.getFields().get("hidden_values_count").getValue()); + } + } + { + SearchResponse searchResponse = highLevelClient.search(searchRequest, RequestOptions.DEFAULT); + assertEquals(3, searchResponse.getHits().getTotalHits().value); + for (SearchHit hit : searchResponse.getHits().getHits()) { + assertEquals(0, (int) hit.getFields().get("hidden_values_count").getValue()); + } + } + } + + public void testFLSOnRuntimeField() throws IOException { + Request createIndex = new Request("PUT", "/fls"); + createIndex.setJsonEntity( + "{\n" + + " \"mappings\" : {\n" + + " \"properties\" : {\n" + + " \"date\" : {\"type\" : \"keyword\"},\n" + + " \"year\" : {\n" + + " \"type\" : \"runtime_script\", \n" + + " \"runtime_type\" : \"keyword\",\n" + + " \"script\" : \"emitValue(doc['date'].value.substring(0,4))\"\n" + + " }\n" + + " }\n" + + " }\n" + + "}\n" + ); + assertOK(adminClient().performRequest(createIndex)); + + Request indexDoc1 = new Request("PUT", "/fls/_doc/1"); + indexDoc1.setJsonEntity("{\n" + " \"date\" : \"2009-11-15T14:12:12\"\n" + "}\n"); + assertOK(adminClient().performRequest(indexDoc1)); + + Request indexDoc2 = new Request("PUT", "/fls/_doc/2"); + indexDoc2.setJsonEntity("{\n" + " \"date\" : \"2016-11-15T14:12:12\"\n" + "}\n"); + assertOK(adminClient().performRequest(indexDoc2)); + + Request indexDoc3 = new Request("PUT", "/fls/_doc/3"); + indexDoc3.addParameter("refresh", "true"); + indexDoc3.setJsonEntity("{\n" + " \"date\" : \"2018-11-15T14:12:12\"\n" + "}\n"); + assertOK(adminClient().performRequest(indexDoc3)); + + // There is no FLS directly on runtime fields + SearchRequest searchRequest = new SearchRequest("fls").source(new SearchSourceBuilder().docValueField("year")); + SearchResponse searchResponse = highLevelClient.search(searchRequest, RequestOptions.DEFAULT); + assertEquals(3, searchResponse.getHits().getTotalHits().value); + for (SearchHit hit : searchResponse.getHits().getHits()) { + Map fields = hit.getFields(); + assertEquals(1, fields.size()); + switch (hit.getId()) { + case "1": + assertEquals("2009", fields.get("year").getValue().toString()); + break; + case "2": + assertEquals("2016", fields.get("year").getValue().toString()); + break; + case "3": + assertEquals("2018", fields.get("year").getValue().toString()); + break; + default: + throw new UnsupportedOperationException(); + } + } + + { + FieldCapabilitiesRequest fieldCapsRequest = new FieldCapabilitiesRequest().indices("fls").fields("year"); + FieldCapabilitiesResponse fieldCapabilitiesResponse = adminHighLevelClient.fieldCaps(fieldCapsRequest, RequestOptions.DEFAULT); + assertNotNull(fieldCapabilitiesResponse.get().get("year")); + } + { + // Though field_caps filters runtime fields out like ordinary fields + FieldCapabilitiesRequest fieldCapsRequest = new FieldCapabilitiesRequest().indices("fls").fields("year"); + FieldCapabilitiesResponse fieldCapabilitiesResponse = highLevelClient.fieldCaps(fieldCapsRequest, RequestOptions.DEFAULT); + assertEquals(0, fieldCapabilitiesResponse.get().size()); + } + } + + private static class HighLevelClient extends RestHighLevelClient { + private HighLevelClient(RestClient restClient) { + super(restClient, (client) -> {}, Collections.emptyList()); + } + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractLongScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractLongScriptFieldScript.java new file mode 100644 index 0000000000000..d7b261a4fae78 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractLongScriptFieldScript.java @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.util.ArrayUtil; +import org.elasticsearch.search.lookup.SearchLookup; + +import java.util.Map; + +/** + * Common base class for script field scripts that return long values. + */ +public abstract class AbstractLongScriptFieldScript extends AbstractScriptFieldScript { + private long[] values = new long[1]; + private int count; + + public AbstractLongScriptFieldScript(Map params, SearchLookup searchLookup, LeafReaderContext ctx) { + super(params, searchLookup, ctx); + } + + /** + * Execute the script for the provided {@code docId}. + */ + public final void runForDoc(int docId) { + count = 0; + setDocument(docId); + execute(); + } + + /** + * Values from the last time {@link #runForDoc(int)} was called. This array + * is mutable and will change with the next call of {@link #runForDoc(int)}. + * It is also oversized and will contain garbage at all indices at and + * above {@link #count()}. + */ + public final long[] values() { + return values; + } + + /** + * The number of results produced the last time {@link #runForDoc(int)} was called. + */ + public final int count() { + return count; + } + + protected final void emitValue(long v) { + if (values.length < count + 1) { + values = ArrayUtil.grow(values, count + 1); + } + values[count++] = v; + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldScript.java new file mode 100644 index 0000000000000..e51fd87c2a057 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldScript.java @@ -0,0 +1,97 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.index.fielddata.ScriptDocValues; +import org.elasticsearch.script.AggregationScript; +import org.elasticsearch.script.DynamicMap; +import org.elasticsearch.script.ScriptCache; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.search.lookup.LeafSearchLookup; +import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.search.lookup.SourceLookup; + +import java.util.HashMap; +import java.util.Map; +import java.util.function.Function; + +import static org.elasticsearch.common.unit.TimeValue.timeValueMillis; + +/** + * Abstract base for scripts to execute to build scripted fields. Inspired by + * {@link AggregationScript} but hopefully with less historical baggage. + */ +public abstract class AbstractScriptFieldScript { + public static ScriptContext newContext(String name, Class factoryClass) { + return new ScriptContext( + name + "_script_field", + factoryClass, + /* + * In an ideal world we wouldn't need the script cache at all + * because we have a hard reference to the script. The trouble + * is that we compile the scripts a few times when performing + * a mapping update. This is unfortunate, but we rely on the + * cache to speed this up. + */ + 100, + timeValueMillis(0), + /* + * Disable compilation rate limits for scripted fields so we + * don't prevent mapping updates because we've performed too + * many recently. That'd just be lame. + */ + ScriptCache.UNLIMITED_COMPILATION_RATE.asTuple() + ); + } + + private static final Map> PARAMS_FUNCTIONS = Map.of( + "_source", + value -> ((SourceLookup) value).loadSourceIfNeeded() + ); + + private final Map params; + private final LeafSearchLookup leafSearchLookup; + + public AbstractScriptFieldScript(Map params, SearchLookup searchLookup, LeafReaderContext ctx) { + this.leafSearchLookup = searchLookup.getLeafSearchLookup(ctx); + params = new HashMap<>(params); + params.put("_source", leafSearchLookup.source()); + params.put("_fields", leafSearchLookup.fields()); + this.params = new DynamicMap(params, PARAMS_FUNCTIONS); + } + + /** + * Set the document to run the script against. + */ + public final void setDocument(int docId) { + this.leafSearchLookup.setDocument(docId); + } + + /** + * Expose the {@code params} of the script to the script itself. + */ + public final Map getParams() { + return params; + } + + /** + * Expose the {@code _source} to the script. + */ + protected final Map getSource() { + return leafSearchLookup.source(); + } + + /** + * Expose field data to the script as {@code doc}. + */ + public final Map> getDoc() { + return leafSearchLookup.doc(); + } + + public abstract void execute(); +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/BooleanScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/BooleanScriptFieldScript.java new file mode 100644 index 0000000000000..2378877af1aae --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/BooleanScriptFieldScript.java @@ -0,0 +1,92 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.common.Booleans; +import org.elasticsearch.painless.spi.Whitelist; +import org.elasticsearch.painless.spi.WhitelistLoader; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptFactory; +import org.elasticsearch.search.lookup.SearchLookup; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +public abstract class BooleanScriptFieldScript extends AbstractScriptFieldScript { + public static final ScriptContext CONTEXT = newContext("boolean_script_field", Factory.class); + + static List whitelist() { + return List.of(WhitelistLoader.loadFromResourceFiles(RuntimeFieldsPainlessExtension.class, "boolean_whitelist.txt")); + } + + public static final String[] PARAMETERS = {}; + + public interface Factory extends ScriptFactory { + LeafFactory newFactory(Map params, SearchLookup searchLookup); + } + + public interface LeafFactory { + BooleanScriptFieldScript newInstance(LeafReaderContext ctx) throws IOException; + } + + private int trues; + private int falses; + + public BooleanScriptFieldScript(Map params, SearchLookup searchLookup, LeafReaderContext ctx) { + super(params, searchLookup, ctx); + } + + /** + * Execute the script for the provided {@code docId}. + */ + public final void runForDoc(int docId) { + trues = 0; + falses = 0; + setDocument(docId); + execute(); + } + + /** + * How many {@code true} values were returned for this document. + */ + public final int trues() { + return trues; + } + + /** + * How many {@code false} values were returned for this document. + */ + public final int falses() { + return falses; + } + + protected final void emitValue(boolean v) { + if (v) { + trues++; + } else { + falses++; + } + } + + public static boolean parse(Object str) { + return Booleans.parseBoolean(str.toString()); + } + + public static class EmitValue { + private final BooleanScriptFieldScript script; + + public EmitValue(BooleanScriptFieldScript script) { + this.script = script; + } + + public void value(boolean v) { + script.emitValue(v); + } + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DateScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DateScriptFieldScript.java new file mode 100644 index 0000000000000..1ac65a35f2788 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DateScriptFieldScript.java @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.common.time.DateFormatter; +import org.elasticsearch.painless.spi.Whitelist; +import org.elasticsearch.painless.spi.WhitelistLoader; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptFactory; +import org.elasticsearch.search.lookup.SearchLookup; + +import java.io.IOException; +import java.time.temporal.ChronoField; +import java.time.temporal.TemporalAccessor; +import java.util.List; +import java.util.Map; + +public abstract class DateScriptFieldScript extends AbstractLongScriptFieldScript { + public static final ScriptContext CONTEXT = newContext("date", Factory.class); + + static List whitelist() { + return List.of(WhitelistLoader.loadFromResourceFiles(RuntimeFieldsPainlessExtension.class, "date_whitelist.txt")); + } + + public static final String[] PARAMETERS = {}; + + public interface Factory extends ScriptFactory { + LeafFactory newFactory(Map params, SearchLookup searchLookup, DateFormatter formatter); + } + + public interface LeafFactory { + DateScriptFieldScript newInstance(LeafReaderContext ctx) throws IOException; + } + + private final DateFormatter formatter; + + public DateScriptFieldScript(Map params, SearchLookup searchLookup, DateFormatter formatter, LeafReaderContext ctx) { + super(params, searchLookup, ctx); + this.formatter = formatter; + } + + public static long toEpochMilli(TemporalAccessor v) { + // TemporalAccessor is a nanos API so we have to convert. + long millis = Math.multiplyExact(v.getLong(ChronoField.INSTANT_SECONDS), 1000); + millis = Math.addExact(millis, v.get(ChronoField.NANO_OF_SECOND) / 1_000_000); + return millis; + } + + public static class EmitValue { + private final DateScriptFieldScript script; + + public EmitValue(DateScriptFieldScript script) { + this.script = script; + } + + public void emitValue(long v) { + script.emitValue(v); + } + } + + public static class Parse { + private final DateScriptFieldScript script; + + public Parse(DateScriptFieldScript script) { + this.script = script; + } + + public long parse(Object str) { + return script.formatter.parseMillis(str.toString()); + } + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScript.java new file mode 100644 index 0000000000000..03172e983965c --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScript.java @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.util.ArrayUtil; +import org.elasticsearch.painless.spi.Whitelist; +import org.elasticsearch.painless.spi.WhitelistLoader; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptFactory; +import org.elasticsearch.search.lookup.SearchLookup; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +public abstract class DoubleScriptFieldScript extends AbstractScriptFieldScript { + public static final ScriptContext CONTEXT = newContext("double_script_field", Factory.class); + + static List whitelist() { + return List.of(WhitelistLoader.loadFromResourceFiles(RuntimeFieldsPainlessExtension.class, "double_whitelist.txt")); + } + + public static final String[] PARAMETERS = {}; + + public interface Factory extends ScriptFactory { + LeafFactory newFactory(Map params, SearchLookup searchLookup); + } + + public interface LeafFactory { + DoubleScriptFieldScript newInstance(LeafReaderContext ctx) throws IOException; + } + + private double[] values = new double[1]; + private int count; + + public DoubleScriptFieldScript(Map params, SearchLookup searchLookup, LeafReaderContext ctx) { + super(params, searchLookup, ctx); + } + + /** + * Execute the script for the provided {@code docId}. + */ + public final void runForDoc(int docId) { + count = 0; + setDocument(docId); + execute(); + } + + /** + * Values from the last time {@link #runForDoc(int)} was called. This array + * is mutable and will change with the next call of {@link #runForDoc(int)}. + * It is also oversized and will contain garbage at all indices at and + * above {@link #count()}. + */ + public final double[] values() { + return values; + } + + /** + * The number of results produced the last time {@link #runForDoc(int)} was called. + */ + public final int count() { + return count; + } + + protected final void emitValue(double v) { + if (values.length < count + 1) { + values = ArrayUtil.grow(values, count + 1); + } + values[count++] = v; + } + + public static class EmitValue { + private final DoubleScriptFieldScript script; + + public EmitValue(DoubleScriptFieldScript script) { + this.script = script; + } + + public void emitValue(double v) { + script.emitValue(v); + } + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/IpScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/IpScriptFieldScript.java new file mode 100644 index 0000000000000..5486fc9307cfb --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/IpScriptFieldScript.java @@ -0,0 +1,113 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.apache.lucene.document.InetAddressPoint; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.network.InetAddresses; +import org.elasticsearch.index.mapper.IpFieldMapper; +import org.elasticsearch.painless.spi.Whitelist; +import org.elasticsearch.painless.spi.WhitelistLoader; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptFactory; +import org.elasticsearch.search.lookup.SearchLookup; + +import java.io.IOException; +import java.net.Inet4Address; +import java.net.Inet6Address; +import java.net.InetAddress; +import java.util.List; +import java.util.Map; + +/** + * Script producing IP addresses. Unlike the other {@linkplain AbstractScriptFieldScript}s + * which deal with their native java objects this converts its values to the same format + * that Lucene uses to store its fields, {@link InetAddressPoint}. There are a few compelling + * reasons to do this: + *
    + *
  • {@link Inet4Address}es and {@link Inet6Address} are not comparable with one another. + * That is correct in some contexts, but not for our queries. Our queries must consider the + * IPv4 address equal to the address that it maps to in IPv6 rfc4291). + *
  • {@link InetAddress}es are not ordered, but we need to implement range queries with + * same same ordering as {@link IpFieldMapper}. That also uses {@link InetAddressPoint} + * so it saves us a lot of trouble to use the same representation. + *
+ */ +public abstract class IpScriptFieldScript extends AbstractScriptFieldScript { + public static final ScriptContext CONTEXT = newContext("ip_script_field", Factory.class); + + static List whitelist() { + return List.of(WhitelistLoader.loadFromResourceFiles(RuntimeFieldsPainlessExtension.class, "ip_whitelist.txt")); + } + + public static final String[] PARAMETERS = {}; + + public interface Factory extends ScriptFactory { + LeafFactory newFactory(Map params, SearchLookup searchLookup); + } + + public interface LeafFactory { + IpScriptFieldScript newInstance(LeafReaderContext ctx) throws IOException; + } + + private BytesRef[] values = new BytesRef[1]; + private int count; + + public IpScriptFieldScript(Map params, SearchLookup searchLookup, LeafReaderContext ctx) { + super(params, searchLookup, ctx); + } + + /** + * Execute the script for the provided {@code docId}. + */ + public final void runForDoc(int docId) { + count = 0; + setDocument(docId); + execute(); + } + + /** + * Values from the last time {@link #runForDoc(int)} was called. This array + * is mutable and will change with the next call of {@link #runForDoc(int)}. + * It is also oversized and will contain garbage at all indices at and + * above {@link #count()}. + *

+ * All values are IPv6 addresses so they are 16 bytes. IPv4 addresses are + * encoded by rfc4291. + */ + public final BytesRef[] values() { + return values; + } + + /** + * The number of results produced the last time {@link #runForDoc(int)} was called. + */ + public final int count() { + return count; + } + + protected final void emitValue(String v) { + if (values.length < count + 1) { + values = ArrayUtil.grow(values, count + 1); + } + values[count++] = new BytesRef(InetAddressPoint.encode(InetAddresses.forString(v))); + } + + public static class EmitValue { + private final IpScriptFieldScript script; + + public EmitValue(IpScriptFieldScript script) { + this.script = script; + } + + public void emitValue(String v) { + script.emitValue(v); + } + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java new file mode 100644 index 0000000000000..1ad0c2b40bb70 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.painless.spi.Whitelist; +import org.elasticsearch.painless.spi.WhitelistLoader; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptFactory; +import org.elasticsearch.search.lookup.SearchLookup; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +public abstract class LongScriptFieldScript extends AbstractLongScriptFieldScript { + public static final ScriptContext CONTEXT = newContext("long_script_field", Factory.class); + + static List whitelist() { + return List.of(WhitelistLoader.loadFromResourceFiles(RuntimeFieldsPainlessExtension.class, "long_whitelist.txt")); + } + + public static final String[] PARAMETERS = {}; + + public interface Factory extends ScriptFactory { + LeafFactory newFactory(Map params, SearchLookup searchLookup); + } + + public interface LeafFactory { + LongScriptFieldScript newInstance(LeafReaderContext ctx) throws IOException; + } + + public LongScriptFieldScript(Map params, SearchLookup searchLookup, LeafReaderContext ctx) { + super(params, searchLookup, ctx); + } + + public static class EmitValue { + private final LongScriptFieldScript script; + + public EmitValue(LongScriptFieldScript script) { + this.script = script; + } + + public void emitValue(long v) { + script.emitValue(v); + } + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java new file mode 100644 index 0000000000000..b174460a513b4 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.plugins.MapperPlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.ScriptPlugin; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.xpack.runtimefields.mapper.RuntimeScriptFieldMapper; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public final class RuntimeFields extends Plugin implements MapperPlugin, ScriptPlugin { + + @Override + public Map getMappers() { + return Collections.singletonMap(RuntimeScriptFieldMapper.CONTENT_TYPE, RuntimeScriptFieldMapper.PARSER); + } + + @Override + public List> getContexts() { + return List.of( + BooleanScriptFieldScript.CONTEXT, + DateScriptFieldScript.CONTEXT, + DoubleScriptFieldScript.CONTEXT, + IpScriptFieldScript.CONTEXT, + LongScriptFieldScript.CONTEXT, + StringScriptFieldScript.CONTEXT + ); + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFieldsPainlessExtension.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFieldsPainlessExtension.java new file mode 100644 index 0000000000000..1229099ab7cea --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFieldsPainlessExtension.java @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.elasticsearch.painless.spi.PainlessExtension; +import org.elasticsearch.painless.spi.Whitelist; +import org.elasticsearch.script.ScriptContext; + +import java.util.List; +import java.util.Map; + +public class RuntimeFieldsPainlessExtension implements PainlessExtension { + @Override + public Map, List> getContextWhitelists() { + return Map.ofEntries( + Map.entry(BooleanScriptFieldScript.CONTEXT, BooleanScriptFieldScript.whitelist()), + Map.entry(DateScriptFieldScript.CONTEXT, DateScriptFieldScript.whitelist()), + Map.entry(DoubleScriptFieldScript.CONTEXT, DoubleScriptFieldScript.whitelist()), + Map.entry(IpScriptFieldScript.CONTEXT, IpScriptFieldScript.whitelist()), + Map.entry(LongScriptFieldScript.CONTEXT, LongScriptFieldScript.whitelist()), + Map.entry(StringScriptFieldScript.CONTEXT, StringScriptFieldScript.whitelist()) + ); + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java new file mode 100644 index 0000000000000..40ab2ecd2ded1 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.painless.spi.Whitelist; +import org.elasticsearch.painless.spi.WhitelistLoader; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptFactory; +import org.elasticsearch.search.lookup.SearchLookup; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public abstract class StringScriptFieldScript extends AbstractScriptFieldScript { + public static final ScriptContext CONTEXT = newContext("string_script_field", Factory.class); + + static List whitelist() { + return List.of(WhitelistLoader.loadFromResourceFiles(RuntimeFieldsPainlessExtension.class, "string_whitelist.txt")); + } + + public static final String[] PARAMETERS = {}; + + public interface Factory extends ScriptFactory { + LeafFactory newFactory(Map params, SearchLookup searchLookup); + } + + public interface LeafFactory { + StringScriptFieldScript newInstance(LeafReaderContext ctx) throws IOException; + } + + private final List results = new ArrayList<>(); + + public StringScriptFieldScript(Map params, SearchLookup searchLookup, LeafReaderContext ctx) { + super(params, searchLookup, ctx); + } + + /** + * Execute the script for the provided {@code docId}. + *

+ * @return a mutable {@link List} that contains the results of the script + * and will be modified the next time you call {@linkplain #resultsForDoc}. + */ + public final List resultsForDoc(int docId) { + results.clear(); + setDocument(docId); + execute(); + return results; + } + + protected final void emitValue(String v) { + results.add(v); + } + + public static class EmitValue { + private final StringScriptFieldScript script; + + public EmitValue(StringScriptFieldScript script) { + this.script = script; + } + + public void emitValue(String v) { + script.emitValue(v); + } + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBinaryFieldData.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBinaryFieldData.java new file mode 100644 index 0000000000000..7db0d6bd79b4d --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBinaryFieldData.java @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.fielddata; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.SortField; +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.LeafFieldData; +import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.MultiValueMode; +import org.elasticsearch.search.sort.BucketedSort; +import org.elasticsearch.search.sort.SortOrder; + +public abstract class ScriptBinaryFieldData implements IndexFieldData { + private final String fieldName; + + protected ScriptBinaryFieldData(String fieldName) { + this.fieldName = fieldName; + } + + @Override + public String getFieldName() { + return fieldName; + } + + @Override + public ScriptBinaryLeafFieldData load(LeafReaderContext context) { + try { + return loadDirect(context); + } catch (Exception e) { + throw ExceptionsHelper.convertToElastic(e); + } + } + + @Override + public SortField sortField(Object missingValue, MultiValueMode sortMode, XFieldComparatorSource.Nested nested, boolean reverse) { + final XFieldComparatorSource source = new BytesRefFieldComparatorSource(this, missingValue, sortMode, nested); + return new SortField(getFieldName(), source, reverse); + } + + @Override + public BucketedSort newBucketedSort( + BigArrays bigArrays, + Object missingValue, + MultiValueMode sortMode, + XFieldComparatorSource.Nested nested, + SortOrder sortOrder, + DocValueFormat format, + int bucketSize, + BucketedSort.ExtraData extra + ) { + throw new IllegalArgumentException("only supported on numeric fields"); + } + + public abstract class ScriptBinaryLeafFieldData implements LeafFieldData { + @Override + public long ramBytesUsed() { + return 0; + } + + @Override + public void close() { + + } + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBooleanDocValues.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBooleanDocValues.java new file mode 100644 index 0000000000000..f91fe3d300eaf --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBooleanDocValues.java @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.fielddata; + +import org.elasticsearch.index.fielddata.AbstractSortedNumericDocValues; +import org.elasticsearch.xpack.runtimefields.BooleanScriptFieldScript; + +import java.io.IOException; + +public final class ScriptBooleanDocValues extends AbstractSortedNumericDocValues { + private final BooleanScriptFieldScript script; + private int cursor; + + ScriptBooleanDocValues(BooleanScriptFieldScript script) { + this.script = script; + } + + @Override + public boolean advanceExact(int docId) { + script.runForDoc(docId); + cursor = 0; + return script.trues() > 0 || script.falses() > 0; + } + + @Override + public long nextValue() throws IOException { + // Emit all false values before all true values + return cursor++ < script.falses() ? 0 : 1; + } + + @Override + public int docValueCount() { + return script.trues() + script.falses(); + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBooleanFieldData.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBooleanFieldData.java new file mode 100644 index 0000000000000..41a8c814bddc2 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBooleanFieldData.java @@ -0,0 +1,99 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.fielddata; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SortedNumericDocValues; +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.IndexFieldDataCache; +import org.elasticsearch.index.fielddata.IndexNumericFieldData; +import org.elasticsearch.index.fielddata.plain.LeafLongFieldData; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.indices.breaker.CircuitBreakerService; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.xpack.runtimefields.BooleanScriptFieldScript; + +import java.io.IOException; + +public final class ScriptBooleanFieldData extends IndexNumericFieldData { + + public static class Builder implements IndexFieldData.Builder { + private final String name; + private final BooleanScriptFieldScript.LeafFactory leafFactory; + + public Builder(String name, BooleanScriptFieldScript.LeafFactory leafFactory) { + this.name = name; + this.leafFactory = leafFactory; + } + + @Override + public ScriptBooleanFieldData build(IndexFieldDataCache cache, CircuitBreakerService breakerService, MapperService mapperService) { + return new ScriptBooleanFieldData(name, leafFactory); + } + } + + private final String fieldName; + private final BooleanScriptFieldScript.LeafFactory leafFactory; + + private ScriptBooleanFieldData(String fieldName, BooleanScriptFieldScript.LeafFactory leafFactory) { + this.fieldName = fieldName; + this.leafFactory = leafFactory; + } + + @Override + public String getFieldName() { + return fieldName; + } + + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.BOOLEAN; + } + + @Override + public ScriptBooleanLeafFieldData load(LeafReaderContext context) { + try { + return loadDirect(context); + } catch (Exception e) { + throw ExceptionsHelper.convertToElastic(e); + } + } + + @Override + public ScriptBooleanLeafFieldData loadDirect(LeafReaderContext context) throws IOException { + return new ScriptBooleanLeafFieldData(new ScriptBooleanDocValues(leafFactory.newInstance(context))); + } + + @Override + public NumericType getNumericType() { + return NumericType.BOOLEAN; + } + + @Override + protected boolean sortRequiresCustomComparator() { + return true; + } + + public static class ScriptBooleanLeafFieldData extends LeafLongFieldData { + private final ScriptBooleanDocValues scriptBooleanDocValues; + + ScriptBooleanLeafFieldData(ScriptBooleanDocValues scriptBooleanDocValues) { + super(0, NumericType.BOOLEAN); + this.scriptBooleanDocValues = scriptBooleanDocValues; + } + + @Override + public SortedNumericDocValues getLongValues() { + return scriptBooleanDocValues; + } + + @Override + public void close() {} + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptDateFieldData.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptDateFieldData.java new file mode 100644 index 0000000000000..4a4e9b58e47ef --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptDateFieldData.java @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.fielddata; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SortedNumericDocValues; +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.IndexFieldDataCache; +import org.elasticsearch.index.fielddata.IndexNumericFieldData; +import org.elasticsearch.index.fielddata.plain.LeafLongFieldData; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.indices.breaker.CircuitBreakerService; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.xpack.runtimefields.DateScriptFieldScript; + +import java.io.IOException; + +public final class ScriptDateFieldData extends IndexNumericFieldData { + + public static class Builder implements IndexFieldData.Builder { + private final String name; + private final DateScriptFieldScript.LeafFactory leafFactory; + + public Builder(String name, DateScriptFieldScript.LeafFactory leafFactory) { + this.name = name; + this.leafFactory = leafFactory; + } + + @Override + public ScriptDateFieldData build(IndexFieldDataCache cache, CircuitBreakerService breakerService, MapperService mapperService) { + return new ScriptDateFieldData(name, leafFactory); + } + } + + private final String fieldName; + private final DateScriptFieldScript.LeafFactory leafFactory; + + private ScriptDateFieldData(String fieldName, DateScriptFieldScript.LeafFactory leafFactory) { + this.fieldName = fieldName; + this.leafFactory = leafFactory; + } + + @Override + public String getFieldName() { + return fieldName; + } + + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.DATE; + } + + @Override + public ScriptDateLeafFieldData load(LeafReaderContext context) { + try { + return loadDirect(context); + } catch (Exception e) { + throw ExceptionsHelper.convertToElastic(e); + } + } + + @Override + public ScriptDateLeafFieldData loadDirect(LeafReaderContext context) throws IOException { + return new ScriptDateLeafFieldData(new ScriptLongDocValues(leafFactory.newInstance(context))); + } + + @Override + public NumericType getNumericType() { + return NumericType.DATE; + } + + @Override + protected boolean sortRequiresCustomComparator() { + return true; + } + + public static class ScriptDateLeafFieldData extends LeafLongFieldData { + private final ScriptLongDocValues scriptLongDocValues; + + ScriptDateLeafFieldData(ScriptLongDocValues scriptLongDocValues) { + super(0, NumericType.DATE); + this.scriptLongDocValues = scriptLongDocValues; + } + + @Override + public SortedNumericDocValues getLongValues() { + return scriptLongDocValues; + } + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptDoubleDocValues.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptDoubleDocValues.java new file mode 100644 index 0000000000000..135229e260eba --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptDoubleDocValues.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.fielddata; + +import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; +import org.elasticsearch.xpack.runtimefields.DoubleScriptFieldScript; + +import java.io.IOException; +import java.util.Arrays; + +public final class ScriptDoubleDocValues extends SortedNumericDoubleValues { + private final DoubleScriptFieldScript script; + private int cursor; + + ScriptDoubleDocValues(DoubleScriptFieldScript script) { + this.script = script; + } + + @Override + public boolean advanceExact(int docId) { + script.runForDoc(docId); + if (script.count() == 0) { + return false; + } + Arrays.sort(script.values(), 0, script.count()); + cursor = 0; + return true; + } + + @Override + public double nextValue() throws IOException { + return script.values()[cursor++]; + } + + @Override + public int docValueCount() { + return script.count(); + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptDoubleFieldData.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptDoubleFieldData.java new file mode 100644 index 0000000000000..a15a79016e434 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptDoubleFieldData.java @@ -0,0 +1,99 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.fielddata; + +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.IndexFieldDataCache; +import org.elasticsearch.index.fielddata.IndexNumericFieldData; +import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; +import org.elasticsearch.index.fielddata.plain.LeafDoubleFieldData; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.indices.breaker.CircuitBreakerService; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.xpack.runtimefields.DoubleScriptFieldScript; + +import java.io.IOException; + +public final class ScriptDoubleFieldData extends IndexNumericFieldData { + + public static class Builder implements IndexFieldData.Builder { + private final String name; + private final DoubleScriptFieldScript.LeafFactory leafFactory; + + public Builder(String name, DoubleScriptFieldScript.LeafFactory leafFactory) { + this.name = name; + this.leafFactory = leafFactory; + } + + @Override + public ScriptDoubleFieldData build(IndexFieldDataCache cache, CircuitBreakerService breakerService, MapperService mapperService) { + return new ScriptDoubleFieldData(name, leafFactory); + } + } + + private final String fieldName; + DoubleScriptFieldScript.LeafFactory leafFactory; + + private ScriptDoubleFieldData(String fieldName, DoubleScriptFieldScript.LeafFactory leafFactory) { + this.fieldName = fieldName; + this.leafFactory = leafFactory; + } + + @Override + public String getFieldName() { + return fieldName; + } + + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.NUMERIC; + } + + @Override + public ScriptDoubleLeafFieldData load(LeafReaderContext context) { + try { + return loadDirect(context); + } catch (Exception e) { + throw ExceptionsHelper.convertToElastic(e); + } + } + + @Override + public ScriptDoubleLeafFieldData loadDirect(LeafReaderContext context) throws IOException { + return new ScriptDoubleLeafFieldData(new ScriptDoubleDocValues(leafFactory.newInstance(context))); + } + + @Override + public NumericType getNumericType() { + return NumericType.DOUBLE; + } + + @Override + protected boolean sortRequiresCustomComparator() { + return true; + } + + public static class ScriptDoubleLeafFieldData extends LeafDoubleFieldData { + private final ScriptDoubleDocValues scriptDoubleDocValues; + + ScriptDoubleLeafFieldData(ScriptDoubleDocValues scriptDoubleDocValues) { + super(0); + this.scriptDoubleDocValues = scriptDoubleDocValues; + } + + @Override + public SortedNumericDoubleValues getDoubleValues() { + return scriptDoubleDocValues; + } + + @Override + public void close() {} + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptIpDocValues.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptIpDocValues.java new file mode 100644 index 0000000000000..0cecde2302d41 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptIpDocValues.java @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.fielddata; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.index.fielddata.SortedBinaryDocValues; +import org.elasticsearch.xpack.runtimefields.IpScriptFieldScript; + +import java.io.IOException; +import java.util.Arrays; + +public final class ScriptIpDocValues extends SortedBinaryDocValues { + private final IpScriptFieldScript script; + private int cursor; + + ScriptIpDocValues(IpScriptFieldScript script) { + this.script = script; + } + + @Override + public boolean advanceExact(int docId) { + script.runForDoc(docId); + if (script.count() == 0) { + return false; + } + Arrays.sort(script.values(), 0, script.count()); + cursor = 0; + return true; + } + + @Override + public BytesRef nextValue() throws IOException { + return script.values()[cursor++]; + } + + @Override + public int docValueCount() { + return script.count(); + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptIpFieldData.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptIpFieldData.java new file mode 100644 index 0000000000000..e9e7e86b3ab32 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptIpFieldData.java @@ -0,0 +1,88 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.fielddata; + +import org.apache.lucene.document.InetAddressPoint; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.network.InetAddresses; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.IndexFieldDataCache; +import org.elasticsearch.index.fielddata.ScriptDocValues; +import org.elasticsearch.index.fielddata.SortedBinaryDocValues; +import org.elasticsearch.index.mapper.IpFieldMapper; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.indices.breaker.CircuitBreakerService; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.xpack.runtimefields.IpScriptFieldScript; + +import java.net.InetAddress; + +public class ScriptIpFieldData extends ScriptBinaryFieldData { + public static class Builder implements IndexFieldData.Builder { + private final String name; + private final IpScriptFieldScript.LeafFactory leafFactory; + + public Builder(String name, IpScriptFieldScript.LeafFactory leafFactory) { + this.name = name; + this.leafFactory = leafFactory; + } + + @Override + public ScriptIpFieldData build(IndexFieldDataCache cache, CircuitBreakerService breakerService, MapperService mapperService) { + return new ScriptIpFieldData(name, leafFactory); + } + } + + private final IpScriptFieldScript.LeafFactory leafFactory; + + private ScriptIpFieldData(String fieldName, IpScriptFieldScript.LeafFactory leafFactory) { + super(fieldName); + this.leafFactory = leafFactory; + } + + @Override + public ScriptBinaryLeafFieldData loadDirect(LeafReaderContext context) throws Exception { + IpScriptFieldScript script = leafFactory.newInstance(context); + return new ScriptBinaryLeafFieldData() { + @Override + public ScriptDocValues getScriptValues() { + return new IpScriptDocValues(getBytesValues()); + } + + @Override + public SortedBinaryDocValues getBytesValues() { + return new ScriptIpDocValues(script); + } + }; + } + + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.IP; + } + + /** + * Doc values implementation for ips. We can't share + * {@link IpFieldMapper.IpFieldType.IpScriptDocValues} because it is based + * on global ordinals and we don't have those. + */ + public static class IpScriptDocValues extends ScriptDocValues.Strings { + public IpScriptDocValues(SortedBinaryDocValues in) { + super(in); + } + + @Override + protected String bytesToString(BytesRef bytes) { + InetAddress addr = InetAddressPoint.decode(BytesReference.toBytes(new BytesArray(bytes))); + return InetAddresses.toAddrString(addr); + } + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptLongDocValues.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptLongDocValues.java new file mode 100644 index 0000000000000..3f7da9cec5a5c --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptLongDocValues.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.fielddata; + +import org.elasticsearch.index.fielddata.AbstractSortedNumericDocValues; +import org.elasticsearch.xpack.runtimefields.AbstractLongScriptFieldScript; + +import java.io.IOException; +import java.util.Arrays; + +public final class ScriptLongDocValues extends AbstractSortedNumericDocValues { + private final AbstractLongScriptFieldScript script; + private int cursor; + + ScriptLongDocValues(AbstractLongScriptFieldScript script) { + this.script = script; + } + + @Override + public boolean advanceExact(int docId) { + script.runForDoc(docId); + if (script.count() == 0) { + return false; + } + Arrays.sort(script.values(), 0, script.count()); + cursor = 0; + return true; + } + + @Override + public long nextValue() throws IOException { + return script.values()[cursor++]; + } + + @Override + public int docValueCount() { + return script.count(); + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptLongFieldData.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptLongFieldData.java new file mode 100644 index 0000000000000..7ad3e377d9254 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptLongFieldData.java @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.fielddata; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SortedNumericDocValues; +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.IndexFieldDataCache; +import org.elasticsearch.index.fielddata.IndexNumericFieldData; +import org.elasticsearch.index.fielddata.plain.LeafLongFieldData; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.indices.breaker.CircuitBreakerService; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.xpack.runtimefields.LongScriptFieldScript; + +import java.io.IOException; + +public final class ScriptLongFieldData extends IndexNumericFieldData { + + public static class Builder implements IndexFieldData.Builder { + private final String name; + private final LongScriptFieldScript.LeafFactory leafFactory; + + public Builder(String name, LongScriptFieldScript.LeafFactory leafFactory) { + this.name = name; + this.leafFactory = leafFactory; + } + + @Override + public ScriptLongFieldData build(IndexFieldDataCache cache, CircuitBreakerService breakerService, MapperService mapperService) { + return new ScriptLongFieldData(name, leafFactory); + } + } + + private final String fieldName; + private final LongScriptFieldScript.LeafFactory leafFactory; + + private ScriptLongFieldData(String fieldName, LongScriptFieldScript.LeafFactory leafFactory) { + this.fieldName = fieldName; + this.leafFactory = leafFactory; + } + + @Override + public String getFieldName() { + return fieldName; + } + + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.NUMERIC; + } + + @Override + public ScriptLongLeafFieldData load(LeafReaderContext context) { + try { + return loadDirect(context); + } catch (Exception e) { + throw ExceptionsHelper.convertToElastic(e); + } + } + + @Override + public ScriptLongLeafFieldData loadDirect(LeafReaderContext context) throws IOException { + return new ScriptLongLeafFieldData(new ScriptLongDocValues(leafFactory.newInstance(context))); + } + + @Override + public NumericType getNumericType() { + return NumericType.LONG; + } + + @Override + protected boolean sortRequiresCustomComparator() { + return true; + } + + public static class ScriptLongLeafFieldData extends LeafLongFieldData { + private final ScriptLongDocValues scriptLongDocValues; + + ScriptLongLeafFieldData(ScriptLongDocValues scriptLongDocValues) { + super(0, NumericType.LONG); + this.scriptLongDocValues = scriptLongDocValues; + } + + @Override + public SortedNumericDocValues getLongValues() { + return scriptLongDocValues; + } + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptStringDocValues.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptStringDocValues.java new file mode 100644 index 0000000000000..ada5dcff13e29 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptStringDocValues.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.fielddata; + +import org.elasticsearch.index.fielddata.SortingBinaryDocValues; +import org.elasticsearch.xpack.runtimefields.StringScriptFieldScript; + +import java.util.List; + +public final class ScriptStringDocValues extends SortingBinaryDocValues { + private final StringScriptFieldScript script; + + ScriptStringDocValues(StringScriptFieldScript script) { + this.script = script; + } + + @Override + public boolean advanceExact(int docId) { + List results = script.resultsForDoc(docId); + count = results.size(); + if (count == 0) { + return false; + } + + grow(); + int i = 0; + for (String value : results) { + values[i++].copyChars(value); + } + sort(); + return true; + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptStringFieldData.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptStringFieldData.java new file mode 100644 index 0000000000000..6c851a6be852a --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptStringFieldData.java @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.fielddata; + +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.IndexFieldDataCache; +import org.elasticsearch.index.fielddata.ScriptDocValues; +import org.elasticsearch.index.fielddata.SortedBinaryDocValues; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.indices.breaker.CircuitBreakerService; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.xpack.runtimefields.StringScriptFieldScript; + +public class ScriptStringFieldData extends ScriptBinaryFieldData { + public static class Builder implements IndexFieldData.Builder { + private final String name; + private final StringScriptFieldScript.LeafFactory leafFactory; + + public Builder(String name, StringScriptFieldScript.LeafFactory leafFactory) { + this.name = name; + this.leafFactory = leafFactory; + } + + @Override + public ScriptStringFieldData build(IndexFieldDataCache cache, CircuitBreakerService breakerService, MapperService mapperService) { + return new ScriptStringFieldData(name, leafFactory); + } + } + + private final StringScriptFieldScript.LeafFactory leafFactory; + + private ScriptStringFieldData(String fieldName, StringScriptFieldScript.LeafFactory leafFactory) { + super(fieldName); + this.leafFactory = leafFactory; + } + + @Override + public ScriptBinaryLeafFieldData loadDirect(LeafReaderContext context) throws Exception { + StringScriptFieldScript script = leafFactory.newInstance(context); + return new ScriptBinaryLeafFieldData() { + @Override + public ScriptDocValues getScriptValues() { + return new ScriptDocValues.Strings(getBytesValues()); + } + + @Override + public SortedBinaryDocValues getBytesValues() { + return new ScriptStringDocValues(script); + } + }; + } + + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.BYTES; + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractScriptMappedFieldType.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractScriptMappedFieldType.java new file mode 100644 index 0000000000000..f35464a2bcf2c --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractScriptMappedFieldType.java @@ -0,0 +1,174 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.mapper; + +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.search.MultiTermQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.spans.SpanMultiTermQueryWrapper; +import org.apache.lucene.search.spans.SpanQuery; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.geo.ShapeRelation; +import org.elasticsearch.common.time.DateMathParser; +import org.elasticsearch.common.unit.Fuzziness; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.TextSearchInfo; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.script.Script; + +import java.io.IOException; +import java.time.ZoneId; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import static org.elasticsearch.search.SearchService.ALLOW_EXPENSIVE_QUERIES; + +/** + * Abstract base {@linkplain MappedFieldType} for scripted fields. + */ +abstract class AbstractScriptMappedFieldType extends MappedFieldType { + protected final Script script; + + AbstractScriptMappedFieldType(String name, Script script, Map meta) { + super(name, false, false, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); + this.script = script; + } + + protected abstract String runtimeType(); + + @Override + public final String typeName() { + return RuntimeScriptFieldMapper.CONTENT_TYPE; + } + + @Override + public final String familyTypeName() { + return runtimeType(); + } + + @Override + public final boolean isSearchable() { + return true; + } + + @Override + public final boolean isAggregatable() { + return true; + } + + public abstract Query termsQuery(List values, QueryShardContext context); + + @Override + public final Query rangeQuery( + Object lowerTerm, + Object upperTerm, + boolean includeLower, + boolean includeUpper, + ShapeRelation relation, + ZoneId timeZone, + DateMathParser parser, + QueryShardContext context + ) { + if (relation == ShapeRelation.DISJOINT) { + String message = "Field [%s] of type [%s] with runtime type [%s] does not support DISJOINT ranges"; + throw new IllegalArgumentException(String.format(Locale.ROOT, message, name(), typeName(), runtimeType())); + } + return rangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, timeZone, parser, context); + } + + protected abstract Query rangeQuery( + Object lowerTerm, + Object upperTerm, + boolean includeLower, + boolean includeUpper, + ZoneId timeZone, + DateMathParser parser, + QueryShardContext context + ); + + public Query fuzzyQuery( + Object value, + Fuzziness fuzziness, + int prefixLength, + int maxExpansions, + boolean transpositions, + QueryShardContext context + ) { + throw new IllegalArgumentException(unsupported("fuzzy", "keyword and text")); + } + + public Query prefixQuery(String value, MultiTermQuery.RewriteMethod method, QueryShardContext context) { + throw new IllegalArgumentException(unsupported("prefix", "keyword, text and wildcard")); + } + + public Query wildcardQuery(String value, MultiTermQuery.RewriteMethod method, QueryShardContext context) { + throw new IllegalArgumentException(unsupported("wildcard", "keyword, text and wildcard")); + } + + public Query regexpQuery( + String value, + int flags, + int maxDeterminizedStates, + MultiTermQuery.RewriteMethod method, + QueryShardContext context + ) { + throw new IllegalArgumentException(unsupported("regexp", "keyword and text")); + } + + public abstract Query existsQuery(QueryShardContext context); + + public Query phraseQuery(TokenStream stream, int slop, boolean enablePositionIncrements) throws IOException { + throw new IllegalArgumentException(unsupported("phrase", "text")); + } + + public Query multiPhraseQuery(TokenStream stream, int slop, boolean enablePositionIncrements) throws IOException { + throw new IllegalArgumentException(unsupported("phrase", "text")); + } + + public Query phrasePrefixQuery(TokenStream stream, int slop, int maxExpansions) throws IOException { + throw new IllegalArgumentException(unsupported("phrase prefix", "text")); + } + + public SpanQuery spanPrefixQuery(String value, SpanMultiTermQueryWrapper.SpanRewriteMethod method, QueryShardContext context) { + throw new IllegalArgumentException(unsupported("span prefix", "text")); + } + + private String unsupported(String query, String supported) { + String thisField = "[" + name() + "] which is of type [script] with runtime_type [" + runtimeType() + "]"; + return "Can only use " + query + " queries on " + supported + " fields - not on " + thisField; + } + + protected final void checkAllowExpensiveQueries(QueryShardContext context) { + if (context.allowExpensiveQueries() == false) { + throw new ElasticsearchException( + "queries cannot be executed against [" + + RuntimeScriptFieldMapper.CONTENT_TYPE + + "] fields while [" + + ALLOW_EXPENSIVE_QUERIES.getKey() + + "] is set to [false]." + ); + } + } + + /** + * The format that this field should use. The default implementation is + * {@code null} because most fields don't support formats. + */ + protected String format() { + return null; + } + + /** + * The locale that this field's format should use. The default + * implementation is {@code null} because most fields don't + * support formats. + */ + protected Locale formatLocale() { + return null; + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeScriptFieldMapper.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeScriptFieldMapper.java new file mode 100644 index 0000000000000..1f57f7445f5d1 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeScriptFieldMapper.java @@ -0,0 +1,298 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.mapper; + +import org.elasticsearch.common.time.DateFormatter; +import org.elasticsearch.common.util.LocaleUtils; +import org.elasticsearch.index.mapper.BooleanFieldMapper; +import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.IpFieldMapper; +import org.elasticsearch.index.mapper.KeywordFieldMapper; +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; +import org.elasticsearch.index.mapper.ParametrizedFieldMapper; +import org.elasticsearch.index.mapper.ParseContext; +import org.elasticsearch.index.mapper.ValueFetcher; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptType; +import org.elasticsearch.xpack.runtimefields.BooleanScriptFieldScript; +import org.elasticsearch.xpack.runtimefields.DateScriptFieldScript; +import org.elasticsearch.xpack.runtimefields.DoubleScriptFieldScript; +import org.elasticsearch.xpack.runtimefields.IpScriptFieldScript; +import org.elasticsearch.xpack.runtimefields.LongScriptFieldScript; +import org.elasticsearch.xpack.runtimefields.StringScriptFieldScript; + +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.function.BiFunction; + +public final class RuntimeScriptFieldMapper extends ParametrizedFieldMapper { + + public static final String CONTENT_TYPE = "runtime_script"; + + public static final TypeParser PARSER = new TypeParser((name, parserContext) -> new Builder(name, new ScriptCompiler() { + @Override + public FactoryType compile(Script script, ScriptContext context) { + return parserContext.scriptService().compile(script, context); + } + })); + + private final String runtimeType; + private final Script script; + private final ScriptCompiler scriptCompiler; + + protected RuntimeScriptFieldMapper( + String simpleName, + AbstractScriptMappedFieldType mappedFieldType, + MultiFields multiFields, + CopyTo copyTo, + String runtimeType, + Script script, + ScriptCompiler scriptCompiler + ) { + super(simpleName, mappedFieldType, multiFields, copyTo); + this.runtimeType = runtimeType; + this.script = script; + this.scriptCompiler = scriptCompiler; + } + + @Override + public ParametrizedFieldMapper.Builder getMergeBuilder() { + return new RuntimeScriptFieldMapper.Builder(simpleName(), scriptCompiler).init(this); + } + + @Override + protected void parseCreateField(ParseContext context) { + // there is no lucene field + } + + @Override + public ValueFetcher valueFetcher(MapperService mapperService, String format) { + throw new UnsupportedOperationException(); + } + + @Override + protected String contentType() { + return CONTENT_TYPE; + } + + public static class Builder extends ParametrizedFieldMapper.Builder { + + static final Map> FIELD_TYPE_RESOLVER = Map.of( + BooleanFieldMapper.CONTENT_TYPE, + (builder, context) -> { + builder.formatAndLocaleNotSupported(); + BooleanScriptFieldScript.Factory factory = builder.scriptCompiler.compile( + builder.script.getValue(), + BooleanScriptFieldScript.CONTEXT + ); + return new ScriptBooleanMappedFieldType( + builder.buildFullName(context), + builder.script.getValue(), + factory, + builder.meta.getValue() + ); + }, + DateFieldMapper.CONTENT_TYPE, + (builder, context) -> { + DateScriptFieldScript.Factory factory = builder.scriptCompiler.compile( + builder.script.getValue(), + DateScriptFieldScript.CONTEXT + ); + String format = builder.format.getValue(); + if (format == null) { + format = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.pattern(); + } + Locale locale = builder.locale.getValue(); + if (locale == null) { + locale = Locale.ROOT; + } + DateFormatter dateTimeFormatter = DateFormatter.forPattern(format).withLocale(locale); + return new ScriptDateMappedFieldType( + builder.buildFullName(context), + builder.script.getValue(), + factory, + dateTimeFormatter, + builder.meta.getValue() + ); + }, + NumberType.DOUBLE.typeName(), + (builder, context) -> { + builder.formatAndLocaleNotSupported(); + DoubleScriptFieldScript.Factory factory = builder.scriptCompiler.compile( + builder.script.getValue(), + DoubleScriptFieldScript.CONTEXT + ); + return new ScriptDoubleMappedFieldType( + builder.buildFullName(context), + builder.script.getValue(), + factory, + builder.meta.getValue() + ); + }, + IpFieldMapper.CONTENT_TYPE, + (builder, context) -> { + builder.formatAndLocaleNotSupported(); + IpScriptFieldScript.Factory factory = builder.scriptCompiler.compile( + builder.script.getValue(), + IpScriptFieldScript.CONTEXT + ); + return new ScriptIpMappedFieldType( + builder.buildFullName(context), + builder.script.getValue(), + factory, + builder.meta.getValue() + ); + }, + KeywordFieldMapper.CONTENT_TYPE, + (builder, context) -> { + builder.formatAndLocaleNotSupported(); + StringScriptFieldScript.Factory factory = builder.scriptCompiler.compile( + builder.script.getValue(), + StringScriptFieldScript.CONTEXT + ); + return new ScriptKeywordMappedFieldType( + builder.buildFullName(context), + builder.script.getValue(), + factory, + builder.meta.getValue() + ); + }, + NumberType.LONG.typeName(), + (builder, context) -> { + builder.formatAndLocaleNotSupported(); + LongScriptFieldScript.Factory factory = builder.scriptCompiler.compile( + builder.script.getValue(), + LongScriptFieldScript.CONTEXT + ); + return new ScriptLongMappedFieldType( + builder.buildFullName(context), + builder.script.getValue(), + factory, + builder.meta.getValue() + ); + } + ); + + private static RuntimeScriptFieldMapper toType(FieldMapper in) { + return (RuntimeScriptFieldMapper) in; + } + + private final Parameter> meta = Parameter.metaParam(); + private final Parameter runtimeType = Parameter.stringParam( + "runtime_type", + true, + mapper -> toType(mapper).runtimeType, + null + ).setValidator(runtimeType -> { + if (runtimeType == null) { + throw new IllegalArgumentException("runtime_type must be specified for " + CONTENT_TYPE + " field [" + name + "]"); + } + }); + private final Parameter