From 7d8be9f7bd6db4f2bc5ceedb31f167e7bd7dbc9f Mon Sep 17 00:00:00 2001 From: Jake Landis Date: Mon, 3 Jul 2023 03:14:40 -0500 Subject: [PATCH 001/111] adjust scope to clear API key cache (#96778) --- .../security/rest/action/apikey/RestClearApiKeyCacheAction.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestClearApiKeyCacheAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestClearApiKeyCacheAction.java index ceb71d762e261..895c93adbad6e 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestClearApiKeyCacheAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestClearApiKeyCacheAction.java @@ -23,7 +23,7 @@ import static org.elasticsearch.rest.RestRequest.Method.POST; -@ServerlessScope(Scope.PUBLIC) +@ServerlessScope(Scope.INTERNAL) public class RestClearApiKeyCacheAction extends SecurityBaseRestHandler { public RestClearApiKeyCacheAction(Settings settings, XPackLicenseState licenseState) { From 81aee669259b2debd3eb1705df204a5ff05916c9 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Mon, 3 Jul 2023 10:39:41 +0200 Subject: [PATCH 002/111] Remove redundant Datastream.TimestampField class (#97298) We can just inline all usage of this thing for now and keep the assertion about the field name for parsing BwC. -> saves some code and questions about the timestamp field --- .../datastreams/DataStreamIT.java | 5 - .../DataStreamsStatsTransportAction.java | 3 +- .../indices/resolve/ResolveIndexAction.java | 4 +- .../datastreams/GetDataStreamAction.java | 5 +- .../metadata/ComposableIndexTemplate.java | 6 - .../cluster/metadata/DataStream.java | 120 ++++-------------- .../uid/PerThreadIDVersionAndSeqNoLookup.java | 4 +- .../index/mapper/MappingLookup.java | 2 +- .../elasticsearch/index/shard/IndexShard.java | 4 +- .../indices/TimestampFieldMapperService.java | 2 +- .../elasticsearch/ingest/IngestService.java | 4 +- .../support/TimeSeriesIndexSearcher.java | 2 +- .../indices/create/AutoCreateActionTests.java | 3 - .../CanMatchPreFilterSearchPhaseTests.java | 32 +++-- .../ComposableIndexTemplateTests.java | 5 +- .../cluster/metadata/DataStreamTests.java | 9 -- .../metadata/ToAndFromJsonMetadataTests.java | 2 - .../common/lucene/uid/VersionLookupTests.java | 4 +- .../common/lucene/uid/VersionsTests.java | 4 +- .../ingest/IngestServiceTests.java | 6 +- .../support/TimeSeriesCancellationTests.java | 4 +- .../support/TimeSeriesIndexSearcherTests.java | 19 ++- .../metadata/DataStreamTestHelper.java | 2 +- .../elasticsearch/xpack/ccr/AutoFollowIT.java | 4 +- .../action/TransportPutFollowActionTests.java | 5 - .../index/engine/frozen/FrozenIndexIT.java | 6 +- .../index/engine/frozen/FrozenIndexTests.java | 10 +- ...pshotsCanMatchOnCoordinatorIntegTests.java | 10 +- .../SearchableSnapshotsIntegTests.java | 4 +- .../GeoLineAggregationBuilder.java | 7 +- 30 files changed, 105 insertions(+), 192 deletions(-) diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java index 83bb6eebb1ec9..e7e54450a861e 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java @@ -165,12 +165,10 @@ public void testBasicScenario() throws Exception { assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(2)); DataStream barDataStream = getDataStreamResponse.getDataStreams().get(0).getDataStream(); assertThat(barDataStream.getName(), equalTo("metrics-bar")); - assertThat(barDataStream.getTimeStampField().getName(), equalTo("@timestamp")); assertThat(barDataStream.getIndices().size(), equalTo(1)); assertThat(barDataStream.getIndices().get(0).getName(), backingIndexEqualTo("metrics-bar", 1)); DataStream fooDataStream = getDataStreamResponse.getDataStreams().get(1).getDataStream(); assertThat(fooDataStream.getName(), equalTo("metrics-foo")); - assertThat(fooDataStream.getTimeStampField().getName(), equalTo("@timestamp")); assertThat(fooDataStream.getIndices().size(), equalTo(1)); assertThat(fooDataStream.getIndices().get(0).getName(), backingIndexEqualTo("metrics-foo", 1)); @@ -462,7 +460,6 @@ public void testComposableTemplateOnlyMatchingWithDataStreamName() throws Except .actionGet(); assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1)); assertThat(getDataStreamResponse.getDataStreams().get(0).getDataStream().getName(), equalTo(dataStreamName)); - assertThat(getDataStreamResponse.getDataStreams().get(0).getDataStream().getTimeStampField().getName(), equalTo("@timestamp")); assertThat(getDataStreamResponse.getDataStreams().get(0).getDataStream().getIndices().size(), equalTo(1)); String backingIndex = getDataStreamResponse.getDataStreams().get(0).getDataStream().getIndices().get(0).getName(); assertThat(backingIndex, backingIndexEqualTo(dataStreamName, 1)); @@ -1087,7 +1084,6 @@ public void testTimestampFieldCustomAttributes() throws Exception { .actionGet(); assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1)); assertThat(getDataStreamResponse.getDataStreams().get(0).getDataStream().getName(), equalTo("logs-foobar")); - assertThat(getDataStreamResponse.getDataStreams().get(0).getDataStream().getTimeStampField().getName(), equalTo("@timestamp")); Map expectedTimestampMapping = Map.of("type", "date", "format", "yyyy-MM", "meta", Map.of("x", "y")); assertBackingIndex( getDataStreamResponse.getDataStreams().get(0).getDataStream().getWriteIndex().getName(), @@ -1499,7 +1495,6 @@ public void testDataStreamMetadata() throws Exception { assertThat(info.getIlmPolicy(), nullValue()); DataStream dataStream = info.getDataStream(); assertThat(dataStream.getName(), equalTo("logs-foobar")); - assertThat(dataStream.getTimeStampField().getName(), equalTo("@timestamp")); assertThat(dataStream.getIndices().size(), equalTo(1)); assertThat(dataStream.getIndices().get(0).getName(), backingIndexEqualTo("logs-foobar", 1)); assertThat(dataStream.getMetadata(), equalTo(Map.of("managed_by", "core-features"))); diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DataStreamsStatsTransportAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DataStreamsStatsTransportAction.java index e948ee6718b43..e63825509d551 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DataStreamsStatsTransportAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DataStreamsStatsTransportAction.java @@ -148,8 +148,7 @@ protected void shardOperation( long maxTimestamp = 0L; try (Engine.Searcher searcher = indexShard.acquireSearcher("data_stream_stats")) { IndexReader indexReader = searcher.getIndexReader(); - String fieldName = dataStream.getTimeStampField().getName(); - byte[] maxPackedValue = PointValues.getMaxPackedValue(indexReader, fieldName); + byte[] maxPackedValue = PointValues.getMaxPackedValue(indexReader, DataStream.TIMESTAMP_FIELD_NAME); if (maxPackedValue != null) { maxTimestamp = LongPoint.decodeDimension(maxPackedValue, 0); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java index a0013f6f65ca9..8b684cc648eae 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java @@ -636,9 +636,7 @@ private static void enrichIndexAbstraction( case DATA_STREAM -> { DataStream dataStream = (DataStream) ia; String[] backingIndices = dataStream.getIndices().stream().map(Index::getName).toArray(String[]::new); - dataStreams.add( - new ResolvedDataStream(dataStream.getName(), backingIndices, dataStream.getTimeStampField().getName()) - ); + dataStreams.add(new ResolvedDataStream(dataStream.getName(), backingIndices, DataStream.TIMESTAMP_FIELD_NAME)); } default -> throw new IllegalStateException("unknown index abstraction type: " + ia.getType()); } diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java index 742d9b59489c2..13e927097468d 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java @@ -238,7 +238,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params, @Nulla throws IOException { builder.startObject(); builder.field(DataStream.NAME_FIELD.getPreferredName(), dataStream.getName()); - builder.field(DataStream.TIMESTAMP_FIELD_FIELD.getPreferredName(), dataStream.getTimeStampField()); + builder.field(DataStream.TIMESTAMP_FIELD_FIELD.getPreferredName()) + .startObject() + .field(DataStream.NAME_FIELD.getPreferredName(), DataStream.TIMESTAMP_FIELD_NAME) + .endObject(); builder.xContentList(DataStream.INDICES_FIELD.getPreferredName(), dataStream.getIndices()); builder.field(DataStream.GENERATION_FIELD.getPreferredName(), dataStream.getGeneration()); if (dataStream.getMetadata() != null) { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java index f93b117c8554e..c465bc492ed06 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java @@ -31,8 +31,6 @@ import java.util.Map; import java.util.Objects; -import static org.elasticsearch.cluster.metadata.DataStream.TimestampField.FIXED_TIMESTAMP_FIELD; - /** * An index template consists of a set of index patterns, an optional template, and a list of * ids corresponding to component templates that should be composed in order when creating a new @@ -411,10 +409,6 @@ public DataStreamTemplate(boolean hidden, boolean allowCustomRouting) { } } - public static String getTimestampField() { - return FIXED_TIMESTAMP_FIELD; - } - /** * A mapping snippet for a backing index with `_data_stream_timestamp` meta field mapper properly configured. */ diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java index acc257f08423a..ec1096531bd56 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java @@ -23,7 +23,6 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.time.DateFormatters; import org.elasticsearch.core.Nullable; @@ -68,11 +67,11 @@ public final class DataStream implements SimpleDiffable, ToXContentO public static final String BACKING_INDEX_PREFIX = ".ds-"; public static final DateFormatter DATE_FORMATTER = DateFormatter.forPattern("uuuu.MM.dd"); - public static final TimestampField TIMESTAMP_FIELD = new DataStream.TimestampField("@timestamp"); + public static final String TIMESTAMP_FIELD_NAME = "@timestamp"; // Timeseries indices' leaf readers should be sorted by desc order of their timestamp field, as it allows search time optimizations public static Comparator TIMESERIES_LEAF_READERS_SORTER = Comparator.comparingLong((LeafReader r) -> { try { - PointValues points = r.getPointValues(DataStream.TimestampField.FIXED_TIMESTAMP_FIELD); + PointValues points = r.getPointValues(TIMESTAMP_FIELD_NAME); if (points != null) { byte[] sortValue = points.getMaxPackedValue(); return LongPoint.decodeDimension(sortValue, 0); @@ -84,10 +83,7 @@ public final class DataStream implements SimpleDiffable, ToXContentO return Long.MIN_VALUE; } } catch (IOException e) { - throw new ElasticsearchException( - "Can't access [" + DataStream.TimestampField.FIXED_TIMESTAMP_FIELD + "] field for the index!", - e - ); + throw new ElasticsearchException("Can't access [" + TIMESTAMP_FIELD_NAME + "] field for the index!", e); } }).reversed(); @@ -200,11 +196,6 @@ public boolean isDataStreamRelated() { return true; } - public TimestampField getTimeStampField() { - // This was always fixed to @timestamp with the idea that one day this field could be configurable. This idea no longer exists. - return TIMESTAMP_FIELD; - } - @Override public List getIndices() { return indices; @@ -775,7 +766,7 @@ public static Diff readDiffFrom(StreamInput in) throws IOException { @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(name); - TIMESTAMP_FIELD.writeTo(out); + out.writeString(TIMESTAMP_FIELD_NAME); out.writeList(indices); out.writeVLong(generation); out.writeGenericMap(metadata); @@ -806,25 +797,32 @@ public void writeTo(StreamOutput out) throws IOException { public static final ParseField LIFECYCLE = new ParseField("lifecycle"); @SuppressWarnings("unchecked") - private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("data_stream", args -> { - assert TIMESTAMP_FIELD == args[1]; - return new DataStream( + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "data_stream", + args -> new DataStream( (String) args[0], - (List) args[2], - (Long) args[3], - (Map) args[4], + (List) args[1], + (Long) args[2], + (Map) args[3], + args[4] != null && (boolean) args[4], args[5] != null && (boolean) args[5], args[6] != null && (boolean) args[6], args[7] != null && (boolean) args[7], - args[8] != null && (boolean) args[8], - args[9] != null ? IndexMode.fromString((String) args[9]) : null, - DataLifecycle.isEnabled() ? (DataLifecycle) args[10] : null - ); - }); + args[8] != null ? IndexMode.fromString((String) args[8]) : null, + DataLifecycle.isEnabled() ? (DataLifecycle) args[9] : null + ) + ); static { PARSER.declareString(ConstructingObjectParser.constructorArg(), NAME_FIELD); - PARSER.declareObject(ConstructingObjectParser.constructorArg(), TimestampField.PARSER, TIMESTAMP_FIELD_FIELD); + final ConstructingObjectParser tsFieldParser = new ConstructingObjectParser<>("timestamp_field", args -> { + if (TIMESTAMP_FIELD_NAME.equals(args[0]) == false) { + throw new IllegalArgumentException("unexpected timestamp field [" + args[0] + "]"); + } + return TIMESTAMP_FIELD_NAME; + }); + tsFieldParser.declareString(ConstructingObjectParser.constructorArg(), NAME_FIELD); + PARSER.declareObject((f, v) -> { assert v == TIMESTAMP_FIELD_NAME; }, tsFieldParser, TIMESTAMP_FIELD_FIELD); PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(), (p, c) -> Index.fromXContent(p), INDICES_FIELD); PARSER.declareLong(ConstructingObjectParser.constructorArg(), GENERATION_FIELD); PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> p.map(), METADATA_FIELD); @@ -854,7 +852,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params, @Nulla throws IOException { builder.startObject(); builder.field(NAME_FIELD.getPreferredName(), name); - builder.field(TIMESTAMP_FIELD_FIELD.getPreferredName(), TIMESTAMP_FIELD); + builder.field(TIMESTAMP_FIELD_FIELD.getPreferredName()) + .startObject() + .field(NAME_FIELD.getPreferredName(), TIMESTAMP_FIELD_NAME) + .endObject(); builder.xContentList(INDICES_FIELD.getPreferredName(), indices); builder.field(GENERATION_FIELD.getPreferredName(), generation); if (metadata != null) { @@ -947,7 +948,7 @@ public DataStream getParentDataStream() { } public static final XContentParserConfiguration TS_EXTRACT_CONFIG = XContentParserConfiguration.EMPTY.withFiltering( - Set.of(TimestampField.FIXED_TIMESTAMP_FIELD), + Set.of(TIMESTAMP_FIELD_NAME), null, false ); @@ -995,71 +996,6 @@ private static Instant getTimestampFromParser(BytesReference source, XContentTyp } } - public static final class TimestampField implements Writeable, ToXContentObject { - - public static final String FIXED_TIMESTAMP_FIELD = "@timestamp"; - - static ParseField NAME_FIELD = new ParseField("name"); - - @SuppressWarnings("unchecked") - private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( - "timestamp_field", - args -> { - if (FIXED_TIMESTAMP_FIELD.equals(args[0]) == false) { - throw new IllegalArgumentException("unexpected timestamp field [" + args[0] + "]"); - } - return TIMESTAMP_FIELD; - } - ); - - static { - PARSER.declareString(ConstructingObjectParser.constructorArg(), NAME_FIELD); - } - - private final String name; - - public TimestampField(String name) { - if (FIXED_TIMESTAMP_FIELD.equals(name) == false) { - throw new IllegalArgumentException("unexpected timestamp field [" + name + "]"); - } - this.name = name; - } - - public TimestampField(StreamInput in) throws IOException { - this(in.readString()); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeString(name); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - builder.field(NAME_FIELD.getPreferredName(), name); - builder.endObject(); - return builder; - } - - public String getName() { - return name; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - TimestampField that = (TimestampField) o; - return name.equals(that.name); - } - - @Override - public int hashCode() { - return Objects.hash(name); - } - } - /** * Modifies the passed Instant object to be used as a bound for a timestamp field in TimeSeries. It needs to be called in both backing * index construction (rollover) and index selection for doc insertion. Failure to do so may lead to errors due to document timestamps diff --git a/server/src/main/java/org/elasticsearch/common/lucene/uid/PerThreadIDVersionAndSeqNoLookup.java b/server/src/main/java/org/elasticsearch/common/lucene/uid/PerThreadIDVersionAndSeqNoLookup.java index 99a2c69985f79..52ddaa9a87589 100644 --- a/server/src/main/java/org/elasticsearch/common/lucene/uid/PerThreadIDVersionAndSeqNoLookup.java +++ b/server/src/main/java/org/elasticsearch/common/lucene/uid/PerThreadIDVersionAndSeqNoLookup.java @@ -96,8 +96,8 @@ final class PerThreadIDVersionAndSeqNoLookup { this.loadedTimestampRange = loadTimestampRange; // Also check for the existence of the timestamp field, because sometimes a segment can only contain tombstone documents, // which don't have any mapped fields (also not the timestamp field) and just some meta fields like _id, _seq_no etc. - if (loadTimestampRange && reader.getFieldInfos().fieldInfo(DataStream.TimestampField.FIXED_TIMESTAMP_FIELD) != null) { - PointValues tsPointValues = reader.getPointValues(DataStream.TimestampField.FIXED_TIMESTAMP_FIELD); + if (loadTimestampRange && reader.getFieldInfos().fieldInfo(DataStream.TIMESTAMP_FIELD_NAME) != null) { + PointValues tsPointValues = reader.getPointValues(DataStream.TIMESTAMP_FIELD_NAME); assert tsPointValues != null : "no timestamp field for reader:" + reader + " and parent:" + reader.getContext().parent.reader(); minTimestamp = LongPoint.decodeDimension(tsPointValues.getMinPackedValue(), 0); maxTimestamp = LongPoint.decodeDimension(tsPointValues.getMaxPackedValue(), 0); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java b/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java index a7b18e1e62139..7c44f33fbafa5 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java @@ -452,7 +452,7 @@ public boolean isDataStreamTimestampFieldEnabled() { * @return {@code true} if contains a timestamp field of type date that is indexed and has doc values, {@code false} otherwise. */ public boolean hasTimestampField() { - final MappedFieldType mappedFieldType = fieldTypesLookup().get(DataStream.TimestampField.FIXED_TIMESTAMP_FIELD); + final MappedFieldType mappedFieldType = fieldTypesLookup().get(DataStream.TIMESTAMP_FIELD_NAME); if (mappedFieldType instanceof DateFieldMapper.DateFieldType) { return mappedFieldType.isIndexed() && mappedFieldType.hasDocValues(); } else { diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java index 2201476d0f482..e3df5967cf77a 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -2068,7 +2068,7 @@ public ShardLongFieldRange getTimestampRange() { if (mapperService() == null) { return ShardLongFieldRange.UNKNOWN; // no mapper service, no idea if the field even exists } - final MappedFieldType mappedFieldType = mapperService().fieldType(DataStream.TimestampField.FIXED_TIMESTAMP_FIELD); + final MappedFieldType mappedFieldType = mapperService().fieldType(DataStream.TIMESTAMP_FIELD_NAME); if (mappedFieldType instanceof DateFieldMapper.DateFieldType == false) { return ShardLongFieldRange.UNKNOWN; // field missing or not a date } @@ -2078,7 +2078,7 @@ public ShardLongFieldRange getTimestampRange() { final ShardLongFieldRange rawTimestampFieldRange; try { - rawTimestampFieldRange = getEngine().getRawFieldRange(DataStream.TimestampField.FIXED_TIMESTAMP_FIELD); + rawTimestampFieldRange = getEngine().getRawFieldRange(DataStream.TIMESTAMP_FIELD_NAME); assert rawTimestampFieldRange != null; } catch (IOException | AlreadyClosedException e) { logger.debug("exception obtaining range for timestamp field", e); diff --git a/server/src/main/java/org/elasticsearch/indices/TimestampFieldMapperService.java b/server/src/main/java/org/elasticsearch/indices/TimestampFieldMapperService.java index 49187bd7a9003..15e409df552bd 100644 --- a/server/src/main/java/org/elasticsearch/indices/TimestampFieldMapperService.java +++ b/server/src/main/java/org/elasticsearch/indices/TimestampFieldMapperService.java @@ -151,7 +151,7 @@ private static boolean hasUsefulTimestampField(IndexMetadata indexMetadata) { } private static DateFieldMapper.DateFieldType fromMapperService(MapperService mapperService) { - final MappedFieldType mappedFieldType = mapperService.fieldType(DataStream.TimestampField.FIXED_TIMESTAMP_FIELD); + final MappedFieldType mappedFieldType = mapperService.fieldType(DataStream.TIMESTAMP_FIELD_NAME); if (mappedFieldType instanceof DateFieldMapper.DateFieldType) { return (DateFieldMapper.DateFieldType) mappedFieldType; } else { diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestService.java b/server/src/main/java/org/elasticsearch/ingest/IngestService.java index c9c37512ae9ad..ef596eb490be6 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestService.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestService.java @@ -30,7 +30,7 @@ import org.elasticsearch.cluster.ClusterStateApplier; import org.elasticsearch.cluster.ClusterStateTaskExecutor; import org.elasticsearch.cluster.ClusterStateTaskListener; -import org.elasticsearch.cluster.metadata.DataStream.TimestampField; +import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; @@ -1056,7 +1056,7 @@ private static void updateIndexRequestSource(final IndexRequest request, final I private static void cacheRawTimestamp(final IndexRequest request, final IngestDocument document) { if (request.getRawTimestamp() == null) { // cache the @timestamp from the ingest document's source map if there is one - Object rawTimestamp = document.getSource().get(TimestampField.FIXED_TIMESTAMP_FIELD); + Object rawTimestamp = document.getSource().get(DataStream.TIMESTAMP_FIELD_NAME); if (rawTimestamp != null) { request.setRawTimestamp(rawTimestamp); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/TimeSeriesIndexSearcher.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/TimeSeriesIndexSearcher.java index 7b568e4a28533..9aafa31e85c17 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/TimeSeriesIndexSearcher.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/TimeSeriesIndexSearcher.java @@ -223,7 +223,7 @@ private static class LeafWalker { this.scorer = scorer; iterator = scorer.iterator(); tsids = DocValues.getSorted(context.reader(), TimeSeriesIdFieldMapper.NAME); - timestamps = DocValues.getSortedNumeric(context.reader(), DataStream.TimestampField.FIXED_TIMESTAMP_FIELD); + timestamps = DocValues.getSortedNumeric(context.reader(), DataStream.TIMESTAMP_FIELD_NAME); } void collectCurrent() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/create/AutoCreateActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/create/AutoCreateActionTests.java index 622c42923e717..f09f565582f4a 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/create/AutoCreateActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/create/AutoCreateActionTests.java @@ -15,7 +15,6 @@ import java.util.List; -import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; @@ -48,13 +47,11 @@ public void testResolveTemplates() { ComposableIndexTemplate result = AutoCreateAction.resolveTemplate(request, metadata); assertThat(result, notNullValue()); assertThat(result.getDataStreamTemplate(), notNullValue()); - assertThat(DataStreamTemplate.getTimestampField(), equalTo("@timestamp")); request = new CreateIndexRequest("logs-barbaz"); result = AutoCreateAction.resolveTemplate(request, metadata); assertThat(result, notNullValue()); assertThat(result.getDataStreamTemplate(), notNullValue()); - assertThat(DataStreamTemplate.getTimestampField(), equalTo("@timestamp")); // An index that matches with a template without a data steam definition request = new CreateIndexRequest("legacy-logs-foobaz"); diff --git a/server/src/test/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhaseTests.java index 7f5e4808b51d2..8b94efaba012c 100644 --- a/server/src/test/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhaseTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhaseTests.java @@ -471,12 +471,16 @@ public void testCanMatchFilteringOnCoordinatorThatCanBeSkipped() throws Exceptio long indexMinTimestamp = randomLongBetween(0, 5000); long indexMaxTimestamp = randomLongBetween(indexMinTimestamp, 5000 * 2); StaticCoordinatorRewriteContextProviderBuilder contextProviderBuilder = new StaticCoordinatorRewriteContextProviderBuilder(); - String timestampFieldName = dataStream.getTimeStampField().getName(); for (Index dataStreamIndex : dataStream.getIndices()) { - contextProviderBuilder.addIndexMinMaxTimestamps(dataStreamIndex, timestampFieldName, indexMinTimestamp, indexMaxTimestamp); + contextProviderBuilder.addIndexMinMaxTimestamps( + dataStreamIndex, + DataStream.TIMESTAMP_FIELD_NAME, + indexMinTimestamp, + indexMaxTimestamp + ); } - RangeQueryBuilder rangeQueryBuilder = new RangeQueryBuilder(timestampFieldName); + RangeQueryBuilder rangeQueryBuilder = new RangeQueryBuilder(DataStream.TIMESTAMP_FIELD_NAME); // We query a range outside of the timestamp range covered by both datastream indices rangeQueryBuilder.from(indexMaxTimestamp + 1).to(indexMaxTimestamp + 2); @@ -538,12 +542,16 @@ public void testCanMatchFilteringOnCoordinatorParsingFails() throws Exception { long indexMinTimestamp = randomLongBetween(0, 5000); long indexMaxTimestamp = randomLongBetween(indexMinTimestamp, 5000 * 2); StaticCoordinatorRewriteContextProviderBuilder contextProviderBuilder = new StaticCoordinatorRewriteContextProviderBuilder(); - String timestampFieldName = dataStream.getTimeStampField().getName(); for (Index dataStreamIndex : dataStream.getIndices()) { - contextProviderBuilder.addIndexMinMaxTimestamps(dataStreamIndex, timestampFieldName, indexMinTimestamp, indexMaxTimestamp); + contextProviderBuilder.addIndexMinMaxTimestamps( + dataStreamIndex, + DataStream.TIMESTAMP_FIELD_NAME, + indexMinTimestamp, + indexMaxTimestamp + ); } - RangeQueryBuilder rangeQueryBuilder = new RangeQueryBuilder(timestampFieldName); + RangeQueryBuilder rangeQueryBuilder = new RangeQueryBuilder(DataStream.TIMESTAMP_FIELD_NAME); // Query with a non default date format rangeQueryBuilder.from("2020-1-01").to("2021-1-01"); @@ -575,16 +583,20 @@ public void testCanMatchFilteringOnCoordinatorThatCanNotBeSkipped() throws Excep long indexMinTimestamp = 10; long indexMaxTimestamp = 20; StaticCoordinatorRewriteContextProviderBuilder contextProviderBuilder = new StaticCoordinatorRewriteContextProviderBuilder(); - String timestampFieldName = dataStream.getTimeStampField().getName(); for (Index dataStreamIndex : dataStream.getIndices()) { - contextProviderBuilder.addIndexMinMaxTimestamps(dataStreamIndex, timestampFieldName, indexMinTimestamp, indexMaxTimestamp); + contextProviderBuilder.addIndexMinMaxTimestamps( + dataStreamIndex, + DataStream.TIMESTAMP_FIELD_NAME, + indexMinTimestamp, + indexMaxTimestamp + ); } BoolQueryBuilder queryBuilder = new BoolQueryBuilder(); // Query inside of the data stream index range if (randomBoolean()) { // Query generation - RangeQueryBuilder rangeQueryBuilder = new RangeQueryBuilder(timestampFieldName); + RangeQueryBuilder rangeQueryBuilder = new RangeQueryBuilder(DataStream.TIMESTAMP_FIELD_NAME); // We query a range within the timestamp range covered by both datastream indices rangeQueryBuilder.from(indexMinTimestamp).to(indexMaxTimestamp); @@ -597,7 +609,7 @@ public void testCanMatchFilteringOnCoordinatorThatCanNotBeSkipped() throws Excep } } else { // We query a range outside of the timestamp range covered by both datastream indices - RangeQueryBuilder rangeQueryBuilder = new RangeQueryBuilder(timestampFieldName).from(indexMaxTimestamp + 1) + RangeQueryBuilder rangeQueryBuilder = new RangeQueryBuilder(DataStream.TIMESTAMP_FIELD_NAME).from(indexMaxTimestamp + 1) .to(indexMaxTimestamp + 2); TermQueryBuilder termQueryBuilder = new TermQueryBuilder("fake", "value"); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java index 343ad52727507..ac35d59195d5a 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java @@ -27,6 +27,7 @@ import java.util.List; import java.util.Map; +import static org.elasticsearch.cluster.metadata.DataStream.TIMESTAMP_FIELD_NAME; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -114,9 +115,7 @@ private static DataLifecycle randomLifecycle() { private static CompressedXContent randomMappings(ComposableIndexTemplate.DataStreamTemplate dataStreamTemplate) { try { if (dataStreamTemplate != null) { - return new CompressedXContent( - "{\"properties\":{\"" + ComposableIndexTemplate.DataStreamTemplate.getTimestampField() + "\":{\"type\":\"date\"}}}" - ); + return new CompressedXContent("{\"properties\":{\"" + TIMESTAMP_FIELD_NAME + "\":{\"type\":\"date\"}}}"); } else { return new CompressedXContent("{\"properties\":{\"" + randomAlphaOfLength(5) + "\":{\"type\":\"keyword\"}}}"); } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java index 629a9560bc3e4..c929a52242f75 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java @@ -137,7 +137,6 @@ public void testRollover() { Tuple newCoordinates = ds.nextWriteIndexAndGeneration(Metadata.EMPTY_METADATA); final DataStream rolledDs = ds.rollover(new Index(newCoordinates.v1(), UUIDs.randomBase64UUID()), newCoordinates.v2(), false); assertThat(rolledDs.getName(), equalTo(ds.getName())); - assertThat(rolledDs.getTimeStampField(), equalTo(ds.getTimeStampField())); assertThat(rolledDs.getGeneration(), equalTo(ds.getGeneration() + 1)); assertThat(rolledDs.getIndices().size(), equalTo(ds.getIndices().size() + 1)); assertTrue(rolledDs.getIndices().containsAll(ds.getIndices())); @@ -163,7 +162,6 @@ public void testRolloverWithConflictingBackingIndexName() { final Tuple newCoordinates = ds.nextWriteIndexAndGeneration(builder.build()); final DataStream rolledDs = ds.rollover(new Index(newCoordinates.v1(), UUIDs.randomBase64UUID()), newCoordinates.v2(), false); assertThat(rolledDs.getName(), equalTo(ds.getName())); - assertThat(rolledDs.getTimeStampField(), equalTo(ds.getTimeStampField())); assertThat(rolledDs.getGeneration(), equalTo(ds.getGeneration() + numConflictingIndices + 1)); assertThat(rolledDs.getIndices().size(), equalTo(ds.getIndices().size() + 1)); assertTrue(rolledDs.getIndices().containsAll(ds.getIndices())); @@ -191,7 +189,6 @@ public void testRolloverUpgradeToTsdbDataStream() { var rolledDs = ds.rollover(new Index(newCoordinates.v1(), UUIDs.randomBase64UUID()), newCoordinates.v2(), true); assertThat(rolledDs.getName(), equalTo(ds.getName())); - assertThat(rolledDs.getTimeStampField(), equalTo(ds.getTimeStampField())); assertThat(rolledDs.getGeneration(), equalTo(ds.getGeneration() + 1)); assertThat(rolledDs.getIndices().size(), equalTo(ds.getIndices().size() + 1)); assertTrue(rolledDs.getIndices().containsAll(ds.getIndices())); @@ -217,7 +214,6 @@ public void testRolloverDowngradeToRegularDataStream() { var rolledDs = ds.rollover(new Index(newCoordinates.v1(), UUIDs.randomBase64UUID()), newCoordinates.v2(), false); assertThat(rolledDs.getName(), equalTo(ds.getName())); - assertThat(rolledDs.getTimeStampField(), equalTo(ds.getTimeStampField())); assertThat(rolledDs.getGeneration(), equalTo(ds.getGeneration() + 1)); assertThat(rolledDs.getIndices().size(), equalTo(ds.getIndices().size() + 1)); assertTrue(rolledDs.getIndices().containsAll(ds.getIndices())); @@ -238,7 +234,6 @@ public void testRemoveBackingIndex() { DataStream updated = original.removeBackingIndex(indices.get(indexToRemove - 1)); assertThat(updated.getName(), equalTo(original.getName())); assertThat(updated.getGeneration(), equalTo(original.getGeneration() + 1)); - assertThat(updated.getTimeStampField(), equalTo(original.getTimeStampField())); assertThat(updated.getIndices().size(), equalTo(numBackingIndices - 1)); for (int k = 0; k < (numBackingIndices - 1); k++) { assertThat(updated.getIndices().get(k), equalTo(original.getIndices().get(k < (indexToRemove - 1) ? k : k + 1))); @@ -321,7 +316,6 @@ public void testAddBackingIndex() { DataStream updated = original.addBackingIndex(builder.build(), indexToAdd); assertThat(updated.getName(), equalTo(original.getName())); assertThat(updated.getGeneration(), equalTo(original.getGeneration() + 1)); - assertThat(updated.getTimeStampField(), equalTo(original.getTimeStampField())); assertThat(updated.getIndices().size(), equalTo(numBackingIndices + 1)); for (int k = 1; k <= numBackingIndices; k++) { assertThat(updated.getIndices().get(k), equalTo(original.getIndices().get(k - 1))); @@ -406,7 +400,6 @@ public void testAddExistingBackingIndex() { DataStream updated = original.addBackingIndex(builder.build(), indexToAdd); assertThat(updated.getName(), equalTo(original.getName())); assertThat(updated.getGeneration(), equalTo(original.getGeneration())); - assertThat(updated.getTimeStampField(), equalTo(original.getTimeStampField())); assertThat(updated.getIndices().size(), equalTo(numBackingIndices)); for (int k = 0; k < numBackingIndices; k++) { assertThat(updated.getIndices().get(k), equalTo(original.getIndices().get(k))); @@ -494,7 +487,6 @@ public void testReplaceBackingIndex() { DataStream updated = original.replaceBackingIndex(indices.get(indexToReplace), newBackingIndex); assertThat(updated.getName(), equalTo(original.getName())); assertThat(updated.getGeneration(), equalTo(original.getGeneration() + 1)); - assertThat(updated.getTimeStampField(), equalTo(original.getTimeStampField())); assertThat(updated.getIndices().size(), equalTo(numBackingIndices)); assertThat(updated.getIndices().get(indexToReplace), equalTo(newBackingIndex)); @@ -580,7 +572,6 @@ public void testSnapshot() { ); assertThat(reconciledDataStream.getName(), equalTo(postSnapshotDataStream.getName())); - assertThat(reconciledDataStream.getTimeStampField(), equalTo(postSnapshotDataStream.getTimeStampField())); assertThat(reconciledDataStream.getGeneration(), equalTo(postSnapshotDataStream.getGeneration())); if (reconciledDataStream.getMetadata() != null) { assertThat( diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java index 6e04a056916fe..1738889ca4c72 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java @@ -197,11 +197,9 @@ public void testSimpleJsonFromAndTo() throws IOException { // data streams assertNotNull(parsedMetadata.dataStreams().get("data-stream1")); assertThat(parsedMetadata.dataStreams().get("data-stream1").getName(), is("data-stream1")); - assertThat(parsedMetadata.dataStreams().get("data-stream1").getTimeStampField().getName(), is("@timestamp")); assertThat(parsedMetadata.dataStreams().get("data-stream1").getIndices(), contains(idx1.getIndex())); assertNotNull(parsedMetadata.dataStreams().get("data-stream2")); assertThat(parsedMetadata.dataStreams().get("data-stream2").getName(), is("data-stream2")); - assertThat(parsedMetadata.dataStreams().get("data-stream2").getTimeStampField().getName(), is("@timestamp")); assertThat(parsedMetadata.dataStreams().get("data-stream2").getIndices(), contains(idx2.getIndex())); // reserved 'operator' metadata diff --git a/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionLookupTests.java b/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionLookupTests.java index 60e5d399ca381..b463a5ddf11a9 100644 --- a/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionLookupTests.java +++ b/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionLookupTests.java @@ -124,14 +124,14 @@ public void testLoadTimestampRange() throws Exception { IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER).setMergePolicy(NoMergePolicy.INSTANCE)); Document doc = new Document(); doc.add(new StringField(IdFieldMapper.NAME, "6", Field.Store.YES)); - doc.add(new LongPoint(DataStream.TimestampField.FIXED_TIMESTAMP_FIELD, 1_000)); + doc.add(new LongPoint(DataStream.TIMESTAMP_FIELD_NAME, 1_000)); doc.add(new NumericDocValuesField(VersionFieldMapper.NAME, 87)); doc.add(new NumericDocValuesField(SeqNoFieldMapper.NAME, randomNonNegativeLong())); doc.add(new NumericDocValuesField(SeqNoFieldMapper.PRIMARY_TERM_NAME, randomLongBetween(1, Long.MAX_VALUE))); writer.addDocument(doc); doc = new Document(); doc.add(new StringField(IdFieldMapper.NAME, "8", Field.Store.YES)); - doc.add(new LongPoint(DataStream.TimestampField.FIXED_TIMESTAMP_FIELD, 1_000_000)); + doc.add(new LongPoint(DataStream.TIMESTAMP_FIELD_NAME, 1_000_000)); doc.add(new NumericDocValuesField(VersionFieldMapper.NAME, 1)); doc.add(new NumericDocValuesField(SeqNoFieldMapper.NAME, randomNonNegativeLong())); doc.add(new NumericDocValuesField(SeqNoFieldMapper.PRIMARY_TERM_NAME, randomLongBetween(1, Long.MAX_VALUE))); diff --git a/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionsTests.java b/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionsTests.java index 6c834d02160b0..6664714f9a302 100644 --- a/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionsTests.java +++ b/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionsTests.java @@ -231,7 +231,7 @@ public void testTimeSeriesLoadDocIdAndVersion() throws Exception { Document doc = new Document(); doc.add(new StringField(IdFieldMapper.NAME, "1", Field.Store.YES)); - doc.add(new LongPoint(DataStream.TimestampField.FIXED_TIMESTAMP_FIELD, 1_000)); + doc.add(new LongPoint(DataStream.TIMESTAMP_FIELD_NAME, 1_000)); doc.add(new NumericDocValuesField(VersionFieldMapper.NAME, 1)); doc.add(new NumericDocValuesField(SeqNoFieldMapper.NAME, 0L)); doc.add(new NumericDocValuesField(SeqNoFieldMapper.PRIMARY_TERM_NAME, 1L)); @@ -239,7 +239,7 @@ public void testTimeSeriesLoadDocIdAndVersion() throws Exception { doc = new Document(); doc.add(new StringField(IdFieldMapper.NAME, "2", Field.Store.YES)); - doc.add(new LongPoint(DataStream.TimestampField.FIXED_TIMESTAMP_FIELD, 10_000)); + doc.add(new LongPoint(DataStream.TIMESTAMP_FIELD_NAME, 10_000)); doc.add(new NumericDocValuesField(VersionFieldMapper.NAME, 1)); doc.add(new NumericDocValuesField(SeqNoFieldMapper.NAME, 0L)); doc.add(new NumericDocValuesField(SeqNoFieldMapper.PRIMARY_TERM_NAME, 1L)); diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java index 21d4abf4d50af..e52296e062d17 100644 --- a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java @@ -32,7 +32,7 @@ import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.AliasMetadata; -import org.elasticsearch.cluster.metadata.DataStream.TimestampField; +import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; import org.elasticsearch.cluster.metadata.Metadata; @@ -1966,7 +1966,7 @@ public void testSetsRawTimestamp() { tag, description, // just always set the timestamp field to 100 - (ingestDocument) -> ingestDocument.setFieldValue(TimestampField.FIXED_TIMESTAMP_FIELD, 100) + (ingestDocument) -> ingestDocument.setFieldValue(DataStream.TIMESTAMP_FIELD_NAME, 100) ) ) ); @@ -2000,7 +2000,7 @@ public void testSetsRawTimestamp() { IndexRequest indexRequest4 = new IndexRequest("idx").setPipeline("_id1").setFinalPipeline("_id2").source(doc1); // feed a document with a timestamp through four scenarios - Map doc2 = Map.of(TimestampField.FIXED_TIMESTAMP_FIELD, 10); + Map doc2 = Map.of(DataStream.TIMESTAMP_FIELD_NAME, 10); // neither a request nor a final pipeline IndexRequest indexRequest5 = new IndexRequest("idx").setPipeline("_none").setFinalPipeline("_none").source(doc2); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/support/TimeSeriesCancellationTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/support/TimeSeriesCancellationTests.java index 28527b9426375..ec2d3214eec1c 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/support/TimeSeriesCancellationTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/support/TimeSeriesCancellationTests.java @@ -50,7 +50,7 @@ public static void setup() throws IOException { iwc.setIndexSort( new Sort( new SortField(TimeSeriesIdFieldMapper.NAME, SortField.Type.STRING), - new SortField(DataStream.TimestampField.FIXED_TIMESTAMP_FIELD, SortField.Type.LONG) + new SortField(DataStream.TIMESTAMP_FIELD_NAME, SortField.Type.LONG) ) ); RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc); @@ -66,7 +66,7 @@ private static void indexRandomDocuments(RandomIndexWriter w, int numDocs) throw String tsid = "tsid" + randomIntBetween(0, 30); long time = randomNonNegativeLong(); doc.add(new SortedDocValuesField(TimeSeriesIdFieldMapper.NAME, new BytesRef(tsid))); - doc.add(new NumericDocValuesField(DataStream.TimestampField.FIXED_TIMESTAMP_FIELD, time)); + doc.add(new NumericDocValuesField(DataStream.TIMESTAMP_FIELD_NAME, time)); w.addDocument(doc); } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/support/TimeSeriesIndexSearcherTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/support/TimeSeriesIndexSearcherTests.java index cae69b2b1ce16..bf1a17f1852b1 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/support/TimeSeriesIndexSearcherTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/support/TimeSeriesIndexSearcherTests.java @@ -69,7 +69,7 @@ public void testCollectInOrderAcrossSegments() throws IOException, InterruptedEx long time = clock.addAndGet(randomIntBetween(0, 10)); doc.clear(); doc.add(new SortedDocValuesField(TimeSeriesIdFieldMapper.NAME, new BytesRef(tsid))); - doc.add(new NumericDocValuesField(DataStream.TimestampField.FIXED_TIMESTAMP_FIELD, time)); + doc.add(new NumericDocValuesField(DataStream.TIMESTAMP_FIELD_NAME, time)); try { iw.addDocument(doc); } catch (IOException e) { @@ -109,7 +109,7 @@ public void testCollectMinScoreAcrossSegments() throws IOException, InterruptedE long time = clock.addAndGet(j % 10); doc.clear(); doc.add(new SortedDocValuesField(TimeSeriesIdFieldMapper.NAME, new BytesRef(tsid))); - doc.add(new NumericDocValuesField(DataStream.TimestampField.FIXED_TIMESTAMP_FIELD, time)); + doc.add(new NumericDocValuesField(DataStream.TIMESTAMP_FIELD_NAME, time)); try { iw.addDocument(doc); } catch (IOException e) { @@ -155,7 +155,7 @@ public void testCollectFromMiddle() throws IOException { // segment 1 // pre add a value doc.add(new SortedDocValuesField(TimeSeriesIdFieldMapper.NAME, new BytesRef("tsid"))); - doc.add(new NumericDocValuesField(DataStream.TimestampField.FIXED_TIMESTAMP_FIELD, 1)); + doc.add(new NumericDocValuesField(DataStream.TIMESTAMP_FIELD_NAME, 1)); iw.addDocument(doc); // segment 1 add value, timestamp is all large then segment 2 @@ -163,7 +163,7 @@ public void testCollectFromMiddle() throws IOException { String tsid = "tsid" + randomIntBetween(0, 1); doc.clear(); doc.add(new SortedDocValuesField(TimeSeriesIdFieldMapper.NAME, new BytesRef(tsid))); - doc.add(new NumericDocValuesField(DataStream.TimestampField.FIXED_TIMESTAMP_FIELD, randomIntBetween(20, 25))); + doc.add(new NumericDocValuesField(DataStream.TIMESTAMP_FIELD_NAME, randomIntBetween(20, 25))); try { iw.addDocument(doc); } catch (IOException e) { @@ -176,13 +176,13 @@ public void testCollectFromMiddle() throws IOException { // pre add a value doc.clear(); doc.add(new SortedDocValuesField(TimeSeriesIdFieldMapper.NAME, new BytesRef("tsid"))); - doc.add(new NumericDocValuesField(DataStream.TimestampField.FIXED_TIMESTAMP_FIELD, 1)); + doc.add(new NumericDocValuesField(DataStream.TIMESTAMP_FIELD_NAME, 1)); iw.addDocument(doc); for (int j = 0; j < DOC_COUNTS; j++) { String tsid = "tsid" + randomIntBetween(0, 1); doc.clear(); doc.add(new SortedDocValuesField(TimeSeriesIdFieldMapper.NAME, new BytesRef(tsid))); - doc.add(new NumericDocValuesField(DataStream.TimestampField.FIXED_TIMESTAMP_FIELD, randomIntBetween(10, 15))); + doc.add(new NumericDocValuesField(DataStream.TIMESTAMP_FIELD_NAME, randomIntBetween(10, 15))); try { iw.addDocument(doc); } catch (IOException e) { @@ -213,7 +213,7 @@ private RandomIndexWriter getIndexWriter(Directory dir) throws IOException { boolean timestampReverse = TIME_SERIES_SORT[1].getOrder() == SortOrder.DESC; Sort sort = new Sort( new SortField(TimeSeriesIdFieldMapper.NAME, SortField.Type.STRING, tsidReverse), - new SortField(DataStream.TimestampField.FIXED_TIMESTAMP_FIELD, SortField.Type.LONG, timestampReverse) + new SortField(DataStream.TIMESTAMP_FIELD_NAME, SortField.Type.LONG, timestampReverse) ); iwc.setIndexSort(sort); return new RandomIndexWriter(random(), dir, iwc); @@ -232,10 +232,7 @@ private BucketCollector getBucketCollector(long totalCount) { @Override public LeafBucketCollector getLeafCollector(AggregationExecutionContext aggCtx) throws IOException { SortedDocValues tsid = DocValues.getSorted(aggCtx.getLeafReaderContext().reader(), TimeSeriesIdFieldMapper.NAME); - NumericDocValues timestamp = DocValues.getNumeric( - aggCtx.getLeafReaderContext().reader(), - DataStream.TimestampField.FIXED_TIMESTAMP_FIELD - ); + NumericDocValues timestamp = DocValues.getNumeric(aggCtx.getLeafReaderContext().reader(), DataStream.TIMESTAMP_FIELD_NAME); return new LeafBucketCollector() { @Override diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java b/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java index a5a9bf30f5c78..eeb5cfda6c80f 100644 --- a/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java +++ b/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java @@ -490,7 +490,7 @@ public static MetadataRolloverService getMetadataRolloverService( RootObjectMapper.Builder root = new RootObjectMapper.Builder("_doc", ObjectMapper.Defaults.SUBOBJECTS); root.add( new DateFieldMapper.Builder( - dataStream.getTimeStampField().getName(), + DataStream.TIMESTAMP_FIELD_NAME, DateFieldMapper.Resolution.MILLISECONDS, DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, ScriptCompiler.NONE, diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java index e69af2d7d51b9..ee461ba9a562a 100644 --- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java +++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java @@ -662,7 +662,7 @@ public void testAutoFollowDatastreamWithClosingFollowerIndex() throws Exception assertAcked(leaderClient().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get()); leaderClient().prepareIndex(datastream) .setCreate(true) - .setSource("foo", "bar", DataStream.TIMESTAMP_FIELD.getName(), randomNonNegativeLong()) + .setSource("foo", "bar", DataStream.TIMESTAMP_FIELD_NAME, randomNonNegativeLong()) .get(); PutAutoFollowPatternAction.Request followRequest = new PutAutoFollowPatternAction.Request(); @@ -698,7 +698,7 @@ public void testAutoFollowDatastreamWithClosingFollowerIndex() throws Exception ); leaderClient().prepareIndex(indexInDatastream) .setCreate(true) - .setSource("foo", "bar", DataStream.TIMESTAMP_FIELD.getName(), randomNonNegativeLong()) + .setSource("foo", "bar", DataStream.TIMESTAMP_FIELD_NAME, randomNonNegativeLong()) .get(); leaderClient().execute( ModifyDataStreamsAction.INSTANCE, diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowActionTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowActionTests.java index 283c0e2de077e..309676cd8adfd 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowActionTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowActionTests.java @@ -34,7 +34,6 @@ public void testCreateNewLocalDataStream() { remoteDataStream ); assertThat(result.getName(), equalTo(remoteDataStream.getName())); - assertThat(result.getTimeStampField(), equalTo(remoteDataStream.getTimeStampField())); assertThat(result.getGeneration(), equalTo(remoteDataStream.getGeneration())); assertThat(result.getIndices().size(), equalTo(1)); assertThat(result.getIndices().get(0), equalTo(backingIndexToFollow)); @@ -51,7 +50,6 @@ public void testUpdateLocalDataStream_followNewBackingIndex() { remoteDataStream ); assertThat(result.getName(), equalTo(remoteDataStream.getName())); - assertThat(result.getTimeStampField(), equalTo(remoteDataStream.getTimeStampField())); assertThat(result.getGeneration(), equalTo(remoteDataStream.getGeneration())); assertThat(result.getIndices().size(), equalTo(3)); assertThat(result.getIndices().get(0).getName(), equalTo(DataStream.getDefaultBackingIndexName("logs-foobar", 1))); @@ -71,7 +69,6 @@ public void testUpdateLocalDataStream_followOlderBackingIndex() { remoteDataStream ); assertThat(result.getName(), equalTo(remoteDataStream.getName())); - assertThat(result.getTimeStampField(), equalTo(remoteDataStream.getTimeStampField())); assertThat(result.getGeneration(), equalTo(remoteDataStream.getGeneration())); assertThat(result.getIndices().size(), equalTo(2)); assertThat(result.getIndices().get(0).getName(), equalTo(DataStream.getDefaultBackingIndexName("logs-foobar", 1))); @@ -87,7 +84,6 @@ public void testUpdateLocalDataStream_followOlderBackingIndex() { remoteDataStream ); assertThat(result.getName(), equalTo(remoteDataStream.getName())); - assertThat(result.getTimeStampField(), equalTo(remoteDataStream.getTimeStampField())); assertThat(result.getGeneration(), equalTo(remoteDataStream.getGeneration())); assertThat(result.getIndices().size(), equalTo(3)); assertThat(result.getIndices().get(0).getName(), equalTo(DataStream.getDefaultBackingIndexName("logs-foobar", 1))); @@ -139,7 +135,6 @@ public void testLocalDataStreamBackingIndicesOrder() { ); assertThat(result.getName(), equalTo(remoteDataStream.getName())); - assertThat(result.getTimeStampField(), equalTo(remoteDataStream.getTimeStampField())); assertThat(result.getGeneration(), equalTo(remoteDataStream.getGeneration())); assertThat(result.getIndices().size(), equalTo(initialLocalBackingIndices.size() + 1)); // the later generation we just followed became the local data stream write index diff --git a/x-pack/plugin/frozen-indices/src/internalClusterTest/java/org/elasticsearch/index/engine/frozen/FrozenIndexIT.java b/x-pack/plugin/frozen-indices/src/internalClusterTest/java/org/elasticsearch/index/engine/frozen/FrozenIndexIT.java index bb9379ec06a76..30f24deac408b 100644 --- a/x-pack/plugin/frozen-indices/src/internalClusterTest/java/org/elasticsearch/index/engine/frozen/FrozenIndexIT.java +++ b/x-pack/plugin/frozen-indices/src/internalClusterTest/java/org/elasticsearch/index/engine/frozen/FrozenIndexIT.java @@ -74,7 +74,7 @@ public void testTimestampRangeRecalculatedOnStalePrimaryAllocation() throws IOEx createIndex("index", 1, 1); final IndexResponse indexResponse = client().prepareIndex("index") - .setSource(DataStream.TimestampField.FIXED_TIMESTAMP_FIELD, "2010-01-06T02:03:04.567Z") + .setSource(DataStream.TIMESTAMP_FIELD_NAME, "2010-01-06T02:03:04.567Z") .get(); ensureGreen("index"); @@ -150,7 +150,7 @@ public void testTimestampFieldTypeExposedByAllIndicesServices() throws Exception jsonBuilder().startObject() .startObject("_doc") .startObject("properties") - .startObject(DataStream.TimestampField.FIXED_TIMESTAMP_FIELD) + .startObject(DataStream.TIMESTAMP_FIELD_NAME) .field("type", "date") .field("format", "dd LLL yyyy HH:mm:ssX") .field("locale", locale) @@ -173,7 +173,7 @@ public void testTimestampFieldTypeExposedByAllIndicesServices() throws Exception ensureGreen("index"); if (randomBoolean()) { - client().prepareIndex("index").setSource(DataStream.TimestampField.FIXED_TIMESTAMP_FIELD, date).get(); + client().prepareIndex("index").setSource(DataStream.TIMESTAMP_FIELD_NAME, date).get(); } for (final IndicesService indicesService : internalCluster().getInstances(IndicesService.class)) { diff --git a/x-pack/plugin/frozen-indices/src/internalClusterTest/java/org/elasticsearch/index/engine/frozen/FrozenIndexTests.java b/x-pack/plugin/frozen-indices/src/internalClusterTest/java/org/elasticsearch/index/engine/frozen/FrozenIndexTests.java index 67aba439ebe60..3a5f2c3027b08 100644 --- a/x-pack/plugin/frozen-indices/src/internalClusterTest/java/org/elasticsearch/index/engine/frozen/FrozenIndexTests.java +++ b/x-pack/plugin/frozen-indices/src/internalClusterTest/java/org/elasticsearch/index/engine/frozen/FrozenIndexTests.java @@ -614,8 +614,8 @@ public void testTranslogStats() { public void testComputesTimestampRangeFromMilliseconds() { final int shardCount = between(1, 3); createIndex("index", Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, shardCount).build()); - client().prepareIndex("index").setSource(DataStream.TimestampField.FIXED_TIMESTAMP_FIELD, "2010-01-05T01:02:03.456Z").get(); - client().prepareIndex("index").setSource(DataStream.TimestampField.FIXED_TIMESTAMP_FIELD, "2010-01-06T02:03:04.567Z").get(); + client().prepareIndex("index").setSource(DataStream.TIMESTAMP_FIELD_NAME, "2010-01-05T01:02:03.456Z").get(); + client().prepareIndex("index").setSource(DataStream.TIMESTAMP_FIELD_NAME, "2010-01-06T02:03:04.567Z").get(); assertAcked(client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest("index")).actionGet()); @@ -637,7 +637,7 @@ public void testComputesTimestampRangeFromNanoseconds() throws IOException { final XContentBuilder mapping = XContentFactory.jsonBuilder() .startObject() .startObject("properties") - .startObject(DataStream.TimestampField.FIXED_TIMESTAMP_FIELD) + .startObject(DataStream.TIMESTAMP_FIELD_NAME) .field("type", "date_nanos") .field("format", "strict_date_optional_time_nanos") .endObject() @@ -646,8 +646,8 @@ public void testComputesTimestampRangeFromNanoseconds() throws IOException { final int shardCount = between(1, 3); createIndex("index", Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, shardCount).build(), mapping); - client().prepareIndex("index").setSource(DataStream.TimestampField.FIXED_TIMESTAMP_FIELD, "2010-01-05T01:02:03.456789012Z").get(); - client().prepareIndex("index").setSource(DataStream.TimestampField.FIXED_TIMESTAMP_FIELD, "2010-01-06T02:03:04.567890123Z").get(); + client().prepareIndex("index").setSource(DataStream.TIMESTAMP_FIELD_NAME, "2010-01-05T01:02:03.456789012Z").get(); + client().prepareIndex("index").setSource(DataStream.TIMESTAMP_FIELD_NAME, "2010-01-06T02:03:04.567890123Z").get(); assertAcked(client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest("index")).actionGet()); diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsCanMatchOnCoordinatorIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsCanMatchOnCoordinatorIntegTests.java index 8fe787c7d4cb2..30b8914648555 100644 --- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsCanMatchOnCoordinatorIntegTests.java +++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsCanMatchOnCoordinatorIntegTests.java @@ -162,7 +162,7 @@ public void testSearchableSnapshotShardsAreSkippedWithoutQueryingAnyNodeWhenThey SearchRequest request = new SearchRequest().indices(indicesToSearch.toArray(new String[0])) .source( new SearchSourceBuilder().query( - QueryBuilders.rangeQuery(DataStream.TimestampField.FIXED_TIMESTAMP_FIELD) + QueryBuilders.rangeQuery(DataStream.TIMESTAMP_FIELD_NAME) .from("2020-11-28T00:00:00.000000000Z", true) .to("2020-11-29T00:00:00.000000000Z") ) @@ -291,7 +291,7 @@ public void testQueryPhaseIsExecutedInAnAvailableNodeWhenAllShardsCanBeSkipped() SearchRequest request = new SearchRequest().indices(indexOutsideSearchRange, searchableSnapshotIndexOutsideSearchRange) .source( new SearchSourceBuilder().query( - QueryBuilders.rangeQuery(DataStream.TimestampField.FIXED_TIMESTAMP_FIELD) + QueryBuilders.rangeQuery(DataStream.TIMESTAMP_FIELD_NAME) .from("2020-11-28T00:00:00.000000000Z", true) .to("2020-11-29T00:00:00.000000000Z") ) @@ -402,7 +402,7 @@ public void testSearchableSnapshotShardsThatHaveMatchingDataAreNotSkippedOnTheCo SearchRequest request = new SearchRequest().indices(searchableSnapshotIndexWithinSearchRange) .source( new SearchSourceBuilder().query( - QueryBuilders.rangeQuery(DataStream.TimestampField.FIXED_TIMESTAMP_FIELD) + QueryBuilders.rangeQuery(DataStream.TIMESTAMP_FIELD_NAME) .from("2020-11-28T00:00:00.000000000Z", true) .to("2020-11-29T00:00:00.000000000Z") ) @@ -445,7 +445,7 @@ private void createIndexWithTimestamp(String indexName, int numShards, Settings XContentFactory.jsonBuilder() .startObject() .startObject("properties") - .startObject(DataStream.TimestampField.FIXED_TIMESTAMP_FIELD) + .startObject(DataStream.TIMESTAMP_FIELD_NAME) .field("type", randomFrom("date", "date_nanos")) .field("format", "strict_date_optional_time_nanos") .endObject() @@ -463,7 +463,7 @@ private void indexDocumentsWithTimestampWithinDate(String indexName, int docCoun indexRequestBuilders.add( client().prepareIndex(indexName) .setSource( - DataStream.TimestampField.FIXED_TIMESTAMP_FIELD, + DataStream.TIMESTAMP_FIELD_NAME, String.format( Locale.ROOT, timestampTemplate, diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java index 8fae2a09ec22d..23e0d9f67a4c5 100644 --- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java +++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java @@ -688,7 +688,7 @@ public void testSnapshotMountedIndexWithTimestampsRecordsTimestampRangeInIndexMe XContentFactory.jsonBuilder() .startObject() .startObject("properties") - .startObject(DataStream.TimestampField.FIXED_TIMESTAMP_FIELD) + .startObject(DataStream.TIMESTAMP_FIELD_NAME) .field("type", dateType) .field("index", indexed) .field("format", "strict_date_optional_time_nanos") @@ -706,7 +706,7 @@ public void testSnapshotMountedIndexWithTimestampsRecordsTimestampRangeInIndexMe indexRequestBuilders.add( client().prepareIndex(indexName) .setSource( - DataStream.TimestampField.FIXED_TIMESTAMP_FIELD, + DataStream.TIMESTAMP_FIELD_NAME, String.format( Locale.ROOT, "2020-11-26T%02d:%02d:%02d.%09dZ", diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregationBuilder.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregationBuilder.java index 02a6fd2887cfa..114c67b82e73f 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregationBuilder.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregationBuilder.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.spatial.search.aggregations; import org.elasticsearch.TransportVersion; +import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.index.query.QueryBuilder; @@ -34,8 +35,6 @@ import java.util.Map; import java.util.Objects; -import static org.elasticsearch.cluster.metadata.DataStream.TIMESTAMP_FIELD; - public class GeoLineAggregationBuilder extends MultiValuesSourceAggregationBuilder.LeafOnly { static final ParseField POINT_FIELD = new ParseField("point"); @@ -158,7 +157,7 @@ private void validateTimeSeriesConfigs(AggregationContext context, Map Date: Mon, 3 Jul 2023 09:55:59 +0100 Subject: [PATCH 003/111] Chunk the GET _ilm/policy response (#97251) Chunk the GET _ilm/policy response --- docs/changelog/97251.yaml | 6 +++ .../core/ilm/action/GetLifecycleAction.java | 40 ++++++++++--------- .../ilm/action/GetLifecycleResponseTests.java | 40 +++++++++++++++++++ .../ilm/action/RestGetLifecycleAction.java | 4 +- 4 files changed, 70 insertions(+), 20 deletions(-) create mode 100644 docs/changelog/97251.yaml diff --git a/docs/changelog/97251.yaml b/docs/changelog/97251.yaml new file mode 100644 index 0000000000000..8cfb50befbb7f --- /dev/null +++ b/docs/changelog/97251.yaml @@ -0,0 +1,6 @@ +pr: 97251 +summary: Chunk the GET _ilm/policy response +area: ILM+SLM +type: enhancement +issues: + - 96569 diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/action/GetLifecycleAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/action/GetLifecycleAction.java index df163ee97d19a..a8403c1c6cc05 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/action/GetLifecycleAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/action/GetLifecycleAction.java @@ -13,18 +13,20 @@ import org.elasticsearch.action.support.master.AcknowledgedRequest; import org.elasticsearch.cluster.metadata.ItemUsage; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ChunkedToXContentObject; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; -import org.elasticsearch.xcontent.ToXContentObject; -import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xpack.core.ilm.LifecyclePolicy; import java.io.IOException; import java.util.Arrays; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Objects; @@ -37,7 +39,7 @@ protected GetLifecycleAction() { super(NAME, GetLifecycleAction.Response::new); } - public static class Response extends ActionResponse implements ToXContentObject { + public static class Response extends ActionResponse implements ChunkedToXContentObject { private List policies; @@ -54,21 +56,6 @@ public List getPolicies() { return policies; } - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - for (LifecyclePolicyResponseItem item : policies) { - builder.startObject(item.getLifecyclePolicy().getName()); - builder.field("version", item.getVersion()); - builder.field("modified_date", item.getModifiedDate()); - builder.field("policy", item.getLifecyclePolicy()); - builder.field("in_use_by", item.getUsage()); - builder.endObject(); - } - builder.endObject(); - return builder; - } - @Override public void writeTo(StreamOutput out) throws IOException { out.writeList(policies); @@ -96,6 +83,23 @@ public String toString() { return Strings.toString(this, true, true); } + @Override + @SuppressWarnings("unchecked") + public Iterator toXContentChunked(ToXContent.Params outerParams) { + return Iterators.concat( + Iterators.single((builder, params) -> builder.startObject()), + policies.stream().map(policy -> (ToXContent) (b, p) -> { + b.startObject(policy.getLifecyclePolicy().getName()); + b.field("version", policy.getVersion()); + b.field("modified_date", policy.getModifiedDate()); + b.field("policy", policy.getLifecyclePolicy()); + b.field("in_use_by", policy.getUsage()); + b.endObject(); + return b; + }).iterator(), + Iterators.single((b, p) -> b.endObject()) + ); + } } public static class Request extends AcknowledgedRequest { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/action/GetLifecycleResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/action/GetLifecycleResponseTests.java index 32355b0d4abcc..05c637a3a66c9 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/action/GetLifecycleResponseTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/action/GetLifecycleResponseTests.java @@ -7,10 +7,14 @@ package org.elasticsearch.xpack.core.ilm.action; import org.elasticsearch.cluster.metadata.ItemUsage; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.Nullable; +import org.elasticsearch.test.AbstractChunkedSerializingTestCase; import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xpack.core.ilm.LifecycleAction; import org.elasticsearch.xpack.core.ilm.LifecycleType; import org.elasticsearch.xpack.core.ilm.MockAction; @@ -18,11 +22,17 @@ import org.elasticsearch.xpack.core.ilm.action.GetLifecycleAction.LifecyclePolicyResponseItem; import org.elasticsearch.xpack.core.ilm.action.GetLifecycleAction.Response; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Map; +import static org.elasticsearch.xcontent.ToXContent.EMPTY_PARAMS; +import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.xpack.core.ilm.LifecyclePolicyTests.randomTestLifecyclePolicy; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; public class GetLifecycleResponseTests extends AbstractWireSerializingTestCase { @@ -43,6 +53,36 @@ protected Response createTestInstance() { return new Response(responseItems); } + @SuppressWarnings("unchecked") + public void testToXContent() throws IOException { + Response response = createTestInstance(); + XContentBuilder builder = jsonBuilder().prettyPrint(); + response.toXContentChunked(EMPTY_PARAMS).forEachRemaining(xcontent -> { + try { + xcontent.toXContent(builder, EMPTY_PARAMS); + } catch (IOException e) { + logger.error(e.getMessage(), e); + fail(e.getMessage()); + } + }); + Map xContentMap = XContentHelper.convertToMap(BytesReference.bytes(builder), false, builder.contentType()).v2(); + assertThat(xContentMap.size(), is(response.getPolicies().size())); + for (LifecyclePolicyResponseItem policy : response.getPolicies()) { + Map policyAsXContent = (Map) xContentMap.get(policy.getLifecyclePolicy().getName()); + assertThat(policyAsXContent, notNullValue()); + assertThat(policyAsXContent.get("version"), is(policy.getVersion())); + assertThat(policyAsXContent.get("modified_date"), is(policy.getModifiedDate())); + assertThat(policyAsXContent.get("policy"), notNullValue()); + } + } + + public void testChunkCount() { + Response response = createTestInstance(); + // we have 2 chunks surrounding the policies - one for { and } respectively + // we have one chunk / policy + AbstractChunkedSerializingTestCase.assertChunkCount(response, ignored -> 2 + response.getPolicies().size()); + } + @Override protected Writeable.Reader instanceReader() { return Response::new; diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestGetLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestGetLifecycleAction.java index 23a01ec834ddc..b92f603da49b5 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestGetLifecycleAction.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestGetLifecycleAction.java @@ -12,7 +12,7 @@ import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestCancellableNodeClient; -import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.rest.action.RestChunkedToXContentListener; import org.elasticsearch.xpack.core.ilm.action.GetLifecycleAction; import java.util.List; @@ -41,7 +41,7 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient return channel -> new RestCancellableNodeClient(client, restRequest.getHttpChannel()).execute( GetLifecycleAction.INSTANCE, getLifecycleRequest, - new RestToXContentListener<>(channel) + new RestChunkedToXContentListener<>(channel) ); } } From 0cb0120f83b77f4f1e28683c0e82ab6a18baac58 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna <93581129+salvatore-campagna@users.noreply.github.com> Date: Mon, 3 Jul 2023 11:51:29 +0200 Subject: [PATCH 004/111] Include more downsampling task statistics (#96930) Add task level statistics for downsampling tasks. Focus is on tracking the task status and providing more information around task progress and indexing stats, including tracking bulk indexing operations. --- docs/changelog/96930.yaml | 6 + .../org/elasticsearch/TransportVersion.java | 5 +- .../downsample/DownsampleIndexerAction.java | 33 ++- .../rollup/action/RollupAfterBulkInfo.java | 99 +++++++ .../rollup/action/RollupBeforeBulkInfo.java | 80 ++++++ .../core/rollup/action/RollupBulkInfo.java | 105 ++++++++ .../core/rollup/action/RollupBulkStats.java | 55 ++++ .../action/RollupShardIndexerStatus.java | 19 ++ .../core/rollup/action/RollupShardStatus.java | 252 +++++++++++++++--- .../core/rollup/action/RollupShardTask.java | 128 ++++++++- .../RollupShardStatusSerializingTests.java | 70 ++++- .../xpack/downsample/RollupShardIndexer.java | 45 +++- .../downsample/TransportDownsampleAction.java | 10 + .../DownsampleActionSingleNodeTests.java | 85 ++++++ 14 files changed, 948 insertions(+), 44 deletions(-) create mode 100644 docs/changelog/96930.yaml create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollupAfterBulkInfo.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollupBeforeBulkInfo.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollupBulkInfo.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollupBulkStats.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollupShardIndexerStatus.java diff --git a/docs/changelog/96930.yaml b/docs/changelog/96930.yaml new file mode 100644 index 0000000000000..e4715091e9874 --- /dev/null +++ b/docs/changelog/96930.yaml @@ -0,0 +1,6 @@ +pr: 96930 +summary: Include more downsampling status statistics +area: TSDB +type: enhancement +issues: + - 96760 diff --git a/server/src/main/java/org/elasticsearch/TransportVersion.java b/server/src/main/java/org/elasticsearch/TransportVersion.java index ce6de2eca42e6..38be8bcf8379e 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersion.java +++ b/server/src/main/java/org/elasticsearch/TransportVersion.java @@ -138,7 +138,6 @@ private static TransportVersion registerTransportVersion(int id, String uniqueId public static final TransportVersion V_8_500_007 = registerTransportVersion(8_500_007, "77261d43-4149-40af-89c5-7e71e0454fce"); // Introduced for stateless plugin public static final TransportVersion V_8_500_008 = registerTransportVersion(8_500_008, "8884ab9d-94cd-4bac-aff8-01f2c394f47c"); - public static final TransportVersion V_8_500_009 = registerTransportVersion(8_500_009, "35091358-fd41-4106-a6e2-d2a1315494c1"); public static final TransportVersion V_8_500_010 = registerTransportVersion(8_500_010, "9818C628-1EEC-439B-B943-468F61460675"); public static final TransportVersion V_8_500_011 = registerTransportVersion(8_500_011, "2209F28D-B52E-4BC4-9889-E780F291C32E"); @@ -147,6 +146,7 @@ private static TransportVersion registerTransportVersion(int id, String uniqueId public static final TransportVersion V_8_500_014 = registerTransportVersion(8_500_014, "D115A2E1-1739-4A02-AB7B-64F6EA157EFB"); public static final TransportVersion V_8_500_015 = registerTransportVersion(8_500_015, "651216c9-d54f-4189-9fe1-48d82d276863"); public static final TransportVersion V_8_500_016 = registerTransportVersion(8_500_016, "492C94FB-AAEA-4C9E-8375-BDB67A398584"); + public static final TransportVersion V_8_500_017 = registerTransportVersion(8_500_017, "0EDCB5BA-049C-443C-8AB1-5FA58FB996FB"); public static final TransportVersion V_8_500_018 = registerTransportVersion(8_500_018, "827C32CE-33D9-4AC3-A773-8FB768F59EAF"); public static final TransportVersion V_8_500_019 = registerTransportVersion(8_500_019, "09bae57f-cab8-423c-aab3-c9778509ffe3"); @@ -162,9 +162,10 @@ private static TransportVersion registerTransportVersion(int id, String uniqueId public static final TransportVersion V_8_500_027 = registerTransportVersion(8_500_027, "B151D967-8E7C-401C-8275-0ABC06335F2D"); public static final TransportVersion V_8_500_028 = registerTransportVersion(8_500_028, "a6592d08-15cb-4e1a-b9b4-b2ba24058444"); public static final TransportVersion V_8_500_029 = registerTransportVersion(8_500_029, "f3bd98af-6187-e161-e315-718a2fecc2db"); + public static final TransportVersion V_8_500_030 = registerTransportVersion(8_500_030, "b72d7f12-8ed3-4a5b-8e6a-4910ea10e0d7"); private static class CurrentHolder { - private static final TransportVersion CURRENT = findCurrent(V_8_500_029); + private static final TransportVersion CURRENT = findCurrent(V_8_500_030); // finds the pluggable current version, or uses the given fallback private static TransportVersion findCurrent(TransportVersion fallback) { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/downsample/DownsampleIndexerAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/downsample/DownsampleIndexerAction.java index 3e8be842db386..770b0f8ceff2d 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/downsample/DownsampleIndexerAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/downsample/DownsampleIndexerAction.java @@ -6,6 +6,7 @@ */ package org.elasticsearch.xpack.core.downsample; +import org.elasticsearch.TransportVersion; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.action.downsample.DownsampleConfig; @@ -40,17 +41,23 @@ private DownsampleIndexerAction() { public static class Request extends BroadcastRequest implements IndicesRequest, ToXContentObject { private DownsampleAction.Request downsampleRequest; + private long indexStartTimeMillis; + private long indexEndTimeMillis; private String[] dimensionFields; private String[] metricFields; private String[] labelFields; public Request( DownsampleAction.Request downsampleRequest, + final long indexStartTimeMillis, + final long indexEndTimeMillis, final String[] dimensionFields, final String[] metricFields, final String[] labelFields ) { super(downsampleRequest.indices()); + this.indexStartTimeMillis = indexStartTimeMillis; + this.indexEndTimeMillis = indexEndTimeMillis; this.downsampleRequest = downsampleRequest; this.dimensionFields = dimensionFields; this.metricFields = metricFields; @@ -61,6 +68,13 @@ public Request() {} public Request(StreamInput in) throws IOException { super(in); + if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_500_030) && in.readBoolean()) { + this.indexStartTimeMillis = in.readVLong(); + this.indexEndTimeMillis = in.readVLong(); + } else { + this.indexStartTimeMillis = 0; + this.indexEndTimeMillis = 0; + } this.downsampleRequest = new DownsampleAction.Request(in); this.dimensionFields = in.readStringArray(); this.metricFields = in.readStringArray(); @@ -81,6 +95,14 @@ public DownsampleAction.Request getDownsampleRequest() { return downsampleRequest; } + public long getIndexStartTimeMillis() { + return indexStartTimeMillis; + } + + public long getIndexEndTimeMillis() { + return indexEndTimeMillis; + } + public String[] getDimensionFields() { return this.dimensionFields; } @@ -109,6 +131,13 @@ public Task createTask(long id, String type, String action, TaskId parentTaskId, @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); + if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_500_030)) { + out.writeBoolean(true); + out.writeVLong(indexStartTimeMillis); + out.writeVLong(indexEndTimeMillis); + } else { + out.writeBoolean(false); + } downsampleRequest.writeTo(out); out.writeStringArray(dimensionFields); out.writeStringArray(metricFields); @@ -215,7 +244,7 @@ public ShardDownsampleRequest(StreamInput in) throws IOException { this.request = new Request(in); } - public ShardDownsampleRequest(ShardId shardId, Request request) { + public ShardDownsampleRequest(final ShardId shardId, final Request request) { super(shardId, request); this.request = request; } @@ -254,6 +283,8 @@ public Task createTask(long id, String type, String action, TaskId parentTaskId, action, parentTaskId, request.downsampleRequest.getSourceIndex(), + request.getIndexStartTimeMillis(), + request.getIndexEndTimeMillis(), request.downsampleRequest.getDownsampleConfig(), headers, shardId() diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollupAfterBulkInfo.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollupAfterBulkInfo.java new file mode 100644 index 0000000000000..fbb1c510f63b8 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollupAfterBulkInfo.java @@ -0,0 +1,99 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.rollup.action; + +import org.elasticsearch.common.io.stream.NamedWriteable; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.xcontent.ConstructingObjectParser; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.ToXContentObject; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentParser; + +import java.io.IOException; + +/** + * This class includes statistics collected by the downsampling task after + * a bulk indexing operation ends. + */ +public record RollupAfterBulkInfo( + long currentTimeMillis, + long executionId, + long lastIngestTookInMillis, + long lastTookInMillis, + boolean hasFailures, + int restStatusCode +) implements NamedWriteable, ToXContentObject { + + public static final String NAME = "rollup_after_bulk_info"; + + private static final ParseField CURRENT_TIME_IN_MILLIS = new ParseField("current_time_in_millis"); + private static final ParseField EXECUTION_ID = new ParseField("execution_id"); + private static final ParseField LAST_INGEST_TOOK_IN_MILLIS = new ParseField("last_ingest_took_in_millis"); + private static final ParseField LAST_TOOK_IN_MILLIS = new ParseField("last_took_in_millis"); + private static final ParseField HAS_FAILURES = new ParseField("has_failures"); + private static final ParseField REST_STATUS_CODE = new ParseField("rest_status_code"); + + private static final ConstructingObjectParser PARSER; + static { + PARSER = new ConstructingObjectParser<>( + NAME, + args -> new RollupAfterBulkInfo( + (Long) args[0], + (Long) args[1], + (Long) args[2], + (Long) args[3], + (Boolean) args[4], + (Integer) args[5] + ) + ); + + PARSER.declareLong(ConstructingObjectParser.constructorArg(), CURRENT_TIME_IN_MILLIS); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), EXECUTION_ID); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), LAST_INGEST_TOOK_IN_MILLIS); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), LAST_TOOK_IN_MILLIS); + PARSER.declareBoolean(ConstructingObjectParser.constructorArg(), HAS_FAILURES); + PARSER.declareInt(ConstructingObjectParser.constructorArg(), REST_STATUS_CODE); + } + + public RollupAfterBulkInfo(final StreamInput in) throws IOException { + this(in.readVLong(), in.readVLong(), in.readVLong(), in.readVLong(), in.readBoolean(), in.readVInt()); + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVLong(currentTimeMillis); + out.writeVLong(executionId); + out.writeVLong(lastIngestTookInMillis); + out.writeVLong(lastTookInMillis); + out.writeBoolean(hasFailures); + out.writeVInt(restStatusCode); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(NAME); + builder.field(CURRENT_TIME_IN_MILLIS.getPreferredName(), currentTimeMillis); + builder.field(EXECUTION_ID.getPreferredName(), executionId); + builder.field(LAST_INGEST_TOOK_IN_MILLIS.getPreferredName(), lastIngestTookInMillis); + builder.field(LAST_TOOK_IN_MILLIS.getPreferredName(), lastTookInMillis); + builder.field(HAS_FAILURES.getPreferredName(), hasFailures); + builder.field(REST_STATUS_CODE.getPreferredName(), restStatusCode); + return builder.endObject(); + } + + public static RollupAfterBulkInfo fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollupBeforeBulkInfo.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollupBeforeBulkInfo.java new file mode 100644 index 0000000000000..4a5955319a8ad --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollupBeforeBulkInfo.java @@ -0,0 +1,80 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.rollup.action; + +import org.elasticsearch.common.io.stream.NamedWriteable; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.xcontent.ConstructingObjectParser; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.ToXContentObject; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentParser; + +import java.io.IOException; + +/** + * This class includes statistics collected by the downsampling task before + * a bulk indexing operation starts. + */ +public record RollupBeforeBulkInfo(long currentTimeMillis, long executionId, long estimatedSizeInBytes, int numberOfActions) + implements + NamedWriteable, + ToXContentObject { + + public static final String NAME = "rollup_before_bulk_info"; + + private static final ParseField CURRENT_TIME_IN_MILLIS = new ParseField("current_time_in_millis"); + private static final ParseField EXECUTION_ID = new ParseField("execution_id"); + private static final ParseField ESTIMATED_SIZE_IN_BYTES = new ParseField("estimated_size_in_bytes"); + private static final ParseField NUMBER_OF_ACTIONS = new ParseField("number_of_actions"); + + private static final ConstructingObjectParser PARSER; + static { + PARSER = new ConstructingObjectParser<>( + NAME, + args -> new RollupBeforeBulkInfo((Long) args[0], (Long) args[1], (Long) args[2], (Integer) args[3]) + ); + + PARSER.declareLong(ConstructingObjectParser.constructorArg(), CURRENT_TIME_IN_MILLIS); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), EXECUTION_ID); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), ESTIMATED_SIZE_IN_BYTES); + PARSER.declareInt(ConstructingObjectParser.constructorArg(), NUMBER_OF_ACTIONS); + } + + public RollupBeforeBulkInfo(final StreamInput in) throws IOException { + this(in.readVLong(), in.readVLong(), in.readVLong(), in.readVInt()); + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVLong(currentTimeMillis); + out.writeVLong(executionId); + out.writeVLong(estimatedSizeInBytes); + out.writeVInt(numberOfActions); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(NAME); + builder.field(CURRENT_TIME_IN_MILLIS.getPreferredName(), currentTimeMillis); + builder.field(EXECUTION_ID.getPreferredName(), executionId); + builder.field(ESTIMATED_SIZE_IN_BYTES.getPreferredName(), estimatedSizeInBytes); + builder.field(NUMBER_OF_ACTIONS.getPreferredName(), numberOfActions); + return builder.endObject(); + } + + public static RollupBeforeBulkInfo fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollupBulkInfo.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollupBulkInfo.java new file mode 100644 index 0000000000000..105001d59f4c7 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollupBulkInfo.java @@ -0,0 +1,105 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.rollup.action; + +import org.elasticsearch.common.io.stream.NamedWriteable; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.xcontent.ConstructingObjectParser; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.ToXContentObject; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentParser; + +import java.io.IOException; + +/** + * This class includes statistics collected by the downsampling task + * for bulk indexing operations. + */ +public record RollupBulkInfo( + long totalBulkCount, + long bulkIngestSumMillis, + long maxBulkIngestMillis, + long minBulkIngestMillis, + long bulkTookSumMillis, + long maxBulkTookMillis, + long minBulkTookMillis +) implements NamedWriteable, ToXContentObject { + + public static final String NAME = "rollup_bulk_info"; + + private static final ParseField TOTAL_BULK_COUNT = new ParseField("total_bulk_count"); + private static final ParseField BULK_INGEST_SUM_MILLIS = new ParseField("bulk_ingest_sum_millis"); + private static final ParseField MAX_BULK_INGEST_MILLIS = new ParseField("max_bulk_ingest_millis"); + private static final ParseField MIN_BULK_INGEST_MILLIS = new ParseField("min_bulk_ingest_millis"); + private static final ParseField BULK_TOOK_SUM_MILLIS = new ParseField("bulk_took_sum_millis"); + private static final ParseField MAX_BULK_TOOK_MILLIS = new ParseField("max_bulk_took_millis"); + private static final ParseField MIN_BULK_TOOK_MILLIS = new ParseField("min_bulk_took_millis"); + + private static final ConstructingObjectParser PARSER; + static { + PARSER = new ConstructingObjectParser<>( + NAME, + args -> new RollupBulkInfo( + (Long) args[0], + (Long) args[1], + (Long) args[2], + (Long) args[3], + (Long) args[4], + (Long) args[5], + (Long) args[6] + ) + ); + + PARSER.declareLong(ConstructingObjectParser.constructorArg(), TOTAL_BULK_COUNT); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), BULK_INGEST_SUM_MILLIS); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), MAX_BULK_INGEST_MILLIS); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), MIN_BULK_INGEST_MILLIS); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), BULK_TOOK_SUM_MILLIS); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), MAX_BULK_TOOK_MILLIS); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), MIN_BULK_TOOK_MILLIS); + } + + public RollupBulkInfo(final StreamInput in) throws IOException { + this(in.readVLong(), in.readVLong(), in.readVLong(), in.readVLong(), in.readVLong(), in.readVLong(), in.readVLong()); + } + + public static RollupBulkInfo fromXContext(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(NAME); + builder.field(TOTAL_BULK_COUNT.getPreferredName(), totalBulkCount); + builder.field(BULK_INGEST_SUM_MILLIS.getPreferredName(), bulkIngestSumMillis); + builder.field(MAX_BULK_INGEST_MILLIS.getPreferredName(), maxBulkIngestMillis); + builder.field(MIN_BULK_INGEST_MILLIS.getPreferredName(), minBulkIngestMillis); + builder.field(BULK_TOOK_SUM_MILLIS.getPreferredName(), bulkTookSumMillis); + builder.field(MAX_BULK_TOOK_MILLIS.getPreferredName(), maxBulkTookMillis); + builder.field(MIN_BULK_TOOK_MILLIS.getPreferredName(), minBulkTookMillis); + return builder.endObject(); + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVLong(totalBulkCount); + out.writeVLong(bulkIngestSumMillis); + out.writeVLong(maxBulkIngestMillis); + out.writeVLong(minBulkIngestMillis); + out.writeVLong(bulkTookSumMillis); + out.writeVLong(maxBulkTookMillis); + out.writeVLong(minBulkTookMillis); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollupBulkStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollupBulkStats.java new file mode 100644 index 0000000000000..fea1a88fe793f --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollupBulkStats.java @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.rollup.action; + +import java.util.concurrent.atomic.AtomicLong; + +public class RollupBulkStats { + private final AtomicLong totalBulkCount = new AtomicLong(0); + private final AtomicLong bulkIngestSumMillis = new AtomicLong(0); + private final AtomicLong maxBulkIngestMillis = new AtomicLong(-1); + private final AtomicLong minBulkIngestMillis = new AtomicLong(-1); + private final AtomicLong bulkTookSumMillis = new AtomicLong(0); + private final AtomicLong maxBulkTookMillis = new AtomicLong(-1); + private final AtomicLong minBulkTookMillis = new AtomicLong(-1); + + public void update(long bulkIngestMillis, long bulkTookMillis) { + this.totalBulkCount.incrementAndGet(); + + this.bulkIngestSumMillis.addAndGet(bulkIngestMillis); + this.maxBulkIngestMillis.updateAndGet(existingValue -> max(bulkIngestMillis, existingValue)); + this.minBulkIngestMillis.updateAndGet(existingValue -> min(bulkIngestMillis, existingValue)); + + this.bulkTookSumMillis.addAndGet(bulkTookMillis); + this.maxBulkTookMillis.updateAndGet(existingValue -> max(bulkTookMillis, existingValue)); + this.minBulkTookMillis.updateAndGet(existingValue -> min(bulkTookMillis, existingValue)); + } + + private static long min(long newValue, long existingValue) { + return existingValue == -1 ? newValue : Math.min(newValue, existingValue); + } + + private static long max(long newValue, long existingValue) { + return existingValue == -1 ? newValue : Math.max(newValue, existingValue); + } + + /** + * @return An instance of {@link RollupBulkInfo} including rollup bulk indexing statistics. + */ + public RollupBulkInfo getRollupBulkInfo() { + return new RollupBulkInfo( + this.totalBulkCount.get(), + this.bulkIngestSumMillis.get(), + Math.max(0, this.maxBulkIngestMillis.get()), + Math.max(0, this.minBulkIngestMillis.get()), + this.bulkTookSumMillis.get(), + Math.max(0, this.maxBulkTookMillis.get()), + Math.max(0, this.minBulkTookMillis.get()) + ); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollupShardIndexerStatus.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollupShardIndexerStatus.java new file mode 100644 index 0000000000000..ad60ab1d92911 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollupShardIndexerStatus.java @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.rollup.action; + +/** + * Status of the rollup indexer task + */ +public enum RollupShardIndexerStatus { + INITIALIZED, + STARTED, + FAILED, + COMPLETED, + CANCELLED +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollupShardStatus.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollupShardStatus.java index 2bbd2d96b9cea..c4d227f057842 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollupShardStatus.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollupShardStatus.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.core.rollup.action; +import org.elasticsearch.TransportVersion; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -21,8 +22,6 @@ import java.time.Instant; import java.util.Objects; -import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; - public class RollupShardStatus implements Task.Status { public static final String NAME = "rollup-index-shard"; private static final ParseField SHARD_FIELD = new ParseField("shard"); @@ -31,6 +30,18 @@ public class RollupShardStatus implements Task.Status { private static final ParseField OUT_NUM_DOCS_SENT_FIELD = new ParseField("out_num_docs_sent"); private static final ParseField OUT_NUM_DOCS_INDEXED_FIELD = new ParseField("out_num_docs_indexed"); private static final ParseField OUT_NUM_DOCS_FAILED_FIELD = new ParseField("out_num_docs_failed"); + private static final ParseField TOTAL_SHARD_DOC_COUNT = new ParseField("total_shard_doc_count"); + private static final ParseField LAST_SOURCE_TIMESTAMP = new ParseField("last_source_timestamp"); + private static final ParseField LAST_TARGET_TIMESTAMP = new ParseField("last_target_timestamp"); + private static final ParseField LAST_INDEXING_TIMESTAMP = new ParseField("last_indexing_timestamp"); + private static final ParseField DOCS_PROCESSED = new ParseField("docs_processed"); + private static final ParseField INDEX_START_TIME_MILLIS = new ParseField("index_start_time"); + private static final ParseField INDEX_END_TIME_MILLIS = new ParseField("index_end_time"); + private static final ParseField DOCS_PROCESSED_PERCENTAGE = new ParseField("docs_processed_percentage"); + private static final ParseField ROLLUP_BULK_INFO = new ParseField("rollup_bulk_info"); + private static final ParseField ROLLUP_BEFORE_BULK_INFO = new ParseField("rollup_before_bulk_info"); + private static final ParseField ROLLUP_AFTER_BULK_INFO = new ParseField("rollup_after_bulk_info"); + private static final ParseField ROLLUP_SHARD_INDEXER_STATUS = new ParseField("rollup_shard_indexer_status"); private final ShardId shardId; private final long rollupStart; @@ -38,27 +49,88 @@ public class RollupShardStatus implements Task.Status { private final long numSent; private final long numIndexed; private final long numFailed; + private final long totalShardDocCount; + private final long lastSourceTimestamp; + private final long lastTargetTimestamp; + private final long lastIndexingTimestamp; + private final long indexStartTimeMillis; + private final long indexEndTimeMillis; + private final long docsProcessed; + private final float docsProcessedPercentage; + private final RollupBulkInfo rollupBulkInfo; + private final RollupBeforeBulkInfo rollupBeforeBulkInfo; + private final RollupAfterBulkInfo rollupAfterBulkInfo; + private final RollupShardIndexerStatus rollupShardIndexerStatus; private static final ConstructingObjectParser PARSER; static { - PARSER = new ConstructingObjectParser<>( - NAME, - args -> new RollupShardStatus( - ShardId.fromString((String) args[0]), - Instant.parse((String) args[1]).toEpochMilli(), - (Long) args[2], - (Long) args[3], - (Long) args[4], - (Long) args[5] - ) - ); + PARSER = new ConstructingObjectParser<>(NAME, args -> { + final ShardId _shardId = ShardId.fromString((String) args[0]); + long _rollupStart = Instant.parse((String) args[1]).toEpochMilli(); + final Long _numReceived = (Long) args[2]; + final Long _numSent = (Long) args[3]; + final Long _numIndexed = (Long) args[4]; + final Long _numFailed = (Long) args[5]; + final Long _totalShardDocCount = (Long) args[6]; + final Long _lastSourceTimestamp = (Long) args[7]; + final Long _lastTargetTimestamp = (Long) args[8]; + final Long _lastIndexingTimestamp = (Long) args[9]; + final Long _indexStartTimeMillis = (Long) args[10]; + final Long _indexEndTimeMillis = (Long) args[11]; + final Long _docsProcessed = (Long) args[12]; + final Float _docsProcessedPercentage = (Float) args[13]; + final RollupBulkInfo _rollupBulkInfo = (RollupBulkInfo) args[14]; + final RollupBeforeBulkInfo _rollupBeforeBulkInfo = (RollupBeforeBulkInfo) args[15]; + final RollupAfterBulkInfo _rollupAfterBulkInfo = (RollupAfterBulkInfo) args[16]; + final RollupShardIndexerStatus _rollupShardIndexerStatus = RollupShardIndexerStatus.valueOf((String) args[17]); + return new RollupShardStatus( + _shardId, + _rollupStart, + _numReceived, + _numSent, + _numIndexed, + _numFailed, + _totalShardDocCount, + _lastSourceTimestamp, + _lastTargetTimestamp, + _lastIndexingTimestamp, + _indexStartTimeMillis, + _indexEndTimeMillis, + _docsProcessed, + _docsProcessedPercentage, + _rollupBulkInfo, + _rollupBeforeBulkInfo, + _rollupAfterBulkInfo, + _rollupShardIndexerStatus + ); + }); - PARSER.declareString(constructorArg(), SHARD_FIELD); - PARSER.declareString(constructorArg(), START_TIME_FIELD); - PARSER.declareLong(constructorArg(), IN_NUM_DOCS_RECEIVED_FIELD); - PARSER.declareLong(constructorArg(), OUT_NUM_DOCS_SENT_FIELD); - PARSER.declareLong(constructorArg(), OUT_NUM_DOCS_INDEXED_FIELD); - PARSER.declareLong(constructorArg(), OUT_NUM_DOCS_FAILED_FIELD); + PARSER.declareString(ConstructingObjectParser.constructorArg(), SHARD_FIELD); + PARSER.declareString(ConstructingObjectParser.constructorArg(), START_TIME_FIELD); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), IN_NUM_DOCS_RECEIVED_FIELD); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), OUT_NUM_DOCS_SENT_FIELD); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), OUT_NUM_DOCS_INDEXED_FIELD); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), OUT_NUM_DOCS_FAILED_FIELD); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), TOTAL_SHARD_DOC_COUNT); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), LAST_SOURCE_TIMESTAMP); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), LAST_TARGET_TIMESTAMP); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), LAST_INDEXING_TIMESTAMP); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), INDEX_START_TIME_MILLIS); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), INDEX_END_TIME_MILLIS); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), DOCS_PROCESSED); + PARSER.declareFloat(ConstructingObjectParser.constructorArg(), DOCS_PROCESSED_PERCENTAGE); + PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> RollupBulkInfo.fromXContext(p), ROLLUP_BULK_INFO); + PARSER.declareObject( + ConstructingObjectParser.optionalConstructorArg(), + (p, c) -> RollupBeforeBulkInfo.fromXContent(p), + ROLLUP_BEFORE_BULK_INFO + ); + PARSER.declareObject( + ConstructingObjectParser.optionalConstructorArg(), + (p, c) -> RollupAfterBulkInfo.fromXContent(p), + ROLLUP_AFTER_BULK_INFO + ); + PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), ROLLUP_SHARD_INDEXER_STATUS); } public RollupShardStatus(StreamInput in) throws IOException { @@ -68,15 +140,73 @@ public RollupShardStatus(StreamInput in) throws IOException { numSent = in.readLong(); numIndexed = in.readLong(); numFailed = in.readLong(); + if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_500_030) && in.readBoolean()) { + totalShardDocCount = in.readVLong(); + lastSourceTimestamp = in.readVLong(); + lastTargetTimestamp = in.readVLong(); + lastIndexingTimestamp = in.readVLong(); + indexStartTimeMillis = in.readVLong(); + indexEndTimeMillis = in.readVLong(); + docsProcessed = in.readVLong(); + docsProcessedPercentage = in.readFloat(); + rollupBulkInfo = new RollupBulkInfo(in); + rollupBeforeBulkInfo = new RollupBeforeBulkInfo(in); + rollupAfterBulkInfo = new RollupAfterBulkInfo(in); + rollupShardIndexerStatus = in.readEnum(RollupShardIndexerStatus.class); + } else { + totalShardDocCount = -1; + lastSourceTimestamp = -1; + lastTargetTimestamp = -1; + lastIndexingTimestamp = -1; + indexStartTimeMillis = -1; + indexEndTimeMillis = -1; + docsProcessed = 0; + docsProcessedPercentage = 0; + rollupBulkInfo = null; + rollupBeforeBulkInfo = null; + rollupAfterBulkInfo = null; + rollupShardIndexerStatus = null; + } } - public RollupShardStatus(ShardId shardId, long rollupStart, long numReceived, long numSent, long numIndexed, long numFailed) { + public RollupShardStatus( + ShardId shardId, + long rollupStart, + long numReceived, + long numSent, + long numIndexed, + long numFailed, + long totalShardDocCount, + long lastSourceTimestamp, + long lastTargetTimestamp, + long lastIndexingTimestamp, + long indexStartTimeMillis, + long indexEndTimeMillis, + long docsProcessed, + float docsProcessedPercentage, + final RollupBulkInfo rollupBulkInfo, + final RollupBeforeBulkInfo rollupBeforeBulkInfo, + final RollupAfterBulkInfo rollupAfterBulkInfo, + final RollupShardIndexerStatus rollupShardIndexerStatus + ) { this.shardId = shardId; this.rollupStart = rollupStart; this.numReceived = numReceived; this.numSent = numSent; this.numIndexed = numIndexed; this.numFailed = numFailed; + this.totalShardDocCount = totalShardDocCount; + this.lastSourceTimestamp = lastSourceTimestamp; + this.lastTargetTimestamp = lastTargetTimestamp; + this.lastIndexingTimestamp = lastIndexingTimestamp; + this.indexStartTimeMillis = indexStartTimeMillis; + this.indexEndTimeMillis = indexEndTimeMillis; + this.docsProcessed = docsProcessed; + this.docsProcessedPercentage = docsProcessedPercentage; + this.rollupBulkInfo = rollupBulkInfo; + this.rollupBeforeBulkInfo = rollupBeforeBulkInfo; + this.rollupAfterBulkInfo = rollupAfterBulkInfo; + this.rollupShardIndexerStatus = rollupShardIndexerStatus; } public static RollupShardStatus fromXContent(XContentParser parser) throws IOException { @@ -92,8 +222,19 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(OUT_NUM_DOCS_SENT_FIELD.getPreferredName(), numSent); builder.field(OUT_NUM_DOCS_INDEXED_FIELD.getPreferredName(), numIndexed); builder.field(OUT_NUM_DOCS_FAILED_FIELD.getPreferredName(), numFailed); - builder.endObject(); - return builder; + builder.field(TOTAL_SHARD_DOC_COUNT.getPreferredName(), totalShardDocCount); + builder.field(LAST_SOURCE_TIMESTAMP.getPreferredName(), lastSourceTimestamp); + builder.field(LAST_TARGET_TIMESTAMP.getPreferredName(), lastTargetTimestamp); + builder.field(LAST_INDEXING_TIMESTAMP.getPreferredName(), lastIndexingTimestamp); + builder.field(INDEX_START_TIME_MILLIS.getPreferredName(), indexStartTimeMillis); + builder.field(INDEX_END_TIME_MILLIS.getPreferredName(), indexEndTimeMillis); + builder.field(DOCS_PROCESSED.getPreferredName(), docsProcessed); + builder.field(DOCS_PROCESSED_PERCENTAGE.getPreferredName(), docsProcessedPercentage); + rollupBulkInfo.toXContent(builder, params); + rollupBeforeBulkInfo.toXContent(builder, params); + rollupAfterBulkInfo.toXContent(builder, params); + builder.field(ROLLUP_SHARD_INDEXER_STATUS.getPreferredName(), rollupShardIndexerStatus); + return builder.endObject(); } @Override @@ -109,29 +250,74 @@ public void writeTo(StreamOutput out) throws IOException { out.writeLong(numSent); out.writeLong(numIndexed); out.writeLong(numFailed); + if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_500_030)) { + out.writeBoolean(true); + out.writeVLong(totalShardDocCount); + out.writeVLong(lastSourceTimestamp); + out.writeVLong(lastTargetTimestamp); + out.writeVLong(lastIndexingTimestamp); + out.writeVLong(indexStartTimeMillis); + out.writeVLong(indexEndTimeMillis); + out.writeVLong(docsProcessed); + out.writeFloat(docsProcessedPercentage); + rollupBulkInfo.writeTo(out); + rollupBeforeBulkInfo.writeTo(out); + rollupAfterBulkInfo.writeTo(out); + out.writeEnum(rollupShardIndexerStatus); + } else { + out.writeBoolean(false); + } } @Override public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; RollupShardStatus that = (RollupShardStatus) o; return rollupStart == that.rollupStart + && numReceived == that.numReceived + && numSent == that.numSent + && numIndexed == that.numIndexed + && numFailed == that.numFailed + && totalShardDocCount == that.totalShardDocCount + && lastSourceTimestamp == that.lastSourceTimestamp + && lastTargetTimestamp == that.lastTargetTimestamp + && lastIndexingTimestamp == that.lastIndexingTimestamp + && indexStartTimeMillis == that.indexStartTimeMillis + && indexEndTimeMillis == that.indexEndTimeMillis + && docsProcessed == that.docsProcessed + && docsProcessedPercentage == that.docsProcessedPercentage && Objects.equals(shardId.getIndexName(), that.shardId.getIndexName()) && Objects.equals(shardId.id(), that.shardId.id()) - && Objects.equals(numReceived, that.numReceived) - && Objects.equals(numSent, that.numSent) - && Objects.equals(numIndexed, that.numIndexed) - && Objects.equals(numFailed, that.numFailed); + && Objects.equals(rollupBulkInfo, that.rollupBulkInfo) + && Objects.equals(rollupBeforeBulkInfo, that.rollupBeforeBulkInfo) + && Objects.equals(rollupAfterBulkInfo, that.rollupAfterBulkInfo) + && Objects.equals(rollupShardIndexerStatus, that.rollupShardIndexerStatus); } @Override public int hashCode() { - return Objects.hash(shardId.getIndexName(), shardId.id(), rollupStart, numReceived, numSent, numIndexed, numFailed); + return Objects.hash( + shardId.getIndexName(), + shardId.id(), + rollupStart, + numReceived, + numSent, + numIndexed, + numFailed, + totalShardDocCount, + lastSourceTimestamp, + lastTargetTimestamp, + lastIndexingTimestamp, + indexStartTimeMillis, + indexEndTimeMillis, + docsProcessed, + docsProcessedPercentage, + rollupBulkInfo, + rollupBeforeBulkInfo, + rollupAfterBulkInfo, + rollupShardIndexerStatus + ); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollupShardTask.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollupShardTask.java index 49a32388845b4..a7277b9c1be99 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollupShardTask.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollupShardTask.java @@ -15,9 +15,14 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; public class RollupShardTask extends CancellableTask { private final String rollupIndex; + private volatile long totalShardDocCount; + private volatile long docsProcessed; + private final long indexStartTimeMillis; + private final long indexEndTimeMillis; private final DownsampleConfig config; private final ShardId shardId; private final long rollupStartTime; @@ -25,6 +30,15 @@ public class RollupShardTask extends CancellableTask { private final AtomicLong numSent = new AtomicLong(0); private final AtomicLong numIndexed = new AtomicLong(0); private final AtomicLong numFailed = new AtomicLong(0); + private final AtomicLong lastSourceTimestamp = new AtomicLong(0); + private final AtomicLong lastTargetTimestamp = new AtomicLong(0); + private final AtomicLong lastIndexingTimestamp = new AtomicLong(0); + private final AtomicReference rollupShardIndexerStatus = new AtomicReference<>( + RollupShardIndexerStatus.INITIALIZED + ); + private final RollupBulkStats rollupBulkStats; + private final AtomicReference lastBeforeBulkInfo = new AtomicReference<>(null); + private final AtomicReference lastAfterBulkInfo = new AtomicReference<>(null); public RollupShardTask( long id, @@ -32,15 +46,20 @@ public RollupShardTask( String action, TaskId parentTask, String rollupIndex, + long indexStartTimeMillis, + long indexEndTimeMillis, DownsampleConfig config, Map headers, ShardId shardId ) { super(id, type, action, RollupField.NAME + "_" + rollupIndex + "[" + shardId.id() + "]", parentTask, headers); this.rollupIndex = rollupIndex; + this.indexStartTimeMillis = indexStartTimeMillis; + this.indexEndTimeMillis = indexEndTimeMillis; this.config = config; this.shardId = shardId; this.rollupStartTime = System.currentTimeMillis(); + this.rollupBulkStats = new RollupBulkStats(); } public String getRollupIndex() { @@ -51,9 +70,32 @@ public DownsampleConfig config() { return config; } + public long getTotalShardDocCount() { + return totalShardDocCount; + } + @Override public Status getStatus() { - return new RollupShardStatus(shardId, rollupStartTime, numReceived.get(), numSent.get(), numIndexed.get(), numFailed.get()); + return new RollupShardStatus( + shardId, + rollupStartTime, + numReceived.get(), + numSent.get(), + numIndexed.get(), + numFailed.get(), + totalShardDocCount, + lastSourceTimestamp.get(), + lastTargetTimestamp.get(), + lastIndexingTimestamp.get(), + indexStartTimeMillis, + indexEndTimeMillis, + docsProcessed, + 100.0F * docsProcessed / totalShardDocCount, + rollupBulkStats.getRollupBulkInfo(), + lastBeforeBulkInfo.get(), + lastAfterBulkInfo.get(), + rollupShardIndexerStatus.get() + ); } public long getNumReceived() { @@ -72,6 +114,50 @@ public long getNumFailed() { return numFailed.get(); } + public long getLastSourceTimestamp() { + return lastSourceTimestamp.get(); + } + + public long getLastTargetTimestamp() { + return lastTargetTimestamp.get(); + } + + public RollupBeforeBulkInfo getLastBeforeBulkInfo() { + return lastBeforeBulkInfo.get(); + } + + public RollupAfterBulkInfo getLastAfterBulkInfo() { + return lastAfterBulkInfo.get(); + } + + public long getDocsProcessed() { + return docsProcessed; + } + + public long getRollupStartTime() { + return rollupStartTime; + } + + public RollupShardIndexerStatus getRollupShardIndexerStatus() { + return rollupShardIndexerStatus.get(); + } + + public long getLastIndexingTimestamp() { + return lastIndexingTimestamp.get(); + } + + public long getIndexStartTimeMillis() { + return indexStartTimeMillis; + } + + public long getIndexEndTimeMillis() { + return indexEndTimeMillis; + } + + public float getDocsProcessedPercentage() { + return getTotalShardDocCount() <= 0 ? 0.0F : 100.0F * docsProcessed / totalShardDocCount; + } + public void addNumReceived(long count) { numReceived.addAndGet(count); } @@ -87,4 +173,44 @@ public void addNumIndexed(long count) { public void addNumFailed(long count) { numFailed.addAndGet(count); } + + public void setLastSourceTimestamp(long timestamp) { + lastSourceTimestamp.set(timestamp); + } + + public void setLastTargetTimestamp(long timestamp) { + lastTargetTimestamp.set(timestamp); + } + + public void setLastIndexingTimestamp(long timestamp) { + lastIndexingTimestamp.set(timestamp); + } + + public void setBeforeBulkInfo(final RollupBeforeBulkInfo beforeBulkInfo) { + lastBeforeBulkInfo.set(beforeBulkInfo); + } + + public void setAfterBulkInfo(final RollupAfterBulkInfo afterBulkInfo) { + lastAfterBulkInfo.set(afterBulkInfo); + } + + public void setRollupShardIndexerStatus(final RollupShardIndexerStatus status) { + this.rollupShardIndexerStatus.set(status); + } + + public void setTotalShardDocCount(int totalShardDocCount) { + this.totalShardDocCount = totalShardDocCount; + } + + public void setDocsProcessed(long docsProcessed) { + this.docsProcessed = docsProcessed; + } + + public void updateRollupBulkInfo(long bulkIngestTookMillis, long bulkTookMillis) { + this.rollupBulkStats.update(bulkIngestTookMillis, bulkTookMillis); + } + + public RollupBulkInfo getRollupBulkInfo() { + return this.rollupBulkStats.getRollupBulkInfo(); + } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/action/RollupShardStatusSerializingTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/action/RollupShardStatusSerializingTests.java index 2802b1765abb0..5a6537ac5b56f 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/action/RollupShardStatusSerializingTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/action/RollupShardStatusSerializingTests.java @@ -9,12 +9,14 @@ import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.AbstractXContentSerializingTestCase; import org.elasticsearch.xcontent.XContentParser; import java.io.IOException; public class RollupShardStatusSerializingTests extends AbstractXContentSerializingTestCase { + @Override protected RollupShardStatus doParseInstance(XContentParser parser) throws IOException { return RollupShardStatus.fromXContent(parser); @@ -27,15 +29,75 @@ protected Reader instanceReader() { @Override protected RollupShardStatus createTestInstance() { - RollupShardStatus rollupShardStatus = new RollupShardStatus( - new ShardId(randomAlphaOfLength(5), randomAlphaOfLength(5), randomInt(5)), - randomMillisUpToYear9999(), + long docsProcessed = randomLongBetween(500_000, 800_000); + long indexEndTimeMillis = System.currentTimeMillis() + randomLongBetween(400_000, 500_000); + long indexStartTimeMillis = System.currentTimeMillis() - randomLongBetween(400_000, 500_000); + long lastIndexingTimestamp = System.currentTimeMillis() + randomLongBetween(200_000, 300_000); + long lastTargetTimestamp = System.currentTimeMillis() - randomLongBetween(200_000, 300_000); + long lastSourceTimestamp = System.currentTimeMillis(); + long totalShardDocCount = randomLongBetween(500_000, 800_000); + long numFailed = randomNonNegativeLong(); + long numIndexed = randomNonNegativeLong(); + long numSent = randomNonNegativeLong(); + long numReceived = randomNonNegativeLong(); + long rollupStart = randomMillisUpToYear9999(); + final ShardId shardId = new ShardId(randomAlphaOfLength(5), randomAlphaOfLength(5), randomInt(5)); + final RollupShardIndexerStatus rollupShardIndexerStatus = randomFrom(RollupShardIndexerStatus.values()); + return new RollupShardStatus( + shardId, + rollupStart, + numReceived, + numSent, + numIndexed, + numFailed, + totalShardDocCount, + lastSourceTimestamp, + lastTargetTimestamp, + lastIndexingTimestamp, + indexStartTimeMillis, + indexEndTimeMillis, + docsProcessed, + 100.0F * docsProcessed / totalShardDocCount, + createTestRollupBulkInfo(), + createTestBeforeBulkInfoInstance(), + createTestAfterBulkInfoInstance(), + rollupShardIndexerStatus + ); + } + + private RollupBulkInfo createTestRollupBulkInfo() { + return new RollupBulkInfo( + randomNonNegativeLong(), + randomNonNegativeLong(), + randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong() ); - return rollupShardStatus; + } + + private RollupBeforeBulkInfo createTestBeforeBulkInfoInstance() { + return new RollupBeforeBulkInfo( + System.currentTimeMillis(), + randomNonNegativeLong(), + randomNonNegativeLong(), + randomIntBetween(1, 10) + ); + } + + private RollupAfterBulkInfo createTestAfterBulkInfoInstance() { + int randomRestStatusCode = randomBoolean() ? RestStatus.OK.getStatus() + : randomBoolean() ? RestStatus.INTERNAL_SERVER_ERROR.getStatus() + : RestStatus.BAD_REQUEST.getStatus(); + return new RollupAfterBulkInfo( + System.currentTimeMillis(), + randomLongBetween(1_000, 5_000), + randomLongBetween(1_000, 5_000), + randomLongBetween(1_000, 5_000), + randomBoolean(), + randomRestStatusCode + ); } @Override diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/RollupShardIndexer.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/RollupShardIndexer.java index 0130c0aaf8403..5a75ad69a2e64 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/RollupShardIndexer.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/RollupShardIndexer.java @@ -18,6 +18,7 @@ import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.downsample.DownsampleConfig; +import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.client.internal.Client; import org.elasticsearch.common.Rounding; @@ -47,6 +48,9 @@ import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.downsample.DownsampleIndexerAction; +import org.elasticsearch.xpack.core.rollup.action.RollupAfterBulkInfo; +import org.elasticsearch.xpack.core.rollup.action.RollupBeforeBulkInfo; +import org.elasticsearch.xpack.core.rollup.action.RollupShardIndexerStatus; import org.elasticsearch.xpack.core.rollup.action.RollupShardTask; import java.io.Closeable; @@ -130,7 +134,9 @@ class RollupShardIndexer { } public DownsampleIndexerAction.ShardDownsampleResponse execute() throws IOException { - long startTime = System.currentTimeMillis(); + long startTime = client.threadPool().relativeTimeInMillis(); + task.setTotalShardDocCount(searcher.getDirectoryReader().numDocs()); + task.setRollupShardIndexerStatus(RollupShardIndexerStatus.STARTED); BulkProcessor2 bulkProcessor = createBulkProcessor(); try (searcher; bulkProcessor) { final TimeSeriesIndexSearcher timeSeriesSearcher = new TimeSeriesIndexSearcher(searcher, List.of(this::checkCancelled)); @@ -147,10 +153,11 @@ public DownsampleIndexerAction.ShardDownsampleResponse execute() throws IOExcept task.getNumSent(), task.getNumIndexed(), task.getNumFailed(), - TimeValue.timeValueMillis(System.currentTimeMillis() - startTime) + TimeValue.timeValueMillis(client.threadPool().relativeTimeInMillis() - startTime) ); if (task.getNumIndexed() != task.getNumSent()) { + task.setRollupShardIndexerStatus(RollupShardIndexerStatus.FAILED); throw new ElasticsearchException( "Shard [" + indexShard.shardId() @@ -163,6 +170,7 @@ public DownsampleIndexerAction.ShardDownsampleResponse execute() throws IOExcept } if (task.getNumFailed() > 0) { + task.setRollupShardIndexerStatus(RollupShardIndexerStatus.FAILED); throw new ElasticsearchException( "Shard [" + indexShard.shardId() @@ -174,6 +182,8 @@ public DownsampleIndexerAction.ShardDownsampleResponse execute() throws IOExcept ); } + task.setRollupShardIndexerStatus(RollupShardIndexerStatus.COMPLETED); + return new DownsampleIndexerAction.ShardDownsampleResponse(indexShard.shardId(), task.getNumIndexed()); } @@ -186,20 +196,44 @@ private void checkCancelled() { task.getNumIndexed(), task.getNumFailed() ); + task.setRollupShardIndexerStatus(RollupShardIndexerStatus.CANCELLED); throw new TaskCancelledException(format("Shard %s rollup cancelled", indexShard.shardId())); } } private BulkProcessor2 createBulkProcessor() { final BulkProcessor2.Listener listener = new BulkProcessor2.Listener() { + @Override public void beforeBulk(long executionId, BulkRequest request) { task.addNumSent(request.numberOfActions()); + task.setBeforeBulkInfo( + new RollupBeforeBulkInfo( + client.threadPool().absoluteTimeInMillis(), + executionId, + request.estimatedSizeInBytes(), + request.numberOfActions() + ) + ); } @Override public void afterBulk(long executionId, BulkRequest request, BulkResponse response) { + long bulkIngestTookMillis = response.getIngestTookInMillis() >= 0 ? response.getIngestTookInMillis() : 0; + long bulkTookMillis = response.getTook().getMillis(); task.addNumIndexed(request.numberOfActions()); + task.setAfterBulkInfo( + new RollupAfterBulkInfo( + client.threadPool().absoluteTimeInMillis(), + executionId, + bulkIngestTookMillis, + bulkTookMillis, + response.hasFailures(), + response.status().getStatus() + ) + ); + task.updateRollupBulkInfo(bulkIngestTookMillis, bulkTookMillis); + if (response.hasFailures()) { List failedItems = Arrays.stream(response.getItems()).filter(BulkItemResponse::isFailed).toList(); task.addNumFailed(failedItems.size()); @@ -283,6 +317,8 @@ public void collect(int docId, long owningBucketOrd) throws IOException { searchExecutionContext.getIndexSettings().getTimestampBounds().startTime() ); } + task.setLastSourceTimestamp(timestamp); + task.setLastTargetTimestamp(lastHistoTimestamp); if (logger.isTraceEnabled()) { logger.trace( @@ -339,6 +375,7 @@ public void collect(int docId, long owningBucketOrd) throws IOException { rollupFieldProducer.collect(docValues, docId); } docsProcessed++; + task.setDocsProcessed(docsProcessed); } }; } @@ -349,7 +386,9 @@ private void indexBucket(XContentBuilder doc) { if (logger.isTraceEnabled()) { logger.trace("Indexing rollup doc: [{}]", Strings.toString(doc)); } - bulkProcessor.addWithBackpressure(request.request(), () -> abort); + IndexRequest indexRequest = request.request(); + task.setLastIndexingTimestamp(System.currentTimeMillis()); + bulkProcessor.addWithBackpressure(indexRequest, () -> abort); } @Override diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java index 7f3fa21811efc..26d93d0ab8559 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java @@ -74,6 +74,8 @@ import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl; import java.io.IOException; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -287,6 +289,14 @@ protected void masterOperation( // 3. Rollup index created. Run rollup indexer DownsampleIndexerAction.Request rollupIndexerRequest = new DownsampleIndexerAction.Request( request, + OffsetDateTime.parse( + sourceIndexMetadata.getSettings().get(IndexSettings.TIME_SERIES_START_TIME.getKey()), + DateTimeFormatter.ISO_DATE_TIME + ).toInstant().toEpochMilli(), + OffsetDateTime.parse( + sourceIndexMetadata.getSettings().get(IndexSettings.TIME_SERIES_END_TIME.getKey()), + DateTimeFormatter.ISO_DATE_TIME + ).toInstant().toEpochMilli(), dimensionFields.toArray(new String[0]), metricFields.toArray(new String[0]), labelFields.toArray(new String[0]) diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java index d5c30dae0d9ac..1d9161c42ccee 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java @@ -51,6 +51,7 @@ import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.aggregations.Aggregation; import org.elasticsearch.search.aggregations.AggregationBuilder; @@ -78,9 +79,11 @@ import org.elasticsearch.xpack.aggregatemetric.AggregateMetricMapperPlugin; import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; import org.elasticsearch.xpack.core.downsample.DownsampleAction; +import org.elasticsearch.xpack.core.downsample.DownsampleIndexerAction; import org.elasticsearch.xpack.core.ilm.LifecycleSettings; import org.elasticsearch.xpack.core.ilm.RolloverAction; import org.elasticsearch.xpack.core.rollup.ConfigTestHelpers; +import org.elasticsearch.xpack.core.rollup.action.RollupShardIndexerStatus; import org.elasticsearch.xpack.core.rollup.action.RollupShardTask; import org.elasticsearch.xpack.ilm.IndexLifecycle; import org.elasticsearch.xpack.rollup.Rollup; @@ -583,12 +586,15 @@ public void testCancelRollupIndexer() throws IOException { IndexService indexService = indexServices.indexServiceSafe(srcIndex); int shardNum = randomIntBetween(0, numOfShards - 1); IndexShard shard = indexService.getShard(shardNum); + RollupShardTask task = new RollupShardTask( randomLong(), "rollup", "action", TaskId.EMPTY_TASK_ID, rollupIndex, + indexService.getIndexSettings().getTimestampBounds().startTime(), + indexService.getIndexSettings().getTimestampBounds().endTime(), config, emptyMap(), shard.shardId() @@ -658,6 +664,8 @@ public void testTooManyBytesInFlight() throws IOException { "action", TaskId.EMPTY_TASK_ID, rollupIndex, + indexService.getIndexSettings().getTimestampBounds().startTime(), + indexService.getIndexSettings().getTimestampBounds().endTime(), config, emptyMap(), shard.shardId() @@ -683,6 +691,83 @@ public void testTooManyBytesInFlight() throws IOException { indexer.execute(); } + public void testRollupStats() throws IOException { + final DownsampleConfig config = new DownsampleConfig(randomInterval()); + final SourceSupplier sourceSupplier = () -> XContentFactory.jsonBuilder() + .startObject() + .field(FIELD_TIMESTAMP, randomDateForInterval(config.getInterval())) + .field(FIELD_DIMENSION_1, randomAlphaOfLength(1)) + .field(FIELD_NUMERIC_1, randomDouble()) + .endObject(); + bulkIndex(sourceSupplier); + prepareSourceIndex(sourceIndex); + + final IndicesService indexServices = getInstanceFromNode(IndicesService.class); + final Index resolvedSourceIndex = resolveIndex(sourceIndex); + final IndexService indexService = indexServices.indexServiceSafe(resolvedSourceIndex); + for (int shardNum = 0; shardNum < numOfShards; shardNum++) { + final IndexShard shard = indexService.getShard(shardNum); + final RollupShardTask task = new RollupShardTask( + randomLong(), + "rollup", + "action", + TaskId.EMPTY_TASK_ID, + rollupIndex, + indexService.getIndexSettings().getTimestampBounds().startTime(), + indexService.getIndexSettings().getTimestampBounds().endTime(), + config, + emptyMap(), + shard.shardId() + ); + + final RollupShardIndexer indexer = new RollupShardIndexer( + task, + client(), + indexService, + shard.shardId(), + rollupIndex, + config, + new String[] { FIELD_NUMERIC_1, FIELD_NUMERIC_2 }, + new String[] {} + ); + + assertEquals(0.0F, task.getDocsProcessedPercentage(), 0.001); + assertEquals(0L, task.getRollupBulkInfo().totalBulkCount()); + assertEquals(0L, task.getRollupBulkInfo().bulkTookSumMillis()); + assertEquals(0L, task.getRollupBulkInfo().bulkIngestSumMillis()); + assertEquals(RollupShardIndexerStatus.INITIALIZED, task.getRollupShardIndexerStatus()); + + final DownsampleIndexerAction.ShardDownsampleResponse executeResponse = indexer.execute(); + + assertEquals(executeResponse.getNumIndexed(), task.getNumIndexed()); + assertEquals(task.getNumReceived(), task.getTotalShardDocCount()); + assertEquals(indexService.getShard(shardNum).docStats().getCount(), task.getTotalShardDocCount()); + assertEquals(100.0F, task.getDocsProcessedPercentage(), 0.001); + assertTrue(task.getRollupBulkInfo().bulkTookSumMillis() >= 0); + assertEquals(task.getRollupBulkInfo().bulkIngestSumMillis(), task.getRollupBulkInfo().maxBulkIngestMillis()); + assertEquals(task.getRollupBulkInfo().bulkIngestSumMillis(), task.getRollupBulkInfo().minBulkIngestMillis()); + assertTrue(task.getRollupBulkInfo().bulkTookSumMillis() >= 0); + assertEquals(task.getRollupBulkInfo().bulkTookSumMillis(), task.getRollupBulkInfo().maxBulkTookMillis()); + assertEquals(task.getRollupBulkInfo().bulkTookSumMillis(), task.getRollupBulkInfo().minBulkTookMillis()); + assertEquals(1L, task.getRollupBulkInfo().totalBulkCount()); + assertEquals(indexService.getIndexSettings().getTimestampBounds().startTime(), task.getIndexStartTimeMillis()); + assertEquals(indexService.getIndexSettings().getTimestampBounds().endTime(), task.getIndexEndTimeMillis()); + assertEquals(RollupShardIndexerStatus.COMPLETED, task.getRollupShardIndexerStatus()); + assertEquals(task.getNumSent(), task.getNumIndexed()); + assertEquals(task.getNumIndexed(), task.getLastBeforeBulkInfo().numberOfActions()); + assertTrue(task.getLastBeforeBulkInfo().estimatedSizeInBytes() > 0); + assertFalse(task.getLastAfterBulkInfo().hasFailures()); + assertEquals(RestStatus.OK.getStatus(), task.getLastAfterBulkInfo().restStatusCode()); + assertTrue(task.getLastAfterBulkInfo().lastTookInMillis() >= 0); + assertTrue(indexService.getIndexSettings().getTimestampBounds().startTime() <= task.getLastIndexingTimestamp()); + assertTrue(indexService.getIndexSettings().getTimestampBounds().startTime() <= task.getLastSourceTimestamp()); + assertTrue(indexService.getIndexSettings().getTimestampBounds().startTime() <= task.getLastTargetTimestamp()); + assertTrue(indexService.getIndexSettings().getTimestampBounds().endTime() >= task.getLastIndexingTimestamp()); + assertTrue(indexService.getIndexSettings().getTimestampBounds().endTime() >= task.getLastSourceTimestamp()); + assertTrue(indexService.getIndexSettings().getTimestampBounds().endTime() >= task.getLastTargetTimestamp()); + } + } + private DateHistogramInterval randomInterval() { return ConfigTestHelpers.randomInterval(); } From 13416f7367154cdac10bcf834fdfb9f70bd0ec97 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Mon, 3 Jul 2023 13:19:26 +0200 Subject: [PATCH 005/111] Improve iterating over many field producers during downsample operation (#97281) Relates to #97141 --- docs/changelog/97281.yaml | 5 +++ .../xpack/downsample/RollupShardIndexer.java | 34 +++++++++++-------- 2 files changed, 24 insertions(+), 15 deletions(-) create mode 100644 docs/changelog/97281.yaml diff --git a/docs/changelog/97281.yaml b/docs/changelog/97281.yaml new file mode 100644 index 0000000000000..5880ffaa8e93b --- /dev/null +++ b/docs/changelog/97281.yaml @@ -0,0 +1,5 @@ +pr: 97281 +summary: Improve iterating over many field producers during downsample operation +area: Downsampling +type: enhancement +issues: [] diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/RollupShardIndexer.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/RollupShardIndexer.java index 5a75ad69a2e64..99063082d0cbd 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/RollupShardIndexer.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/RollupShardIndexer.java @@ -27,7 +27,6 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.core.IOUtils; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.core.Tuple; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.fielddata.FormattedDocValues; @@ -284,9 +283,9 @@ private class TimeSeriesBucketCollector extends BucketCollector { TimeSeriesBucketCollector(BulkProcessor2 bulkProcessor) { this.bulkProcessor = bulkProcessor; - List rollupFieldProducers = fieldValueFetchers.stream() + AbstractDownsampleFieldProducer[] rollupFieldProducers = fieldValueFetchers.stream() .map(FieldValueFetcher::rollupFieldProducer) - .toList(); + .toArray(AbstractDownsampleFieldProducer[]::new); this.rollupBucketBuilder = new RollupBucketBuilder(rollupFieldProducers); } @@ -297,9 +296,12 @@ public LeafBucketCollector getLeafCollector(final AggregationExecutionContext ag docCountProvider.setLeafReaderContext(ctx); // For each field, return a tuple with the rollup field producer and the field value leaf - final List> fieldValueTuples = fieldValueFetchers.stream() - .map(fetcher -> Tuple.tuple(fetcher.rollupFieldProducer(), fetcher.getLeaf(ctx))) - .toList(); + final AbstractDownsampleFieldProducer[] fieldProducers = new AbstractDownsampleFieldProducer[fieldValueFetchers.size()]; + final FormattedDocValues[] formattedDocValues = new FormattedDocValues[fieldValueFetchers.size()]; + for (int i = 0; i < fieldProducers.length; i++) { + fieldProducers[i] = fieldValueFetchers.get(i).rollupFieldProducer(); + formattedDocValues[i] = fieldValueFetchers.get(i).getLeaf(ctx); + } return new LeafBucketCollector() { @Override @@ -369,9 +371,9 @@ public void collect(int docId, long owningBucketOrd) throws IOException { final int docCount = docCountProvider.getDocCount(docId); rollupBucketBuilder.collectDocCount(docCount); // Iterate over all field values and collect the doc_values for this docId - for (Tuple tuple : fieldValueTuples) { - AbstractDownsampleFieldProducer rollupFieldProducer = tuple.v1(); - FormattedDocValues docValues = tuple.v2(); + for (int i = 0; i < fieldProducers.length; i++) { + AbstractDownsampleFieldProducer rollupFieldProducer = fieldProducers[i]; + FormattedDocValues docValues = formattedDocValues[i]; rollupFieldProducer.collect(docValues, docId); } docsProcessed++; @@ -422,10 +424,10 @@ private class RollupBucketBuilder { private int tsidOrd = -1; private long timestamp; private int docCount; - private final List rollupFieldProducers; - private final List groupedProducers; + private final AbstractDownsampleFieldProducer[] rollupFieldProducers; + private final DownsampleFieldSerializer[] groupedProducers; - RollupBucketBuilder(List rollupFieldProducers) { + RollupBucketBuilder(AbstractDownsampleFieldProducer[] rollupFieldProducers) { this.rollupFieldProducers = rollupFieldProducers; /* * The rollup field producers for aggregate_metric_double all share the same name (this is @@ -433,7 +435,7 @@ private class RollupBucketBuilder { * name. If grouping yields multiple rollup field producers, we delegate serialization to * the AggregateMetricFieldSerializer class. */ - groupedProducers = rollupFieldProducers.stream() + groupedProducers = Arrays.stream(rollupFieldProducers) .collect(groupingBy(AbstractDownsampleFieldProducer::name)) .entrySet() .stream() @@ -444,7 +446,7 @@ private class RollupBucketBuilder { return new AggregateMetricFieldSerializer(e.getKey(), e.getValue()); } }) - .toList(); + .toArray(DownsampleFieldSerializer[]::new); } /** @@ -462,7 +464,9 @@ public void resetTsid(BytesRef tsid, int tsidOrd, long timestamp) { public void resetTimestamp(long timestamp) { this.timestamp = timestamp; this.docCount = 0; - this.rollupFieldProducers.forEach(AbstractDownsampleFieldProducer::reset); + for (AbstractDownsampleFieldProducer producer : rollupFieldProducers) { + producer.reset(); + } if (logger.isTraceEnabled()) { logger.trace( "New bucket for _tsid: [{}], @timestamp: [{}]", From 7f9b9a567726d00b51a4980d9c8396dfc41fa705 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Witek?= Date: Mon, 3 Jul 2023 13:27:28 +0200 Subject: [PATCH 006/111] [Transform] Allow passing additional settings during transform internal index creation. (#97312) --- .../TransformConfigManagerTests.java | 2 +- .../xpack/transform/Transform.java | 10 +++++--- .../persistence/TransformInternalIndex.java | 17 +++++++++---- .../TransformPersistentTasksExecutor.java | 10 +++++++- .../TransformInternalIndexTests.java | 25 +++++++++++++------ ...TransformPersistentTasksExecutorTests.java | 1 + 6 files changed, 48 insertions(+), 17 deletions(-) diff --git a/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/persistence/TransformConfigManagerTests.java b/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/persistence/TransformConfigManagerTests.java index 02dd785af73d3..42247f9481b14 100644 --- a/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/persistence/TransformConfigManagerTests.java +++ b/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/persistence/TransformConfigManagerTests.java @@ -752,7 +752,7 @@ private static ClusterState createClusterStateWithTransformIndex(String... index for (String index : indexes) { IndexMetadata.Builder builder = new IndexMetadata.Builder(index).settings( Settings.builder() - .put(TransformInternalIndex.settings()) + .put(TransformInternalIndex.settings(Settings.EMPTY)) .put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), Version.CURRENT) .build() ).numberOfReplicas(0).numberOfShards(1).putMapping(Strings.toString(TransformInternalIndex.mappings())); diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java index e0fa01a09ae5d..b72138cde8fb9 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java @@ -117,7 +117,6 @@ import java.time.Clock; import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.function.Supplier; @@ -278,13 +277,14 @@ public List> getPersistentTasksExecutor( // the transform services should have been created assert transformServices.get() != null; - return Collections.singletonList( + return List.of( new TransformPersistentTasksExecutor( client, transformServices.get(), threadPool, clusterService, settingsModule.getSettings(), + getTransformInternalIndexAdditionalSettings(), expressionResolver ) ); @@ -327,7 +327,7 @@ public UnaryOperator> getIndexTemplateMetadat @Override public Collection getSystemIndexDescriptors(Settings settings) { try { - return Collections.singletonList(TransformInternalIndex.getSystemIndexDescriptor()); + return List.of(TransformInternalIndex.getSystemIndexDescriptor(getTransformInternalIndexAdditionalSettings())); } catch (IOException e) { throw new UncheckedIOException(e); } @@ -443,4 +443,8 @@ public String getFeatureDescription() { public boolean includeNodeInfo() { return true; } + + public Settings getTransformInternalIndexAdditionalSettings() { + return Settings.EMPTY; + } } diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndex.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndex.java index 1e62db8d6d316..783aadedcd699 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndex.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndex.java @@ -88,13 +88,13 @@ public final class TransformInternalIndex { public static final String BOOLEAN = "boolean"; public static final String FLATTENED = "flattened"; - public static SystemIndexDescriptor getSystemIndexDescriptor() throws IOException { + public static SystemIndexDescriptor getSystemIndexDescriptor(Settings transformInternalIndexAdditionalSettings) throws IOException { return SystemIndexDescriptor.builder() .setIndexPattern(TransformInternalIndexConstants.INDEX_NAME_PATTERN) .setPrimaryIndex(TransformInternalIndexConstants.LATEST_INDEX_NAME) .setDescription("Contains Transform configuration data") .setMappings(mappings()) - .setSettings(settings()) + .setSettings(settings(transformInternalIndexAdditionalSettings)) .setVersionMetaKey("version") .setOrigin(TRANSFORM_ORIGIN) .build(); @@ -149,11 +149,13 @@ private static XContentBuilder auditMappings() throws IOException { return builder; } - public static Settings settings() { + public static Settings settings(Settings additionalSettings) { + assert additionalSettings != null; return Settings.builder() // the configurations are expected to be small .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "0-1") + .put(additionalSettings) .build(); } @@ -389,7 +391,12 @@ private static void waitForLatestVersionedIndexShardsActive(Client client, Actio * Without this check the data nodes will create an internal index with dynamic * mappings when indexing a document, but we want our own well defined mappings. */ - public static void createLatestVersionedIndexIfRequired(ClusterService clusterService, Client client, ActionListener listener) { + public static void createLatestVersionedIndexIfRequired( + ClusterService clusterService, + Client client, + Settings transformInternalIndexAdditionalSettings, + ActionListener listener + ) { ClusterState state = clusterService.state(); // The check for existence is against local cluster state, so very cheap if (hasLatestVersionedIndex(state)) { @@ -405,7 +412,7 @@ public static void createLatestVersionedIndexIfRequired(ClusterService clusterSe // Creating the index involves communication with the master node, so it's more expensive but much rarer try { CreateIndexRequest request = new CreateIndexRequest(TransformInternalIndexConstants.LATEST_INDEX_VERSIONED_NAME).settings( - settings() + settings(transformInternalIndexAdditionalSettings) ) .mapping(mappings()) .origin(TRANSFORM_ORIGIN) diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutor.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutor.java index 9d8173ca562bc..6e35bcbd18a5d 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutor.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutor.java @@ -74,6 +74,7 @@ public class TransformPersistentTasksExecutor extends PersistentTasksExecutor Check the latest internal index (IMPORTANT: according to _this_ node, which might be newer than master) is installed - TransformInternalIndex.createLatestVersionedIndexIfRequired(clusterService, buildTask.getParentTaskClient(), templateCheckListener); + TransformInternalIndex.createLatestVersionedIndexIfRequired( + clusterService, + buildTask.getParentTaskClient(), + transformInternalIndexAdditionalSettings, + templateCheckListener + ); } private static IndexerState currentIndexerState(TransformState previousState) { diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndexTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndexTests.java index f408e4785ff7a..a6a224c1b053f 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndexTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndexTests.java @@ -31,6 +31,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; @@ -43,6 +44,8 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; @@ -64,7 +67,7 @@ public static ClusterState randomTransformClusterState(boolean shardsReady) { try { IndexMetadata.Builder builder = new IndexMetadata.Builder(TransformInternalIndexConstants.LATEST_INDEX_VERSIONED_NAME).settings( Settings.builder() - .put(TransformInternalIndex.settings()) + .put(TransformInternalIndex.settings(Settings.EMPTY)) .put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), Version.CURRENT) .build() ).numberOfReplicas(0).numberOfShards(1).putMapping(Strings.toString(TransformInternalIndex.mappings())); @@ -122,7 +125,7 @@ public void testCreateLatestVersionedIndexIfRequired_GivenNotRequired() { AtomicBoolean gotResponse = new AtomicBoolean(false); ActionListener testListener = ActionTestUtils.assertNoFailureListener(aVoid -> gotResponse.set(true)); - TransformInternalIndex.createLatestVersionedIndexIfRequired(clusterService, client, testListener); + TransformInternalIndex.createLatestVersionedIndexIfRequired(clusterService, client, Settings.EMPTY, testListener); assertTrue(gotResponse.get()); verifyNoMoreInteractions(client); @@ -153,7 +156,7 @@ public void testCreateLatestVersionedIndexIfRequired_GivenRequired() { AtomicBoolean gotResponse = new AtomicBoolean(false); ActionListener testListener = ActionTestUtils.assertNoFailureListener(aVoid -> gotResponse.set(true)); - TransformInternalIndex.createLatestVersionedIndexIfRequired(clusterService, client, testListener); + TransformInternalIndex.createLatestVersionedIndexIfRequired(clusterService, client, Settings.EMPTY, testListener); assertTrue(gotResponse.get()); verify(client, times(1)).threadPool(); @@ -190,7 +193,7 @@ public void testCreateLatestVersionedIndexIfRequired_GivenShardInitializationPen AtomicBoolean gotResponse = new AtomicBoolean(false); ActionListener testListener = ActionTestUtils.assertNoFailureListener(aVoid -> gotResponse.set(true)); - TransformInternalIndex.createLatestVersionedIndexIfRequired(clusterService, client, testListener); + TransformInternalIndex.createLatestVersionedIndexIfRequired(clusterService, client, Settings.EMPTY, testListener); assertTrue(gotResponse.get()); verify(client, times(1)).threadPool(); @@ -228,7 +231,7 @@ public void testCreateLatestVersionedIndexIfRequired_GivenConcurrentCreation() { AtomicBoolean gotResponse = new AtomicBoolean(false); ActionListener testListener = ActionTestUtils.assertNoFailureListener(aVoid -> gotResponse.set(true)); - TransformInternalIndex.createLatestVersionedIndexIfRequired(clusterService, client, testListener); + TransformInternalIndex.createLatestVersionedIndexIfRequired(clusterService, client, Settings.EMPTY, testListener); assertTrue(gotResponse.get()); verify(client, times(1)).threadPool(); @@ -275,7 +278,7 @@ public void testCreateLatestVersionedIndexIfRequired_GivenConcurrentCreationShar AtomicBoolean gotResponse = new AtomicBoolean(false); ActionListener testListener = ActionTestUtils.assertNoFailureListener(aVoid -> gotResponse.set(true)); - TransformInternalIndex.createLatestVersionedIndexIfRequired(clusterService, client, testListener); + TransformInternalIndex.createLatestVersionedIndexIfRequired(clusterService, client, Settings.EMPTY, testListener); assertTrue(gotResponse.get()); verify(client, times(2)).threadPool(); @@ -314,7 +317,7 @@ public void testEnsureLatestIndexAndTemplateInstalled_GivenRequired() { AtomicBoolean gotResponse = new AtomicBoolean(false); ActionListener testListener = ActionTestUtils.assertNoFailureListener(aVoid -> gotResponse.set(true)); - TransformInternalIndex.createLatestVersionedIndexIfRequired(clusterService, client, testListener); + TransformInternalIndex.createLatestVersionedIndexIfRequired(clusterService, client, Settings.EMPTY, testListener); assertTrue(gotResponse.get()); verify(client, times(1)).threadPool(); @@ -325,4 +328,12 @@ public void testEnsureLatestIndexAndTemplateInstalled_GivenRequired() { verify(indicesClient, times(1)).create(any(), any()); verifyNoMoreInteractions(indicesClient); } + + public void testSettings() { + Settings settings = TransformInternalIndex.settings(Settings.EMPTY); + assertThat(settings.get(IndexSettings.INDEX_FAST_REFRESH_SETTING.getKey()), is(nullValue())); + + settings = TransformInternalIndex.settings(Settings.builder().put(IndexSettings.INDEX_FAST_REFRESH_SETTING.getKey(), true).build()); + assertThat(settings.getAsBoolean(IndexSettings.INDEX_FAST_REFRESH_SETTING.getKey(), false), is(true)); + } } diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutorTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutorTests.java index df6f7c69b55cd..fb5604fd01beb 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutorTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutorTests.java @@ -429,6 +429,7 @@ private TransformPersistentTasksExecutor buildTaskExecutor() { threadPool, clusterService, Settings.EMPTY, + Settings.EMPTY, TestIndexNameExpressionResolver.newInstance() ); } From 5eeaecd9cf14ef6bffeda77ff8fe50ef527a220c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Mon, 3 Jul 2023 13:46:17 +0200 Subject: [PATCH 007/111] Use collector managers in DfsPhase (#96689) This change switches the part of the DfsPhase that runs a knn query from a single thread collector to a potentially multi-threaded collector manager and optional profiling wrapper. This doesn't in itself enable concurrent execution but lays the foundation for a switch of the executor in the ContextIndexSearcher to actually enable this. --- docs/changelog/96689.yaml | 5 + .../search/profile/dfs/DfsProfilerIT.java | 1 - .../elasticsearch/search/dfs/DfsPhase.java | 43 ++++--- .../search/profile/dfs/DfsProfiler.java | 4 +- .../query/ProfileCollectorManager.java | 23 ++-- .../search/dfs/DfsPhaseTests.java | 117 ++++++++++++++++++ .../query/ProfileCollectorManagerTests.java | 52 ++++---- 7 files changed, 189 insertions(+), 56 deletions(-) create mode 100644 docs/changelog/96689.yaml create mode 100644 server/src/test/java/org/elasticsearch/search/dfs/DfsPhaseTests.java diff --git a/docs/changelog/96689.yaml b/docs/changelog/96689.yaml new file mode 100644 index 0000000000000..220624b8c1eca --- /dev/null +++ b/docs/changelog/96689.yaml @@ -0,0 +1,5 @@ +pr: 96689 +summary: Use a collector manager in DfsPhase Knn Search +area: Search +type: enhancement +issues: [] diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/profile/dfs/DfsProfilerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/profile/dfs/DfsProfilerIT.java index 46ce6c21474e7..fce19a316b34a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/profile/dfs/DfsProfilerIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/profile/dfs/DfsProfilerIT.java @@ -111,7 +111,6 @@ public void testProfileDfs() throws Exception { CollectorResult result = queryProfileShardResult.getCollectorResult(); assertThat(result.getName(), is(not(emptyOrNullString()))); assertThat(result.getTime(), greaterThan(0L)); - assertThat(result.getTime(), greaterThan(0L)); } ProfileResult statsResult = searchProfileDfsPhaseResult.getDfsShardResult(); assertThat(statsResult.getQueryName(), equalTo("statistics")); diff --git a/server/src/main/java/org/elasticsearch/search/dfs/DfsPhase.java b/server/src/main/java/org/elasticsearch/search/dfs/DfsPhase.java index 78d2dd5fc9b44..84794b82ad00a 100644 --- a/server/src/main/java/org/elasticsearch/search/dfs/DfsPhase.java +++ b/server/src/main/java/org/elasticsearch/search/dfs/DfsPhase.java @@ -16,19 +16,20 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.TermStatistics; +import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.TopScoreDocCollector; import org.elasticsearch.index.query.ParsedQuery; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.internal.ContextIndexSearcher; import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.search.profile.Profilers; import org.elasticsearch.search.profile.Timer; import org.elasticsearch.search.profile.dfs.DfsProfiler; import org.elasticsearch.search.profile.dfs.DfsTimingType; import org.elasticsearch.search.profile.query.CollectorResult; -import org.elasticsearch.search.profile.query.InternalProfileCollector; -import org.elasticsearch.search.profile.query.InternalProfileCollectorManager; +import org.elasticsearch.search.profile.query.ProfileCollectorManager; import org.elasticsearch.search.profile.query.QueryProfiler; -import org.elasticsearch.search.query.SingleThreadCollectorManager; import org.elasticsearch.search.rescore.RescoreContext; import org.elasticsearch.search.vectors.KnnSearchBuilder; import org.elasticsearch.search.vectors.KnnVectorQueryBuilder; @@ -43,7 +44,6 @@ /** * DFS phase of a search request, used to make scoring 100% accurate by collecting additional info from each shard before the query phase. * The additional information is used to better compare the scores coming from all the shards, which depend on local factors (e.g. idf). - * * When a kNN search is provided alongside the query, the DFS phase is also used to gather the top k candidates from each shard. Then the * global top k hits are passed on to the query phase. */ @@ -189,24 +189,27 @@ private void executeKnnVectorQuery(SearchContext context) throws IOException { List knnResults = new ArrayList<>(knnVectorQueryBuilders.size()); for (int i = 0; i < knnSearch.size(); i++) { Query knnQuery = searchExecutionContext.toQuery(knnVectorQueryBuilders.get(i)).query(); - TopScoreDocCollector topScoreDocCollector = TopScoreDocCollector.create(knnSearch.get(i).k(), Integer.MAX_VALUE); - CollectorManager collectorManager = new SingleThreadCollectorManager(topScoreDocCollector); - if (context.getProfilers() != null) { - InternalProfileCollectorManager ipcm = new InternalProfileCollectorManager( - new InternalProfileCollector(collectorManager.newCollector(), CollectorResult.REASON_SEARCH_TOP_HITS) - ); - QueryProfiler knnProfiler = context.getProfilers().getDfsProfiler().addQueryProfiler(ipcm); - collectorManager = ipcm; - // Set the current searcher profiler to gather query profiling information for gathering top K docs - context.searcher().setProfiler(knnProfiler); - } - context.searcher().search(knnQuery, collectorManager); - knnResults.add(new DfsKnnResults(topScoreDocCollector.topDocs().scoreDocs)); + knnResults.add(singleKnnSearch(knnQuery, knnSearch.get(i).k(), context.getProfilers(), context.searcher())); } + context.dfsResult().knnResults(knnResults); + } + + static DfsKnnResults singleKnnSearch(Query knnQuery, int k, Profilers profilers, ContextIndexSearcher searcher) throws IOException { + CollectorManager cm = TopScoreDocCollector.createSharedManager(k, null, Integer.MAX_VALUE); + + if (profilers != null) { + ProfileCollectorManager ipcm = new ProfileCollectorManager<>(cm, CollectorResult.REASON_SEARCH_TOP_HITS); + QueryProfiler knnProfiler = profilers.getDfsProfiler().addQueryProfiler(ipcm); + cm = ipcm; + // Set the current searcher profiler to gather query profiling information for gathering top K docs + searcher.setProfiler(knnProfiler); + } + TopDocs topDocs = searcher.search(knnQuery, cm); + // Set profiler back after running KNN searches - if (context.getProfilers() != null) { - context.searcher().setProfiler(context.getProfilers().getCurrentQueryProfiler()); + if (profilers != null) { + searcher.setProfiler(profilers.getCurrentQueryProfiler()); } - context.dfsResult().knnResults(knnResults); + return new DfsKnnResults(topDocs.scoreDocs); } } diff --git a/server/src/main/java/org/elasticsearch/search/profile/dfs/DfsProfiler.java b/server/src/main/java/org/elasticsearch/search/profile/dfs/DfsProfiler.java index 4c0c56327813f..c04336fe53096 100644 --- a/server/src/main/java/org/elasticsearch/search/profile/dfs/DfsProfiler.java +++ b/server/src/main/java/org/elasticsearch/search/profile/dfs/DfsProfiler.java @@ -12,7 +12,7 @@ import org.elasticsearch.search.profile.ProfileResult; import org.elasticsearch.search.profile.SearchProfileDfsPhaseResult; import org.elasticsearch.search.profile.Timer; -import org.elasticsearch.search.profile.query.InternalProfileCollectorManager; +import org.elasticsearch.search.profile.query.ProfileCollectorManager; import org.elasticsearch.search.profile.query.QueryProfileShardResult; import org.elasticsearch.search.profile.query.QueryProfiler; @@ -51,7 +51,7 @@ public Timer startTimer(DfsTimingType dfsTimingType) { return newTimer; } - public QueryProfiler addQueryProfiler(InternalProfileCollectorManager collectorManager) { + public QueryProfiler addQueryProfiler(ProfileCollectorManager collectorManager) { QueryProfiler queryProfiler = new QueryProfiler(); queryProfiler.setCollectorManager(collectorManager::getCollectorTree); knnQueryProfilers.add(queryProfiler); diff --git a/server/src/main/java/org/elasticsearch/search/profile/query/ProfileCollectorManager.java b/server/src/main/java/org/elasticsearch/search/profile/query/ProfileCollectorManager.java index 548016a01f9c7..b9f5a36491cd6 100644 --- a/server/src/main/java/org/elasticsearch/search/profile/query/ProfileCollectorManager.java +++ b/server/src/main/java/org/elasticsearch/search/profile/query/ProfileCollectorManager.java @@ -13,6 +13,7 @@ import java.io.IOException; import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -21,15 +22,15 @@ * in an {@link InternalProfileCollector}. It delegates all the profiling to the generated collectors via {@link #getCollectorTree()} * and joins them up when its {@link #reduce} method is called. The profile result can */ -public final class ProfileCollectorManager implements CollectorManager { +public final class ProfileCollectorManager implements CollectorManager { - private final CollectorManager collectorManager; + private final CollectorManager collectorManager; private final String reason; private CollectorResult collectorTree; @SuppressWarnings("unchecked") - public ProfileCollectorManager(CollectorManager collectorManager, String reason) { - this.collectorManager = (CollectorManager) collectorManager; + public ProfileCollectorManager(CollectorManager collectorManager, String reason) { + this.collectorManager = (CollectorManager) collectorManager; this.reason = reason; } @@ -38,22 +39,26 @@ public InternalProfileCollector newCollector() throws IOException { return new InternalProfileCollector(collectorManager.newCollector(), reason); } - public Void reduce(Collection profileCollectors) throws IOException { + public T reduce(Collection profileCollectors) throws IOException { + assert profileCollectors.size() > 0 : "at least one collector expected"; List unwrapped = profileCollectors.stream() .map(InternalProfileCollector::getWrappedCollector) .collect(Collectors.toList()); - collectorManager.reduce(unwrapped); + T returnValue = collectorManager.reduce(unwrapped); List resultsPerProfiler = profileCollectors.stream() .map(ipc -> ipc.getCollectorTree()) .collect(Collectors.toList()); - this.collectorTree = new CollectorResult(this.getClass().getSimpleName(), "segment_search", 0, resultsPerProfiler); - return null; + + long totalTime = resultsPerProfiler.stream().map(CollectorResult::getTime).reduce(0L, Long::sum); + String collectorName = resultsPerProfiler.get(0).getName(); + this.collectorTree = new CollectorResult(collectorName, reason, totalTime, Collections.emptyList()); + return returnValue; } public CollectorResult getCollectorTree() { if (this.collectorTree == null) { - throw new IllegalStateException("A collectorTree hasn't been set yet, call reduce() before attempting to retrieve it"); + throw new IllegalStateException("A collectorTree hasn't been set yet. Call reduce() before attempting to retrieve it"); } return this.collectorTree; } diff --git a/server/src/test/java/org/elasticsearch/search/dfs/DfsPhaseTests.java b/server/src/test/java/org/elasticsearch/search/dfs/DfsPhaseTests.java new file mode 100644 index 0000000000000..19f1d251d7a9a --- /dev/null +++ b/server/src/test/java/org/elasticsearch/search/dfs/DfsPhaseTests.java @@ -0,0 +1,117 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.search.dfs; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.KnnFloatVectorField; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.KnnFloatVectorQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.index.RandomIndexWriter; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.search.internal.ContextIndexSearcher; +import org.elasticsearch.search.profile.Profilers; +import org.elasticsearch.search.profile.SearchProfileDfsPhaseResult; +import org.elasticsearch.search.profile.query.CollectorResult; +import org.elasticsearch.search.profile.query.QueryProfileShardResult; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.TestThreadPool; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.util.List; +import java.util.concurrent.ThreadPoolExecutor; + +public class DfsPhaseTests extends ESTestCase { + + ThreadPoolExecutor threadPoolExecutor; + private TestThreadPool threadPool; + + @Before + public final void init() { + int numThreads = randomIntBetween(2, 4); + threadPool = new TestThreadPool(DfsPhaseTests.class.getName()); + threadPoolExecutor = EsExecutors.newFixed( + "test", + numThreads, + 10, + EsExecutors.daemonThreadFactory("test"), + threadPool.getThreadContext(), + randomBoolean() + ); + } + + @After + public void cleanup() { + threadPoolExecutor.shutdown(); + terminate(threadPool); + } + + public void testSingleKnnSearch() throws IOException { + try (Directory dir = newDirectory(); RandomIndexWriter w = new RandomIndexWriter(random(), dir, newIndexWriterConfig())) { + int numDocs = randomIntBetween(900, 1000); + for (int i = 0; i < numDocs; i++) { + Document d = new Document(); + d.add(new KnnFloatVectorField("float_vector", new float[] { i, 0, 0 })); + w.addDocument(d); + } + w.flush(); + + IndexReader reader = w.getReader(); + ContextIndexSearcher searcher = new ContextIndexSearcher( + reader, + IndexSearcher.getDefaultSimilarity(), + IndexSearcher.getDefaultQueryCache(), + IndexSearcher.getDefaultQueryCachingPolicy(), + randomBoolean(), + this.threadPoolExecutor + ) { + @Override + protected LeafSlice[] slices(List leaves) { + // get a thread per segment + return slices(leaves, 1, 1); + } + }; + + Query query = new KnnFloatVectorQuery("float_vector", new float[] { 0, 0, 0 }, numDocs, null); + + int k = 10; + // run without profiling enabled + DfsKnnResults dfsKnnResults = DfsPhase.singleKnnSearch(query, k, null, searcher); + assertEquals(k, dfsKnnResults.scoreDocs().length); + + // run with profiling enabled + Profilers profilers = new Profilers(searcher); + dfsKnnResults = DfsPhase.singleKnnSearch(query, k, profilers, searcher); + assertEquals(k, dfsKnnResults.scoreDocs().length); + SearchProfileDfsPhaseResult searchProfileDfsPhaseResult = profilers.getDfsProfiler().buildDfsPhaseResults(); + List queryProfileShardResult = searchProfileDfsPhaseResult.getQueryProfileShardResult(); + assertNotNull(queryProfileShardResult); + CollectorResult collectorResult = queryProfileShardResult.get(0).getCollectorResult(); + assertEquals("SimpleTopScoreDocCollector", (collectorResult.getName())); + assertEquals("search_top_hits", (collectorResult.getReason())); + assertTrue(collectorResult.getTime() > 0); + List children = collectorResult.getCollectorResults(); + if (children.size() > 0) { + long totalTime = 0L; + for (CollectorResult child : children) { + assertEquals("SimpleTopScoreDocCollector", (child.getName())); + assertEquals("search_top_hits", (child.getReason())); + totalTime += child.getTime(); + } + assertEquals(totalTime, collectorResult.getTime()); + } + reader.close(); + } + } +} diff --git a/server/src/test/java/org/elasticsearch/search/profile/query/ProfileCollectorManagerTests.java b/server/src/test/java/org/elasticsearch/search/profile/query/ProfileCollectorManagerTests.java index b018f16442df6..209efe59333c8 100644 --- a/server/src/test/java/org/elasticsearch/search/profile/query/ProfileCollectorManagerTests.java +++ b/server/src/test/java/org/elasticsearch/search/profile/query/ProfileCollectorManagerTests.java @@ -12,8 +12,6 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.StringField; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.sandbox.search.ProfilerCollectorResult; -import org.apache.lucene.search.Collector; import org.apache.lucene.search.CollectorManager; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; @@ -27,6 +25,7 @@ import org.elasticsearch.test.ESTestCase; import java.io.IOException; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -48,29 +47,41 @@ private static class TestCollector extends DummyTotalHitCountCollector { */ public void testBasic() throws IOException { final SetOnce reduceCalled = new SetOnce<>(); - ProfileCollectorManager pcm = new ProfileCollectorManager(new CollectorManager<>() { + ProfileCollectorManager pcm = new ProfileCollectorManager<>(new CollectorManager() { - private static int counter = 0; + private int counter = 0; @Override - public Collector newCollector() { + public TestCollector newCollector() { return new TestCollector(counter++); } @Override - public Void reduce(Collection collectors) { + public Integer reduce(Collection collectors) { reduceCalled.set(true); - return null; + return counter; } }, CollectorResult.REASON_SEARCH_TOP_HITS); - for (int i = 0; i < randomIntBetween(5, 10); i++) { - InternalProfileCollector internalProfileCollector = pcm.newCollector(); - assertEquals(i, ((TestCollector) internalProfileCollector.getWrappedCollector()).id); + int runs = randomIntBetween(5, 10); + List collectors = new ArrayList<>(); + for (int i = 0; i < runs; i++) { + collectors.add(pcm.newCollector()); + assertEquals(i, ((TestCollector) collectors.get(i).getWrappedCollector()).id); } - pcm.reduce(Collections.emptyList()); + Integer returnValue = pcm.reduce(collectors); + assertEquals(runs, returnValue.intValue()); assertTrue(reduceCalled.get()); } + public void testReduceEmpty() { + ProfileCollectorManager pcm = new ProfileCollectorManager<>( + TopScoreDocCollector.createSharedManager(10, null, 1000), + CollectorResult.REASON_SEARCH_TOP_HITS + ); + AssertionError ae = expectThrows(AssertionError.class, () -> pcm.reduce(Collections.emptyList())); + assertEquals("at least one collector expected", ae.getMessage()); + } + /** * This test checks functionality with potentially more than one slice on a real searcher, * wrapping a {@link TopScoreDocCollector} into {@link ProfileCollectorManager} and checking the @@ -88,7 +99,6 @@ public void testManagerWithSearcher() throws IOException { writer.flush(); IndexReader reader = writer.getReader(); IndexSearcher searcher = newSearcher(reader); - int numSlices = searcher.getSlices() == null ? 1 : searcher.getSlices().length; searcher.setSimilarity(new BM25Similarity()); CollectorManager topDocsManager = TopScoreDocCollector.createSharedManager(10, null, 1000); @@ -96,21 +106,15 @@ public void testManagerWithSearcher() throws IOException { assertEquals(numDocs, topDocs.totalHits.value); String profileReason = "profiler_reason"; - ProfileCollectorManager profileCollectorManager = new ProfileCollectorManager(topDocsManager, profileReason); + ProfileCollectorManager profileCollectorManager = new ProfileCollectorManager<>(topDocsManager, profileReason); searcher.search(new MatchAllDocsQuery(), profileCollectorManager); - CollectorResult parent = profileCollectorManager.getCollectorTree(); - assertEquals("ProfileCollectorManager", parent.getName()); - assertEquals("segment_search", parent.getReason()); - assertEquals(0, parent.getTime()); - List delegateCollectorResults = parent.getProfiledChildren(); - assertEquals(numSlices, delegateCollectorResults.size()); - for (ProfilerCollectorResult pcr : delegateCollectorResults) { - assertEquals("SimpleTopScoreDocCollector", pcr.getName()); - assertEquals(profileReason, pcr.getReason()); - assertTrue(pcr.getTime() > 0); - } + CollectorResult result = profileCollectorManager.getCollectorTree(); + assertEquals("profiler_reason", result.getReason()); + assertEquals("SimpleTopScoreDocCollector", result.getName()); + assertTrue(result.getTime() > 0); + reader.close(); } directory.close(); From 63e64ae61b867d5309a3a999861b51bffb3bf730 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Mon, 3 Jul 2023 14:24:57 +0200 Subject: [PATCH 008/111] Cleanup Stream usage in various spots (#97306) Lots of spots where we did weird things around streams like redundant stream creation, redundant collecting before adding all the collected elements to another collection or so, redundant streams for joining strings and using less efficient `Collectors.toList` and in a few cases also incorrectly relying on the result being mutable. --- .../precommit/LicenseHeadersTask.java | 9 +- .../internal/docker/DockerSupportService.java | 5 +- .../precommit/ForbiddenPatternsTask.java | 25 +-- .../precommit/JavaModulePrecommitTask.java | 3 +- .../precommit/SplitPackagesAuditTask.java | 3 +- .../TestingConventionsCheckTask.java | 8 +- .../internal/test/DistroTestPlugin.java | 11 +- .../rest/transform/RestTestTransformer.java | 4 +- .../AdoptiumJdkToolchainResolver.java | 5 +- .../internal/vagrant/VagrantBasePlugin.java | 2 +- .../internal/vagrant/VagrantShellTask.java | 3 +- .../elasticsearch/gradle/reaper/Reaper.java | 3 +- .../gradle/LazyPropertyList.java | 4 +- .../gradle/testclusters/RunTask.java | 2 +- .../testclusters/TestClustersAware.java | 2 +- .../client/benchmark/AbstractBenchmark.java | 3 +- .../org/elasticsearch/geoip/GeoIpCli.java | 7 +- .../server/cli/SystemJvmOptions.java | 5 +- .../provider/EmbeddedImplClassLoader.java | 2 +- .../provider/InMemoryModuleFinder.java | 4 +- .../elasticsearch/dissect/DissectParser.java | 1 - .../ssl/SslClientAuthenticationMode.java | 3 +- .../common/ssl/SslDiagnostics.java | 4 +- .../common/ssl/SslVerificationMode.java | 3 +- .../AutoDateHistogramAggregatorTests.java | 4 +- .../toxcontent/UserTreeToXContent.java | 8 +- .../join/mapper/ParentJoinFieldMapper.java | 3 +- .../percolator/QueryAnalyzer.java | 5 +- .../rankeval/DiscountedCumulativeGain.java | 7 +- ...nsportNodesReloadSecureSettingsAction.java | 2 +- ...ReservedComposableIndexTemplateAction.java | 13 +- .../xcontent/SuggestingErrorOnUnknown.java | 4 +- .../index/query/TermsQueryBuilder.java | 10 +- .../search/stats/ShardFieldUsageTracker.java | 2 +- .../index/translog/Translog.java | 3 +- .../elasticsearch/plugins/ModuleSupport.java | 4 +- .../search/aggregations/InternalOrder.java | 8 +- .../search/rescore/QueryRescorer.java | 6 +- .../cluster/node/DiscoveryNodesTests.java | 4 +- .../allocation/FailedNodeRoutingTests.java | 8 +- .../range/RangeAggregationBuilderTests.java | 2 +- .../java/fixture/azure/AzureHttpHandler.java | 3 +- .../CoordinationStateTestCluster.java | 8 +- .../blobstore/BlobStoreTestUtil.java | 8 +- .../snapshots/mockstore/MockRepository.java | 3 +- .../test/ESSingleNodeTestCase.java | 2 +- .../org/elasticsearch/test/VersionUtils.java | 7 +- .../test/rest/ESRestTestCase.java | 14 +- .../rest/yaml/ESClientYamlSuiteTestCase.java | 3 +- .../MovingPercentilesPipelineAggregator.java | 4 +- .../protocol/xpack/XPackInfoResponse.java | 3 +- .../core/ilm/PauseFollowerIndexStep.java | 3 +- .../xpack/core/ilm/PhaseCacheManagement.java | 2 +- .../xpack/core/ilm/SegmentCountStep.java | 2 +- .../core/ilm/WaitForFollowShardTasksStep.java | 2 +- .../trainedmodel/InferenceHelpers.java | 3 +- .../xpack/core/ml/job/config/Detector.java | 7 +- .../xpack/core/ml/utils/NameResolver.java | 3 +- .../core/rollup/action/RollableIndexCaps.java | 6 +- .../core/rollup/action/RollupJobCaps.java | 2 +- .../MountSearchableSnapshotRequest.java | 3 +- .../SearchableSnapshotShardStats.java | 3 +- .../privilege/PutPrivilegesRequest.java | 4 +- .../core/security/authc/Authentication.java | 6 +- .../DefaultAuthenticationFailureHandler.java | 6 +- .../security/authz/AuthorizationEngine.java | 3 +- .../permission/ResourcePrivilegesMap.java | 4 +- .../core/slm/SnapshotLifecycleStats.java | 2 +- .../slm/SnapshotRetentionConfiguration.java | 5 +- .../xpack/core/ssl/SSLService.java | 7 +- .../core/LocalStateCompositeXPackPlugin.java | 192 ++++++++---------- .../deprecation/DeprecationInfoAction.java | 4 +- .../deprecation/IndexDeprecationChecks.java | 3 +- .../deprecation/NodeDeprecationChecks.java | 30 ++- .../TransportDeprecationInfoAction.java | 4 +- .../xpack/eql/qa/mixed_node/EqlSearchIT.java | 2 +- .../xpack/eql/analysis/AnalysisUtils.java | 4 +- .../org/elasticsearch/xpack/fleet/Fleet.java | 7 +- .../action/TransportGetLifecycleAction.java | 2 +- .../action/TransportGetPipelineAction.java | 2 +- .../xpack/ml/MlInitializationService.java | 3 +- .../TransportGetTrainedModelsStatsAction.java | 2 +- .../ml/action/TransportOpenJobAction.java | 3 +- .../action/TransportSetUpgradeModeAction.java | 2 +- .../MlMemoryAutoscalingDecider.java | 23 +-- .../datafeed/DatafeedConfigAutoUpdater.java | 3 +- .../AbstractAggregationDataExtractor.java | 4 +- .../RollupDataExtractorFactory.java | 2 +- .../scroll/TimeBasedExtractedFields.java | 2 +- .../DataFrameAnalyticsConfigProvider.java | 3 +- .../assignment/planning/AssignmentPlan.java | 9 +- .../planning/ZoneAwareAssignmentPlanner.java | 4 +- .../OverallBucketsAggregator.java | 4 +- .../normalizer/BucketNormalizable.java | 10 +- ...ernalItemSetMapReduceAggregationTests.java | 6 +- .../xpack/ql/index/IndexResolver.java | 2 +- .../xpack/ql/type/DataTypes.java | 6 +- .../action/TransportRollupSearchAction.java | 2 +- .../PinnedQueryBuilder.java | 3 +- .../SearchableSnapshotsStatsResponse.java | 4 +- ...rchableSnapshotsNodeCachesStatsAction.java | 2 - ...actDocumentAndFieldLevelSecurityTests.java | 2 +- .../xpack/security/Security.java | 12 +- .../authc/ldap/support/LdapUtils.java | 4 +- .../authc/saml/SamlMetadataCommand.java | 2 +- .../authc/service/ElasticServiceAccounts.java | 5 +- .../filter/SecurityIpFilterRule.java | 2 +- .../authc/saml/SamlMetadataCommandTests.java | 2 +- .../security/authc/saml/SamlRealmTests.java | 2 +- .../structurefinder/FieldStatsCalculator.java | 3 +- .../TimestampFormatFinder.java | 2 +- .../TransportGetTransformStatsAction.java | 7 +- .../action/TransportStopTransformAction.java | 2 +- .../pivot/AggregationResultUtils.java | 17 +- .../elasticsearch/xpack/watcher/Watcher.java | 6 +- .../trigger/schedule/ScheduleRegistry.java | 2 +- .../xpack/test/rest/XPackRestTestHelper.java | 2 +- 117 files changed, 321 insertions(+), 442 deletions(-) diff --git a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/LicenseHeadersTask.java b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/LicenseHeadersTask.java index 58cb62b7a4466..76b117f8f5308 100644 --- a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/LicenseHeadersTask.java +++ b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/LicenseHeadersTask.java @@ -50,15 +50,11 @@ import java.nio.file.Files; import java.util.ArrayList; import java.util.Arrays; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.stream.Collectors; import javax.inject.Inject; import java.io.Serializable; -import javax.inject.Inject; - /** * Checks files for license headers.. */ @@ -193,9 +189,8 @@ public void runRat() { boolean unApprovedLicenses = stats.getNumUnApproved() > 0; if (unknownLicenses || unApprovedLicenses) { getLogger().error("The following files contain unapproved license headers:"); - unapprovedFiles(repFile).stream().forEachOrdered(unapprovedFile -> getLogger().error(unapprovedFile)); - throw new GradleException("Check failed. License header problems were found. Full details: " + - repFile.getAbsolutePath()); + unapprovedFiles(repFile).forEach(getLogger()::error); + throw new GradleException("Check failed. License header problems were found. Full details: " + repFile.getAbsolutePath()); } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerSupportService.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerSupportService.java index 0922ce0437e3b..99c993055cf51 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerSupportService.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerSupportService.java @@ -34,6 +34,7 @@ import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; +import java.util.stream.Stream; import javax.inject.Inject; @@ -287,7 +288,7 @@ static Map parseOsRelease(final List osReleaseLines) { */ private Optional getDockerPath() { // Check if the Docker binary exists - return List.of(DOCKER_BINARIES).stream().filter(path -> new File(path).exists()).findFirst(); + return Stream.of(DOCKER_BINARIES).filter(path -> new File(path).exists()).findFirst(); } /** @@ -298,7 +299,7 @@ private Optional getDockerPath() { */ private Optional getDockerComposePath() { // Check if the Docker binary exists - return List.of(DOCKER_COMPOSE_BINARIES).stream().filter(path -> new File(path).exists()).findFirst(); + return Stream.of(DOCKER_COMPOSE_BINARIES).filter(path -> new File(path).exists()).findFirst(); } private void throwDockerRequiredException(final String message) { diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ForbiddenPatternsTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ForbiddenPatternsTask.java index abb933d142780..b5582e4acf14a 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ForbiddenPatternsTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ForbiddenPatternsTask.java @@ -114,24 +114,19 @@ public void checkInvalidPatterns() throws IOException { } catch (UncheckedIOException e) { throw new IllegalArgumentException("Failed to read " + f + " as UTF_8", e); } - List invalidLines = IntStream.range(0, lines.size()) - .filter(i -> allPatterns.matcher(lines.get(i)).find()) - .boxed() - .collect(Collectors.toList()); URI baseUri = getRootDir().orElse(projectLayout.getProjectDirectory().getAsFile()).get().toURI(); String path = baseUri.relativize(f.toURI()).toString(); - failures.addAll( - invalidLines.stream() - .map(l -> new AbstractMap.SimpleEntry<>(l + 1, lines.get(l))) - .flatMap( - kv -> patterns.entrySet() - .stream() - .filter(p -> Pattern.compile(p.getValue()).matcher(kv.getValue()).find()) - .map(p -> "- " + p.getKey() + " on line " + kv.getKey() + " of " + path) - ) - .collect(Collectors.toList()) - ); + IntStream.range(0, lines.size()) + .filter(i -> allPatterns.matcher(lines.get(i)).find()) + .mapToObj(l -> new AbstractMap.SimpleEntry<>(l + 1, lines.get(l))) + .flatMap( + kv -> patterns.entrySet() + .stream() + .filter(p -> Pattern.compile(p.getValue()).matcher(kv.getValue()).find()) + .map(p -> "- " + p.getKey() + " on line " + kv.getKey() + " of " + path) + ) + .forEach(failures::add); } if (failures.isEmpty() == false) { throw new GradleException("Found invalid patterns:\n" + String.join("\n", failures)); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/JavaModulePrecommitTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/JavaModulePrecommitTask.java index 14f6461aba7cf..cee10cf70b19c 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/JavaModulePrecommitTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/JavaModulePrecommitTask.java @@ -159,8 +159,7 @@ private static ModuleReference esModuleFor(File filePath) { return ModuleFinder.of(filePath.toPath()) .findAll() .stream() - .sorted(Comparator.comparing(ModuleReference::descriptor)) - .findFirst() + .min(Comparator.comparing(ModuleReference::descriptor)) .orElseThrow(() -> new GradleException("module not found in " + filePath)); } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/SplitPackagesAuditTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/SplitPackagesAuditTask.java index b93862384580f..ec279589a6bed 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/SplitPackagesAuditTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/SplitPackagesAuditTask.java @@ -48,7 +48,6 @@ import java.util.Set; import java.util.TreeSet; import java.util.function.Consumer; -import java.util.stream.Collectors; import javax.inject.Inject; @@ -234,7 +233,7 @@ private void filterSplitPackages(Map> splitPackages) { String lastPackageName = null; Set currentClasses = null; boolean filterErrorsFound = false; - for (String fqcn : getParameters().getIgnoreClasses().get().stream().sorted().collect(Collectors.toList())) { + for (String fqcn : getParameters().getIgnoreClasses().get().stream().sorted().toList()) { int lastDot = fqcn.lastIndexOf('.'); if (lastDot == -1) { LOGGER.error("Missing package in classname in split package ignores: " + fqcn); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/TestingConventionsCheckTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/TestingConventionsCheckTask.java index de800f0ff9a15..87f15351c1565 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/TestingConventionsCheckTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/TestingConventionsCheckTask.java @@ -38,6 +38,7 @@ import java.util.List; import java.util.function.Predicate; import java.util.stream.Collectors; +import java.util.stream.Stream; import javax.inject.Inject; @@ -128,7 +129,7 @@ private void assertNoMissmatchingTest(List> testClassesCandid var mismatchingBaseClasses = testClassesCandidate.stream() .filter(testClassDefaultPredicate) .filter(TestingConventionsCheckWorkAction::seemsLikeATest) - .collect(Collectors.toList()); + .toList(); if (mismatchingBaseClasses.isEmpty() == false) { throw new GradleException( "Following test classes do not extend any supported base class:\n\t" @@ -141,7 +142,7 @@ private void assertMatchesSuffix(List suffixes, List matchingBase // ensure base class matching do match suffix var matchingBaseClassNotMatchingSuffix = matchingBaseClass.stream() .filter(c -> suffixes.stream().allMatch(s -> c.getName().endsWith(s) == false)) - .collect(Collectors.toList()); + .toList(); if (matchingBaseClassNotMatchingSuffix.isEmpty() == false) { throw new GradleException( "Following test classes do not match naming convention to use suffix " @@ -202,8 +203,7 @@ private static boolean matchesTestMethodNamingConvention(Method method) { } private static boolean isAnnotated(Method method, Class annotation) { - return List.of(method.getAnnotations()) - .stream() + return Stream.of(method.getAnnotations()) .anyMatch(presentAnnotation -> annotation.isAssignableFrom(presentAnnotation.getClass())); } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java index e900eaada4c28..ada263853c07d 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java @@ -389,12 +389,11 @@ private List configureDistributions(Project project) List currentDistros = new ArrayList<>(); for (Architecture architecture : Architecture.values()) { - ALL_INTERNAL.stream() - .forEach( - type -> currentDistros.add( - createDistro(distributions, architecture, type, null, true, VersionProperties.getElasticsearch()) - ) - ); + ALL_INTERNAL.forEach( + type -> currentDistros.add( + createDistro(distributions, architecture, type, null, true, VersionProperties.getElasticsearch()) + ) + ); } for (Architecture architecture : Architecture.values()) { diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/RestTestTransformer.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/RestTestTransformer.java index 4645d895ad973..7237e3ce27ce8 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/RestTestTransformer.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/RestTestTransformer.java @@ -38,13 +38,13 @@ public List transformRestTests(LinkedList tests, List setupTransforms = transformations.stream() .filter(transform -> transform instanceof RestTestTransformGlobalSetup) .map(transform -> (RestTestTransformGlobalSetup) transform) - .collect(Collectors.toList()); + .toList(); // Collect any global teardown transformations List teardownTransforms = transformations.stream() .filter(transform -> transform instanceof RestTestTransformGlobalTeardown) .map(transform -> (RestTestTransformGlobalTeardown) transform) - .collect(Collectors.toList()); + .toList(); // Collect any transformations that are identified by an object key. Map> objectKeyFinders = transformations.stream() diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/AdoptiumJdkToolchainResolver.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/AdoptiumJdkToolchainResolver.java index 1dd8b0b9d9e2e..bddf95cae77d4 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/AdoptiumJdkToolchainResolver.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/AdoptiumJdkToolchainResolver.java @@ -75,9 +75,8 @@ private Optional resolveAvailableVersion(AdoptiumVersionReq return Optional.of( Lists.newArrayList(versionsNode.iterator()) .stream() - .map(node -> toVersionInfo(node)) - .sorted(Comparator.comparing(AdoptiumVersionInfo::semver).reversed()) - .findFirst() + .map(this::toVersionInfo) + .max(Comparator.comparing(AdoptiumVersionInfo::semver)) .get() ); } catch (FileNotFoundException e) { diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/vagrant/VagrantBasePlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/vagrant/VagrantBasePlugin.java index 10fe51bbef729..b7b4d20d51f56 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/vagrant/VagrantBasePlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/vagrant/VagrantBasePlugin.java @@ -89,7 +89,7 @@ void checkVersion(Project project, String tool, Pattern versionRegex, int... min } String version = matcher.group(1); - List versionParts = Stream.of(version.split("\\.")).map(Integer::parseInt).collect(Collectors.toList()); + List versionParts = Stream.of(version.split("\\.")).map(Integer::parseInt).toList(); for (int i = 0; i < minVersion.length; ++i) { int found = versionParts.get(i); if (found > minVersion[i]) { diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/vagrant/VagrantShellTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/vagrant/VagrantShellTask.java index 1a26a4897d7c6..8c2bd05a95938 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/vagrant/VagrantShellTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/vagrant/VagrantShellTask.java @@ -17,7 +17,6 @@ import java.util.Arrays; import java.util.List; import java.util.function.UnaryOperator; -import java.util.stream.Collectors; import static org.elasticsearch.gradle.internal.vagrant.VagrantMachine.convertLinuxPath; import static org.elasticsearch.gradle.internal.vagrant.VagrantMachine.convertWindowsPath; @@ -71,7 +70,7 @@ public void runScript() { script.add("try {"); script.add("cd " + convertWindowsPath(buildLayout.getRootDirectory(), buildLayout.getRootDirectory().toString())); extension.getVmEnv().forEach((k, v) -> script.add("$Env:" + k + " = \"" + v + "\"")); - script.addAll(getWindowsScript().stream().map(s -> " " + s).collect(Collectors.toList())); + script.addAll(getWindowsScript().stream().map(s -> " " + s).toList()); script.addAll( Arrays.asList( " exit $LASTEXITCODE", diff --git a/build-tools/reaper/src/main/java/org/elasticsearch/gradle/reaper/Reaper.java b/build-tools/reaper/src/main/java/org/elasticsearch/gradle/reaper/Reaper.java index e6c5b61e0a76c..c2adaf59ead1b 100644 --- a/build-tools/reaper/src/main/java/org/elasticsearch/gradle/reaper/Reaper.java +++ b/build-tools/reaper/src/main/java/org/elasticsearch/gradle/reaper/Reaper.java @@ -16,7 +16,6 @@ import java.nio.file.Paths; import java.util.Comparator; import java.util.List; -import java.util.stream.Collectors; import java.util.stream.Stream; /** @@ -62,7 +61,7 @@ public static void main(String[] args) throws Exception { private void reap() { try (Stream stream = Files.list(inputDir)) { - final List inputFiles = stream.filter(p -> p.getFileName().toString().endsWith(".cmd")).collect(Collectors.toList()); + final List inputFiles = stream.filter(p -> p.getFileName().toString().endsWith(".cmd")).toList(); for (Path inputFile : inputFiles) { System.out.println("Process file: " + inputFile); diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/LazyPropertyList.java b/build-tools/src/main/java/org/elasticsearch/gradle/LazyPropertyList.java index d901d78be882b..f44e3ea2979cd 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/LazyPropertyList.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/LazyPropertyList.java @@ -56,7 +56,7 @@ public Object[] toArray() { @Override public T1[] toArray(T1[] a) { - return delegate.stream().peek(this::validate).map(PropertyListEntry::getValue).collect(Collectors.toList()).toArray(a); + return delegate.stream().peek(this::validate).map(PropertyListEntry::getValue).toList().toArray(a); } @Override @@ -79,7 +79,7 @@ public boolean remove(Object o) { @Override public boolean containsAll(Collection c) { - return delegate.stream().map(PropertyListEntry::getValue).collect(Collectors.toList()).containsAll(c); + return delegate.stream().map(PropertyListEntry::getValue).collect(Collectors.toSet()).containsAll(c); } @Override diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/RunTask.java b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/RunTask.java index 424043c752402..d0b2581895db7 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/RunTask.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/RunTask.java @@ -136,7 +136,7 @@ public void beforeStart() { entry -> entry.getValue().toString() ) ); - boolean singleNode = getClusters().stream().flatMap(c -> c.getNodes().stream()).count() == 1; + boolean singleNode = getClusters().stream().mapToLong(c -> c.getNodes().size()).sum() == 1; final Function getDataPath; if (singleNode) { getDataPath = n -> dataDir; diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersAware.java b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersAware.java index 18f88b0dc4afc..9537162b5d109 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersAware.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersAware.java @@ -26,7 +26,7 @@ default void useCluster(ElasticsearchCluster cluster) { } cluster.getNodes() - .all(node -> node.getDistributions().stream().forEach(distro -> dependsOn(getProject().provider(() -> distro.maybeFreeze())))); + .all(node -> node.getDistributions().forEach(distro -> dependsOn(getProject().provider(() -> distro.maybeFreeze())))); cluster.getNodes().all(node -> dependsOn((Callable>) node::getPluginAndModuleConfigurations)); getClusters().add(cluster); } diff --git a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/AbstractBenchmark.java b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/AbstractBenchmark.java index 92f6e61636f50..6218b4d9a0d36 100644 --- a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/AbstractBenchmark.java +++ b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/AbstractBenchmark.java @@ -18,7 +18,6 @@ import java.lang.management.ManagementFactory; import java.util.Arrays; import java.util.List; -import java.util.stream.Collectors; public abstract class AbstractBenchmark { private static final int SEARCH_BENCHMARK_ITERATIONS = 10_000; @@ -92,7 +91,7 @@ private void runSearchBenchmark(String[] args) throws Exception { String benchmarkTargetHost = args[1]; String indexName = args[2]; String searchBody = args[3]; - List throughputRates = Arrays.asList(args[4].split(",")).stream().map(Integer::valueOf).collect(Collectors.toList()); + List throughputRates = Arrays.stream(args[4].split(",")).map(Integer::valueOf).toList(); T client = client(benchmarkTargetHost); diff --git a/distribution/tools/geoip-cli/src/main/java/org/elasticsearch/geoip/GeoIpCli.java b/distribution/tools/geoip-cli/src/main/java/org/elasticsearch/geoip/GeoIpCli.java index 2327b9e2235d5..8aea17585a5af 100644 --- a/distribution/tools/geoip-cli/src/main/java/org/elasticsearch/geoip/GeoIpCli.java +++ b/distribution/tools/geoip-cli/src/main/java/org/elasticsearch/geoip/GeoIpCli.java @@ -33,7 +33,6 @@ import java.security.MessageDigest; import java.util.Arrays; import java.util.Locale; -import java.util.stream.Collectors; import java.util.stream.Stream; import java.util.zip.GZIPOutputStream; @@ -74,7 +73,7 @@ private void copyTgzToTarget(Path source, Path target) throws IOException { return; } try (Stream files = Files.list(source)) { - for (Path path : files.filter(p -> p.getFileName().toString().endsWith(".tgz")).collect(Collectors.toList())) { + for (Path path : files.filter(p -> p.getFileName().toString().endsWith(".tgz")).toList()) { Files.copy(path, target.resolve(path.getFileName()), StandardCopyOption.REPLACE_EXISTING); } } @@ -82,7 +81,7 @@ private void copyTgzToTarget(Path source, Path target) throws IOException { private void packDatabasesToTgz(Terminal terminal, Path source, Path target) throws IOException { try (Stream files = Files.list(source)) { - for (Path path : files.filter(p -> p.getFileName().toString().endsWith(".mmdb")).collect(Collectors.toList())) { + for (Path path : files.filter(p -> p.getFileName().toString().endsWith(".mmdb")).toList()) { String fileName = path.getFileName().toString(); Path compressedPath = target.resolve(fileName.replaceAll("mmdb$", "") + "tgz"); terminal.println("Found " + fileName + ", will compress it to " + compressedPath.getFileName()); @@ -111,7 +110,7 @@ private void createOverviewJson(Terminal terminal, Path directory) throws IOExce XContentGenerator generator = XContentType.JSON.xContent().createGenerator(os) ) { generator.writeStartArray(); - for (Path db : files.filter(p -> p.getFileName().toString().endsWith(".tgz")).collect(Collectors.toList())) { + for (Path db : files.filter(p -> p.getFileName().toString().endsWith(".tgz")).toList()) { terminal.println("Adding " + db.getFileName() + " to overview.json"); MessageDigest md5 = MessageDigests.md5(); try (InputStream dis = new DigestInputStream(new BufferedInputStream(Files.newInputStream(db)), md5)) { diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java index 88a5727bc0b7d..a55a303517d6f 100644 --- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java +++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java @@ -10,11 +10,12 @@ import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; final class SystemJvmOptions { static List systemJvmOptions() { - return List.of( + return Stream.of( /* * Cache ttl in seconds for positive DNS lookups noting that this overrides the JDK security property networkaddress.cache.ttl; * can be set to -1 to cache forever. @@ -61,7 +62,7 @@ static List systemJvmOptions() { */ "--add-opens=java.base/java.io=org.elasticsearch.preallocate", maybeOverrideDockerCgroup() - ).stream().filter(e -> e.isEmpty() == false).collect(Collectors.toList()); + ).filter(e -> e.isEmpty() == false).collect(Collectors.toList()); } /* diff --git a/libs/core/src/main/java/org/elasticsearch/core/internal/provider/EmbeddedImplClassLoader.java b/libs/core/src/main/java/org/elasticsearch/core/internal/provider/EmbeddedImplClassLoader.java index e9e4a7760b990..134170f52ee14 100644 --- a/libs/core/src/main/java/org/elasticsearch/core/internal/provider/EmbeddedImplClassLoader.java +++ b/libs/core/src/main/java/org/elasticsearch/core/internal/provider/EmbeddedImplClassLoader.java @@ -108,7 +108,7 @@ private EmbeddedImplClassLoader(ClassLoader parent, Map pre .collect(toUnmodifiableMap(k -> k.getKey().prefix(), Map.Entry::getValue)); Map map = new HashMap<>(); for (var jarMeta : prefixToCodeBase.keySet()) { - jarMeta.packages().stream().forEach(pkg -> { + jarMeta.packages().forEach(pkg -> { var prev = map.put(pkg, jarMeta); assert prev == null; }); diff --git a/libs/core/src/main/java/org/elasticsearch/core/internal/provider/InMemoryModuleFinder.java b/libs/core/src/main/java/org/elasticsearch/core/internal/provider/InMemoryModuleFinder.java index aa7858ec206ba..f9c399e9522b3 100644 --- a/libs/core/src/main/java/org/elasticsearch/core/internal/provider/InMemoryModuleFinder.java +++ b/libs/core/src/main/java/org/elasticsearch/core/internal/provider/InMemoryModuleFinder.java @@ -72,8 +72,8 @@ static ModuleDescriptor filterRequires(ModuleDescriptor md, Set missingM md.requires().stream().filter(req -> missingModules.contains(req.name()) == false).forEach(builder::requires); md.exports().forEach(builder::exports); md.opens().forEach(builder::opens); - md.provides().stream().forEach(builder::provides); - md.uses().stream().forEach(builder::uses); + md.provides().forEach(builder::provides); + md.uses().forEach(builder::uses); builder.packages(md.packages()); return builder.build(); } diff --git a/libs/dissect/src/main/java/org/elasticsearch/dissect/DissectParser.java b/libs/dissect/src/main/java/org/elasticsearch/dissect/DissectParser.java index 147efc65099cb..f3f53f1b3c5ea 100644 --- a/libs/dissect/src/main/java/org/elasticsearch/dissect/DissectParser.java +++ b/libs/dissect/src/main/java/org/elasticsearch/dissect/DissectParser.java @@ -129,7 +129,6 @@ public DissectParser(String pattern, String appendSeparator) { Set appendKeyNames = dissectPairs.stream() .filter(dissectPair -> APPEND_MODIFIERS.contains(dissectPair.key().getModifier())) .map(KEY_NAME) - .distinct() .collect(Collectors.toSet()); if (appendKeyNames.size() > 0) { List modifiedMatchPairs = new ArrayList<>(dissectPairs.size()); diff --git a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslClientAuthenticationMode.java b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslClientAuthenticationMode.java index 05b464934523b..03754caf58049 100644 --- a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslClientAuthenticationMode.java +++ b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslClientAuthenticationMode.java @@ -11,7 +11,6 @@ import java.util.LinkedHashMap; import java.util.Locale; import java.util.Map; -import java.util.stream.Collectors; import javax.net.ssl.SSLParameters; @@ -82,7 +81,7 @@ static Map buildLookup() { public static SslClientAuthenticationMode parse(String value) { final SslClientAuthenticationMode mode = LOOKUP.get(value.toLowerCase(Locale.ROOT)); if (mode == null) { - final String allowedValues = LOOKUP.keySet().stream().collect(Collectors.joining(",")); + final String allowedValues = String.join(",", LOOKUP.keySet()); throw new SslConfigException( "could not resolve ssl client authentication, unknown value [" + value + "], recognised values are [" + allowedValues + "]" ); diff --git a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslDiagnostics.java b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslDiagnostics.java index b2fc59d69768a..e3fb59f79b41a 100644 --- a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslDiagnostics.java +++ b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslDiagnostics.java @@ -236,9 +236,7 @@ public String getTrustDiagnosticFailure( if (hostnames.isEmpty()) { message.append("; the certificate does not have any DNS/IP subject alternative names"); } else { - message.append("; the certificate has subject alternative names [") - .append(hostnames.stream().collect(Collectors.joining(","))) - .append("]"); + message.append("; the certificate has subject alternative names [").append(String.join(",", hostnames)).append("]"); } } } catch (CertificateParsingException e) { diff --git a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslVerificationMode.java b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslVerificationMode.java index 9b0f93c8360f6..1329ca276d678 100644 --- a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslVerificationMode.java +++ b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslVerificationMode.java @@ -11,7 +11,6 @@ import java.util.LinkedHashMap; import java.util.Locale; import java.util.Map; -import java.util.stream.Collectors; /** * Represents the verification mode to be used for SSL connections. @@ -84,7 +83,7 @@ private static Map buildLookup() { public static SslVerificationMode parse(String value) { final SslVerificationMode mode = LOOKUP.get(value.toLowerCase(Locale.ROOT)); if (mode == null) { - final String allowedValues = LOOKUP.keySet().stream().collect(Collectors.joining(",")); + final String allowedValues = String.join(",", LOOKUP.keySet()); throw new SslConfigException( "could not resolve ssl client verification mode, unknown value [" + value diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/histogram/AutoDateHistogramAggregatorTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/histogram/AutoDateHistogramAggregatorTests.java index d0e9b1ccd28eb..c8e6be2587a08 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/histogram/AutoDateHistogramAggregatorTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/histogram/AutoDateHistogramAggregatorTests.java @@ -1072,7 +1072,7 @@ private void indexSampleData(List dataset, RandomIndexWriter inde private Map bucketCountsAsMap(InternalAutoDateHistogram result) { Map map = Maps.newLinkedHashMapWithExpectedSize(result.getBuckets().size()); - result.getBuckets().stream().forEach(b -> { + result.getBuckets().forEach(b -> { Object old = map.put(b.getKeyAsString(), Math.toIntExact(b.getDocCount())); assertNull(old); }); @@ -1081,7 +1081,7 @@ private Map bucketCountsAsMap(InternalAutoDateHistogram result) private Map maxAsMap(InternalAutoDateHistogram result) { Map map = Maps.newLinkedHashMapWithExpectedSize(result.getBuckets().size()); - result.getBuckets().stream().forEach(b -> { + result.getBuckets().forEach(b -> { Max max = b.getAggregations().get("max"); Object old = map.put(b.getKeyAsString(), max.value()); assertNull(old); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/toxcontent/UserTreeToXContent.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/toxcontent/UserTreeToXContent.java index e77174e0f469e..2756419e68e16 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/toxcontent/UserTreeToXContent.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/toxcontent/UserTreeToXContent.java @@ -670,14 +670,10 @@ private void decorations(ANode node, ScriptScope scope) { if (decorations.isEmpty() == false) { builder.startArray(Fields.DECORATIONS); - List> dkeys = decorations.keySet() + decorations.keySet() .stream() .sorted(Comparator.comparing(Class::getName)) - .collect(Collectors.toList()); - - for (Class dkey : dkeys) { - DecorationToXContent.ToXContent(decorations.get(dkey), builder); - } + .forEachOrdered(dkey -> DecorationToXContent.ToXContent(decorations.get(dkey), builder)); builder.endArray(); } } diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java index 8a7f6fd8ae96b..c04a4b0db77dc 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java @@ -45,7 +45,6 @@ import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.stream.Collectors; /** * A {@link FieldMapper} that creates hierarchical joins (parent-join) between documents in the same index. @@ -324,7 +323,7 @@ protected void doValidate(MappingLookup mappingLookup) { .map(mappingLookup::getFieldType) .filter(ft -> ft instanceof JoinFieldType) .map(MappedFieldType::name) - .collect(Collectors.toList()); + .toList(); if (joinFields.size() > 1) { throw new IllegalArgumentException("Only one [parent-join] field can be defined per index, got " + joinFields); } diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/QueryAnalyzer.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/QueryAnalyzer.java index 7caad5976cdea..84fb889c6dd5a 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/QueryAnalyzer.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/QueryAnalyzer.java @@ -125,8 +125,7 @@ Result getResult() { partialResults.addAll(terms); } if (children.isEmpty() == false) { - List childResults = children.stream().map(ResultBuilder::getResult).collect(Collectors.toList()); - partialResults.addAll(childResults); + children.stream().map(ResultBuilder::getResult).forEach(partialResults::add); } if (partialResults.isEmpty()) { return verified ? Result.MATCH_NONE : Result.UNKNOWN; @@ -243,7 +242,7 @@ private static byte[] prepad(byte[] original) { } private static Result handleConjunction(List conjunctionsWithUnknowns) { - List conjunctions = conjunctionsWithUnknowns.stream().filter(r -> r.isUnknown() == false).collect(Collectors.toList()); + List conjunctions = conjunctionsWithUnknowns.stream().filter(r -> r.isUnknown() == false).toList(); if (conjunctions.isEmpty()) { if (conjunctionsWithUnknowns.isEmpty()) { throw new IllegalArgumentException("Must have at least one conjunction sub result"); diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGain.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGain.java index 6275a4f87d5e4..6b57acb8589ae 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGain.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGain.java @@ -19,7 +19,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; -import java.util.Comparator; import java.util.List; import java.util.Objects; import java.util.OptionalInt; @@ -133,8 +132,10 @@ public EvalQueryQuality evaluate(String taskId, SearchHit[] hits, List allRatings = ratedDocs.stream().mapToInt(RatedDocument::getRating).boxed().collect(Collectors.toList()); - Collections.sort(allRatings, Comparator.nullsLast(Collections.reverseOrder())); + List allRatings = ratedDocs.stream() + .map(RatedDocument::getRating) + .sorted(Collections.reverseOrder()) + .collect(Collectors.toList()); idcg = computeDCG(allRatings.subList(0, Math.min(ratingsInSearchHits.size(), allRatings.size()))); if (idcg != 0) { result = dcg / idcg; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/TransportNodesReloadSecureSettingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/TransportNodesReloadSecureSettingsAction.java index a5944e39fc5e7..079a12a806a2d 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/TransportNodesReloadSecureSettingsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/TransportNodesReloadSecureSettingsAction.java @@ -126,7 +126,7 @@ protected NodesReloadSecureSettingsResponse.NodeResponse nodeOperation( final Settings settingsWithKeystore = Settings.builder().put(environment.settings(), false).setSecureSettings(keystore).build(); final List exceptions = new ArrayList<>(); // broadcast the new settings object (with the open embedded keystore) to all reloadable plugins - pluginsService.filterPlugins(ReloadablePlugin.class).stream().forEach(p -> { + pluginsService.filterPlugins(ReloadablePlugin.class).forEach(p -> { try { p.reload(settingsWithKeystore); } catch (final Exception e) { diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateAction.java index 8b16f983048fc..7f3e058916572 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateAction.java @@ -24,7 +24,6 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -166,10 +165,10 @@ public TransformState transform(Object source, TransformState prevState) throws } Set composableEntities = composables.stream().map(r -> reservedComposableIndexName(r.name())).collect(Collectors.toSet()); - Set composablesToDelete = new HashSet<>( - prevState.keys().stream().filter(k -> k.startsWith(COMPOSABLE_PREFIX)).collect(Collectors.toSet()) - ); - composablesToDelete.removeAll(composableEntities); + Set composablesToDelete = prevState.keys() + .stream() + .filter(k -> k.startsWith(COMPOSABLE_PREFIX) && composableEntities.contains(k) == false) + .collect(Collectors.toSet()); // 3. delete composable index templates (this will fail on attached data streams, unless we added a higher priority one) if (composablesToDelete.isEmpty() == false) { @@ -183,9 +182,7 @@ public TransformState transform(Object source, TransformState prevState) throws } Set componentEntities = components.stream().map(r -> reservedComponentName(r.name())).collect(Collectors.toSet()); - Set componentsToDelete = new HashSet<>( - prevState.keys().stream().filter(k -> k.startsWith(COMPONENT_PREFIX)).collect(Collectors.toSet()) - ); + Set componentsToDelete = prevState.keys().stream().filter(k -> k.startsWith(COMPONENT_PREFIX)).collect(Collectors.toSet()); componentsToDelete.removeAll(componentEntities); // 5. delete component templates (this will check if there are any related composable index templates and fail) diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/SuggestingErrorOnUnknown.java b/server/src/main/java/org/elasticsearch/common/xcontent/SuggestingErrorOnUnknown.java index 17ac0e6a7c06d..00af25ed5b952 100644 --- a/server/src/main/java/org/elasticsearch/common/xcontent/SuggestingErrorOnUnknown.java +++ b/server/src/main/java/org/elasticsearch/common/xcontent/SuggestingErrorOnUnknown.java @@ -17,8 +17,6 @@ import java.util.List; import java.util.Locale; -import static java.util.stream.Collectors.toList; - public class SuggestingErrorOnUnknown implements ErrorOnUnknown { @Override public String errorMessage(String parserName, String unknownField, Iterable candidates) { @@ -55,7 +53,7 @@ public static String suggest(String unknownField, Iterable candidates) { } return a.v2().compareTo(b.v2()); }); - List keys = scored.stream().map(Tuple::v2).collect(toList()); + List keys = scored.stream().map(Tuple::v2).toList(); StringBuilder builder = new StringBuilder(" did you mean "); if (keys.size() == 1) { builder.append("[").append(keys.get(0)).append("]"); diff --git a/server/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java index b0b56ff58d376..7fa50386fcc85 100644 --- a/server/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java @@ -97,7 +97,7 @@ public TermsQueryBuilder(String fieldName, String... values) { * @param values The terms */ public TermsQueryBuilder(String fieldName, int... values) { - this(fieldName, values != null ? Arrays.stream(values).mapToObj(s -> s).toList() : (Iterable) null); + this(fieldName, values != null ? Arrays.stream(values).boxed().toList() : null); } /** @@ -107,7 +107,7 @@ public TermsQueryBuilder(String fieldName, int... values) { * @param values The terms */ public TermsQueryBuilder(String fieldName, long... values) { - this(fieldName, values != null ? Arrays.stream(values).mapToObj(s -> s).toList() : (Iterable) null); + this(fieldName, values != null ? Arrays.stream(values).boxed().toList() : null); } /** @@ -117,7 +117,7 @@ public TermsQueryBuilder(String fieldName, long... values) { * @param values The terms */ public TermsQueryBuilder(String fieldName, float... values) { - this(fieldName, values != null ? IntStream.range(0, values.length).mapToObj(i -> values[i]).toList() : (Iterable) null); + this(fieldName, values != null ? IntStream.range(0, values.length).mapToObj(i -> values[i]).toList() : null); } /** @@ -127,7 +127,7 @@ public TermsQueryBuilder(String fieldName, float... values) { * @param values The terms */ public TermsQueryBuilder(String fieldName, double... values) { - this(fieldName, values != null ? Arrays.stream(values).mapToObj(s -> s).toList() : (Iterable) null); + this(fieldName, values != null ? Arrays.stream(values).boxed().toList() : null); } /** @@ -137,7 +137,7 @@ public TermsQueryBuilder(String fieldName, double... values) { * @param values The terms */ public TermsQueryBuilder(String fieldName, Object... values) { - this(fieldName, values != null ? Arrays.asList(values) : (Iterable) null); + this(fieldName, values != null ? Arrays.asList(values) : null); } /** diff --git a/server/src/main/java/org/elasticsearch/index/search/stats/ShardFieldUsageTracker.java b/server/src/main/java/org/elasticsearch/index/search/stats/ShardFieldUsageTracker.java index 97178bfb80ad1..54d6d35301c67 100644 --- a/server/src/main/java/org/elasticsearch/index/search/stats/ShardFieldUsageTracker.java +++ b/server/src/main/java/org/elasticsearch/index/search/stats/ShardFieldUsageTracker.java @@ -110,7 +110,7 @@ public class FieldUsageStatsTrackingSession implements FieldUsageNotifier, Relea @Override public void close() { - usages.entrySet().stream().forEach(e -> { + usages.entrySet().forEach(e -> { InternalFieldStats fieldStats = perFieldStats.computeIfAbsent(e.getKey(), f -> new InternalFieldStats()); PerField pf = e.getValue(); boolean any = false; diff --git a/server/src/main/java/org/elasticsearch/index/translog/Translog.java b/server/src/main/java/org/elasticsearch/index/translog/Translog.java index 8759f1cf4b9d1..871de3c9f9f87 100644 --- a/server/src/main/java/org/elasticsearch/index/translog/Translog.java +++ b/server/src/main/java/org/elasticsearch/index/translog/Translog.java @@ -59,7 +59,6 @@ import java.util.function.LongSupplier; import java.util.regex.Matcher; import java.util.regex.Pattern; -import java.util.stream.Collectors; import java.util.stream.Stream; import static org.elasticsearch.core.Strings.format; @@ -375,7 +374,7 @@ private static boolean calledFromOutsideOrViaTragedyClose() { return false; } }). // find all inner callers including Translog subclasses - collect(Collectors.toList()); + toList(); // the list of inner callers should be either empty or should contain closeOnTragicEvent method return frames.isEmpty() || frames.stream().anyMatch(f -> f.getMethodName().equals("closeOnTragicEvent")); } diff --git a/server/src/main/java/org/elasticsearch/plugins/ModuleSupport.java b/server/src/main/java/org/elasticsearch/plugins/ModuleSupport.java index 0213748f642d4..e5f0004431b0e 100644 --- a/server/src/main/java/org/elasticsearch/plugins/ModuleSupport.java +++ b/server/src/main/java/org/elasticsearch/plugins/ModuleSupport.java @@ -73,8 +73,8 @@ static ModuleDescriptor createModuleDescriptor( Predicate isPackageInParentLayers ) throws IOException { var builder = ModuleDescriptor.newOpenModule(name); // open module, for now - requires.stream().forEach(builder::requires); - uses.stream().forEach(builder::uses); + requires.forEach(builder::requires); + uses.forEach(builder::uses); // scan the names of the entries in the JARs Set pkgs = new HashSet<>(); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/InternalOrder.java b/server/src/main/java/org/elasticsearch/search/aggregations/InternalOrder.java index 3d170694b62ee..f167d15473b0b 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/InternalOrder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/InternalOrder.java @@ -30,8 +30,6 @@ import java.util.Objects; import java.util.function.ToLongFunction; -import static java.util.stream.Collectors.toList; - /** * Implementations for {@link Bucket} ordering strategies. */ @@ -187,7 +185,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws public Comparator partiallyBuiltBucketComparator(ToLongFunction ordinalReader, Aggregator aggregator) { List> comparators = orderElements.stream() .map(oe -> oe.partiallyBuiltBucketComparator(ordinalReader, aggregator)) - .collect(toList()); + .toList(); return (lhs, rhs) -> { for (Comparator c : comparators) { int result = c.compare(lhs, rhs); @@ -201,7 +199,7 @@ public Comparator partiallyBuiltBucketComparator(ToLongFun @Override public Comparator comparator() { - List> comparators = orderElements.stream().map(BucketOrder::comparator).collect(toList()); + List> comparators = orderElements.stream().map(BucketOrder::comparator).toList(); return (lhs, rhs) -> { for (Comparator c : comparators) { int result = c.compare(lhs, rhs); @@ -217,7 +215,7 @@ public Comparator comparator() { Comparator> delayedBucketComparator() { List>> comparators = orderElements.stream() .map(BucketOrder::delayedBucketComparator) - .collect(toList()); + .toList(); return (lhs, rhs) -> { for (Comparator> c : comparators) { int result = c.compare(lhs, rhs); diff --git a/server/src/main/java/org/elasticsearch/search/rescore/QueryRescorer.java b/server/src/main/java/org/elasticsearch/search/rescore/QueryRescorer.java index 1eab3251c6184..389e3a56cf152 100644 --- a/server/src/main/java/org/elasticsearch/search/rescore/QueryRescorer.java +++ b/server/src/main/java/org/elasticsearch/search/rescore/QueryRescorer.java @@ -21,7 +21,7 @@ import java.util.List; import java.util.Set; -import static java.util.stream.Collectors.toSet; +import static java.util.stream.Collectors.toUnmodifiableSet; public final class QueryRescorer implements Rescorer { @@ -57,9 +57,7 @@ protected float combine(float firstPassScore, boolean secondPassMatches, float s TopDocs topNFirstPass = topN(topDocs, rescoreContext.getWindowSize()); // Save doc IDs for which rescoring was applied to be used in score explanation - Set topNDocIDs = Collections.unmodifiableSet( - Arrays.stream(topNFirstPass.scoreDocs).map(scoreDoc -> scoreDoc.doc).collect(toSet()) - ); + Set topNDocIDs = Arrays.stream(topNFirstPass.scoreDocs).map(scoreDoc -> scoreDoc.doc).collect(toUnmodifiableSet()); rescoreContext.setRescoredDocs(topNDocIDs); // Rescore them: diff --git a/server/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodesTests.java b/server/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodesTests.java index a8a2d05ac4b8e..13472eff2312a 100644 --- a/server/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodesTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodesTests.java @@ -250,13 +250,13 @@ public void testDeltas() { DiscoveryNode masterB = randomBoolean() ? null : RandomPicks.randomFrom(random(), nodesB); DiscoveryNodes.Builder builderA = DiscoveryNodes.builder(); - nodesA.stream().forEach(builderA::add); + nodesA.forEach(builderA::add); final String masterAId = masterA == null ? null : masterA.getId(); builderA.masterNodeId(masterAId); builderA.localNodeId(RandomPicks.randomFrom(random(), nodesA).getId()); DiscoveryNodes.Builder builderB = DiscoveryNodes.builder(); - nodesB.stream().forEach(builderB::add); + nodesB.forEach(builderB::add); final String masterBId = masterB == null ? null : masterB.getId(); builderB.masterNodeId(masterBId); builderB.localNodeId(RandomPicks.randomFrom(random(), nodesB).getId()); diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/FailedNodeRoutingTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/FailedNodeRoutingTests.java index aad7f7473d3cf..0e13bfeecfdb4 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/FailedNodeRoutingTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/FailedNodeRoutingTests.java @@ -158,9 +158,9 @@ public void testRandomClusterPromotesNewestReplica() throws InterruptedException // Pick a random subset of primaries to fail List shardsToFail = new ArrayList<>(); List failedPrimaries = randomSubsetOf(primaries); - failedPrimaries.stream().forEach(sr -> { - shardsToFail.add(new FailedShard(randomFrom(sr), "failed primary", new Exception(), randomBoolean())); - }); + failedPrimaries.forEach( + sr -> shardsToFail.add(new FailedShard(randomFrom(sr), "failed primary", new Exception(), randomBoolean())) + ); logger.info("--> state before failing shards: {}", state); state = cluster.applyFailedShards(state, shardsToFail); @@ -173,7 +173,7 @@ public void testRandomClusterPromotesNewestReplica() throws InterruptedException Version newPrimaryVersion = getNodeVersion(newPrimary, compareState); logger.info("--> new primary is on version {}: {}", newPrimaryVersion, newPrimary); - compareState.routingTable().shardRoutingTable(newPrimary.shardId()).shardsWithState(STARTED).stream().forEach(sr -> { + compareState.routingTable().shardRoutingTable(newPrimary.shardId()).shardsWithState(STARTED).forEach(sr -> { Version candidateVer = getNodeVersion(sr, compareState); if (candidateVer != null) { logger.info("--> candidate on {} node; shard routing: {}", candidateVer, sr); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregationBuilderTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregationBuilderTests.java index 6c3d5b03ffdc5..354ace81e08b6 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregationBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregationBuilderTests.java @@ -82,7 +82,7 @@ protected RangeAggregationBuilder mutateInstance(RangeAggregationBuilder builder default -> fail(); } RangeAggregationBuilder mutant = new RangeAggregationBuilder(name).keyed(keyed).field(field); - ranges.stream().forEach(mutant::addRange); + ranges.forEach(mutant::addRange); return mutant; } diff --git a/test/fixtures/azure-fixture/src/main/java/fixture/azure/AzureHttpHandler.java b/test/fixtures/azure-fixture/src/main/java/fixture/azure/AzureHttpHandler.java index 7d5f16ae1cdcf..14ce4a8318370 100644 --- a/test/fixtures/azure-fixture/src/main/java/fixture/azure/AzureHttpHandler.java +++ b/test/fixtures/azure-fixture/src/main/java/fixture/azure/AzureHttpHandler.java @@ -34,7 +34,6 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.regex.Matcher; import java.util.regex.Pattern; -import java.util.stream.Collectors; import static org.elasticsearch.repositories.azure.AzureFixtureHelper.assertValidBlockId; @@ -77,7 +76,7 @@ public void handle(final HttpExchange exchange) throws IOException { final List blockIds = Arrays.stream(blockList.split("")) .filter(line -> line.contains("")) .map(line -> line.substring(0, line.indexOf(""))) - .collect(Collectors.toList()); + .toList(); final ByteArrayOutputStream blob = new ByteArrayOutputStream(); for (String blockId : blockIds) { diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/coordination/CoordinationStateTestCluster.java b/test/framework/src/main/java/org/elasticsearch/cluster/coordination/CoordinationStateTestCluster.java index e8687d8d02623..76d662e594644 100644 --- a/test/framework/src/main/java/org/elasticsearch/cluster/coordination/CoordinationStateTestCluster.java +++ b/test/framework/src/main/java/org/elasticsearch/cluster/coordination/CoordinationStateTestCluster.java @@ -187,7 +187,7 @@ public CoordinationStateTestCluster(List nodes, ElectionStrategy this.electionStrategy = electionStrategy; messages = new ArrayList<>(); - clusterNodes = nodes.stream().map(node -> new ClusterNode(node, electionStrategy)).collect(Collectors.toList()); + clusterNodes = nodes.stream().map(node -> new ClusterNode(node, electionStrategy)).toList(); initialConfiguration = randomVotingConfig(); initialValue = randomLong(); @@ -200,7 +200,7 @@ void reply(Message m, Object payload) { } void broadcast(DiscoveryNode sourceNode, Object payload) { - messages.addAll(clusterNodes.stream().map(cn -> new Message(sourceNode, cn.localNode, payload)).collect(Collectors.toList())); + clusterNodes.stream().map(cn -> new Message(sourceNode, cn.localNode, payload)).forEach(messages::add); } Optional getNode(DiscoveryNode node) { @@ -251,9 +251,7 @@ public void runRandomly() { } else if (rarely() && rarely()) { randomFrom(clusterNodes).reboot(); } else if (rarely()) { - final List masterNodes = clusterNodes.stream() - .filter(cn -> cn.state.electionWon()) - .collect(Collectors.toList()); + final List masterNodes = clusterNodes.stream().filter(cn -> cn.state.electionWon()).toList(); if (masterNodes.isEmpty() == false) { final ClusterNode clusterNode = randomFrom(masterNodes); final long term = rarely() ? randomLongBetween(0, maxTerm + 1) : clusterNode.state.getCurrentTerm(); diff --git a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreTestUtil.java b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreTestUtil.java index db0ff3efcda4c..d4d7fdcb40c65 100644 --- a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreTestUtil.java +++ b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreTestUtil.java @@ -187,11 +187,7 @@ private static void assertShardIndexGenerations(BlobContainer repoRoot, ShardGen } private static void assertIndexUUIDs(BlobStoreRepository repository, RepositoryData repositoryData) throws IOException { - final List expectedIndexUUIDs = repositoryData.getIndices() - .values() - .stream() - .map(IndexId::getId) - .collect(Collectors.toList()); + final List expectedIndexUUIDs = repositoryData.getIndices().values().stream().map(IndexId::getId).toList(); final BlobContainer indicesContainer = repository.blobContainer().children().get("indices"); final List foundIndexUUIDs; if (indicesContainer == null) { @@ -231,7 +227,7 @@ private static void assertSnapshotUUIDs( ) throws IOException { final BlobContainer repoRoot = repository.blobContainer(); final Collection snapshotIds = repositoryData.getSnapshotIds(); - final List expectedSnapshotUUIDs = snapshotIds.stream().map(SnapshotId::getUUID).collect(Collectors.toList()); + final List expectedSnapshotUUIDs = snapshotIds.stream().map(SnapshotId::getUUID).toList(); for (String prefix : new String[] { BlobStoreRepository.SNAPSHOT_PREFIX, BlobStoreRepository.METADATA_PREFIX }) { final Collection foundSnapshotUUIDs = repoRoot.listBlobs() .keySet() diff --git a/test/framework/src/main/java/org/elasticsearch/snapshots/mockstore/MockRepository.java b/test/framework/src/main/java/org/elasticsearch/snapshots/mockstore/MockRepository.java index 6bd565c712cf4..575a31b087c7f 100644 --- a/test/framework/src/main/java/org/elasticsearch/snapshots/mockstore/MockRepository.java +++ b/test/framework/src/main/java/org/elasticsearch/snapshots/mockstore/MockRepository.java @@ -60,7 +60,6 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; -import java.util.stream.Collectors; public class MockRepository extends FsRepository { private static final Logger logger = LogManager.getLogger(MockRepository.class); @@ -517,7 +516,7 @@ public DeleteResult delete() throws IOException { final Map blobs = listBlobs(); long deleteBlobCount = blobs.size(); long deleteByteCount = 0L; - for (String blob : blobs.values().stream().map(BlobMetadata::name).collect(Collectors.toList())) { + for (String blob : blobs.values().stream().map(BlobMetadata::name).toList()) { maybeIOExceptionOrBlock(blob); deleteBlobsIgnoringIfNotExists(Iterators.single(blob)); deleteByteCount += blobs.get(blob).length(); diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java index 39c8c1ce55fe8..d3669086d2d13 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java @@ -219,7 +219,7 @@ protected boolean addMockHttpTransport() { protected List filteredWarnings() { return Stream.concat( super.filteredWarnings().stream(), - List.of("[index.data_path] setting was deprecated in Elasticsearch and will be removed in a future release.").stream() + Stream.of("[index.data_path] setting was deprecated in Elasticsearch and will be removed in a future release.") ).collect(Collectors.toList()); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java b/test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java index 81e9939c2d485..6ec3614043565 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java @@ -192,7 +192,7 @@ public static Version randomVersion(Random random) { /** Returns a random {@link Version} from all available versions, that is compatible with the given version. */ public static Version randomCompatibleVersion(Random random, Version version) { - final List compatible = ALL_VERSIONS.stream().filter(version::isCompatible).collect(Collectors.toList()); + final List compatible = ALL_VERSIONS.stream().filter(version::isCompatible).toList(); return compatible.get(random.nextInt(compatible.size())); } @@ -228,10 +228,7 @@ public static Version compatibleFutureVersion(Version version) { /** Returns the maximum {@link Version} that is compatible with the given version. */ public static Version maxCompatibleVersion(Version version) { - final List compatible = ALL_VERSIONS.stream() - .filter(version::isCompatible) - .filter(version::onOrBefore) - .collect(Collectors.toList()); + final List compatible = ALL_VERSIONS.stream().filter(version::isCompatible).filter(version::onOrBefore).toList(); assert compatible.size() > 0; return compatible.get(compatible.size() - 1); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index d092c431f2b8d..f85a93e300043 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -839,13 +839,13 @@ private void checkForUnexpectedlyRecreatedObjects() throws IOException { if (hasIlm && false == preserveILMPoliciesUponCompletion()) { Set unexpectedIlmPlicies = getAllUnexpectedIlmPolicies(preserveILMPolicyIds()); assertTrue( - "Expected no ILM policies after deletions, but found " + unexpectedIlmPlicies.stream().collect(Collectors.joining(", ")), + "Expected no ILM policies after deletions, but found " + String.join(", ", unexpectedIlmPlicies), unexpectedIlmPlicies.isEmpty() ); } Set unexpectedTemplates = getAllUnexpectedTemplates(); assertTrue( - "Expected no templates after deletions, but found " + unexpectedTemplates.stream().collect(Collectors.joining(", ")), + "Expected no templates after deletions, but found " + String.join(", ", unexpectedTemplates), unexpectedTemplates.isEmpty() ); } @@ -893,12 +893,10 @@ private Set getAllUnexpectedTemplates() throws IOException { Request compReq = new Request("GET", "_component_template"); String componentTemplates = EntityUtils.toString(adminClient().performRequest(compReq).getEntity()); Map cTemplates = XContentHelper.convertToMap(JsonXContent.jsonXContent, componentTemplates, false); - unexpectedTemplates.addAll( - ((List) cTemplates.get("component_templates")).stream() - .map(ct -> (String) ((Map) ct).get("name")) - .filter(name -> isXPackTemplate(name) == false) - .collect(Collectors.toList()) - ); + ((List) cTemplates.get("component_templates")).stream() + .map(ct -> (String) ((Map) ct).get("name")) + .filter(name -> isXPackTemplate(name) == false) + .forEach(unexpectedTemplates::add); } // Always check for legacy templates: Request getLegacyTemplatesRequest = new Request("GET", "_template"); diff --git a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java index cb60cb2758dd5..28f68081b73a4 100644 --- a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java +++ b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java @@ -61,7 +61,6 @@ import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; -import java.util.stream.Collectors; import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; @@ -326,7 +325,7 @@ private static void addSuite(Path root, Path file, Map> files) } filesSet.add(file); - List fileNames = filesSet.stream().map(p -> p.getFileName().toString()).collect(Collectors.toList()); + List fileNames = filesSet.stream().map(p -> p.getFileName().toString()).toList(); if (Collections.frequency(fileNames, file.getFileName().toString()) > 1) { Logger logger = LogManager.getLogger(ESClientYamlSuiteTestCase.class); logger.warn( diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/movingPercentiles/MovingPercentilesPipelineAggregator.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/movingPercentiles/MovingPercentilesPipelineAggregator.java index ff49c9681b20a..72a0dd4b6cb07 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/movingPercentiles/MovingPercentilesPipelineAggregator.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/movingPercentiles/MovingPercentilesPipelineAggregator.java @@ -76,7 +76,7 @@ private void reduceTDigest( List values = buckets.stream() .map(b -> resolveTDigestBucketValue(histo, b, bucketsPaths()[0])) .filter(v -> v != null) - .collect(Collectors.toList()); + .toList(); int index = 0; for (InternalMultiBucketAggregation.InternalBucket bucket : buckets) { @@ -126,7 +126,7 @@ private void reduceHDR( List values = buckets.stream() .map(b -> resolveHDRBucketValue(histo, b, bucketsPaths()[0])) .filter(v -> v != null) - .collect(Collectors.toList()); + .toList(); int index = 0; for (InternalMultiBucketAggregation.InternalBucket bucket : buckets) { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/XPackInfoResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/XPackInfoResponse.java index 820181c5c3218..8f872dcb056a1 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/XPackInfoResponse.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/XPackInfoResponse.java @@ -31,7 +31,6 @@ import java.util.Objects; import java.util.Set; import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; @@ -342,7 +341,7 @@ public int hashCode() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - List names = new ArrayList<>(this.featureSets.keySet()).stream().sorted().collect(Collectors.toList()); + List names = new ArrayList<>(this.featureSets.keySet()).stream().sorted().toList(); for (String name : names) { builder.field(name, featureSets.get(name), params); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/PauseFollowerIndexStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/PauseFollowerIndexStep.java index 3c436b591ae7c..a7605f94eac44 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/PauseFollowerIndexStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/PauseFollowerIndexStep.java @@ -16,7 +16,6 @@ import org.elasticsearch.xpack.core.ccr.action.ShardFollowTask; import java.util.List; -import java.util.stream.Collectors; final class PauseFollowerIndexStep extends AbstractUnfollowIndexStep { @@ -46,7 +45,7 @@ void innerPerformAction(String followerIndex, ClusterState currentClusterState, ShardFollowTask shardFollowTask = (ShardFollowTask) persistentTask.getParams(); return shardFollowTask.getFollowShardId().getIndexName().equals(followerIndex); }) - .collect(Collectors.toList()); + .toList(); if (shardFollowTasks.isEmpty()) { listener.onResponse(null); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/PhaseCacheManagement.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/PhaseCacheManagement.java index 9be37e9193b64..26966195989bb 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/PhaseCacheManagement.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/PhaseCacheManagement.java @@ -156,7 +156,7 @@ public static boolean updateIndicesForPolicy( .stream() .filter(meta -> newPolicy.getName().equals(meta.getLifecyclePolicyName())) .filter(meta -> isIndexPhaseDefinitionUpdatable(xContentRegistry, client, meta, newPolicy.getPolicy(), licenseState)) - .collect(Collectors.toList()); + .toList(); final List refreshedIndices = new ArrayList<>(indicesThatCanBeUpdated.size()); for (IndexMetadata index : indicesThatCanBeUpdated) { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SegmentCountStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SegmentCountStep.java index df1fb0be534a2..82f502c96e8b5 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SegmentCountStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SegmentCountStep.java @@ -79,7 +79,7 @@ public void evaluateCondition(Metadata metadata, Index index, Listener listener, .stream() .flatMap(iss -> Arrays.stream(iss.shards())) .filter(shardSegments -> shardSegments.getSegments().size() > maxNumSegments) - .collect(Collectors.toList()); + .toList(); if (unmergedShards.size() > 0) { Map unmergedShardCounts = unmergedShards.stream() .collect(Collectors.toMap(ShardSegments::getShardRouting, ss -> ss.getSegments().size())); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForFollowShardTasksStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForFollowShardTasksStep.java index 604872e11c584..f1fbdde1e9a5d 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForFollowShardTasksStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForFollowShardTasksStep.java @@ -63,7 +63,7 @@ static void handleResponse(FollowStatsAction.StatsResponses responses, Listener .stream() .map(FollowStatsAction.StatsResponse::status) .filter(shardFollowStatus -> shardFollowStatus.leaderGlobalCheckpoint() != shardFollowStatus.followerGlobalCheckpoint()) - .collect(Collectors.toList()); + .toList(); // Follow stats api needs to return stats for follower index and all shard follow tasks should be synced: boolean conditionMet = responses.getStatsResponses().size() > 0 && unSyncedShardFollowStatuses.isEmpty(); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/InferenceHelpers.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/InferenceHelpers.java index 4d2a3d08d38c3..e60e950f3cfd4 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/InferenceHelpers.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/InferenceHelpers.java @@ -20,7 +20,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; import java.util.stream.IntStream; public final class InferenceHelpers { @@ -68,7 +67,7 @@ public static Tuple> topClasses( List labels = classificationLabels == null ? // If we don't have the labels we should return the top classification values anyways, they will just be numeric - IntStream.range(0, probabilities.length).boxed().map(String::valueOf).collect(Collectors.toList()) : classificationLabels; + IntStream.range(0, probabilities.length).mapToObj(String::valueOf).toList() : classificationLabels; int count = numToInclude < 0 ? probabilities.length : Math.min(numToInclude, probabilities.length); List topClassEntries = new ArrayList<>(count); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Detector.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Detector.java index 3b5182ca0cbab..9cd8afe44587c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Detector.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Detector.java @@ -200,10 +200,9 @@ private static ObjectParser createParser(boolean ignoreUnknownFie * ", \ */ public static final Character[] PROHIBITED_FIELDNAME_CHARACTERS = { '"', '\\' }; - public static final String PROHIBITED = String.join( - ",", - Arrays.stream(PROHIBITED_FIELDNAME_CHARACTERS).map(c -> Character.toString(c)).collect(Collectors.toList()) - ); + public static final String PROHIBITED = Arrays.stream(PROHIBITED_FIELDNAME_CHARACTERS) + .map(c -> Character.toString(c)) + .collect(Collectors.joining(",")); private final String detectorDescription; private final DetectorFunction function; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/NameResolver.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/NameResolver.java index 220b55ebf9e2a..3c92a792cc90f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/NameResolver.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/NameResolver.java @@ -18,7 +18,6 @@ import java.util.SortedSet; import java.util.TreeSet; import java.util.function.Function; -import java.util.stream.Collectors; /** * Expands an expression into the set of matching names. @@ -92,7 +91,7 @@ public SortedSet expand(String expression, boolean allowNoMatch, Optiona .filter(key -> Regex.simpleMatch(token, key)) .map(this::lookup) .flatMap(List::stream) - .collect(Collectors.toList()); + .toList(); if (expanded.isEmpty() && allowNoMatch == false) { throw notFoundExceptionSupplier.apply(token); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollableIndexCaps.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollableIndexCaps.java index d18b196449b4c..411f8b0db497e 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollableIndexCaps.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollableIndexCaps.java @@ -14,11 +14,9 @@ import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; -import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Objects; -import java.util.stream.Collectors; /** * Represents the rollup capabilities of a non-rollup index. E.g. what values/aggregations @@ -35,9 +33,7 @@ public class RollableIndexCaps implements Writeable, ToXContentObject { public RollableIndexCaps(String indexName, List caps) { this.indexName = indexName; - this.jobCaps = Collections.unmodifiableList( - Objects.requireNonNull(caps).stream().sorted(Comparator.comparing(RollupJobCaps::getJobID)).collect(Collectors.toList()) - ); + this.jobCaps = caps.stream().sorted(Comparator.comparing(RollupJobCaps::getJobID)).toList(); } public RollableIndexCaps(StreamInput in) throws IOException { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollupJobCaps.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollupJobCaps.java index f11e56edf5449..ceccfd4028fae 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollupJobCaps.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollupJobCaps.java @@ -190,7 +190,7 @@ private static Map createRollupFieldCaps(final RollupJo final List> metrics = metricConfig.getMetrics() .stream() .map(metric -> singletonMap("agg", (Object) metric)) - .collect(Collectors.toList()); + .toList(); metrics.forEach(m -> { List> caps = tempFieldCaps.getOrDefault(metricConfig.getField(), new ArrayList<>()); caps.add(m); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/searchablesnapshots/MountSearchableSnapshotRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/searchablesnapshots/MountSearchableSnapshotRequest.java index 5bd0024f2fb63..eb1017fcd4399 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/searchablesnapshots/MountSearchableSnapshotRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/searchablesnapshots/MountSearchableSnapshotRequest.java @@ -27,7 +27,6 @@ import java.util.Arrays; import java.util.Locale; import java.util.Objects; -import java.util.stream.Collectors; import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.common.settings.Settings.readSettingsFromStream; @@ -71,7 +70,7 @@ public class MountSearchableSnapshotRequest extends MasterNodeRequest p.list().stream().map(s -> (String) s).collect(Collectors.toList()).toArray(Strings.EMPTY_ARRAY), + p -> p.list().stream().map(s -> (String) s).toArray(String[]::new), IGNORE_INDEX_SETTINGS_FIELD, ObjectParser.ValueType.STRING_ARRAY ); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/searchablesnapshots/SearchableSnapshotShardStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/searchablesnapshots/SearchableSnapshotShardStats.java index 887ca78e32f12..068a7dda5f7f3 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/searchablesnapshots/SearchableSnapshotShardStats.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/searchablesnapshots/SearchableSnapshotShardStats.java @@ -24,7 +24,6 @@ import java.util.Objects; import static java.util.Collections.unmodifiableList; -import static java.util.stream.Collectors.toList; public class SearchableSnapshotShardStats implements Writeable, ToXContentObject { @@ -97,7 +96,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws { List stats = inputStats.stream() .sorted(Comparator.comparing(CacheIndexInputStats::getFileExt)) - .collect(toList()); + .toList(); for (CacheIndexInputStats stat : stats) { stat.toXContent(builder, params); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/PutPrivilegesRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/PutPrivilegesRequest.java index 2556246589c5f..77d4ac677e17e 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/PutPrivilegesRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/PutPrivilegesRequest.java @@ -111,9 +111,7 @@ public void setPrivileges(Collection privileges) @Override public Collection getApplicationNames() { - return Collections.unmodifiableSet( - privileges.stream().map(ApplicationPrivilegeDescriptor::getApplication).collect(Collectors.toSet()) - ); + return privileges.stream().map(ApplicationPrivilegeDescriptor::getApplication).collect(Collectors.toUnmodifiableSet()); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java index 82e4ef11762e4..819e1fa09a022 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java @@ -1393,10 +1393,10 @@ static BytesReference maybeRemoveRemoteIndicesFromRoleDescriptors(BytesReference final Map roleDescriptorsMap = convertRoleDescriptorsBytesToMap(roleDescriptorsBytes); final AtomicBoolean removedAtLeastOne = new AtomicBoolean(false); - roleDescriptorsMap.entrySet().stream().forEach(entry -> { - if (entry.getValue() instanceof Map) { + roleDescriptorsMap.forEach((key, value) -> { + if (value instanceof Map) { @SuppressWarnings("unchecked") - Map roleDescriptor = (Map) entry.getValue(); + Map roleDescriptor = (Map) value; boolean removed = roleDescriptor.remove(RoleDescriptor.Fields.REMOTE_INDICES.getPreferredName()) != null; if (removed) { removedAtLeastOne.set(true); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/DefaultAuthenticationFailureHandler.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/DefaultAuthenticationFailureHandler.java index 6fc83354aa2a7..b5469fadd95b6 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/DefaultAuthenticationFailureHandler.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/DefaultAuthenticationFailureHandler.java @@ -186,12 +186,12 @@ private ElasticsearchSecurityException createAuthenticationError(final String me ese = authenticationError(message, t, args); containsNegotiateWithToken = false; } - defaultFailureResponseHeaders.entrySet().stream().forEach((e) -> { - if (containsNegotiateWithToken && e.getKey().equalsIgnoreCase("WWW-Authenticate")) { + defaultFailureResponseHeaders.forEach((key, value) -> { + if (containsNegotiateWithToken && key.equalsIgnoreCase("WWW-Authenticate")) { return; } // If it is already present then it will replace the existing header. - ese.addHeader(e.getKey(), e.getValue()); + ese.addHeader(key, value); }); return ese; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/AuthorizationEngine.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/AuthorizationEngine.java index 46b3e303fe88f..17d80274e161e 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/AuthorizationEngine.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/AuthorizationEngine.java @@ -632,8 +632,7 @@ public String getFailureContext(RequestInfo requestInfo, RestrictedIndices restr || Arrays.equals(IndicesAndAliasesResolverField.NO_INDICES_OR_ALIASES_ARRAY, indices)) { return null; } - Set deniedIndices = Arrays.asList(indices) - .stream() + Set deniedIndices = Arrays.stream(indices) .filter(index -> false == indicesAccessControl.hasIndexPermissions(index)) .collect(Collectors.toSet()); return getFailureDescription(deniedIndices, restrictedIndices); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ResourcePrivilegesMap.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ResourcePrivilegesMap.java index 3750cdc94ab26..6e4557ad06e12 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ResourcePrivilegesMap.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ResourcePrivilegesMap.java @@ -74,9 +74,7 @@ public Builder addResourcePrivilege(String resource, Map privil public Builder addResourcePrivilegesMap(ResourcePrivilegesMap resourcePrivilegesMap) { resourcePrivilegesMap.getResourceToResourcePrivileges() - .entrySet() - .stream() - .forEach(e -> this.addResourcePrivilege(e.getKey(), e.getValue().getPrivileges())); + .forEach((key, value) -> this.addResourcePrivilege(key, value.getPrivileges())); return this; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecycleStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecycleStats.java index 10ab898b34f71..4370671faf901 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecycleStats.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecycleStats.java @@ -230,7 +230,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws List metrics = getMetrics().values() .stream() .sorted(Comparator.comparing(SnapshotPolicyStats::getPolicyId)) // maintain a consistent order when serializing - .collect(Collectors.toList()); + .toList(); long totalTaken = metrics.stream().mapToLong(s -> s.snapshotsTaken.count()).sum(); long totalFailed = metrics.stream().mapToLong(s -> s.snapshotsFailed.count()).sum(); long totalDeleted = metrics.stream().mapToLong(s -> s.snapshotsDeleted.count()).sum(); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotRetentionConfiguration.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotRetentionConfiguration.java index d8b7ffc16691d..54c153c04ea31 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotRetentionConfiguration.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotRetentionConfiguration.java @@ -31,7 +31,6 @@ import java.util.Set; import java.util.function.LongSupplier; import java.util.function.Predicate; -import java.util.stream.Collectors; import static org.elasticsearch.core.Strings.format; @@ -132,9 +131,7 @@ public Integer getMaximumSnapshotCount() { */ public Predicate getSnapshotDeletionPredicate(final List allSnapshots) { final int totalSnapshotCount = allSnapshots.size(); - final List sortedSnapshots = allSnapshots.stream() - .sorted(Comparator.comparingLong(SnapshotInfo::startTime)) - .collect(Collectors.toList()); + final List sortedSnapshots = allSnapshots.stream().sorted(Comparator.comparingLong(SnapshotInfo::startTime)).toList(); int successCount = 0; long latestSuccessfulTimestamp = Long.MIN_VALUE; for (SnapshotInfo snapshot : allSnapshots) { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/SSLService.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/SSLService.java index 5e31067ea2445..0d5c90795fd30 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/SSLService.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/SSLService.java @@ -407,9 +407,10 @@ public SslConfiguration sslConfiguration(Settings settingsToUse) { } public Set getTransportProfileContextNames() { - return Collections.unmodifiableSet( - this.sslConfigurations.keySet().stream().filter(k -> k.startsWith("transport.profiles.")).collect(Collectors.toSet()) - ); + return this.sslConfigurations.keySet() + .stream() + .filter(k -> k.startsWith("transport.profiles.")) + .collect(Collectors.toUnmodifiableSet()); } /** diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java index 6e7383ea314f6..af0f0187dc003 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java @@ -208,8 +208,7 @@ public Collection createComponents( AllocationService allocationService, IndicesService indicesService ) { - List components = new ArrayList<>(); - components.addAll( + List components = new ArrayList<>( super.createComponents( client, clusterService, @@ -228,68 +227,61 @@ public Collection createComponents( ) ); - filterPlugins(Plugin.class).stream() - .forEach( - p -> components.addAll( - p.createComponents( - client, - clusterService, - threadPool, - resourceWatcherService, - scriptService, - xContentRegistry, - environment, - nodeEnvironment, - namedWriteableRegistry, - expressionResolver, - repositoriesServiceSupplier, - tracer, - allocationService, - indicesService - ) + filterPlugins(Plugin.class).forEach( + p -> components.addAll( + p.createComponents( + client, + clusterService, + threadPool, + resourceWatcherService, + scriptService, + xContentRegistry, + environment, + nodeEnvironment, + namedWriteableRegistry, + expressionResolver, + repositoriesServiceSupplier, + tracer, + allocationService, + indicesService ) - ); + ) + ); return components; } @Override public Collection getRestHeaders() { - List headers = new ArrayList<>(); - headers.addAll(super.getRestHeaders()); - filterPlugins(ActionPlugin.class).stream().forEach(p -> headers.addAll(p.getRestHeaders())); + List headers = new ArrayList<>(super.getRestHeaders()); + filterPlugins(ActionPlugin.class).forEach(p -> headers.addAll(p.getRestHeaders())); return headers; } @Override public List> getSettings() { - ArrayList> settings = new ArrayList<>(); - settings.addAll(super.getSettings()); - - filterPlugins(Plugin.class).stream().forEach(p -> settings.addAll(p.getSettings())); + ArrayList> settings = new ArrayList<>(super.getSettings()); + filterPlugins(Plugin.class).forEach(p -> settings.addAll(p.getSettings())); return settings; } @Override public List getSettingsFilter() { - List filters = new ArrayList<>(); - filters.addAll(super.getSettingsFilter()); - filterPlugins(Plugin.class).stream().forEach(p -> filters.addAll(p.getSettingsFilter())); + List filters = new ArrayList<>(super.getSettingsFilter()); + filterPlugins(Plugin.class).forEach(p -> filters.addAll(p.getSettingsFilter())); return filters; } @Override public List> getActions() { - List> actions = new ArrayList<>(); - actions.addAll(super.getActions()); - filterPlugins(ActionPlugin.class).stream().forEach(p -> actions.addAll(p.getActions())); + List> actions = new ArrayList<>(super.getActions()); + filterPlugins(ActionPlugin.class).forEach(p -> actions.addAll(p.getActions())); return actions; } @Override public List getActionFilters() { - List filters = new ArrayList<>(); - filters.addAll(super.getActionFilters()); - filterPlugins(ActionPlugin.class).stream().forEach(p -> filters.addAll(p.getActionFilters())); + List filters = new ArrayList<>(super.getActionFilters()); + filterPlugins(ActionPlugin.class).forEach(p -> filters.addAll(p.getActionFilters())); return filters; } @@ -303,8 +295,7 @@ public List getRestHandlers( IndexNameExpressionResolver indexNameExpressionResolver, Supplier nodesInCluster ) { - List handlers = new ArrayList<>(); - handlers.addAll( + List handlers = new ArrayList<>( super.getRestHandlers( settings, restController, @@ -315,27 +306,25 @@ public List getRestHandlers( nodesInCluster ) ); - filterPlugins(ActionPlugin.class).stream() - .forEach( - p -> handlers.addAll( - p.getRestHandlers( - settings, - restController, - clusterSettings, - indexScopedSettings, - settingsFilter, - indexNameExpressionResolver, - nodesInCluster - ) + filterPlugins(ActionPlugin.class).forEach( + p -> handlers.addAll( + p.getRestHandlers( + settings, + restController, + clusterSettings, + indexScopedSettings, + settingsFilter, + indexNameExpressionResolver, + nodesInCluster ) - ); + ) + ); return handlers; } @Override public List getNamedWriteables() { - List entries = new ArrayList<>(); - entries.addAll(super.getNamedWriteables()); + List entries = new ArrayList<>(super.getNamedWriteables()); for (Plugin p : plugins) { entries.addAll(p.getNamedWriteables()); } @@ -344,8 +333,7 @@ public List getNamedWriteables() { @Override public List getNamedXContent() { - List entries = new ArrayList<>(); - entries.addAll(super.getNamedXContent()); + List entries = new ArrayList<>(super.getNamedXContent()); for (Plugin p : plugins) { entries.addAll(p.getNamedXContent()); } @@ -358,29 +346,30 @@ public List getNamedXContent() { public Settings additionalSettings() { Settings.Builder builder = Settings.builder(); builder.put(super.additionalSettings()); - filterPlugins(Plugin.class).stream().forEach(p -> builder.put(p.additionalSettings())); + filterPlugins(Plugin.class).forEach(p -> builder.put(p.additionalSettings())); return builder.build(); } @Override public List> getContexts() { List> contexts = new ArrayList<>(); - filterPlugins(ScriptPlugin.class).stream().forEach(p -> contexts.addAll(p.getContexts())); + filterPlugins(ScriptPlugin.class).forEach(p -> contexts.addAll(p.getContexts())); return contexts; } @Override public Map getProcessors(Processor.Parameters parameters) { Map processors = new HashMap<>(); - filterPlugins(IngestPlugin.class).stream().forEach(p -> processors.putAll(p.getProcessors(parameters))); + filterPlugins(IngestPlugin.class).forEach(p -> processors.putAll(p.getProcessors(parameters))); return processors; } @Override public List getTransportInterceptors(NamedWriteableRegistry namedWriteableRegistry, ThreadContext threadContext) { List interceptors = new ArrayList<>(); - filterPlugins(NetworkPlugin.class).stream() - .forEach(p -> interceptors.addAll(p.getTransportInterceptors(namedWriteableRegistry, threadContext))); + filterPlugins(NetworkPlugin.class).forEach( + p -> interceptors.addAll(p.getTransportInterceptors(namedWriteableRegistry, threadContext)) + ); return interceptors; } @@ -393,16 +382,14 @@ public Map> getTransports( NamedWriteableRegistry namedWriteableRegistry, NetworkService networkService ) { - Map> transports = new HashMap<>(); - transports.putAll( + Map> transports = new HashMap<>( super.getTransports(settings, threadPool, pageCacheRecycler, circuitBreakerService, namedWriteableRegistry, networkService) ); - filterPlugins(NetworkPlugin.class).stream() - .forEach( - p -> transports.putAll( - p.getTransports(settings, threadPool, pageCacheRecycler, circuitBreakerService, namedWriteableRegistry, networkService) - ) - ); + filterPlugins(NetworkPlugin.class).forEach( + p -> transports.putAll( + p.getTransports(settings, threadPool, pageCacheRecycler, circuitBreakerService, namedWriteableRegistry, networkService) + ) + ); return transports; } @@ -422,31 +409,30 @@ public Map> getHttpTransports( Tracer tracer ) { Map> transports = new HashMap<>(); - filterPlugins(NetworkPlugin.class).stream() - .forEach( - p -> transports.putAll( - p.getHttpTransports( - settings, - threadPool, - bigArrays, - pageCacheRecycler, - circuitBreakerService, - xContentRegistry, - networkService, - dispatcher, - perRequestThreadContext, - clusterSettings, - tracer - ) + filterPlugins(NetworkPlugin.class).forEach( + p -> transports.putAll( + p.getHttpTransports( + settings, + threadPool, + bigArrays, + pageCacheRecycler, + circuitBreakerService, + xContentRegistry, + networkService, + dispatcher, + perRequestThreadContext, + clusterSettings, + tracer ) - ); + ) + ); return transports; } @Override public List getBootstrapChecks() { List checks = new ArrayList<>(); - filterPlugins(Plugin.class).stream().forEach(p -> checks.addAll(p.getBootstrapChecks())); + filterPlugins(Plugin.class).forEach(p -> checks.addAll(p.getBootstrapChecks())); return Collections.unmodifiableList(checks); } @@ -459,7 +445,7 @@ public UnaryOperator getRestHandlerInterceptor(ThreadContext thread .map(RestInterceptorActionPlugin.class::cast) .map(p -> p.getRestHandlerInterceptor(threadContext)) .filter(Objects::nonNull) - .collect(Collectors.toList()); + .toList(); if (items.size() > 1) { throw new UnsupportedOperationException("Only the security ActionPlugin should override this"); @@ -473,7 +459,7 @@ public UnaryOperator getRestHandlerInterceptor(ThreadContext thread @Override public List> getExecutorBuilders(final Settings settings) { List> builders = new ArrayList<>(); - filterPlugins(Plugin.class).stream().forEach(p -> builders.addAll(p.getExecutorBuilders(settings))); + filterPlugins(Plugin.class).forEach(p -> builders.addAll(p.getExecutorBuilders(settings))); return builders; } @@ -490,14 +476,14 @@ public UnaryOperator> getIndexTemplateMetadat @Override public Map getElectionStrategies() { Map electionStrategies = new HashMap<>(); - filterPlugins(ClusterCoordinationPlugin.class).stream().forEach(p -> electionStrategies.putAll(p.getElectionStrategies())); + filterPlugins(ClusterCoordinationPlugin.class).forEach(p -> electionStrategies.putAll(p.getElectionStrategies())); return electionStrategies; } @Override public Collection getAdditionalIndexSettingProviders(IndexSettingProvider.Parameters parameters) { Set providers = new HashSet<>(); - filterPlugins(Plugin.class).stream().forEach(p -> providers.addAll(p.getAdditionalIndexSettingProviders(parameters))); + filterPlugins(Plugin.class).forEach(p -> providers.addAll(p.getAdditionalIndexSettingProviders(parameters))); providers.addAll(super.getAdditionalIndexSettingProviders(parameters)); return providers; @@ -506,21 +492,19 @@ public Collection getAdditionalIndexSettingProviders(Index @Override public Map> getTokenizers() { Map> tokenizers = new HashMap<>(); - filterPlugins(AnalysisPlugin.class).stream().forEach(p -> tokenizers.putAll(p.getTokenizers())); + filterPlugins(AnalysisPlugin.class).forEach(p -> tokenizers.putAll(p.getTokenizers())); return tokenizers; } @Override public void onIndexModule(IndexModule indexModule) { super.onIndexModule(indexModule); - filterPlugins(Plugin.class).stream().forEach(p -> p.onIndexModule(indexModule)); + filterPlugins(Plugin.class).forEach(p -> p.onIndexModule(indexModule)); } @Override public Function> getFieldFilter() { - List>> items = filterPlugins(MapperPlugin.class).stream() - .map(p -> p.getFieldFilter()) - .collect(Collectors.toList()); + List>> items = filterPlugins(MapperPlugin.class).stream().map(p -> p.getFieldFilter()).toList(); if (items.size() > 1) { throw new UnsupportedOperationException("Only the security MapperPlugin should override this"); } else if (items.size() == 1) { @@ -536,7 +520,7 @@ public BiConsumer getJoinValidator() { // There can be only one. List> items = filterPlugins(ClusterCoordinationPlugin.class).stream() .map(p -> p.getJoinValidator()) - .collect(Collectors.toList()); + .toList(); if (items.size() > 1) { throw new UnsupportedOperationException("Only the security DiscoveryPlugin should override this"); } else if (items.size() == 1) { @@ -598,7 +582,7 @@ public BiConsumer addPreRestoreVersionCheck() { List> checks = filterPlugins(RepositoryPlugin.class).stream() .map(RepositoryPlugin::addPreRestoreVersionCheck) .filter(Objects::nonNull) - .collect(Collectors.toList()); + .toList(); return checks.isEmpty() ? null : (s, v) -> checks.forEach(c -> c.accept(s, v)); } @@ -611,7 +595,7 @@ public void close() throws IOException { public Optional getEngineFactory(IndexSettings indexSettings) { List> enginePlugins = filterPlugins(EnginePlugin.class).stream() .map(p -> p.getEngineFactory(indexSettings)) - .collect(Collectors.toList()); + .toList(); if (enginePlugins.size() == 0) { return Optional.empty(); } else if (enginePlugins.size() == 1) { @@ -639,7 +623,7 @@ public Collection createAllocationDeciders(Settings settings, deciders.addAll( filterPlugins(ClusterPlugin.class).stream() .flatMap(p -> p.createAllocationDeciders(settings, clusterSettings).stream()) - .collect(Collectors.toList()) + .toList() ); deciders.addAll(super.createAllocationDeciders(settings, clusterSettings)); return deciders; @@ -648,21 +632,21 @@ public Collection createAllocationDeciders(Settings settings, @Override public Map getExistingShardsAllocators() { final Map allocators = new HashMap<>(); - filterPlugins(ClusterPlugin.class).stream().forEach(p -> allocators.putAll(p.getExistingShardsAllocators())); + filterPlugins(ClusterPlugin.class).forEach(p -> allocators.putAll(p.getExistingShardsAllocators())); return allocators; } @Override public Map getDirectoryFactories() { final Map factories = new HashMap<>(); - filterPlugins(IndexStorePlugin.class).stream().forEach(p -> factories.putAll(p.getDirectoryFactories())); + filterPlugins(IndexStorePlugin.class).forEach(p -> factories.putAll(p.getDirectoryFactories())); return factories; } @Override public Map getRecoveryStateFactories() { final Map factories = new HashMap<>(); - filterPlugins(IndexStorePlugin.class).stream().forEach(p -> factories.putAll(p.getRecoveryStateFactories())); + filterPlugins(IndexStorePlugin.class).forEach(p -> factories.putAll(p.getRecoveryStateFactories())); return factories; } diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationInfoAction.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationInfoAction.java index 1a6f11fa32514..8ec978d175f22 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationInfoAction.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationInfoAction.java @@ -134,10 +134,10 @@ private static Map> getMergedIssuesToNodesMap( Map> issueToListOfNodesMap = new HashMap<>(); for (List> similarIssues : issuesToMerge) { DeprecationIssue leastCommonDenominator = DeprecationIssue.getIntersectionOfRemovableSettings( - similarIssues.stream().map(Tuple::v1).collect(Collectors.toList()) + similarIssues.stream().map(Tuple::v1).toList() ); issueToListOfNodesMap.computeIfAbsent(leastCommonDenominator, (key) -> new ArrayList<>()) - .addAll(similarIssues.stream().map(Tuple::v2).collect(Collectors.toList())); + .addAll(similarIssues.stream().map(Tuple::v2).toList()); } return issueToListOfNodesMap; } diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecks.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecks.java index 05463ef5e6048..316f29dde7999 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecks.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecks.java @@ -23,7 +23,6 @@ import java.util.function.BiConsumer; import java.util.function.BiFunction; import java.util.function.Function; -import java.util.stream.Collectors; /** * Index-specific deprecation checks @@ -212,7 +211,7 @@ static DeprecationIssue deprecatedCamelCasePattern(IndexMetadata indexMetadata) ); if (fields.size() > 0) { - String detailsMessageBeginning = fields.stream().collect(Collectors.joining(" ")); + String detailsMessageBeginning = String.join(" ", fields); return new DeprecationIssue( DeprecationIssue.Level.CRITICAL, "Date fields use deprecated camel case formats", diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecks.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecks.java index 8f8ed8bf00c3a..b265dd5e44710 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecks.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecks.java @@ -246,21 +246,19 @@ private static DeprecationIssue deprecatedAffixSetting( Settings clusterSettings, Settings nodeSettings ) { - List> deprecatedConcreteNodeSettings = deprecatedAffixSetting.getAllConcreteSettings(nodeSettings) + var deprecatedConcreteNodeSettings = deprecatedAffixSetting.getAllConcreteSettings(nodeSettings) .sorted(Comparator.comparing(Setting::getKey)) - .collect(Collectors.toList()); - List> deprecatedConcreteClusterSettings = deprecatedAffixSetting.getAllConcreteSettings(clusterSettings) + .toList(); + var deprecatedConcreteClusterSettings = deprecatedAffixSetting.getAllConcreteSettings(clusterSettings) .sorted(Comparator.comparing(Setting::getKey)) - .collect(Collectors.toList()); + .toList(); if (deprecatedConcreteNodeSettings.isEmpty() && deprecatedConcreteClusterSettings.isEmpty()) { return null; } - List deprecatedNodeSettingKeys = deprecatedConcreteNodeSettings.stream().map(Setting::getKey).collect(Collectors.toList()); - List deprecatedClusterSettingKeys = deprecatedConcreteClusterSettings.stream() - .map(Setting::getKey) - .collect(Collectors.toList()); + List deprecatedNodeSettingKeys = deprecatedConcreteNodeSettings.stream().map(Setting::getKey).toList(); + List deprecatedClusterSettingKeys = deprecatedConcreteClusterSettings.stream().map(Setting::getKey).toList(); final String concatSettingNames = Stream.concat(deprecatedNodeSettingKeys.stream(), deprecatedClusterSettingKeys.stream()) .distinct() @@ -289,10 +287,10 @@ private static DeprecationIssue deprecatedAffixGroupedSetting( ) { List> deprecatedConcreteNodeSettings = deprecatedAffixSetting.getAllConcreteSettings(nodeSettings) .sorted(Comparator.comparing(Setting::getKey)) - .collect(Collectors.toList()); + .toList(); List> deprecatedConcreteClusterSettings = deprecatedAffixSetting.getAllConcreteSettings(clusterSettings) .sorted(Comparator.comparing(Setting::getKey)) - .collect(Collectors.toList()); + .toList(); if (deprecatedConcreteNodeSettings.isEmpty() && deprecatedConcreteClusterSettings.isEmpty()) { return null; @@ -305,19 +303,19 @@ private static DeprecationIssue deprecatedAffixGroupedSetting( .map(key -> key + "*") .collect(Collectors.joining(",")); // The actual group setting that are present in the settings objects, with full setting name prepended. - List allNodeSubSettingKeys = deprecatedConcreteNodeSettings.stream().map(affixSetting -> { + List allNodeSubSettingKeys = deprecatedConcreteNodeSettings.stream().flatMap(affixSetting -> { String groupPrefix = affixSetting.getKey(); Settings groupSettings = affixSetting.get(nodeSettings); Set subSettings = groupSettings.keySet(); - return subSettings.stream().map(key -> groupPrefix + key).collect(Collectors.toList()); - }).flatMap(List::stream).sorted().collect(Collectors.toList()); + return subSettings.stream().map(key -> groupPrefix + key); + }).sorted().toList(); - List allClusterSubSettingKeys = deprecatedConcreteClusterSettings.stream().map(affixSetting -> { + List allClusterSubSettingKeys = deprecatedConcreteClusterSettings.stream().flatMap(affixSetting -> { String groupPrefix = affixSetting.getKey(); Settings groupSettings = affixSetting.get(clusterSettings); Set subSettings = groupSettings.keySet(); - return subSettings.stream().map(key -> groupPrefix + key).collect(Collectors.toList()); - }).flatMap(List::stream).sorted().collect(Collectors.toList()); + return subSettings.stream().map(key -> groupPrefix + key); + }).sorted().toList(); final String allSubSettings = Stream.concat(allNodeSubSettingKeys.stream(), allClusterSubSettingKeys.stream()) .distinct() diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java index 0ffcd0c01d18d..1851816e8d143 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java @@ -151,9 +151,7 @@ static void pluginSettingIssues( DeprecationChecker.Components components, ActionListener>> listener ) { - List enabledCheckers = checkers.stream() - .filter(c -> c.enabled(components.settings())) - .collect(Collectors.toList()); + List enabledCheckers = checkers.stream().filter(c -> c.enabled(components.settings())).toList(); if (enabledCheckers.isEmpty()) { listener.onResponse(Collections.emptyMap()); return; diff --git a/x-pack/plugin/eql/qa/mixed-node/src/javaRestTest/java/org/elasticsearch/xpack/eql/qa/mixed_node/EqlSearchIT.java b/x-pack/plugin/eql/qa/mixed-node/src/javaRestTest/java/org/elasticsearch/xpack/eql/qa/mixed_node/EqlSearchIT.java index a0360f9f90332..5446e9c27a81a 100644 --- a/x-pack/plugin/eql/qa/mixed-node/src/javaRestTest/java/org/elasticsearch/xpack/eql/qa/mixed_node/EqlSearchIT.java +++ b/x-pack/plugin/eql/qa/mixed-node/src/javaRestTest/java/org/elasticsearch/xpack/eql/qa/mixed_node/EqlSearchIT.java @@ -441,7 +441,7 @@ private void assertResponse(String query, List expected, Map actualList = new ArrayList<>(); - events.stream().forEach(m -> actualList.add(m.get("_id"))); + events.forEach(m -> actualList.add(m.get("_id"))); if (false == expected.equals(actualList)) { NotEqualMessageBuilder message = new NotEqualMessageBuilder(); diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/AnalysisUtils.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/AnalysisUtils.java index 8ab2365d3b2db..4f9abb1328b69 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/AnalysisUtils.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/AnalysisUtils.java @@ -21,8 +21,6 @@ import java.util.Objects; import java.util.Set; -import static java.util.stream.Collectors.toList; - public final class AnalysisUtils { private AnalysisUtils() {} @@ -67,7 +65,7 @@ static Attribute resolveAgainstList(UnresolvedAttribute u, Collection "Reference [" + u.qualifiedName() + "] is ambiguous (to disambiguate use quotes or qualifiers); matches any of " - + matches.stream().map(a -> "\"" + a.qualifier() + "\".\"" + a.name() + "\"").sorted().collect(toList()) + + matches.stream().map(a -> "\"" + a.qualifier() + "\".\"" + a.name() + "\"").sorted().toList() ); } diff --git a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java index 5adea642ed16f..95b3c86259aa2 100644 --- a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java +++ b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java @@ -25,7 +25,6 @@ import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.IndexScopedSettings; @@ -66,7 +65,6 @@ import java.util.List; import java.util.Map; import java.util.function.Supplier; -import java.util.stream.Collectors; import static org.elasticsearch.xpack.core.ClientHelper.FLEET_ORIGIN; @@ -306,10 +304,7 @@ public void cleanUpFeature(ClusterService clusterService, Client client, ActionL if (dataStreamDescriptors.isEmpty() == false) { try { Request request = new Request( - dataStreamDescriptors.stream() - .map(SystemDataStreamDescriptor::getDataStreamName) - .collect(Collectors.toList()) - .toArray(Strings.EMPTY_ARRAY) + dataStreamDescriptors.stream().map(SystemDataStreamDescriptor::getDataStreamName).toArray(String[]::new) ); EnumSet