diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties index 50c1e3c32c8f7..01f00cf42a5ef 100644 --- a/build-tools-internal/version.properties +++ b/build-tools-internal/version.properties @@ -2,7 +2,7 @@ elasticsearch = 8.1.0 lucene = 9.0.0-snapshot-cc2a31f2be8 bundled_jdk_vendor = adoptium -bundled_jdk = 17.0.1+12 +bundled_jdk = 17+35 checkstyle = 8.42 diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml index a3305fa987d4b..709f633e74820 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml @@ -276,7 +276,7 @@ set bad start_time and end_time: time_series_dimension: true - do: - catch: /time series index @timestamp value \[1632625781000\] must be larger than 1632625782000/ + catch: /time series index @timestamp value \[2021-09-26T03:09:41Z\] must be larger than 2021-09-26T03:09:42Z/ index: index: test_index body: { @@ -285,10 +285,169 @@ set bad start_time and end_time: } - do: - catch: /time series index @timestamp value \[1632625793000\] must be smaller than 1632625792000/ + catch: /time series index @timestamp value \[2021-09-26T03:09:53Z\] must be smaller than 2021-09-26T03:09:52Z/ index: index: test_index body: { "@timestamp": 1632625793000, "metricset": "pod" } + +--- +check start_time and end_time with data_nano: + - skip: + version: " - 8.0.99" + reason: introduced in 8.1.0 + - do: + indices.create: + index: test_index + body: + settings: + index: + mode: time_series + routing_path: [metricset] + time_series: + start_time: 2021-09-26T03:09:42Z + end_time: 2021-09-26T03:09:52Z + mappings: + properties: + "@timestamp": + type: date_nanos + metricset: + type: keyword + time_series_dimension: true + + - do: + index: + refresh: true + index: test_index + body: { + "@timestamp": "2021-09-26T03:09:51.123456789Z", + "metricset": "pod" + } + + - do: + search: + index: test_index + body: + docvalue_fields: [ '@timestamp' ] + - match: { hits.total.value: 1 } + - match: { "hits.hits.0.fields.@timestamp": [ "2021-09-26T03:09:51.123456789Z" ] } + + - do: + catch: /time series index @timestamp value \[2010-09-26T03:09:52.123456789Z\] must be larger than 2021-09-26T03:09:42Z/ + index: + index: test_index + body: { + "@timestamp": "2010-09-26T03:09:52.123456789Z", + "metricset": "pod" + } + + - do: + catch: /time series index @timestamp value \[2031-09-26T03:09:52.123456789Z\] must be smaller than 2021-09-26T03:09:52Z/ + index: + index: test_index + body: { + "@timestamp": "2031-09-26T03:09:52.123456789Z", + "metricset": "pod" + } + +--- +check start_time boundary with data_nano: + - skip: + version: " - 8.0.99" + reason: introduced in 8.1.0 + - do: + indices.create: + index: test_index + body: + settings: + index: + mode: time_series + routing_path: [metricset] + time_series: + start_time: 2021-09-26T03:09:42Z + end_time: 2021-09-26T03:09:52Z + mappings: + properties: + "@timestamp": + type: date_nanos + metricset: + type: keyword + time_series_dimension: true + + - do: + index: + refresh: true + index: test_index + body: { + "@timestamp": "2021-09-26T03:09:42.123456789Z", + "metricset": "pod" + } + + - do: + search: + index: test_index + body: + docvalue_fields: [ '@timestamp' ] + - match: { hits.total.value: 1 } + - match: { "hits.hits.0.fields.@timestamp": [ "2021-09-26T03:09:42.123456789Z" ] } + + - do: + catch: /time series index @timestamp value \[2021-09-26T03:09:41.123456789Z\] must be larger than 2021-09-26T03:09:42Z/ + index: + index: test_index + body: { + "@timestamp": "2021-09-26T03:09:41.123456789Z", + "metricset": "pod" + } + +--- +check end_time boundary with data_nano: + - skip: + version: " - 8.0.99" + reason: introduced in 8.1.0 + - do: + indices.create: + index: test_index + body: + settings: + index: + mode: time_series + routing_path: [metricset] + time_series: + start_time: 2021-09-26T03:09:42Z + end_time: 2021-09-26T03:09:52Z + mappings: + properties: + "@timestamp": + type: date_nanos + metricset: + type: keyword + time_series_dimension: true + + - do: + index: + refresh: true + index: test_index + body: { + "@timestamp": "2021-09-26T03:09:51.123456789Z", + "metricset": "pod" + } + + - do: + search: + index: test_index + body: + docvalue_fields: [ '@timestamp' ] + - match: { hits.total.value: 1 } + - match: { "hits.hits.0.fields.@timestamp": [ "2021-09-26T03:09:51.123456789Z" ] } + + - do: + catch: /time series index @timestamp value \[2021-09-26T03:09:52.123456789Z\] must be smaller than 2021-09-26T03:09:52Z/ + index: + index: test_index + body: { + "@timestamp": "2021-09-26T03:09:52.123456789Z", + "metricset": "pod" + } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java b/server/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java index d61a21d0930aa..f71ff55ec68a4 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java @@ -18,7 +18,6 @@ import org.elasticsearch.cluster.routing.UnassignedInfo.AllocationStatus; import org.elasticsearch.cluster.routing.allocation.ExistingShardsAllocator; import org.elasticsearch.cluster.service.MasterService; -import org.elasticsearch.common.Randomness; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.Index; @@ -1068,11 +1067,6 @@ public boolean isIgnoredEmpty() { return ignored.isEmpty(); } - public void shuffle() { - nodes.ensureMutable(); - Randomness.shuffle(unassigned); - } - /** * Drains all unassigned shards and returns it. * This method will not drain ignored shards. diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java index 882cc976f1924..82c2841f40e70 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java @@ -117,8 +117,6 @@ public ClusterState applyStartedShards(ClusterState clusterState, List TIME_SERIES_END_TIME = Setting.dateSetting( "index.time_series.end_time", - DateUtils.MAX_NANOSECOND_INSTANT, + Instant.ofEpochMilli(DateUtils.MAX_MILLIS_BEFORE_9999), new Setting.Validator<>() { @Override public void validate(Instant value) {} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java index 5b7b0e5fe6273..9bbe0f2f41889 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java @@ -14,12 +14,14 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.mapper.DateFieldMapper.Resolution; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentType; import java.io.IOException; import java.io.UncheckedIOException; +import java.time.Instant; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -215,19 +217,35 @@ private void validateTimestamp(IndexableField field, DocumentParserContext conte return; } - long value = field.numericValue().longValue(); + long originValue = field.numericValue().longValue(); + long value = originValue; + + Resolution resolution; if (context.mappingLookup().getMapper(DEFAULT_PATH).typeName().equals(DateFieldMapper.DATE_NANOS_CONTENT_TYPE)) { + resolution = Resolution.NANOSECONDS; value /= NSEC_PER_MSEC; + } else { + resolution = Resolution.MILLISECONDS; } long startTime = context.indexSettings().getTimeSeriesStartTime(); if (value < startTime) { - throw new IllegalArgumentException("time series index @timestamp value [" + value + "] must be larger than " + startTime); + throw new IllegalArgumentException( + "time series index @timestamp value [" + + resolution.toInstant(originValue) + + "] must be larger than " + + Instant.ofEpochMilli(startTime) + ); } long endTime = context.indexSettings().getTimeSeriesEndTime(); if (value >= endTime) { - throw new IllegalArgumentException("time series index @timestamp value [" + value + "] must be smaller than " + endTime); + throw new IllegalArgumentException( + "time series index @timestamp value [" + + resolution.toInstant(originValue) + + "] must be smaller than " + + Instant.ofEpochMilli(endTime) + ); } } diff --git a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java index a0bcf9f866f85..4eb9d058aa2dc 100644 --- a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java +++ b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.ActionResponse; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.IndexScopedSettings; @@ -41,7 +42,9 @@ import java.io.UncheckedIOException; import java.util.Collection; import java.util.List; +import java.util.Map; import java.util.function.Supplier; +import java.util.function.UnaryOperator; import static org.elasticsearch.index.engine.EngineConfig.INDEX_CODEC_SETTING; import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME; @@ -191,4 +194,13 @@ public String getFeatureName() { public String getFeatureDescription() { return "Enables Logstash Central Management pipeline storage"; } + + @Override + public UnaryOperator> getIndexTemplateMetadataUpgrader() { + return templates -> { + // .logstash is a system index now. deleting the legacy template + templates.remove("logstash-index-template"); + return templates; + }; + } } diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/Monitoring.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/Monitoring.java index 500fdc47bb6c3..c487e8bca5701 100644 --- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/Monitoring.java +++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/Monitoring.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.ActionResponse; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; @@ -72,6 +73,7 @@ import java.util.Map; import java.util.Set; import java.util.function.Supplier; +import java.util.function.UnaryOperator; import static org.elasticsearch.common.settings.Setting.boolSetting; @@ -236,4 +238,17 @@ public void reload(Settings settings) throws Exception { exporters.setExportersSetting(settingsForChangedExporter); } } + + @Override + public UnaryOperator> getIndexTemplateMetadataUpgrader() { + return map -> { + // this template was not migrated to typeless due to the possibility of the old /_monitoring/bulk API being used + // see {@link org.elasticsearch.xpack.core.monitoring.exporter.MonitoringTemplateUtils#OLD_TEMPLATE_VERSION} + // however the bulk API is not typed (the type field is for the docs, a field inside the docs) so it's safe to remove this + // old template and rely on the updated, typeless, .monitoring-alerts-7 template + map.remove(".monitoring-alerts"); + return map; + }; + + } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java index f125fd186592e..f4858acc9cf46 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java @@ -1531,6 +1531,8 @@ public UnaryOperator> getIndexTemplateMetadat return templates -> { // .security index is not managed by using templates anymore templates.remove("security_audit_log"); + // .security is a system index now. deleting another legacy template that's not used anymore + templates.remove("security-index-template"); return templates; }; } diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java index 0b770f857cc8f..dccde9f18a796 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java @@ -716,6 +716,11 @@ public void onIndexModule(IndexModule module) { public UnaryOperator> getIndexTemplateMetadataUpgrader() { return map -> { map.keySet().removeIf(name -> name.startsWith("watch_history_")); + // watcher migrated to using system indices so these legacy templates are not needed anymore + map.remove(".watches"); + map.remove(".triggered_watches"); + // post 7.x we moved to typeless watch-history-10 + map.remove(".watch-history-9"); return map; }; } diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/WatcherRestartIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/WatcherRestartIT.java index a255f96a10221..54921de6b9320 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/WatcherRestartIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/WatcherRestartIT.java @@ -9,10 +9,14 @@ import org.apache.http.util.EntityUtils; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; import java.nio.charset.StandardCharsets; +import java.util.Map; +import java.util.concurrent.TimeUnit; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; public class WatcherRestartIT extends AbstractUpgradeTestCase { @@ -25,6 +29,34 @@ public void testWatcherRestart() throws Exception { ensureWatcherStarted(); } + public void testEnsureWatcherDeletesLegacyTemplates() throws Exception { + client().performRequest(new Request("POST", "/_watcher/_start")); + ensureWatcherStarted(); + + if (CLUSTER_TYPE.equals(ClusterType.UPGRADED)) { + // legacy index template created in previous releases should not be present anymore + assertBusy(() -> { + Request request = new Request("GET", "/_template/*watch*"); + try { + Response response = client().performRequest(request); + Map responseLevel = entityAsMap(response); + assertNotNull(responseLevel); + + assertThat(responseLevel.containsKey(".watches"), is(false)); + assertThat(responseLevel.containsKey(".triggered_watches"), is(false)); + assertThat(responseLevel.containsKey(".watch-history-9"), is(false)); + } catch (ResponseException e) { + // Not found is fine + assertThat( + "Unexpected failure getting templates: " + e.getResponse().getStatusLine(), + e.getResponse().getStatusLine().getStatusCode(), + is(404) + ); + } + }, 30, TimeUnit.SECONDS); + } + } + private void ensureWatcherStopped() throws Exception { assertBusy(() -> { Response stats = client().performRequest(new Request("GET", "_watcher/stats"));