Merge branch 'upstream/master' into reuse-date-stream-timestamp

* upstream/master: (109 commits)
  Migrate custom role providers to licensed feature (elastic#79127)
  Remove stale AwaitsFix in InternalEngineTests (elastic#79323)
  Fix errors in RefreshListenersTests (elastic#79324)
  Reeable BwC Tests after elastic#79318 (elastic#79320)
  Mute BwC Tests for elastic#79318 (elastic#79319)
  Reenable BwC Tests after elastic#79308 (elastic#79313)
  Disable BwC Tests for elastic#79308 (elastic#79310)
  Adjust BWC for node-level field cap requests (elastic#79301)
  Allow total memory to be overridden (elastic#78750)
  Fix SnapshotBasedIndexRecoveryIT#testRecoveryIsCancelledAfterDeletingTheIndex (elastic#79269)
  Disable BWC tests
  Mute GeoIpDownloaderCliIT.testStartWithNoDatabases (elastic#79299)
  Add alias support to fleet search API (elastic#79285)
  Create a coordinating node level reader for tsdb (elastic#79197)
  Route documents to the correct shards in tsdb (elastic#77731)
  Inject migrate action regardless of allocate action (elastic#79090)
  Migrate to data tiers should always ensure a TIER_PREFERENCE is set (elastic#79100)
  Skip building of BWC distributions when building release artifacts (elastic#79180)
  Default ENFORCE_DEFAULT_TIER_PREFERENCE to true (elastic#79275)
  Deprecation of transient cluster settings (elastic#78794)
  ...

# Conflicts:
#	server/src/main/java/org/elasticsearch/index/IndexMode.java
#	server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java

.ci/bwcVersions

@@ -43,5 +43,6 @@ BWC_VERSION:
   - "7.14.2"
   - "7.15.0"
   - "7.15.1"
+  - "7.15.2"
   - "7.16.0"
   - "8.0.0"

CONTRIBUTING.md

@@ -193,14 +193,14 @@ need them.
    2. Click "Use the Eclipse Code Formatter"
    3. Under "Eclipse formatter config", select "Eclipse workspace/project
       folder or config file"
-   4. Click "Browse", and navigate to the file `build-tools-internal/formatterConfig.xml`
+   4. Click "Browse", and navigate to the file `build-conventions/formatterConfig.xml`
    5. **IMPORTANT** - make sure "Optimize Imports" is **NOT** selected.
    6. Click "OK"
 
 Note that only some sub-projects in the Elasticsearch project are currently
 fully-formatted. You can see a list of project that **are not**
 automatically formatted in
-[build-tools-internal/src/main/groovy/elasticsearch.formatting.gradle](build-tools-internal/src/main/groovy/elasticsearch.formatting.gradle).
+[FormattingPrecommitPlugin.java](build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/FormattingPrecommitPlugin.java).
 
 ### Importing the project into Eclipse
 
@@ -234,15 +234,15 @@ Next you'll want to import our auto-formatter:
  - Select **Window > Preferences**
  - Select **Java > Code Style > Formatter**
  - Click **Import**
- - Import the file at **build-tools-internal/formatterConfig.xml**
+ - Import the file at **build-conventions/formatterConfig.xml**
  - Make sure it is the **Active profile**
 
 Finally, set up import order:
 
  - Select **Window > Preferences**
  - Select **Java > Code Style > Organize Imports**
  - Click **Import...**
- - Import the file at **build-tools-internal/elastic.importorder**
+ - Import the file at **build-conventions/elastic.importorder**
  - Set the **Number of imports needed for `.*`** to ***9999***
  - Set the **Number of static imports needed for `.*`** to ***9999*** as well
  - Apply that
@@ -279,11 +279,12 @@ form.
 Java files in the Elasticsearch codebase are automatically formatted using
 the [Spotless Gradle] plugin. All new projects are automatically formatted,
 while existing projects are gradually being opted-in. The formatting check
-can be run explicitly with:
+is run automatically via the `precommit` task, but it can be run explicitly with:
 
     ./gradlew spotlessJavaCheck
 
-The code can be formatted with:
+It is usually more useful, and just as fast, to just reformat the project. You
+can do this with:
 
     ./gradlew spotlessApply
 
@@ -304,10 +305,9 @@ Please follow these formatting guidelines:
 * Wildcard imports (`import foo.bar.baz.*`) are forbidden and will cause
   the build to fail.
 * If *absolutely* necessary, you can disable formatting for regions of code
-  with the `// tag::NAME` and `// end::NAME` directives, but note that
-  these are intended for use in documentation, so please make it clear what
-  you have done, and only do this where the benefit clearly outweighs the
-  decrease in consistency.
+  with the `// @formatter:off` and `// @formatter:on` directives, but
+  only do this where the benefit clearly outweighs the decrease in formatting
+  consistency.
 * Note that Javadoc and block comments i.e. `/* ... */` are not formatted,
   but line comments i.e `// ...` are.
 * Negative boolean expressions must use the form `foo == false` instead of

TESTING.asciidoc

@@ -283,6 +283,36 @@ memory or some of the containers will fail to start. You can tell that you
 are short of memory if containers are exiting quickly after starting with
 code 137 (128 + 9, where 9 means SIGKILL).
 
+== Debugging tests
+
+If you would like to debug your tests themselves, simply pass the `--debug-jvm`
+flag to the testing task and connect a debugger on the default port of `5005`.
+
+---------------------------------------------------------------------------
+./gradlew :server:test --debug-jvm
+---------------------------------------------------------------------------
+
+For REST tests, if you'd like to debug the Elasticsearch server itself, and
+not your test code, use the `--debug-server-jvm` flag and use the
+"Debug Elasticsearch" run configuration in IntelliJ to listen on the default
+port of `5007`.
+
+---------------------------------------------------------------------------
+./gradlew :rest-api-spec:yamlRestTest --debug-server-jvm
+---------------------------------------------------------------------------
+
+NOTE: In the case of test clusters using multiple nodes, multiple debuggers
+will need to be attached on incrementing ports. For example, for a 3 node
+cluster ports `5007`, `5008`, and `5009` will attempt to attach to a listening
+debugger.
+
+You can also use a combination of both flags to debug both tests and server.
+This is only applicable to Java REST tests.
+
+---------------------------------------------------------------------------
+./gradlew :modules:kibana:javaRestTest --debug-jvm --debug-server-jvm
+---------------------------------------------------------------------------
+
 == Testing the REST layer
 
 The REST layer is tested through specific tests that are executed against

benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/AllocationBenchmark.java

@@ -14,7 +14,7 @@
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.RoutingTable;
-import org.elasticsearch.cluster.routing.ShardRoutingState;
+import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.common.settings.Settings;
 import org.openjdk.jmh.annotations.Benchmark;
@@ -31,6 +31,8 @@
 
 import java.util.Collections;
 import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
 
 @Fork(3)
 @Warmup(iterations = 10)
@@ -154,7 +156,10 @@ public ClusterState measureAllocation() {
         while (clusterState.getRoutingNodes().hasUnassignedShards()) {
             clusterState = strategy.applyStartedShards(
                 clusterState,
-                clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.INITIALIZING)
+                StreamSupport.stream(clusterState.getRoutingNodes().spliterator(), false)
+                    .flatMap(shardRoutings -> StreamSupport.stream(shardRoutings.spliterator(), false))
+                    .filter(ShardRouting::initializing)
+                    .collect(Collectors.toList())
             );
             clusterState = strategy.reroute(clusterState, "reroute");
         }

benchmarks/src/main/java/org/elasticsearch/benchmark/search/aggregations/TermsReduceBenchmark.java

@@ -11,7 +11,6 @@
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.search.TotalHits;
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.action.OriginalIndices;
 import org.elasticsearch.action.search.QueryPhaseResultConsumer;
 import org.elasticsearch.action.search.SearchPhaseController;
 import org.elasticsearch.action.search.SearchProgressListener;
@@ -183,9 +182,7 @@ public SearchPhaseController.ReducedQueryPhase reduceAggs(TermsList candidateLis
                 new DocValueFormat[] { DocValueFormat.RAW }
             );
             result.aggregations(candidateList.get(i));
-            result.setSearchShardTarget(
-                new SearchShardTarget("node", new ShardId(new Index("index", "index"), i), null, OriginalIndices.NONE)
-            );
+            result.setSearchShardTarget(new SearchShardTarget("node", new ShardId(new Index("index", "index"), i), null));
             shards.add(result);
         }
         SearchRequest request = new SearchRequest();

build-conventions/build.gradle

@@ -49,6 +49,10 @@ gradlePlugin {
             id = 'elasticsearch.versions'
             implementationClass = 'org.elasticsearch.gradle.internal.conventions.VersionPropertiesPlugin'
         }
+        formatting {
+            id = 'elasticsearch.formatting'
+            implementationClass = 'org.elasticsearch.gradle.internal.conventions.precommit.FormattingPrecommitPlugin'
+        }
     }
 }
 
@@ -62,6 +66,9 @@ dependencies {
     api 'gradle.plugin.com.github.jengelman.gradle.plugins:shadow:7.0.0'
     api 'org.apache.rat:apache-rat:0.11'
     compileOnly "com.puppycrawl.tools:checkstyle:8.42"
+    api('com.diffplug.spotless:spotless-plugin-gradle:5.16.0') {
+      exclude module: "groovy-xml"
+    }
 }
 
 project.getPlugins().withType(JavaBasePlugin.class) {

build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/VersionPropertiesBuildService.java

@@ -15,13 +15,11 @@
 import org.gradle.api.provider.ProviderFactory;
 import org.gradle.api.services.BuildService;
 import org.gradle.api.services.BuildServiceParameters;
-import org.gradle.initialization.layout.BuildLayout;
-import org.gradle.initialization.layout.BuildLayoutFactory;
 
-import javax.inject.Inject;
 import java.io.File;
 import java.io.IOException;
 import java.util.Properties;
+import javax.inject.Inject;
 
 abstract class VersionPropertiesBuildService implements BuildService<VersionPropertiesBuildService.Params>, AutoCloseable {
 
@@ -33,15 +31,24 @@ public VersionPropertiesBuildService(ProviderFactory providerFactory) {
         try {
             File propertiesInputFile = new File(infoPath, "version.properties");
             properties = VersionPropertiesLoader.loadBuildSrcVersion(propertiesInputFile, providerFactory);
-            properties.computeIfAbsent("minimumJava", s -> resolveMinimumJavaVersion(infoPath));
+            properties.computeIfAbsent("minimumRuntimeJava", s -> resolveMinimumRuntimeJavaVersion(infoPath));
+            properties.computeIfAbsent("minimumCompilerJava", s -> resolveMinimumCompilerJavaVersion(infoPath));
         } catch (IOException e) {
             throw new GradleException("Cannot load VersionPropertiesBuildService", e);
         }
     }
 
-    private JavaVersion resolveMinimumJavaVersion(File infoPath) {
+    private JavaVersion resolveMinimumRuntimeJavaVersion(File infoPath) {
+        return resolveJavaVersion(infoPath, "src/main/resources/minimumRuntimeVersion");
+    }
+
+    private JavaVersion resolveMinimumCompilerJavaVersion(File infoPath) {
+        return resolveJavaVersion(infoPath, "src/main/resources/minimumCompilerVersion");
+    }
+
+    private JavaVersion resolveJavaVersion(File infoPath, String path) {
         final JavaVersion minimumJavaVersion;
-        File minimumJavaInfoSource = new File(infoPath, "src/main/resources/minimumRuntimeVersion");
+        File minimumJavaInfoSource = new File(infoPath, path);
         try {
             String versionString = FileUtils.readFileToString(minimumJavaInfoSource);
             minimumJavaVersion = JavaVersion.toVersion(versionString);