Merge remote-tracking branch 'origin/master' into mapper/builder-generics

Author: romseygeek

.ci/bwcVersions

@@ -17,6 +17,8 @@ BWC_VERSION:
   - "7.6.0"
   - "7.6.1"
   - "7.6.2"
+  - "7.7.0"
+  - "7.7.1"
   - "7.8.0"
   - "7.9.0"
   - "8.0.0"

build.gradle

@@ -113,7 +113,7 @@ task updateCIBwcVersions() {
     File yml = file(".ci/bwcVersions")
     yml.text = ""
     yml << "BWC_VERSION:\n"
-    versions.indexCompatible.each {
+    BuildParams.bwcVersions.indexCompatible.each {
       yml << "  - \"$it\"\n"
     }
   }

buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ValidateJsonAgainstSchemaTask.java

@@ -29,10 +29,8 @@
 import org.gradle.api.DefaultTask;
 import org.gradle.api.UncheckedIOException;
 import org.gradle.api.file.FileCollection;
-import org.gradle.api.tasks.Input;
 import org.gradle.api.tasks.InputFile;
 import org.gradle.api.tasks.InputFiles;
-import org.gradle.api.tasks.Optional;
 import org.gradle.api.tasks.OutputFile;
 import org.gradle.api.tasks.TaskAction;
 import org.gradle.work.ChangeType;
@@ -44,9 +42,7 @@
 import java.io.PrintWriter;
 import java.nio.file.Files;
 import java.nio.file.StandardOpenOption;
-import java.util.Arrays;
 import java.util.Collection;
-import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
 import java.util.Map;
@@ -59,7 +55,6 @@
 public class ValidateJsonAgainstSchemaTask extends DefaultTask {
 
     private final ObjectMapper mapper = new ObjectMapper();
-    private Set<String> ignore = new HashSet<>();
     private File jsonSchema;
     private FileCollection inputFiles;
 
@@ -82,16 +77,6 @@ public void setJsonSchema(File jsonSchema) {
         this.jsonSchema = jsonSchema;
     }
 
-    @Input
-    @Optional
-    public Set<String> getIgnore() {
-        return ignore;
-    }
-
-    public void ignore(String... ignore) {
-        this.ignore.addAll(Arrays.asList(ignore));
-    }
-
     @OutputFile
     public File getReport() {
         return new File(getProject().getBuildDir(), "reports/validateJson.txt");
@@ -110,9 +95,7 @@ public void validate(InputChanges inputChanges) throws IOException {
             .filter(f -> f.getChangeType() != ChangeType.REMOVED)
             .forEach(fileChange -> {
                 File file = fileChange.getFile();
-                if (ignore.contains(file.getName())) {
-                    getLogger().debug("Ignoring file [{}] due to configuration", file.getName());
-                } else if (file.isDirectory() == false) {
+                if (file.isDirectory() == false) {
                     // validate all files and hold on to errors for a complete report if there are failures
                     getLogger().debug("Validating JSON [{}]", file.getName());
                     try {

buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/BwcVersions.java

@@ -19,14 +19,12 @@
 package org.elasticsearch.gradle;
 
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.NoSuchElementException;
 import java.util.Set;
 import java.util.SortedSet;
 import java.util.TreeSet;
@@ -89,7 +87,6 @@ public class BwcVersions {
     private static final Pattern LINE_PATTERN = Pattern.compile(
         "\\W+public static final Version V_(\\d+)_(\\d+)_(\\d+)(_alpha\\d+|_beta\\d+|_rc\\d+)? .*"
     );
-    private static final List<Version> IGNORED_VERSIONS = Arrays.asList(Version.fromString("7.7.0"));
 
     private final Version currentVersion;
     private final Map<Integer, List<Version>> groupByMajor;
@@ -123,7 +120,6 @@ protected BwcVersions(List<String> versionLines, Version currentVersionProperty)
                         Integer.parseInt(match.group(3))
                     )
                 )
-                .filter(v -> !IGNORED_VERSIONS.contains(v)) // remove any specifically ignored versions
                 .collect(Collectors.toCollection(TreeSet::new)),
             currentVersionProperty
         );
@@ -276,16 +272,7 @@ public List<Version> getUnreleased() {
                 // we found that the previous minor is staged but not yet released
                 // in this case, the minor before that has a bugfix, should there be such a minor
                 if (greatestMinor >= 2) {
-                    int major = groupByMinor.values()
-                        .stream()
-                        .flatMap(Collection::stream)
-                        .findFirst()
-                        .map(Version::getMajor)
-                        .orElseThrow(NoSuchElementException::new);
-                    // Don't bother searching for a version we've ignored
-                    if (IGNORED_VERSIONS.contains(new Version(major, greatestMinor - 2, 0)) == false) {
-                        unreleased.add(getLatestVersionByKey(groupByMinor, greatestMinor - 2));
-                    }
+                    unreleased.add(getLatestVersionByKey(groupByMinor, greatestMinor - 2));
                 }
             }
         }

client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java

@@ -129,10 +129,10 @@
 import org.elasticsearch.search.aggregations.bucket.range.RangeAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.sampler.InternalSampler;
 import org.elasticsearch.search.aggregations.bucket.sampler.ParsedSampler;
-import org.elasticsearch.search.aggregations.bucket.significant.ParsedSignificantLongTerms;
-import org.elasticsearch.search.aggregations.bucket.significant.ParsedSignificantStringTerms;
-import org.elasticsearch.search.aggregations.bucket.significant.SignificantLongTerms;
-import org.elasticsearch.search.aggregations.bucket.significant.SignificantStringTerms;
+import org.elasticsearch.search.aggregations.bucket.terms.ParsedSignificantLongTerms;
+import org.elasticsearch.search.aggregations.bucket.terms.ParsedSignificantStringTerms;
+import org.elasticsearch.search.aggregations.bucket.terms.SignificantLongTerms;
+import org.elasticsearch.search.aggregations.bucket.terms.SignificantStringTerms;
 import org.elasticsearch.search.aggregations.bucket.terms.DoubleTerms;
 import org.elasticsearch.search.aggregations.bucket.terms.LongTerms;
 import org.elasticsearch.search.aggregations.bucket.terms.ParsedDoubleTerms;

client/rest-high-level/src/main/java/org/elasticsearch/client/graph/GraphExploreRequest.java

@@ -28,7 +28,7 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.search.aggregations.bucket.sampler.SamplerAggregationBuilder;
-import org.elasticsearch.search.aggregations.bucket.significant.SignificantTerms;
+import org.elasticsearch.search.aggregations.bucket.terms.SignificantTerms;
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregator;
 
 import java.io.IOException;

docs/reference/ilm/actions/ilm-allocate.asciidoc

@@ -7,16 +7,16 @@ Phases allowed: warm, cold.
 Updates the index settings to change which nodes are allowed to host the index shards
 and change the number of replicas.
 
+The allocate action is not allowed in the hot phase. 
+The initial allocation for the index must be done manually or via 
+<<indices-templates, index templates>>.
+
 You can configure this action to modify both the allocation rules and number of replicas, 
 only the allocation rules, or only the number of replicas. 
-
 For more information about how {es} uses replicas for scaling, see
 <<scalability>>. See <<shard-allocation-filtering>> for more information about
 controlling where {es} allocates shards of a particular index.
 
-NOTE: The allocate action is not allowed in the hot phase. 
-The initial allocation for the index must be done manually or via index templates.
-{ilm-init} doesn't handle index allocation during the hot phase.
 
 [[ilm-allocate-options]]
 ==== Options

docs/reference/ilm/ilm-index-lifecycle.asciidoc

@@ -36,6 +36,15 @@ the index must be older than the minimum age of the next phase.
 The minimum age defaults to zero, which causes {ilm-init} to move indices to the next phase
 as soon as all actions in the current phase complete. 
 
+If an index has unallocated shards and the <<cluster-health,cluster health status>> is yellow, 
+the index can still transition to the next phase according to its {ilm} policy.
+However, because {es} can only perform certain clean up tasks on a green
+cluster, there might be unexpected side effects. 
+
+To avoid increased disk usage and reliability issues,
+address any cluster health problems in a timely fashion.
+
+
 [discrete]
 [[ilm-phase-execution]]
 === Phase execution

docs/reference/ilm/ilm-tutorial.asciidoc

@@ -10,8 +10,7 @@
 This tutorial demonstrates how to use {ilm} 
 ({ilm-init}) to manage indices that contain time-series data.
 
-When you continuously index timestamped documents into {es} using
-Filebeat, Logstash, or some other mechanism,
+When you continuously index timestamped documents into {es},
 you typically use an index alias so you can periodically roll over to a new index.
 This enables you to implement a hot-warm-cold architecture to meet your performance
 requirements for your newest data, control costs over time, enforce retention policies,
@@ -28,10 +27,12 @@ as expected.
 
 For an introduction to rolling indices, see <<index-rollover>>. 
 
-NOTE: {filebeat} includes a default {ilm-init} policy that initiates the rollover action when
-the index size reaches 50GB or becomes 30 days old. 
-You can use this policy as a starting point, or replace it with a custom policy.
-See {kibana-ref}/example-using-index-lifecycle-policy.html[Use {ilm-init} to manage Filebeat time-based indices].
+IMPORTANT: When you enable {ilm} for {beats} or the {ls} {es} output plugin, 
+lifecycle policies are set up automatically. 
+You do not need bootstrap the initial index or take any other actions.
+You can modify the default policies through 
+{kibana-ref}/example-using-index-lifecycle-policy.html[{kib} Management]
+or the {ilm-init} APIs.
 
 
 [discrete]

docs/reference/ilm/index.asciidoc

@@ -13,6 +13,10 @@ For example, you could use {ilm-init} to:
 * Create a new index each day, week, or month and archive previous ones
 * Delete stale indices to enforce data retention standards
 
+When you enable {ilm} for {beats} or the {ls} {es} output plugin, 
+{ilm-init} is configured automatically. 
+You can modify the default policies through {kib} Management or the {ilm-init} APIs.
+
 [TIP]
 To automatically back up your indices and manage snapshots, 
 use <<getting-started-snapshot-lifecycle-management,snapshot lifecycle policies>>.
@@ -48,4 +52,3 @@ include::using-policies-rollover.asciidoc[]
 include::ilm-with-existing-indices.asciidoc[]
 
 include::ilm-and-snapshots.asciidoc[]
-