Merge remote-tracking branch 'elastic/master' into pr/37940

* elastic/master:
  ILM setPriority corrections for a 0 value (elastic#38001)
  Temporarily disable BWC for retention lease stats (elastic#38049)
  Skip Shrink when numberOfShards not changed (elastic#37953)
  Add dispatching to `HandledTransportAction` (elastic#38050)
  Update httpclient for JDK 11 TLS engine (elastic#37994)
  Reduce flaxiness of ccr recovery timeouts test (elastic#38035)
  Fix ILM status to allow unknown fields (elastic#38043)
  Fix ILM Lifecycle Policy to allow unknown fields (elastic#38041)
  Update verify repository to allow unknown fields (elastic#37619)
  [ML] Datafeed deprecation checks (elastic#38026)
  Deprecate minimum_master_nodes (elastic#37868)
  Remove types from watcher docs (elastic#38002)
  Add test coverage for Painless general casting of boolean and Boolean (elastic#37780)
  Fixed test bug, lastFollowTime is null if there are no follower indices.
  Add ECS schema for user-agent ingest processor (elastic#37727) (elastic#37984)
  Extract TransportRequestDeduplication from ShardStateAction (elastic#37870)

build.gradle

@@ -159,8 +159,8 @@ task verifyVersions {
  * the enabled state of every bwc task. It should be set back to true
  * after the backport of the backcompat code is complete.
  */
-final boolean bwc_tests_enabled = true
-final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */
+final boolean bwc_tests_enabled = false
+final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/37991" /* place a PR link here when committing bwc changes */
 if (bwc_tests_enabled == false) {
   if (bwc_tests_disabled_issue.isEmpty()) {
     throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -373,18 +373,15 @@ class ClusterFormationTasks {
                 'path.repo'                    : "${node.sharedDir}/repo",
                 'path.shared_data'             : "${node.sharedDir}/",
                 // Define a node attribute so we can test that it exists
-                'node.attr.testattr'           : 'test'
+                'node.attr.testattr'           : 'test',
+                // Don't wait for state, just start up quickly. This will also allow new and old nodes in the BWC case to become the master
+                'discovery.initial_state_timeout' : '0s'
         ]
         int minimumMasterNodes = node.config.minimumMasterNodes.call()
-        if (minimumMasterNodes > 0) {
+        if (node.nodeVersion.before("7.0.0") && minimumMasterNodes > 0) {
             esConfig['discovery.zen.minimum_master_nodes'] = minimumMasterNodes
         }
-        if (minimumMasterNodes > 1) {
-            // don't wait for state.. just start up quickly
-            // this will also allow new and old nodes in the BWC case to become the master
-            esConfig['discovery.initial_state_timeout'] = '0s'
-        }
-        if (esConfig.containsKey('discovery.zen.master_election.wait_for_joins_timeout') == false) {
+        if (node.nodeVersion.before("7.0.0") && esConfig.containsKey('discovery.zen.master_election.wait_for_joins_timeout') == false) {
             // If a node decides to become master based on partial information from the pinging, don't let it hang for 30 seconds to correct
             // its mistake. Instead, only wait 5s to do another round of pinging.
             // This is necessary since we use 30s as the default timeout in REST requests waiting for cluster formation

buildSrc/version.properties

@@ -21,16 +21,13 @@ joda              = 2.10.1
 # test dependencies
 randomizedrunner  = 2.7.1
 junit             = 4.12
-httpclient        = 4.5.2
-# When updating httpcore, please also update server/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
-httpcore          = 4.4.5
-# When updating httpasyncclient, please also update server/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
-httpasyncclient   = 4.1.2
+httpclient        = 4.5.7
+httpcore          = 4.4.11
+httpasyncclient   = 4.1.4
 commonslogging    = 1.1.3
-commonscodec      = 1.10
+commonscodec      = 1.11
 hamcrest          = 1.3
 securemock        = 1.2
-# When updating mocksocket, please also update server/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
 mocksocket        = 1.2
 
 # benchmark dependencies

client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/AllocateAction.java

@@ -40,7 +40,7 @@ public class AllocateAction implements LifecycleAction, ToXContentObject {
     static final ParseField REQUIRE_FIELD = new ParseField("require");
 
     @SuppressWarnings("unchecked")
-    private static final ConstructingObjectParser<AllocateAction, Void> PARSER = new ConstructingObjectParser<>(NAME,
+    private static final ConstructingObjectParser<AllocateAction, Void> PARSER = new ConstructingObjectParser<>(NAME, true,
             a -> new AllocateAction((Integer) a[0], (Map<String, String>) a[1], (Map<String, String>) a[2], (Map<String, String>) a[3]));
 
     static {

client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/DeleteAction.java

@@ -30,7 +30,7 @@
 public class DeleteAction implements LifecycleAction, ToXContentObject {
     public static final String NAME = "delete";
 
-    private static final ObjectParser<DeleteAction, Void> PARSER = new ObjectParser<>(NAME, DeleteAction::new);
+    private static final ObjectParser<DeleteAction, Void> PARSER = new ObjectParser<>(NAME, true, DeleteAction::new);
 
     public static DeleteAction parse(XContentParser parser) {
         return PARSER.apply(parser, null);

client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/ForceMergeAction.java

@@ -33,7 +33,7 @@ public class ForceMergeAction implements LifecycleAction, ToXContentObject {
     private static final ParseField MAX_NUM_SEGMENTS_FIELD = new ParseField("max_num_segments");
 
     private static final ConstructingObjectParser<ForceMergeAction, Void> PARSER = new ConstructingObjectParser<>(NAME,
-        false, a -> {
+        true, a -> {
         int maxNumSegments = (int) a[0];
         return new ForceMergeAction(maxNumSegments);
     });

client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/FreezeAction.java

@@ -29,7 +29,7 @@
 public class FreezeAction implements LifecycleAction, ToXContentObject {
     public static final String NAME = "freeze";
 
-    private static final ObjectParser<FreezeAction, Void> PARSER = new ObjectParser<>(NAME, FreezeAction::new);
+    private static final ObjectParser<FreezeAction, Void> PARSER = new ObjectParser<>(NAME, true, FreezeAction::new);
 
     public static FreezeAction parse(XContentParser parser) {
         return PARSER.apply(parser, null);

client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/LifecycleManagementStatusResponse.java

@@ -34,7 +34,7 @@ public class LifecycleManagementStatusResponse {
     private static final String OPERATION_MODE = "operation_mode";
     @SuppressWarnings("unchecked")
     private static final ConstructingObjectParser<LifecycleManagementStatusResponse, Void> PARSER = new ConstructingObjectParser<>(
-        OPERATION_MODE, a -> new LifecycleManagementStatusResponse((String) a[0]));
+        OPERATION_MODE, true, a -> new LifecycleManagementStatusResponse((String) a[0]));
 
     static {
         PARSER.declareString(ConstructingObjectParser.constructorArg(), new ParseField(OPERATION_MODE));

client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicy.java

@@ -44,7 +44,7 @@ public class LifecyclePolicy implements ToXContentObject {
     static final ParseField PHASES_FIELD = new ParseField("phases");
 
     @SuppressWarnings("unchecked")
-    public static ConstructingObjectParser<LifecyclePolicy, String> PARSER = new ConstructingObjectParser<>("lifecycle_policy", false,
+    public static ConstructingObjectParser<LifecyclePolicy, String> PARSER = new ConstructingObjectParser<>("lifecycle_policy", true,
         (a, name) -> {
             List<Phase> phases = (List<Phase>) a[0];
             Map<String, Phase> phaseMap = phases.stream().collect(Collectors.toMap(Phase::getName, Function.identity()));

client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicyMetadata.java

@@ -38,7 +38,8 @@ public class LifecyclePolicyMetadata implements ToXContentObject {
     static final ParseField MODIFIED_DATE = new ParseField("modified_date");
 
     @SuppressWarnings("unchecked")
-    public static final ConstructingObjectParser<LifecyclePolicyMetadata, String> PARSER = new ConstructingObjectParser<>("policy_metadata",
+    public static final ConstructingObjectParser<LifecyclePolicyMetadata, String> PARSER = new ConstructingObjectParser<>(
+            "policy_metadata", true,
             a -> {
                 LifecyclePolicy policy = (LifecyclePolicy) a[0];
                 return new LifecyclePolicyMetadata(policy, (long) a[1], ZonedDateTime.parse((String) a[2]).toInstant().toEpochMilli());

client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/Phase.java

@@ -44,7 +44,7 @@ public class Phase implements ToXContentObject {
     static final ParseField ACTIONS_FIELD = new ParseField("actions");
 
     @SuppressWarnings("unchecked")
-    private static final ConstructingObjectParser<Phase, String> PARSER = new ConstructingObjectParser<>("phase", false,
+    private static final ConstructingObjectParser<Phase, String> PARSER = new ConstructingObjectParser<>("phase", true,
         (a, name) -> new Phase(name, (TimeValue) a[0], ((List<LifecycleAction>) a[1]).stream()
             .collect(Collectors.toMap(LifecycleAction::getName, Function.identity()))));
     static {

client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/PhaseExecutionInfo.java

@@ -40,7 +40,7 @@ public class PhaseExecutionInfo implements ToXContentObject {
     private static final ParseField MODIFIED_DATE_IN_MILLIS_FIELD = new ParseField("modified_date_in_millis");
 
     private static final ConstructingObjectParser<PhaseExecutionInfo, String> PARSER = new ConstructingObjectParser<>(
-        "phase_execution_info", false,
+        "phase_execution_info", true,
         (a, name) -> new PhaseExecutionInfo((String) a[0], (Phase) a[1], (long) a[2], (long) a[3]));
     static {
         PARSER.declareString(ConstructingObjectParser.constructorArg(), POLICY_NAME_FIELD);

client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/ReadOnlyAction.java

@@ -29,7 +29,7 @@
 public class ReadOnlyAction implements LifecycleAction, ToXContentObject {
     public static final String NAME = "readonly";
 
-    private static final ObjectParser<ReadOnlyAction, Void> PARSER = new ObjectParser<>(NAME, false, ReadOnlyAction::new);
+    private static final ObjectParser<ReadOnlyAction, Void> PARSER = new ObjectParser<>(NAME, true, ReadOnlyAction::new);
 
     public static ReadOnlyAction parse(XContentParser parser) {
         return PARSER.apply(parser, null);

client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/RolloverAction.java

@@ -38,7 +38,7 @@ public class RolloverAction implements LifecycleAction, ToXContentObject {
     private static final ParseField MAX_DOCS_FIELD = new ParseField("max_docs");
     private static final ParseField MAX_AGE_FIELD = new ParseField("max_age");
 
-    private static final ConstructingObjectParser<RolloverAction, Void> PARSER = new ConstructingObjectParser<>(NAME,
+    private static final ConstructingObjectParser<RolloverAction, Void> PARSER = new ConstructingObjectParser<>(NAME, true,
         a -> new RolloverAction((ByteSizeValue) a[0], (TimeValue) a[1], (Long) a[2]));
     static {
         PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(),

client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/SetPriorityAction.java

@@ -55,7 +55,7 @@ public static SetPriorityAction parse(XContentParser parser) {
     }
 
     public SetPriorityAction(@Nullable Integer recoveryPriority) {
-        if (recoveryPriority != null && recoveryPriority <= 0) {
+        if (recoveryPriority != null && recoveryPriority < 0) {
             throw new IllegalArgumentException("[" + RECOVERY_PRIORITY_FIELD.getPreferredName() + "] must be 0 or greater");
         }
         this.recoveryPriority = recoveryPriority;

client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/ShrinkAction.java

@@ -33,7 +33,7 @@ public class ShrinkAction implements LifecycleAction, ToXContentObject {
     private static final ParseField NUMBER_OF_SHARDS_FIELD = new ParseField("number_of_shards");
 
     private static final ConstructingObjectParser<ShrinkAction, Void> PARSER =
-        new ConstructingObjectParser<>(NAME, a -> new ShrinkAction((Integer) a[0]));
+        new ConstructingObjectParser<>(NAME, true, a -> new ShrinkAction((Integer) a[0]));
 
     static {
         PARSER.declareInt(ConstructingObjectParser.constructorArg(), NUMBER_OF_SHARDS_FIELD);

client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/UnfollowAction.java

@@ -31,7 +31,7 @@
 public class UnfollowAction implements LifecycleAction, ToXContentObject {
     public static final String NAME = "unfollow";
 
-    private static final ObjectParser<UnfollowAction, Void> PARSER = new ObjectParser<>(NAME, UnfollowAction::new);
+    private static final ObjectParser<UnfollowAction, Void> PARSER = new ObjectParser<>(NAME, true, UnfollowAction::new);
 
     public UnfollowAction() {}
 

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/ILMDocumentationIT.java

@@ -591,7 +591,7 @@ public void testRetryPolicy() throws Exception {
         {
             Map<String, Phase> phases = new HashMap<>();
             Map<String, LifecycleAction> warmActions = new HashMap<>();
-            warmActions.put(ShrinkAction.NAME, new ShrinkAction(1));
+            warmActions.put(ShrinkAction.NAME, new ShrinkAction(3));
             phases.put("warm", new Phase("warm", TimeValue.ZERO, warmActions));
 
             LifecyclePolicy policy = new LifecyclePolicy("my_policy",
@@ -602,7 +602,7 @@ public void testRetryPolicy() throws Exception {
 
             CreateIndexRequest createIndexRequest = new CreateIndexRequest("my_index")
                 .settings(Settings.builder()
-                    .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+                    .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 2)
                     .put("index.lifecycle.name", "my_policy")
                     .build());
             client.indices().create(createIndexRequest, RequestOptions.DEFAULT);

client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/AllocateActionTests.java

@@ -24,6 +24,7 @@
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.function.Predicate;
 
 public class AllocateActionTests extends AbstractXContentTestCase<AllocateAction> {
 
@@ -65,7 +66,14 @@ protected AllocateAction doParseInstance(XContentParser parser) {
 
     @Override
     protected boolean supportsUnknownFields() {
-        return false;
+        return true;
+    }
+
+    @Override
+    protected Predicate<String> getRandomFieldsExcludeFilter() {
+        // this whole structure expects to be maps of strings, so more complex objects would just mess that up.
+        // setting it this way allows for new fields at the root
+        return (field) -> field.isEmpty() == false;
     }
 
     public void testAllMapsNullOrEmpty() {

client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/DeleteActionTests.java

@@ -35,6 +35,6 @@ protected DeleteAction doParseInstance(XContentParser parser) {
 
     @Override
     protected boolean supportsUnknownFields() {
-        return false;
+        return true;
     }
 }

client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/ForceMergeActionTests.java

@@ -39,7 +39,7 @@ protected ForceMergeAction doParseInstance(XContentParser parser) {
 
     @Override
     protected boolean supportsUnknownFields() {
-        return false;
+        return true;
     }
 
     @Override

client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/FreezeActionTests.java

@@ -35,6 +35,6 @@ protected FreezeAction doParseInstance(XContentParser parser) {
 
     @Override
     protected boolean supportsUnknownFields() {
-        return false;
+        return true;
     }
 }

client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/GetLifecyclePolicyResponseTests.java

@@ -30,6 +30,7 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.function.Predicate;
 
 import static org.elasticsearch.client.indexlifecycle.LifecyclePolicyTests.createRandomPolicy;
 
@@ -54,7 +55,23 @@ protected GetLifecyclePolicyResponse doParseInstance(XContentParser parser) thro
 
     @Override
     protected boolean supportsUnknownFields() {
-        return false;
+        return true;
+    }
+
+    @Override
+    protected Predicate<String> getRandomFieldsExcludeFilter() {
+        return (field) ->
+            // phases is a list of Phase parsable entries only
+            field.endsWith(".phases")
+            // these are all meant to be maps of strings, so complex objects will confuse the parser
+            || field.endsWith(".include")
+            || field.endsWith(".exclude")
+            || field.endsWith(".require")
+            // actions are meant to be a list of LifecycleAction parsable entries only
+            || field.endsWith(".actions")
+            // field.isEmpty() means do not insert an object at the root of the json. This parser expects
+            // every root level named object to be parsable as a specific type
+            || field.isEmpty();
     }
 
     @Override

client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/LifecycleManagementStatusResponseTests.java

@@ -21,6 +21,7 @@
 
 import org.elasticsearch.common.xcontent.DeprecationHandler;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.ESTestCase;
@@ -30,8 +31,31 @@
 import java.util.EnumSet;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.test.AbstractXContentTestCase.xContentTester;
+
 public class LifecycleManagementStatusResponseTests extends ESTestCase {
 
+    public void testFromXContent() throws IOException {
+        xContentTester(this::createParser,
+            LifecycleManagementStatusResponseTests::createTestInstance,
+            LifecycleManagementStatusResponseTests::toXContent,
+            LifecycleManagementStatusResponse::fromXContent)
+            .supportsUnknownFields(true)
+            .assertToXContentEquivalence(false)
+            .test();
+    }
+
+    private static XContentBuilder toXContent(LifecycleManagementStatusResponse response, XContentBuilder builder) throws IOException {
+        builder.startObject();
+        builder.field("operation_mode", response.getOperationMode());
+        builder.endObject();
+        return builder;
+    }
+
+    private static LifecycleManagementStatusResponse createTestInstance() {
+        return new LifecycleManagementStatusResponse(randomFrom(OperationMode.values()).name());
+    }
+
     public void testAllValidStatuses() {
         EnumSet.allOf(OperationMode.class)
             .forEach(e -> assertEquals(new LifecycleManagementStatusResponse(e.name()).getOperationMode(), e));

client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicyMetadataTests.java

@@ -29,6 +29,7 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.function.Predicate;
 
 import static org.elasticsearch.client.indexlifecycle.LifecyclePolicyTests.createRandomPolicy;
 
@@ -50,7 +51,21 @@ protected LifecyclePolicyMetadata doParseInstance(XContentParser parser) throws
 
     @Override
     protected boolean supportsUnknownFields() {
-        return false;
+        return true;
+    }
+
+    @Override
+    protected Predicate<String> getRandomFieldsExcludeFilter() {
+        return (field) ->
+            // phases is a list of Phase parsable entries only
+            field.endsWith(".phases")
+            // these are all meant to be maps of strings, so complex objects will confuse the parser
+            || field.endsWith(".include")
+            || field.endsWith(".exclude")
+            || field.endsWith(".require")
+            // actions are meant to be a list of LifecycleAction parsable entries only
+            || field.endsWith(".actions");
+
     }
 
     @Override

client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicyTests.java

@@ -34,6 +34,7 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.function.Function;
+import java.util.function.Predicate;
 import java.util.stream.Collectors;
 
 import static org.hamcrest.Matchers.equalTo;
@@ -55,7 +56,13 @@ protected LifecyclePolicy doParseInstance(XContentParser parser) {
 
     @Override
     protected boolean supportsUnknownFields() {
-        return false;
+        return true;
+    }
+
+    @Override
+    protected Predicate<String> getRandomFieldsExcludeFilter() {
+        // these items all have some specific parsing that does not allow them to have additional objects within them.
+        return (field) -> field.contains("allocate.") || field.equals("phases") || field.endsWith("actions");
     }
 
     @Override