Merge branch 'upstream/master' into reuse-date-stream-timestamp

* upstream/master: (34 commits)
  Add extensionName() to security extension (elastic#79329)
  More robust and consistent allowAll indicesAccessControl (elastic#79415)
  Fix circuit breaker leak in MultiTerms aggregation (elastic#79362)
  guard geoline aggregation from parents aggegator that emit empty buckets (elastic#79129)
  Vector tiles: increase the size of the envelope used to clip geometries (elastic#79030)
  Revert "[ML] Add queue_capacity setting to start deployment API (elastic#79369)" (elastic#79374)
  Convert token service license object to LicensedFeature (elastic#79284)
  [TEST] Fix ShardPathTests for MDP (elastic#79393)
  Fix fleet search API with no checkpints (elastic#79400)
  Reduce BWC version for transient settings (elastic#79396)
  EQL: Rename a test class for eclipse (elastic#79254)
  Use search_coordination threadpool in field caps (elastic#79378)
  Use query param instead of a system property for opting in for new cluster health response code (elastic#79351)
  Add new kNN search endpoint (elastic#79013)
  Disable BWC tests
  Convert auditing license object to LicensedFeature (elastic#79280)
  Update BWC versions after backport of elastic#78551
  Enable InstantiatingObjectParser to pass context as a first argument (elastic#79206)
  Move xcontent filtering tests (elastic#79298)
  Update links to Fleet/Agent docs (elastic#79303)
  ...

build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/FormattingPrecommitPlugin.java

@@ -89,7 +89,6 @@ private Object[] getTargets(String projectPath) {
             return new String[] {
                 "src/*/java/org/elasticsearch/action/admin/cluster/repositories/**/*.java",
                 "src/*/java/org/elasticsearch/action/admin/cluster/snapshots/**/*.java",
-                "src/test/java/org/elasticsearch/common/xcontent/support/AbstractFilteringTestCase.java",
                 "src/test/java/org/elasticsearch/common/xcontent/support/XContentMapValuesTests.java",
                 "src/*/java/org/elasticsearch/index/IndexMode.java",
                 "src/*/java/org/elasticsearch/index/IndexRouting.java",
@@ -98,6 +97,10 @@ private Object[] getTargets(String projectPath) {
                 "src/*/java/org/elasticsearch/repositories/**/*.java",
                 "src/*/java/org/elasticsearch/search/aggregations/**/*.java",
                 "src/*/java/org/elasticsearch/snapshots/**/*.java" };
+        } else if (projectPath.equals(":test:framework")) {
+            return new String[] {
+                "src/test/java/org/elasticsearch/common/xcontent/support/AbstractFilteringTestCase.java",
+            };
         } else {
             // Normally this isn"t necessary, but we have Java sources in
             // non-standard places
@@ -203,7 +206,6 @@ private Object[] getTargets(String projectPath) {
         ":test:fixtures:geoip-fixture",
         ":test:fixtures:krb5kdc-fixture",
         ":test:fixtures:old-elasticsearch",
-        ":test:framework",
         ":test:logger-usage",
         ":x-pack:docs",
         ":x-pack:license-tools",

build.gradle

@@ -132,9 +132,9 @@ tasks.register("verifyVersions") {
  * after the backport of the backcompat code is complete.
  */
 
-boolean bwc_tests_enabled = true
+boolean bwc_tests_enabled = false
 // place a PR link here when committing bwc changes:
-String bwc_tests_disabled_issue = ""
+String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/79385"
 /*
  * FIPS 140-2 behavior was fixed in 7.11.0. Before that there is no way to run elasticsearch in a
  * JVM that is properly configured to be in fips mode with BCFIPS. For now we need to disable

client/rest-high-level/src/test/java/org/elasticsearch/client/ClusterClientIT.java

@@ -37,13 +37,13 @@
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
-import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.indices.recovery.RecoverySettings;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.transport.RemoteClusterService;
 import org.elasticsearch.transport.SniffConnectionStrategy;
+import org.elasticsearch.xcontent.XContentType;
 
 import java.io.IOException;
 import java.util.HashMap;
@@ -316,7 +316,7 @@ public void testClusterHealthNotFoundIndex() throws IOException {
         assertThat(response.getStatus(), equalTo(ClusterHealthStatus.RED));
         assertNoIndices(response);
         assertWarnings("The HTTP status code for a cluster health timeout will be changed from 408 to 200 in a " +
-            "future version. Set the [es.cluster_health.request_timeout_200] system property to [true] to suppress this message and " +
+            "future version. Set the [return_200_for_cluster_health_timeout] query parameter to [true] to suppress this message and " +
             "opt in to the future behaviour now.");
     }
 

distribution/src/bin/elasticsearch

@@ -56,7 +56,7 @@ if [[ $ATTEMPT_SECURITY_AUTO_CONFIG = true ]]; then
   if ES_MAIN_CLASS=org.elasticsearch.xpack.security.cli.ConfigInitialNode \
     ES_ADDITIONAL_SOURCES="x-pack-env;x-pack-security-env" \
     ES_ADDITIONAL_CLASSPATH_DIRECTORIES=lib/tools/security-cli \
-    "`dirname "$0"`"/elasticsearch-cli "$@" <<<"$KEYSTORE_PASSWORD"; then
+    bin/elasticsearch-cli "$@" <<<"$KEYSTORE_PASSWORD"; then
       :
   else
     retval=$?

docs/reference/cluster/health.asciidoc

@@ -97,6 +97,11 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms]
     provided or better, i.e. `green` > `yellow` > `red`. By default, will not
     wait for any status.
 
+`return_200_for_cluster_health_timeout`::
+   (Optional, Boolean) A boolean value which controls whether to return HTTP 200
+   status code instead of HTTP 408 in case of a cluster health timeout from
+   the server side. Defaults to false.
+
 [[cluster-health-api-response-body]]
 ==== {api-response-body-title}
 

docs/reference/getting-started.asciidoc

@@ -507,7 +507,7 @@ include::{es-repo-dir}/tab-widgets/quick-start-cleanup-widget.asciidoc[]
 
 * Use {fleet} and {agent} to collect logs and metrics directly from your data
 sources and send them to {es}. See the
-{fleet-guide}/fleet-quick-start.html[{fleet} quick start guide].
+{observability-guide}/ingest-logs-metrics-uptime.html[Ingest logs, metrics, and uptime data with {agent}].
 
 * Use {kib} to explore, visualize, and manage your {es} data. See the
 {kibana-ref}/get-started.html[{kib} quick start guide].

docs/reference/ingest.asciidoc

@@ -432,8 +432,7 @@ If you run {agent} standalone, you can apply pipelines using an
 <<index-default-pipeline,`index.default_pipeline`>> or
 <<index-final-pipeline,`index.final_pipeline`>> index setting. Alternatively,
 you can specify the `pipeline` policy setting in your `elastic-agent.yml`
-configuration. See {fleet-guide}/run-elastic-agent-standalone.html[Run {agent}
-standalone].
+configuration. See {fleet-guide}/install-standalone-elastic-agent.html[Install standalone {agent}s].
 
 [discrete]
 [[access-source-fields]]

docs/reference/migration/migrate_8_0/scripting.asciidoc

@@ -6,21 +6,42 @@
 //Installation and Upgrade Guide
 
 //tag::notable-breaking-changes[]
-.The `JodaCompatibleDateTime` class has been removed.
+.The `JodaCompatibleZonedDateTime` class has been removed.
 [%collapsible]
 ====
 *Details* +
 As a transition from Joda datetime to Java datetime, scripting used
-an intermediate class called `JodaCompatibleDateTime`. This class has
-been removed and is replaced by `ZonedDateTime`. Any use of casting
-to a `JodaCompatibleDateTime` in a script will result in a compilation
+an intermediate class called `JodaCompatibleZonedDateTime`. This class
+has been removed and is replaced by `ZonedDateTime`. Any use of casting
+to a `JodaCompatibleZonedDateTime` or use of method calls only available
+in `JodaCompatibleZonedDateTime` in a script will result in a compilation
 error, and may not allow the upgraded node to start.
 
 *Impact* +
 Before upgrading, replace `getDayOfWeek` with `getDayOfWeekEnum().value` in any
 scripts. Any use of `getDayOfWeek` expecting a return value of `int` will result
 in a compilation error or runtime error and may not allow the upgraded node to
 start.
+
+The following `JodaCompatibleZonedDateTime` methods must be replaced using
+`ZonedDateTime` methods prior to upgrade:
+
+* `getMillis()` -> `toInstant().toEpochMilli()`
+* `getCenturyOfEra()` -> `get(ChronoField.YEAR_OF_ERA) / 100`
+* `getEra()` -> `get(ChronoField.ERA)`
+* `getHourOfDay()` -> `getHour()`
+* `getMillisOfDay()` -> `get(ChronoField.MILLI_OF_DAY)`
+* `getMillisOfSecond()` -> `get(ChronoField.MILLI_OF_SECOND)`
+* `getMinuteOfDay()` -> `get(ChronoField.MINUTE_OF_DAY)`
+* `getMinuteOfHour()` -> `getMinute()`
+* `getMonthOfYear()` -> `getMonthValue()`
+* `getSecondOfDay()` -> `get(ChronoField.SECOND_OF_DAY)`
+* `getSecondOfMinute()` -> `getSecond()`
+* `getWeekOfWeekyear()` -> `get(DateFormatters.WEEK_FIELDS_ROOT.weekBasedYear())`
+* `getYearOfCentury()` -> `get(ChronoField.YEAR_OF_ERA) % 100`
+* `getYearOfEra()` -> `get(ChronoField.YEAR_OF_ERA)`
+* `toString(String)` -> a DateTimeFormatter
+* `toString(String, Locale)` -> a DateTimeFormatter
 ====
 
 .Stored scripts no longer support empty scripts or search templates.

docs/reference/modules/indices/recovery.asciidoc

@@ -100,3 +100,10 @@ sent in parallel to the target node for each recovery. Defaults to `5`.
 +
 Do not increase this setting without carefully verifying that your cluster has
 the resources available to handle the extra load that will result.
+
+`indices.recovery.max_concurrent_snapshot_file_downloads_per_node`::
+(<<cluster-update-settings,Dynamic>>, Expert) Number of snapshot file downloads requests
+execyted in parallel in the target node for all recoveries. Defaults to `25`.
++
+Do not increase this setting without carefully verifying that your cluster has
+the resources available to handle the extra load that will result.

libs/x-content/src/main/java/org/elasticsearch/xcontent/InstantiatingObjectParser.java

@@ -23,8 +23,12 @@
  * <p>
  * The main differences being that it is using Builder to construct the parser and takes a class of the target object instead of the object
  * builder. The target object must have exactly one constructor with the number and order of arguments matching the number of order of
- * declared fields. If there are more then 2 constructors with the same number of arguments, one of them needs to be marked with
+ * declared fields. If there are more than 2 constructors with the same number of arguments, one of them needs to be marked with
  * {@linkplain ParserConstructor} annotation.
+ *
+ * It is also possible for the constructor to accept Context as the first parameter, in this case as in the case with multiple constructors
+ * it is required for the constructor to be marked with {@linkplain ParserConstructor} annotation.
+ *
  * <pre>{@code
  *   public static class Thing{
  *       public Thing(String animal, String vegetable, int mineral) {
@@ -37,14 +41,35 @@
  *
  *   }
  *
- *   private static final InstantiatingObjectParser<Thing, SomeContext> PARSER = new InstantiatingObjectParser<>("thing", Thing.class);
+ *   private static final InstantiatingObjectParser<Thing, SomeContext> PARSER;
+ *   static {
+ *       InstantiatingObjectParser.Builder<Thing, SomeContext> parser =
+ *           InstantiatingObjectParser,builder<>("thing", true, Thing.class);
+ *       parser.declareString(constructorArg(), new ParseField("animal"));
+ *       parser.declareString(constructorArg(), new ParseField("vegetable"));
+ *       parser.declareInt(optionalConstructorArg(), new ParseField("mineral"));
+ *       parser.declareInt(Thing::setFruit, new ParseField("fruit"));
+ *       parser.declareInt(Thing::setBug, new ParseField("bug"));
+ *       PARSER = parser.build()
+ *   }
+ * }</pre>
+ * <pre>{@code
+ *
+ *   public static class AnotherThing {
+ *       @ParserConstructor
+ *       public AnotherThing(SomeContext continent, String animal, String vegetable, int mineral) {
+ *           ....
+ *       }
+ *   }
+ *
+ *   private static final InstantiatingObjectParser<AnotherThing, SomeContext> PARSER;
  *   static {
- *       PARSER.declareString(constructorArg(), new ParseField("animal"));
- *       PARSER.declareString(constructorArg(), new ParseField("vegetable"));
- *       PARSER.declareInt(optionalConstructorArg(), new ParseField("mineral"));
- *       PARSER.declareInt(Thing::setFruit, new ParseField("fruit"));
- *       PARSER.declareInt(Thing::setBug, new ParseField("bug"));
- *       PARSER.finalizeFields()
+ *       InstantiatingObjectParser.Builder<AnotherThing, SomeContext> parser =
+ *           InstantiatingObjectParser,builder<>("thing", true, AnotherThing.class);
+ *       parser.declareString(constructorArg(), new ParseField("animal"));
+ *       parser.declareString(constructorArg(), new ParseField("vegetable"));
+ *       parser.declareInt(optionalConstructorArg(), new ParseField("mineral"));
+ *       PARSER = parser.build()
  *   }
  * }</pre>
  */
@@ -72,7 +97,7 @@ public Builder(String name, Class<Value> valueClass) {
         }
 
         public Builder(String name, boolean ignoreUnknownFields, Class<Value> valueClass) {
-            this.constructingObjectParser = new ConstructingObjectParser<>(name, ignoreUnknownFields, this::build);
+            this.constructingObjectParser = new ConstructingObjectParser<>(name, ignoreUnknownFields, this::buildInstance);
             this.valueClass = valueClass;
         }
 
@@ -87,9 +112,15 @@ public InstantiatingObjectParser<Value, Context> build() {
                         throw new IllegalArgumentException("More then one public constructor with @ParserConstructor annotation exist in " +
                             "the class " + valueClass.getName());
                     }
-                    if (c.getParameterCount() != neededArguments) {
-                        throw new IllegalArgumentException("Annotated constructor doesn't have " + neededArguments +
-                            " arguments in the class " + valueClass.getName());
+                    if (c.getParameterCount() < neededArguments || c.getParameterCount() > neededArguments + 1) {
+                        throw new IllegalArgumentException(
+                            "Annotated constructor doesn't have "
+                                + neededArguments
+                                + " or "
+                                + (neededArguments + 1)
+                                + " arguments in the class "
+                                + valueClass.getName()
+                        );
                     }
                     constructor = c;
                 }
@@ -154,13 +185,20 @@ public void declareExclusiveFieldSet(String... exclusiveSet) {
             constructingObjectParser.declareExclusiveFieldSet(exclusiveSet);
         }
 
-        private Value build(Object[] args) {
+        private Value buildInstance(Object[] args, Context context) {
             if (constructor == null) {
                 throw new IllegalArgumentException("InstantiatingObjectParser for type " + valueClass.getName() + " has to be finalized " +
                     "before the first use");
             }
             try {
-                return constructor.newInstance(args);
+                if (constructor.getParameterCount() != args.length) {
+                    Object[] newArgs = new Object[args.length + 1];
+                    System.arraycopy(args, 0, newArgs, 1, args.length);
+                    newArgs[0] = context;
+                    return constructor.newInstance(newArgs);
+                } else {
+                    return constructor.newInstance(args);
+                }
             } catch (Exception ex) {
                 throw new IllegalArgumentException("Cannot instantiate an object of " + valueClass.getName(), ex);
             }

libs/x-content/src/test/java/org/elasticsearch/xcontent/InstantiatingObjectParserTests.java

@@ -8,11 +8,8 @@
 
 package org.elasticsearch.xcontent;
 
-import org.elasticsearch.xcontent.InstantiatingObjectParser;
-import org.elasticsearch.xcontent.ParseField;
-import org.elasticsearch.xcontent.ParserConstructor;
-import org.elasticsearch.xcontent.json.JsonXContent;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xcontent.json.JsonXContent;
 
 import java.io.IOException;
 import java.util.Objects;
@@ -217,8 +214,10 @@ public void testAnnotationWrongArgumentNumber() {
         InstantiatingObjectParser.Builder<Annotations, Void> builder = InstantiatingObjectParser.builder("foo", Annotations.class);
         builder.declareInt(constructorArg(), new ParseField("a"));
         builder.declareString(constructorArg(), new ParseField("b"));
+        builder.declareInt(constructorArg(), new ParseField("c"));
+        builder.declareString(constructorArg(), new ParseField("d"));
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, builder::build);
-        assertThat(e.getMessage(), containsString("Annotated constructor doesn't have 2 arguments in the class"));
+        assertThat(e.getMessage(), containsString("Annotated constructor doesn't have 4 or 5 arguments in the class"));
     }
 
     public void testDoubleDeclarationThrowsException() throws IOException {
@@ -240,4 +239,80 @@ class DoubleFieldDeclaration {
         assertThat(exception, instanceOf(IllegalArgumentException.class));
         assertThat(exception.getMessage(), startsWith("Parser already registered for name=[name]"));
     }
+
+    public static class ContextArgument {
+        final String context;
+        final int a;
+        final String b;
+        final long c;
+
+        public ContextArgument() {
+            this(1, "2", 3);
+        }
+
+        public ContextArgument(int a, String b) {
+            this(a, b, -1);
+        }
+
+
+        public ContextArgument(int a, String b, long c) {
+            this(null, a, b, c);
+        }
+
+        public ContextArgument(String context, int a, String b, long c) {
+            this.context = context;
+            this.a = a;
+            this.b = b;
+            this.c = c;
+        }
+
+        @ParserConstructor
+        public ContextArgument(String context, int a, String b, String c) {
+            this.context = context;
+            this.a = a;
+            this.b = b;
+            this.c = Long.parseLong(c);
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            ContextArgument that = (ContextArgument) o;
+            return a == that.a &&
+                c == that.c &&
+                Objects.equals(b, that.b);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(a, b, c);
+        }
+    }
+
+    public void testContextAsArgument() throws IOException {
+        InstantiatingObjectParser.Builder<ContextArgument, String> builder = InstantiatingObjectParser.builder(
+            "foo",
+            ContextArgument.class
+        );
+        builder.declareInt(constructorArg(), new ParseField("a"));
+        builder.declareString(constructorArg(), new ParseField("b"));
+        builder.declareString(constructorArg(), new ParseField("c"));
+        InstantiatingObjectParser<ContextArgument, String> parser = builder.build();
+        try (XContentParser contentParser = createParser(JsonXContent.jsonXContent, "{\"a\": 5, \"b\":\"6\", \"c\": \"7\"}")) {
+            assertThat(parser.parse(contentParser, "context"), equalTo(new ContextArgument("context", 5, "6", 7)));
+        }
+    }
+
+    public void testContextAsArgumentWrongArgumentNumber() {
+        InstantiatingObjectParser.Builder<ContextArgument, String> builder = InstantiatingObjectParser.builder(
+            "foo",
+            ContextArgument.class
+        );
+        builder.declareInt(constructorArg(), new ParseField("a"));
+        builder.declareString(constructorArg(), new ParseField("b"));
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, builder::build);
+        assertThat(e.getMessage(), containsString("Annotated constructor doesn't have 2 or 3 arguments in the class"));
+    }
+
 }