Merge branch 'main' into check-missing-shards-after-rewrite

.github/CODEOWNERS

@@ -65,6 +65,30 @@ server/src/main/java/org/elasticsearch/bootstrap @elastic/es-core-infra
 server/src/main/java/org/elasticsearch/node @elastic/es-core-infra
 server/src/main/java/org/elasticsearch/plugins @elastic/es-core-infra
 server/src/main/java/org/elasticsearch/threadpool @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/breaker @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/bytes @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/cli @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/collect @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/component @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/compress @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/document @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/file @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/hash @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/io @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/logging @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/metrics @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/network @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/path @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/recycler @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/regex @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/scheduler @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/settings @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/text @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/time @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/transport @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/unit @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/util @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/xcontent @elastic/es-core-infra
 
 # Security
 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege @elastic/es-security

docs/changelog/116021.yaml

@@ -0,0 +1,6 @@
+pr: 116021
+summary: Fields caps does not honour ignore_unavailable
+area: Search
+type: bug
+issues:
+ - 107767

docs/changelog/116346.yaml

@@ -0,0 +1,5 @@
+pr: 116346
+summary: "[ESQL] Fix Binary Comparisons on Date Nanos"
+area: ES|QL
+type: bug
+issues: []

docs/reference/connector/docs/_connectors-create-native.asciidoc

@@ -22,7 +22,7 @@ PUT _connector/my-{service-name-stub}-connector
   "index_name": "my-elasticsearch-index",
   "name": "Content synced from {service-name}",
   "service_type": "{service-name-stub}",
-  "is_native": "true"
+  "is_native": true
 }
 ----
 // TEST[skip:can't test in isolation]

docs/reference/search/retriever.asciidoc

@@ -554,8 +554,8 @@ It then re-ranks the results based on semantic similarity to the text in the `in
 [[rule-retriever]]
 ==== Query Rules Retriever
 
-The `rule` retriever enables fine-grained control over search results by applying contextual <<query-rules>> to pin or exclude documents for specific queries.
-This retriever has similar functionality to the <<query-dsl-rule-query>>, but works out of the box with other retrievers.
+The `rule` retriever enables fine-grained control over search results by applying contextual <<query-rules,query rules>> to pin or exclude documents for specific queries.
+This retriever has similar functionality to the <<query-dsl-rule-query, rule query>>, but works out of the box with other retrievers.
 
 ===== Prerequisites
 

muted-tests.yml

@@ -207,9 +207,6 @@ tests:
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=ml/inference_crud/Test delete given model with alias referenced by pipeline}
   issue: https://github.com/elastic/elasticsearch/issues/116133
-- class: org.elasticsearch.xpack.inference.InferenceRestIT
-  method: test {p0=inference/30_semantic_text_inference/Calculates embeddings using the default ELSER 2 endpoint}
-  issue: https://github.com/elastic/elasticsearch/issues/114412
 - class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT
   method: test {categorize.Categorize SYNC}
   issue: https://github.com/elastic/elasticsearch/issues/113054
@@ -237,18 +234,12 @@ tests:
 - class: org.elasticsearch.search.basic.SearchWithRandomDisconnectsIT
   method: testSearchWithRandomDisconnects
   issue: https://github.com/elastic/elasticsearch/issues/116175
-- class: org.elasticsearch.indexing.IndexActionIT
-  method: testAutoGenerateIdNoDuplicates
-  issue: https://github.com/elastic/elasticsearch/issues/115716
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=ml/start_stop_datafeed/Test start datafeed given index pattern with no matching indices}
   issue: https://github.com/elastic/elasticsearch/issues/116220
 - class: org.elasticsearch.search.basic.SearchWhileRelocatingIT
   method: testSearchAndRelocateConcurrentlyRandomReplicas
   issue: https://github.com/elastic/elasticsearch/issues/116145
-- class: org.elasticsearch.reservedstate.service.FileSettingsServiceTests
-  method: testProcessFileChanges
-  issue: https://github.com/elastic/elasticsearch/issues/115280
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=ml/filter_crud/Test update filter}
   issue: https://github.com/elastic/elasticsearch/issues/116271
@@ -261,18 +252,6 @@ tests:
 - class: org.elasticsearch.xpack.security.operator.OperatorPrivilegesIT
   method: testEveryActionIsEitherOperatorOnlyOrNonOperator
   issue: https://github.com/elastic/elasticsearch/issues/102992
-- class: org.elasticsearch.datastreams.DataStreamsClientYamlTestSuiteIT
-  issue: https://github.com/elastic/elasticsearch/issues/116291
-- class: org.elasticsearch.xpack.test.rest.XPackRestIT
-  issue: https://github.com/elastic/elasticsearch/issues/114723
-- class: org.elasticsearch.xpack.search.AsyncSearchSecurityIT
-  issue: https://github.com/elastic/elasticsearch/issues/116293
-- class: org.elasticsearch.xpack.downsample.DownsampleRestIT
-  issue: https://github.com/elastic/elasticsearch/issues/116326
-- class: org.elasticsearch.xpack.downsample.DownsampleWithBasicRestIT
-  issue: https://github.com/elastic/elasticsearch/issues/116327
-- class: org.elasticsearch.validation.DotPrefixClientYamlTestSuiteIT
-  issue: https://github.com/elastic/elasticsearch/issues/116328
 - class: org.elasticsearch.action.search.SearchQueryThenFetchAsyncActionTests
   method: testBottomFieldSort
   issue: https://github.com/elastic/elasticsearch/issues/116249
@@ -288,6 +267,12 @@ tests:
 - class: org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsIntegTests
   method: testCreateAndRestoreSearchableSnapshot
   issue: https://github.com/elastic/elasticsearch/issues/116377
+- class: org.elasticsearch.threadpool.SimpleThreadPoolIT
+  method: testThreadPoolMetrics
+  issue: https://github.com/elastic/elasticsearch/issues/108320
+- class: org.elasticsearch.xpack.test.rest.XPackRestIT
+  method: test {p0=ml/jobs_crud/Test put job deprecated bucket span}
+  issue: https://github.com/elastic/elasticsearch/issues/116419
 
 # Examples:
 #

server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java

@@ -14,13 +14,15 @@
 import org.apache.logging.log4j.Level;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteUtils;
+import org.elasticsearch.action.admin.indices.close.CloseIndexRequest;
 import org.elasticsearch.action.fieldcaps.FieldCapabilities;
 import org.elasticsearch.action.fieldcaps.FieldCapabilitiesFailure;
 import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest;
 import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse;
 import org.elasticsearch.action.fieldcaps.TransportFieldCapabilitiesAction;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.support.ActiveShardCount;
+import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.client.Cancellable;
 import org.elasticsearch.client.Request;
@@ -45,6 +47,7 @@
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.indices.IndexClosedException;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.plugins.FieldPredicate;
 import org.elasticsearch.plugins.MapperPlugin;
@@ -319,6 +322,63 @@ public void testWithIndexAlias() {
         assertEquals(response1, response2);
     }
 
+    public void testNoIndices() {
+        boolean ignoreUnavailable = false;
+        IndicesOptions options = IndicesOptions.fromOptions(ignoreUnavailable, true, true, false, true, true, false, false);
+        client().admin().indices().close(new CloseIndexRequest("old_index")).actionGet();
+        FieldCapabilitiesResponse response = client().prepareFieldCaps().setFields("*").setIndicesOptions(options).get();
+        assertIndices(response, "new_index");
+    }
+
+    public void testNoIndicesIgnoreUnavailable() {
+        boolean ignoreUnavailable = true;
+        IndicesOptions options = IndicesOptions.fromOptions(ignoreUnavailable, true, true, false, true, true, false, false);
+        client().admin().indices().close(new CloseIndexRequest("old_index")).actionGet();
+        FieldCapabilitiesResponse response = client().prepareFieldCaps().setFields("*").setIndicesOptions(options).get();
+        assertIndices(response, "new_index");
+    }
+
+    public void testOneClosedIndex() {
+        boolean ignoreUnavailable = false;
+        IndicesOptions options = IndicesOptions.fromOptions(ignoreUnavailable, true, true, false, true, true, false, false);
+        client().admin().indices().close(new CloseIndexRequest("old_index")).actionGet();
+        IndexClosedException ex = expectThrows(
+            IndexClosedException.class,
+            client().prepareFieldCaps("old_index").setFields("*").setIndicesOptions(options)
+        );
+        assertEquals("closed", ex.getMessage());
+    }
+
+    public void testOneClosedIndexIgnoreUnavailable() {
+        boolean ignoreUnavailable = true;
+        IndicesOptions options = IndicesOptions.fromOptions(ignoreUnavailable, true, true, false, true, true, false, false);
+        client().admin().indices().close(new CloseIndexRequest("old_index")).actionGet();
+        FieldCapabilitiesResponse response = client().prepareFieldCaps("old_index").setFields("*").setIndicesOptions(options).get();
+        assertIndices(response);
+    }
+
+    public void testTwoIndicesOneClosed() {
+        boolean ignoreUnavailable = false;
+        IndicesOptions options = IndicesOptions.fromOptions(ignoreUnavailable, true, true, false, true, true, false, false);
+        client().admin().indices().close(new CloseIndexRequest("old_index")).actionGet();
+        IndexClosedException ex = expectThrows(
+            IndexClosedException.class,
+            client().prepareFieldCaps("old_index", "new_index").setFields("*").setIndicesOptions(options)
+        );
+        assertEquals("closed", ex.getMessage());
+    }
+
+    public void testTwoIndicesOneClosedIgnoreUnavailable() {
+        boolean ignoreUnavailable = true;
+        IndicesOptions options = IndicesOptions.fromOptions(ignoreUnavailable, true, true, false, true, true, false, false);
+        client().admin().indices().close(new CloseIndexRequest("old_index")).actionGet();
+        FieldCapabilitiesResponse response = client().prepareFieldCaps("old_index", "new_index")
+            .setFields("*")
+            .setIndicesOptions(options)
+            .get();
+        assertIndices(response, "new_index");
+    }
+
     public void testWithIndexFilter() throws InterruptedException {
         assertAcked(prepareCreate("index-1").setMapping("timestamp", "type=date", "field1", "type=keyword"));
         assertAcked(prepareCreate("index-2").setMapping("timestamp", "type=date", "field1", "type=long"));

server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java

@@ -35,7 +35,7 @@
 
 public final class FieldCapabilitiesRequest extends ActionRequest implements IndicesRequest.Replaceable, ToXContentObject {
     public static final String NAME = "field_caps_request";
-    public static final IndicesOptions DEFAULT_INDICES_OPTIONS = IndicesOptions.strictExpandOpen();
+    public static final IndicesOptions DEFAULT_INDICES_OPTIONS = IndicesOptions.strictExpandOpenAndForbidClosed();
 
     private String[] indices = Strings.EMPTY_ARRAY;
     private IndicesOptions indicesOptions = DEFAULT_INDICES_OPTIONS;

server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestBuilder.java

@@ -10,6 +10,7 @@
 package org.elasticsearch.action.fieldcaps;
 
 import org.elasticsearch.action.ActionRequestBuilder;
+import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.internal.ElasticsearchClient;
 import org.elasticsearch.index.query.QueryBuilder;
 
@@ -38,6 +39,11 @@ public FieldCapabilitiesRequestBuilder setincludeEmptyFields(boolean includeEmpt
         return this;
     }
 
+    public FieldCapabilitiesRequestBuilder setIndicesOptions(IndicesOptions indicesOptions) {
+        request().indicesOptions(indicesOptions);
+        return this;
+    }
+
     public FieldCapabilitiesRequestBuilder setIndexFilter(QueryBuilder indexFilter) {
         request().indexFilter(indexFilter);
         return this;

server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java

@@ -48,8 +48,8 @@
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.Executor;
-import java.util.concurrent.LinkedTransferQueue;
 import java.util.concurrent.Semaphore;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -751,7 +751,7 @@ protected final ShardSearchRequest buildShardSearchRequest(SearchShardIterator s
 
     private static final class PendingExecutions {
         private final Semaphore semaphore;
-        private final LinkedTransferQueue<Consumer<Releasable>> queue = new LinkedTransferQueue<>();
+        private final ConcurrentLinkedQueue<Consumer<Releasable>> queue = new ConcurrentLinkedQueue<>();
 
         PendingExecutions(int permits) {
             assert permits > 0 : "not enough permits: " + permits;
@@ -770,11 +770,10 @@ void submit(Consumer<Releasable> task) {
                     }
                 }
             }
-
         }
 
         private void executeAndRelease(Consumer<Releasable> task) {
-            while (task != null) {
+            do {
                 final SubscribableListener<Void> onDone = new SubscribableListener<>();
                 task.accept(() -> onDone.onResponse(null));
                 if (onDone.isDone()) {
@@ -797,13 +796,21 @@ public void onFailure(Exception e) {
                     });
                     return;
                 }
-            }
+            } while (task != null);
         }
 
         private Consumer<Releasable> pollNextTaskOrReleasePermit() {
             var task = queue.poll();
             if (task == null) {
                 semaphore.release();
+                while (queue.peek() != null && semaphore.tryAcquire()) {
+                    task = queue.poll();
+                    if (task == null) {
+                        semaphore.release();
+                    } else {
+                        return task;
+                    }
+                }
             }
             return task;
         }