Expand wildcards to closed indices in /_cat/indices (#18545)

Closed indices are already displayed when no indices are explicitly selected. This commit ensures that closed indices are also shown when wildcard filtering is used. It also addresses another issue that is caused by the fact that the cat action is based internally on 3 different cluster states (one when we query the cluster state to get all indices, one when we query cluster health, and one when we query indices stats). We currently fail the cat request when the user specifies a concrete index as parameter that does not exist. The implementation works as intended in that regard. It checks this not only for the first cluster state request, but also the subsequent indices stats one. This means that if the index is deleted before the cat action has queried the indices stats, it rightfully fails. In case the user provides wildcards (or no parameter at all), however, we fail the indices stats as we pass the resolved concrete indices to the indices stats request and fail to distinguish whether these indices have been resolved by wildcards or explicitly requested by the user. This means that if an index has been deleted before the indices stats request gets to execute, we fail the overall cat request. The fix is to let the indices stats request do the resolving again and not pass the concrete indices.

Closes #16419
Closes #17395

core/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java

@@ -74,27 +74,41 @@ public void doRequest(final RestRequest request, final RestChannel channel, fina
         clusterStateRequest.clear().indices(indices).metaData(true);
         clusterStateRequest.local(request.paramAsBoolean("local", clusterStateRequest.local()));
         clusterStateRequest.masterNodeTimeout(request.paramAsTime("master_timeout", clusterStateRequest.masterNodeTimeout()));
+        final IndicesOptions strictExpandIndicesOptions = IndicesOptions.strictExpand();
+        clusterStateRequest.indicesOptions(strictExpandIndicesOptions);
 
         client.admin().cluster().state(clusterStateRequest, new RestActionListener<ClusterStateResponse>(channel) {
             @Override
             public void processResponse(final ClusterStateResponse clusterStateResponse) {
-                ClusterState state = clusterStateResponse.getState();
-                final IndicesOptions concreteIndicesOptions = IndicesOptions.fromOptions(false, true, true, true);
-                final String[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, concreteIndicesOptions, indices);
-                final String[] openIndices = indexNameExpressionResolver.concreteIndices(state, IndicesOptions.lenientExpandOpen(), indices);
-                ClusterHealthRequest clusterHealthRequest = Requests.clusterHealthRequest(openIndices);
+                final ClusterState state = clusterStateResponse.getState();
+                final String[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, strictExpandIndicesOptions, indices);
+                // concreteIndices should contain exactly the indices in state.metaData() that were selected by clusterStateRequest using
+                // IndicesOptions.strictExpand(). We select the indices again here so that they can be displayed in the resulting table
+                // in the requesting order.
+                assert concreteIndices.length == state.metaData().getIndices().size();
+
+                // Indices that were successfully resolved during the cluster state request might be deleted when the subsequent cluster
+                // health and indices stats requests execute. We have to distinguish two cases:
+                // 1) the deleted index was explicitly passed as parameter to the /_cat/indices request. In this case we want the subsequent
+                //    requests to fail.
+                // 2) the deleted index was resolved as part of a wildcard or _all. In this case, we want the subsequent requests not to
+                //    fail on the deleted index (as we want to ignore wildcards that cannot be resolved).
+                // This behavior can be ensured by letting the cluster health and indices stats requests re-resolve the index names with the
+                // same indices options that we used for the initial cluster state request (strictExpand). Unfortunately cluster health
+                // requests hard-code their indices options and the best we can do is apply strictExpand to the indices stats request.
+                ClusterHealthRequest clusterHealthRequest = Requests.clusterHealthRequest(indices);
                 clusterHealthRequest.local(request.paramAsBoolean("local", clusterHealthRequest.local()));
                 client.admin().cluster().health(clusterHealthRequest, new RestActionListener<ClusterHealthResponse>(channel) {
                     @Override
                     public void processResponse(final ClusterHealthResponse clusterHealthResponse) {
                         IndicesStatsRequest indicesStatsRequest = new IndicesStatsRequest();
-                        indicesStatsRequest.indices(concreteIndices);
-                        indicesStatsRequest.indicesOptions(concreteIndicesOptions);
+                        indicesStatsRequest.indices(indices);
+                        indicesStatsRequest.indicesOptions(strictExpandIndicesOptions);
                         indicesStatsRequest.all();
                         client.admin().indices().stats(indicesStatsRequest, new RestResponseListener<IndicesStatsResponse>(channel) {
                             @Override
                             public RestResponse buildResponse(IndicesStatsResponse indicesStatsResponse) throws Exception {
-                                Table tab = buildTable(request, concreteIndices, clusterHealthResponse, indicesStatsResponse, clusterStateResponse.getState().metaData());
+                                Table tab = buildTable(request, concreteIndices, clusterHealthResponse, indicesStatsResponse, state.metaData());
                                 return RestTable.buildResponse(tab, channel);
                             }
                         });

rest-api-spec/src/main/resources/rest-api-spec/test/cat.indices/10_basic.yaml

@@ -24,7 +24,7 @@
   - match:
       $body: |
                /^(green  \s+
-                  (open|close) \s+
+                  open   \s+
                   index1 \s+
                   1      \s+
                   0      \s+
@@ -49,3 +49,24 @@
                   (\d\d\d\d\-\d\d\-\d\dT\d\d:\d\d:\d\d.\d\d\dZ) \s*
                 )
                 $/
+  - do:
+      indices.close:
+        index: index1
+
+  - do:
+      cat.indices:
+        index: index*
+
+  - match:
+      $body: |
+               /^(       \s+
+                  close  \s+
+                  index1 \s+
+                         \s+
+                         \s+
+                         \s+
+                         \s+
+                         \s+
+                         \s*
+                )
+                $/