Make watch history indices hidden

This commit updates the template used for watch history indices with
hidden index setting so that new indices will be created as hidden.

Additionally, some failing tests were encountered where a search would
find the documents in the history index but a subsequent search would
fail to find the documents. This is most likely due to different
refresh times between the primary and replica shards. The failures
were resolved by using an assertsBusy to retrieve the documents.

Relates elastic#50251

server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java

@@ -118,7 +118,6 @@ public class MetaDataCreateIndexService {
      * These index patterns will be converted to hidden indices, at which point they should be removed from this list.
      */
     private static final CharacterRunAutomaton DOT_INDICES_EXCLUSIONS = new CharacterRunAutomaton(Regex.simpleMatchToAutomaton(
-        ".watch-history-*",
         ".data-frame-notifications-*",
         ".transform-notifications-*"
     ));

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/WatcherIndexTemplateRegistryField.java

@@ -15,8 +15,9 @@ public final class WatcherIndexTemplateRegistryField {
     // version 8: fix slack attachment property not to be dynamic, causing field type issues
     // version 9: add a user field defining which user executed the watch
     // version 10: add support for foreach path in actions
+    // version 11: watch history indices are hidden
     // Note: if you change this, also inform the kibana team around the watcher-ui
-    public static final int INDEX_TEMPLATE_VERSION = 10;
+    public static final int INDEX_TEMPLATE_VERSION = 11;
     public static final String HISTORY_TEMPLATE_NAME = ".watch-history-" + INDEX_TEMPLATE_VERSION;
     public static final String HISTORY_TEMPLATE_NAME_NO_ILM = ".watch-history-no-ilm-" + INDEX_TEMPLATE_VERSION;
     public static final String TRIGGERED_TEMPLATE_NAME = ".triggered_watches";

x-pack/plugin/core/src/main/resources/watch-history-no-ilm.json

@@ -5,6 +5,7 @@
     "index.number_of_shards": 1,
     "index.number_of_replicas": 0,
     "index.auto_expand_replicas": "0-1",
+    "index.hidden": true,
     "index.format": 6
   },
   "mappings": {

x-pack/plugin/core/src/main/resources/watch-history.json

@@ -6,6 +6,7 @@
     "index.number_of_replicas": 0,
     "index.auto_expand_replicas": "0-1",
     "index.lifecycle.name": "watch-history-ilm-policy",
+    "index.hidden": true,
     "index.format": 6
   },
   "mappings": {

x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/email/EmailAttachmentTests.java

@@ -187,10 +187,12 @@ public void testThatEmailAttachmentsAreSent() throws Exception {
         timeWarp().trigger("_test_id");
         refresh();
 
-        SearchResponse searchResponse = client().prepareSearch(HistoryStoreField.INDEX_PREFIX_WITH_TEMPLATE + "*")
+        assertBusy(() -> {
+            SearchResponse searchResponse = client().prepareSearch(HistoryStoreField.INDEX_PREFIX_WITH_TEMPLATE + "*")
                 .setQuery(QueryBuilders.termQuery("watch_id", "_test_id"))
                 .execute().actionGet();
-        assertHitCount(searchResponse, 1);
+            assertHitCount(searchResponse, 1);
+        });
 
         if (!latch.await(5, TimeUnit.SECONDS)) {
             fail("waited too long for email to be received");

x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookHttpsIntegrationTests.java

@@ -6,6 +6,7 @@
 package org.elasticsearch.xpack.watcher.actions.webhook;
 
 import com.sun.net.httpserver.HttpsServer;
+import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.bootstrap.JavaVersion;
 import org.elasticsearch.common.settings.Settings;
@@ -34,6 +35,7 @@
 import java.security.PrivilegedAction;
 import java.util.List;
 
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
 import static org.elasticsearch.xpack.watcher.client.WatchSourceBuilders.watchBuilder;
 import static org.elasticsearch.xpack.watcher.input.InputBuilders.simpleInput;
@@ -99,10 +101,16 @@ public void testHttps() throws Exception {
         assertThat(webServer.requests().get(0).getUri().getPath(), equalTo("/test/_id"));
         assertThat(webServer.requests().get(0).getBody(), equalTo("{key=value}"));
 
-        SearchResponse response =
+        final SetOnce<SearchResponse> responseSetOnce = new SetOnce<>();
+        assertBusy(() -> {
+            SearchResponse response =
                 searchWatchRecords(b -> b.setQuery(QueryBuilders.termQuery(WatchRecord.STATE.getPreferredName(), "executed")));
+            assertNoFailures(response);
+            assertHitCount(response, 1L);
+            responseSetOnce.set(response);
+        });
+        final SearchResponse response = responseSetOnce.get();
 
-        assertNoFailures(response);
         XContentSource source = xContentSource(response.getHits().getAt(0).getSourceRef());
         String body = source.getValue("result.actions.0.webhook.response.body");
         assertThat(body, notNullValue());

x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryActionConditionTests.java

@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.watcher.history;
 
+import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse;
@@ -104,11 +105,15 @@ public void testActionConditionWithHardFailures() throws Exception {
 
         assertWatchWithMinimumActionsCount(id, ExecutionState.EXECUTED, 1);
 
-        // only one action should have failed via condition
-        final SearchResponse response = searchHistory(SearchSourceBuilder.searchSource().query(termQuery("watch_id", id)));
-        assertThat(response.getHits().getTotalHits().value, is(1L));
+        final SetOnce<SearchResponse> responseSetOnce = new SetOnce<>();
+        assertBusy(() -> {
+            final SearchResponse response = searchHistory(SearchSourceBuilder.searchSource().query(termQuery("watch_id", id)));
+            assertThat(response.getHits().getTotalHits().value, is(1L));
+            responseSetOnce.set(response);
+        });
 
-        final SearchHit hit = response.getHits().getAt(0);
+        // only one action should have failed via condition
+        final SearchHit hit = responseSetOnce.get().getHits().getAt(0);
         final List<Object> actions = getActionsFromHit(hit.getSourceAsMap());
 
         for (int i = 0; i < actionConditionsWithFailure.size(); ++i) {
@@ -199,11 +204,15 @@ public void testActionCondition() throws Exception {
 
         assertWatchWithMinimumActionsCount(id, ExecutionState.EXECUTED, 1);
 
-        // all actions should be successful
-        final SearchResponse response = searchHistory(SearchSourceBuilder.searchSource().query(termQuery("watch_id", id)));
-        assertThat(response.getHits().getTotalHits().value, is(1L));
+        final SetOnce<SearchResponse> responseSetOnce = new SetOnce<>();
+        assertBusy(() -> {
+            final SearchResponse response = searchHistory(SearchSourceBuilder.searchSource().query(termQuery("watch_id", id)));
+            assertThat(response.getHits().getTotalHits().value, is(1L));
+            responseSetOnce.set(response);
+        });
 
-        final SearchHit hit = response.getHits().getAt(0);
+        // all actions should be successful
+        final SearchHit hit = responseSetOnce.get().getHits().getAt(0);
         final List<Object> actions = getActionsFromHit(hit.getSourceAsMap());
 
         for (int i = 0; i < actionConditions.size(); ++i) {

x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateEmailMappingsTests.java

@@ -20,6 +20,8 @@
 import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase;
 import org.junit.After;
 
+import java.util.concurrent.atomic.AtomicReference;
+
 import static org.elasticsearch.search.aggregations.AggregationBuilders.terms;
 import static org.elasticsearch.search.builder.SearchSourceBuilder.searchSource;
 import static org.elasticsearch.xpack.watcher.actions.ActionBuilders.emailAction;
@@ -88,14 +90,21 @@ public void testEmailFields() throws Exception {
         // the action should fail as no email server is available
         assertWatchWithMinimumActionsCount("_id", ExecutionState.EXECUTED, 1);
 
-        SearchResponse response = client().prepareSearch(HistoryStoreField.INDEX_PREFIX_WITH_TEMPLATE + "*").setSource(searchSource()
-                .aggregation(terms("from").field("result.actions.email.message.from"))
-                .aggregation(terms("to").field("result.actions.email.message.to"))
-                .aggregation(terms("cc").field("result.actions.email.message.cc"))
-                .aggregation(terms("bcc").field("result.actions.email.message.bcc"))
-                .aggregation(terms("reply_to").field("result.actions.email.message.reply_to")))
-                .get();
-
+        final AtomicReference<SearchResponse> responseRef = new AtomicReference<>();
+        assertBusy(() -> {
+            SearchResponse response = client().prepareSearch(HistoryStoreField.INDEX_PREFIX_WITH_TEMPLATE + "*").setSource(searchSource()
+                    .aggregation(terms("from").field("result.actions.email.message.from"))
+                    .aggregation(terms("to").field("result.actions.email.message.to"))
+                    .aggregation(terms("cc").field("result.actions.email.message.cc"))
+                    .aggregation(terms("bcc").field("result.actions.email.message.bcc"))
+                    .aggregation(terms("reply_to").field("result.actions.email.message.reply_to")))
+                    .get();
+            assertThat(response, notNullValue());
+            assertThat(response.getHits().getTotalHits().value, is(1L));
+            responseRef.set(response);
+        });
+
+        final SearchResponse response = responseRef.get();
         assertThat(response, notNullValue());
         assertThat(response.getHits().getTotalHits().value, is(1L));
         Aggregations aggs = response.getAggregations();

x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateIndexActionMappingsTests.java

@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.watcher.history;
 
+import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse;
 import org.elasticsearch.search.aggregations.Aggregations;
@@ -47,12 +48,17 @@ public void testIndexActionFields() throws Exception {
         flush();
         refresh();
 
-        SearchResponse response = client().prepareSearch(HistoryStoreField.INDEX_PREFIX_WITH_TEMPLATE + "*").setSource(searchSource()
-                .aggregation(terms("index_action_indices").field("result.actions.index.response.index")))
-                .get();
+        final SetOnce<SearchResponse> responseReference = new SetOnce<>();
+        assertBusy(() -> {
+            SearchResponse response = client().prepareSearch(HistoryStoreField.INDEX_PREFIX_WITH_TEMPLATE + "*").setSource(searchSource()
+                    .aggregation(terms("index_action_indices").field("result.actions.index.response.index")))
+                    .get();
+            assertThat(response, notNullValue());
+            assertThat(response.getHits().getTotalHits().value, is(1L));
+            responseReference.set(response);
+        });
 
-        assertThat(response, notNullValue());
-        assertThat(response.getHits().getTotalHits().value, is(1L));
+        final SearchResponse response = responseReference.get();
         Aggregations aggs = response.getAggregations();
         assertThat(aggs, notNullValue());
 

x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/WatchMetadataTests.java

@@ -61,10 +61,12 @@ public void testWatchMetadata() throws Exception {
         timeWarp().trigger("_name");
 
         refresh();
-        SearchResponse searchResponse = client().prepareSearch(HistoryStoreField.INDEX_PREFIX_WITH_TEMPLATE + "*")
+        assertBusy(() -> {
+            SearchResponse searchResponse = client().prepareSearch(HistoryStoreField.INDEX_PREFIX_WITH_TEMPLATE + "*")
                 .setQuery(termQuery("metadata.foo", "bar"))
                 .get();
-        assertThat(searchResponse.getHits().getTotalHits().value, greaterThan(0L));
+            assertThat(searchResponse.getHits().getTotalHits().value, greaterThan(0L));
+        });
     }
 
     public void testWatchMetadataAvailableAtExecution() throws Exception {