Fix flaky test SlowIntegrationTests.testDelayInSecurityIndexInitializ…

…ation (#3763)

Signed-off-by: Craig Perkins <cwperx@amazon.com>

DEVELOPER_GUIDE.md

@@ -232,11 +232,30 @@ curl -XGET https://localhost:9200/_plugins/_security/authinfo -u 'admin:admin' -
 
 Launch IntelliJ IDEA, choose **Project from Existing Sources**, and select directory with Gradle build script (`build.gradle`).
 
-## Running integration tests
+## Running tests
 
 Locally these can be run with `./gradlew test` with detailed results being available at `${project-root}/build/reports/tests/test/index.html`. You can also run tests through an IDEs JUnit test runner.
 
-Integration tests are automatically run on all pull requests for all supported versions of the JDK. These must pass for change(s) to be merged. Detailed logs of these test results are available by going to the GitHub Actions workflow summary view and downloading the workflow run of the tests. If you see multiple tests listed with different JDK versions, you can download the version with whichever JDK you are interested in. After extracting the test file on your local machine, integration tests results can be found at `./tests/tests/index.html`.
+Tests are automatically run on all pull requests for all supported versions of the JDK. These must pass for change(s) to be merged. Detailed logs of these test results are available by going to the GitHub Actions workflow summary view and downloading the workflow run of the tests. If you see multiple tests listed with different JDK versions, you can download the version with whichever JDK you are interested in. After extracting the test file on your local machine, integration tests results can be found at `./tests/tests/index.html`.
+
+### Running an individual test multiple times
+
+This repo has a `@Repeat` annotation which you can import to annotate a test to run many times repeatedly. To use the annotation, add the following code to your test suite.
+
+```
+@Rule
+public RepeatRule repeatRule = new RepeatRule();
+
+@Test
+@Repeat(10)
+public void testMethod() {
+    ...
+}
+```
+
+## Running tests in the integrationTest package
+
+Tests in the integrationTest package can be run with `./gradlew integrationTest`.
 
 ### Bulk test runs
 

build.gradle

@@ -687,6 +687,9 @@ dependencies {
     testImplementation "org.springframework:spring-beans:${spring_version}"
     testImplementation 'org.junit.jupiter:junit-jupiter:5.10.1'
     testImplementation 'org.junit.jupiter:junit-jupiter-api:5.10.1'
+    testImplementation('org.awaitility:awaitility:4.2.0') {
+        exclude(group: 'org.hamcrest', module: 'hamcrest')
+    }
     // Only osx-x86_64, osx-aarch_64, linux-x86_64, linux-aarch_64, windows-x86_64 are available
     if (osdetector.classifier in ["osx-x86_64", "osx-aarch_64", "linux-x86_64", "linux-aarch_64", "windows-x86_64"]) {
         testImplementation "io.netty:netty-tcnative-classes:2.0.61.Final"

src/test/java/org/opensearch/security/SlowIntegrationTests.java

@@ -26,14 +26,20 @@
 
 package org.opensearch.security;
 
+import java.io.IOException;
+
 import com.google.common.collect.Lists;
+import org.awaitility.Awaitility;
 import org.junit.Assert;
 import org.junit.Test;
 
 import org.opensearch.action.admin.cluster.health.ClusterHealthRequest;
 import org.opensearch.action.admin.cluster.node.info.NodesInfoRequest;
+import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
+import org.opensearch.action.admin.indices.create.CreateIndexRequest;
 import org.opensearch.cluster.health.ClusterHealthStatus;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.common.unit.TimeValue;
 import org.opensearch.node.Node;
 import org.opensearch.node.PluginAwareNode;
 import org.opensearch.security.ssl.util.SSLConfigConstants;
@@ -42,8 +48,12 @@
 import org.opensearch.security.test.SingleClusterTest;
 import org.opensearch.security.test.helper.cluster.ClusterConfiguration;
 import org.opensearch.security.test.helper.file.FileHelper;
+import org.opensearch.security.test.helper.rest.RestHelper;
 import org.opensearch.transport.Netty4ModulePlugin;
 
+import static org.hamcrest.Matchers.equalTo;
+import static org.junit.Assert.assertThrows;
+
 public class SlowIntegrationTests extends SingleClusterTest {
 
     @Test
@@ -210,4 +220,41 @@ public void testNodeClientDisallowedWithNonServerCertificate2() throws Exception
             Assert.fail(e.toString());
         }
     }
+
+    @Test
+    public void testDelayInSecurityIndexInitialization() throws Exception {
+        final Settings settings = Settings.builder()
+            .put(ConfigConstants.SECURITY_ALLOW_DEFAULT_INIT_SECURITYINDEX, true)
+            .put("cluster.routing.allocation.exclude._ip", "127.0.0.1")
+            .build();
+        assertThrows(IOException.class, () -> {
+            setup(Settings.EMPTY, null, settings, false);
+            clusterHelper.nodeClient()
+                .admin()
+                .indices()
+                .create(new CreateIndexRequest("test-index").timeout(TimeValue.timeValueSeconds(10)))
+                .actionGet();
+            clusterHelper.waitForCluster(ClusterHealthStatus.GREEN, TimeValue.timeValueSeconds(5), ClusterConfiguration.DEFAULT.getNodes());
+        });
+        // Ideally, we would want to remove this cluster setting, but default settings cannot be removed. So overriding with a reserved
+        // IP address
+        clusterHelper.nodeClient()
+            .admin()
+            .cluster()
+            .updateSettings(
+                new ClusterUpdateSettingsRequest().transientSettings(
+                    Settings.builder().put("cluster.routing.allocation.exclude._ip", "192.0.2.0").build()
+                )
+            );
+        this.clusterInfo = clusterHelper.waitForCluster(ClusterHealthStatus.GREEN, TimeValue.timeValueSeconds(10), 3);
+        RestHelper rh = nonSslRestHelper();
+        Awaitility.await()
+            .alias("Wait until Security is initialized")
+            .until(
+                () -> rh.executeGetRequest("/_plugins/_security/health", encodeBasicHeader("admin", "admin"))
+                    .getTextFromJsonBody("/status"),
+                equalTo("UP")
+            );
+    }
+
 }

src/test/java/org/opensearch/security/util/Repeat.java

@@ -0,0 +1,25 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+package org.opensearch.security.util;
+
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+import static java.lang.annotation.ElementType.ANNOTATION_TYPE;
+import static java.lang.annotation.ElementType.METHOD;
+
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ METHOD, ANNOTATION_TYPE })
+public @interface Repeat {
+    int value() default 1;
+}

src/test/java/org/opensearch/security/util/RepeatRule.java

@@ -0,0 +1,48 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+package org.opensearch.security.util;
+
+import org.junit.rules.TestRule;
+import org.junit.runner.Description;
+import org.junit.runners.model.Statement;
+
+public class RepeatRule implements TestRule {
+
+    private static class RepeatStatement extends Statement {
+        private final Statement statement;
+        private final int repeat;
+
+        public RepeatStatement(Statement statement, int repeat) {
+            this.statement = statement;
+            this.repeat = repeat;
+        }
+
+        @Override
+        public void evaluate() throws Throwable {
+            for (int i = 0; i < repeat; i++) {
+                statement.evaluate();
+            }
+        }
+
+    }
+
+    @Override
+    public Statement apply(Statement statement, Description description) {
+        Statement result = statement;
+        Repeat repeat = description.getAnnotation(Repeat.class);
+        if (repeat != null) {
+            int times = repeat.value();
+            result = new RepeatStatement(statement, times);
+        }
+        return result;
+    }
+}