Skip to content

Commit

Permalink
Abstract waitForInit to AbstractSecurityUnitTest and increase configu…
Browse files Browse the repository at this point in the history 
…ration load timeout to 10s

Signed-off-by: Craig Perkins <cwperx@amazon.com>
  • Loading branch information
@cwperks
cwperks committed Jul 19, 2022
1 parent f153c27 commit 187a2ef
Show file tree
Hide file tree
Showing 9 changed files with 46 additions and 85 deletions.
4 changes: 2 additions & 2 deletions src/main/java/org/opensearch/security/configuration/ConfigurationRepository.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -363,13 +363,13 @@ public Map<CType, SecurityDynamicConfiguration<?>> getConfigurationsFromIndex(Co
} else {
LOGGER.debug("security index exists and was created with ES 7 (new layout)");
}
retVal.putAll(validate(cl.load(configTypes.toArray(new CType[0]), 5, TimeUnit.SECONDS, acceptInvalid), configTypes.size()));
retVal.putAll(validate(cl.load(configTypes.toArray(new CType[0]), 10, TimeUnit.SECONDS, acceptInvalid), configTypes.size()));


} else {
//wait (and use new layout)
LOGGER.debug("security index not exists (yet)");
retVal.putAll(validate(cl.load(configTypes.toArray(new CType[0]), 5, TimeUnit.SECONDS, acceptInvalid), configTypes.size()));
retVal.putAll(validate(cl.load(configTypes.toArray(new CType[0]), 10, TimeUnit.SECONDS, acceptInvalid), configTypes.size()));
}

} catch (Exception e) {
Expand Down
12 changes: 0 additions & 12 deletions src/test/java/org/opensearch/security/RolesInjectorIntegTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,18 +74,6 @@ public Collection<Object> createComponents(Client client, ClusterService cluster
}
}

//Wait for the security plugin to load roles.
private void waitForInit(Client client) throws Exception {
try {
client.admin().cluster().health(new ClusterHealthRequest()).actionGet();
} catch (OpenSearchSecurityException ex) {
if(ex.getMessage().contains("OpenSearch Security not initialized")) {
Thread.sleep(500);
waitForInit(client);
}
}
}

@Test
public void testRolesInject() throws Exception {
setup(Settings.EMPTY, new DynamicSecurityConfig().setSecurityRoles("roles.yml"), Settings.EMPTY);
Expand Down
13 changes: 0 additions & 13 deletions src/test/java/org/opensearch/security/RolesValidationIntegTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
import org.junit.Test;

import org.opensearch.OpenSearchSecurityException;
import org.opensearch.action.admin.cluster.health.ClusterHealthRequest;
import org.opensearch.action.admin.indices.create.CreateIndexRequest;
import org.opensearch.action.admin.indices.create.CreateIndexResponse;
import org.opensearch.action.admin.indices.exists.indices.IndicesExistsRequest;
Expand Down Expand Up @@ -71,18 +70,6 @@ public Collection<Object> createComponents(Client client, ClusterService cluster
}
}

//Wait for the security plugin to load roles.
private void waitForInit(Client client) throws Exception {
try {
client.admin().cluster().health(new ClusterHealthRequest()).actionGet();
} catch (OpenSearchSecurityException ex) {
if(ex.getMessage().contains("OpenSearch Security not initialized")) {
Thread.sleep(500);
waitForInit(client);
}
}
}

@Test
public void testRolesValidation() throws Exception {
setup(Settings.EMPTY, new DynamicSecurityConfig().setSecurityRoles("roles.yml"), Settings.EMPTY);
Expand Down
13 changes: 0 additions & 13 deletions src/test/java/org/opensearch/security/TransportUserInjectorIntegTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
import org.junit.Test;

import org.opensearch.OpenSearchSecurityException;
import org.opensearch.action.admin.cluster.health.ClusterHealthRequest;
import org.opensearch.action.admin.indices.create.CreateIndexRequest;
import org.opensearch.action.admin.indices.create.CreateIndexResponse;
import org.opensearch.client.Client;
Expand Down Expand Up @@ -67,18 +66,6 @@ public Collection<Object> createComponents(Client client, ClusterService cluster
}
}

//Wait for the security plugin to load roles.
private void waitForInit(Client client) throws Exception {
try {
client.admin().cluster().health(new ClusterHealthRequest()).actionGet();
} catch (OpenSearchSecurityException ex) {
if(ex.getMessage().contains("OpenSearch Security not initialized")) {
Thread.sleep(500);
waitForInit(client);
}
}
}

@Test
public void testSecurityUserInjection() throws Exception {
final Settings clusterNodeSettings = Settings.builder()
Expand Down
4 changes: 2 additions & 2 deletions src/test/java/org/opensearch/security/auditlog/integration/TestAuditlogImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ public static List<AuditMessage> doThenWaitForMessages(final Runnable action, fi
throw new MessagesNotFoundException(expectedCount, messages);
}
if (messages.size() != expectedCount) {
throw new RuntimeException("Unexpected number of messages, was expecting " + expectedCount + ", recieved " + messages.size());
throw new RuntimeException("Unexpected number of messages, was expecting " + expectedCount + ", received " + messages.size());
}
} catch (final InterruptedException e) {
throw new RuntimeException("Unexpected exception", e);
Expand Down Expand Up @@ -119,7 +119,7 @@ public List<AuditMessage> getFoundMessages() {

private static String createDetailMessage(final int expectedCount, final List<AuditMessage> foundMessages) {
return new StringBuilder()
.append("Did not recieve all " + expectedCount + " audit messages after a short wait. ")
.append("Did not receive all " + expectedCount + " audit messages after a short wait. ")
.append("Missing " + (expectedCount - foundMessages.size()) + " messages.")
.append("Messages found during this time: \n\n")
.append(foundMessages.stream()
Expand Down
22 changes: 2 additions & 20 deletions src/test/java/org/opensearch/security/ccstest/CrossClusterSearchTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -982,24 +982,6 @@ public void testCcsWithDiffCertsWithNodesDnDynamicallyAdded() throws Exception {
assertThat(ccs.getBody(), containsString("cross_cluster_two:twitter"));
}

//Wait for the security plugin to load roles.
private void waitOrThrow(Client client) throws Exception {
int failures = 0;
while(failures < 5) {
try {
client.admin().cluster().health(new ClusterHealthRequest()).actionGet();
break;
} catch (OpenSearchSecurityException ex) {
if (ex.getMessage().contains("OpenSearch Security not initialized")) {
Thread.sleep(500);
failures++;
} else {
throw ex;
}
}
}
}

@Test
public void testCcsWithRoleInjection() throws Exception {
setupCcs(new DynamicSecurityConfig().setSecurityRoles("roles.yml"));
Expand Down Expand Up @@ -1041,7 +1023,7 @@ public void testCcsWithRoleInjection() throws Exception {
RolesInjectorIntegTest.RolesInjectorPlugin.injectedRoles = "invalid_user|invalid_role";
try (Node node = new PluginAwareNode(false, tcSettings, Netty4Plugin.class,
OpenSearchSecurityPlugin.class, RolesInjectorIntegTest.RolesInjectorPlugin.class).start()) {
waitOrThrow(node.client());
waitForInit(node.client());
Client remoteClient = node.client().getRemoteClusterClient("cross_cluster_two");
GetRequest getReq = new GetRequest("twitter", "0");
getReq.realtime(true);
Expand All @@ -1061,7 +1043,7 @@ public void testCcsWithRoleInjection() throws Exception {
RolesInjectorIntegTest.RolesInjectorPlugin.injectedRoles = "valid_user|opendistro_security_all_access";
try (Node node = new PluginAwareNode(false, tcSettings, Netty4Plugin.class,
OpenSearchSecurityPlugin.class, RolesInjectorIntegTest.RolesInjectorPlugin.class).start()) {
waitOrThrow(node.client());
waitForInit(node.client());
Client remoteClient = node.client().getRemoteClusterClient("cross_cluster_two");
GetRequest getReq = new GetRequest("twitter", "0");
getReq.realtime(true);
Expand Down
30 changes: 8 additions & 22 deletions src/test/java/org/opensearch/security/dlic/dlsfls/CCReplicationTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -156,24 +156,6 @@ protected void doExecute(Task task, MockReplicationRequest request, ActionListen
}
}

//Wait for the security plugin to load roles.
private void waitOrThrow(Client client, String index) throws Exception {
int failures = 0;
while(failures < 5) {
try {
client.execute(MockReplicationAction.INSTANCE, new MockReplicationRequest(index)).actionGet();
break;
} catch (OpenSearchSecurityException ex) {
if (ex.getMessage().contains("OpenSearch Security not initialized")) {
Thread.sleep(500);
failures++;
} else {
throw ex;
}
}
}
}

void populateData(Client tc) {
tc.index(new IndexRequest("hr-dls").id("0").setRefreshPolicy(RefreshPolicy.IMMEDIATE)
.source("{\"User\": \"testuser\",\"Date\":\"2021-01-18T17:27:20Z\",\"Designation\":\"HR\"}", XContentType.JSON)).actionGet();
Expand Down Expand Up @@ -212,7 +194,8 @@ public void testReplication() throws Exception {
// Set roles for the user
MockReplicationPlugin.injectedRoles = "ccr_user|opendistro_security_human_resources_trainee";
try (Node node = new PluginAwareNode(false, tcSettings, Netty4Plugin.class, OpenSearchSecurityPlugin.class, MockReplicationPlugin.class).start()) {
waitOrThrow(node.client(), "hr-dls");
waitForInit(node.client());
node.client().execute(MockReplicationAction.INSTANCE, new MockReplicationRequest("hr-dls")).actionGet();
Assert.fail("Expecting exception");
} catch (OpenSearchSecurityException ex) {
log.warn(ex.getMessage());
Expand All @@ -222,7 +205,8 @@ public void testReplication() throws Exception {
}

try (Node node = new PluginAwareNode(false, tcSettings, Netty4Plugin.class, OpenSearchSecurityPlugin.class, MockReplicationPlugin.class).start()) {
waitOrThrow(node.client(), "hr-fls");
waitForInit(node.client());
node.client().execute(MockReplicationAction.INSTANCE, new MockReplicationRequest("hr-fls")).actionGet();
Assert.fail("Expecting exception");
} catch (OpenSearchSecurityException ex) {
log.warn(ex.getMessage());
Expand All @@ -232,7 +216,8 @@ public void testReplication() throws Exception {
}

try (Node node = new PluginAwareNode(false, tcSettings, Netty4Plugin.class, OpenSearchSecurityPlugin.class, MockReplicationPlugin.class).start()) {
waitOrThrow(node.client(), "hr-masking");
waitForInit(node.client());
node.client().execute(MockReplicationAction.INSTANCE, new MockReplicationRequest("hr-masking")).actionGet();
Assert.fail("Expecting exception");
} catch (OpenSearchSecurityException ex) {
log.warn(ex.getMessage());
Expand All @@ -242,7 +227,8 @@ public void testReplication() throws Exception {
}

try (Node node = new PluginAwareNode(false, tcSettings, Netty4Plugin.class, OpenSearchSecurityPlugin.class, MockReplicationPlugin.class).start()) {
waitOrThrow(node.client(), "hr-normal");
waitForInit(node.client());
node.client().execute(MockReplicationAction.INSTANCE, new MockReplicationRequest("hr-normal")).actionGet();
AcknowledgedResponse res = node.client().execute(MockReplicationAction.INSTANCE, new MockReplicationRequest("hr-normal")).actionGet();
Assert.assertTrue(res.isAcknowledged());
}
Expand Down
1 change: 1 addition & 0 deletions src/test/java/org/opensearch/security/dlic/rest/api/TenantInfoActionTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ public void testTenantInfoAPIAccess() throws Exception {
public void testTenantInfoAPIUpdate() throws Exception {
Settings settings = Settings.builder().put(ConfigConstants.SECURITY_UNSUPPORTED_RESTAPI_ALLOW_SECURITYCONFIG_MODIFICATION, true).build();
setup(settings);

rh.keystore = "restapi/kirk-keystore.jks";
rh.sendHTTPClientCredentials = true;
rh.sendAdminCertificate = true;
Expand Down
32 changes: 31 additions & 1 deletion src/test/java/org/opensearch/security/test/AbstractSecurityUnitTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
import java.util.Base64;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.concurrent.atomic.AtomicLong;

import javax.net.ssl.SSLContext;
Expand All @@ -57,6 +58,8 @@
import org.junit.rules.TestName;
import org.junit.rules.TestWatcher;

import org.opensearch.OpenSearchSecurityException;
import org.opensearch.action.admin.cluster.health.ClusterHealthRequest;
import org.opensearch.action.admin.cluster.node.info.NodesInfoRequest;
import org.opensearch.action.admin.indices.create.CreateIndexRequest;
import org.opensearch.action.index.IndexRequest;
Expand All @@ -83,7 +86,7 @@

/*
* There are real thread leaks during test execution, not all threads are
* properly waited on or interupted. While this normally doesn't create test
* properly waited on or interrupted. While this normally doesn't create test
* failures, retries mitigate this. Remove this attribute to explore these
* issues.
*/
Expand Down Expand Up @@ -164,6 +167,33 @@ protected RestHighLevelClient getRestClient(ClusterInfo info, String keyStoreNam
}
}

/** Wait for the security plugin to load roles. */
public void waitForInit(Client client) {
int maxRetries = 5;
Optional<Exception> retainedException = Optional.empty();
for (int i = 0; i < maxRetries; i++) {
try {
client.admin().cluster().health(new ClusterHealthRequest()).actionGet();
retainedException = Optional.empty();
return;
} catch (OpenSearchSecurityException ex) {
if(ex.getMessage().contains("OpenSearch Security not initialized")) {
retainedException = Optional.of(ex);
try {
Thread.sleep(500);
} catch (InterruptedException e) { /* ignored */ }
} else {
// plugin is initialized, but another error received.
// Example could be user does not have permissions for cluster:monitor/health
retainedException = Optional.empty();
}
}
}
if (retainedException.isPresent()) {
throw new RuntimeException(retainedException.get());
}
}

protected void initialize(ClusterHelper clusterHelper, ClusterInfo clusterInfo, DynamicSecurityConfig securityConfig) throws IOException {
try (Client tc = clusterHelper.nodeClient()) {
Assert.assertEquals(clusterInfo.numNodes,
Expand Down

0 comments on commit 187a2ef

Please sign in to comment.