Skip to content

Commit

Permalink
added correlationAlert integ tests (opensearch-project#1099)
Browse files Browse the repository at this point in the history 
* added correlationAlert integ tests

Signed-off-by: Riya Saxena <riysaxen@amazon.com>

* added licences

Signed-off-by: Riya Saxena <riysaxen@amazon.com>

* fixed imports

Signed-off-by: Riya Saxena <riysaxen@amazon.com>

* deleted SecureCorrelationAlerts Tests, will add later

Signed-off-by: Riya Saxena <riysaxen@amazon.com>

---------

Signed-off-by: Riya Saxena <riysaxen@amazon.com>
  • Loading branch information
@riysaxen-amzn
riysaxen-amzn authored Jul 2, 2024
1 parent b99121e commit e8d7879
Show file tree
Hide file tree
Showing 6 changed files with 762 additions and 315 deletions.
359 changes: 347 additions & 12 deletions src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java
View file

Large diffs are not rendered by default.

12 changes: 12 additions & 0 deletions src/test/java/org/opensearch/securityanalytics/TestHelpers.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import org.opensearch.script.ScriptType;
import org.opensearch.securityanalytics.model.CorrelationQuery;
import org.opensearch.securityanalytics.model.CorrelationRule;
import org.opensearch.securityanalytics.model.CorrelationRuleTrigger;
import org.opensearch.securityanalytics.model.CustomLogType;
import org.opensearch.securityanalytics.model.Detector;
import org.opensearch.securityanalytics.model.DetectorInput;
Expand Down Expand Up @@ -230,6 +231,17 @@ public static CorrelationRule randomCorrelationRule(String name) {
), 300000L, null);
}

public static CorrelationRule randomCorrelationRuleWithTrigger(String name) {
name = name.isEmpty()? "><script>prompt(document.domain)</script>": name;
List<Action> actions = new ArrayList<Action>();
CorrelationRuleTrigger trigger = new CorrelationRuleTrigger("trigger-123", "Trigger 1", "high", actions);
return new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, name,
List.of(
new CorrelationQuery("vpc_flow1", "dstaddr:192.168.1.*", "network", null),
new CorrelationQuery("ad_logs1", "azure.platformlogs.result_type:50126", "ad_ldap", null)
), 300000L, trigger);
}

public static String randomRule() {
return "title: Remote Encrypting File System Abuse\n" +
"id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" +
Expand Down
303 changes: 0 additions & 303 deletions src/test/java/org/opensearch/securityanalytics/correlation/CorrelationEngineRestApiIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,6 @@
import org.opensearch.securityanalytics.SecurityAnalyticsPlugin;
import org.opensearch.securityanalytics.SecurityAnalyticsRestTestCase;
import org.opensearch.securityanalytics.TestHelpers;
import org.opensearch.securityanalytics.model.CorrelationQuery;
import org.opensearch.securityanalytics.model.CorrelationRule;
import org.opensearch.securityanalytics.model.CustomLogType;
import org.opensearch.securityanalytics.model.Detector;
import org.opensearch.securityanalytics.model.DetectorInput;
Expand All @@ -32,7 +30,6 @@
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.function.BooleanSupplier;
import java.util.stream.Collectors;

import static org.opensearch.securityanalytics.TestHelpers.*;

Expand Down Expand Up @@ -954,304 +951,4 @@ public void testBasicCorrelationEngineWorkflowWithCustomLogTypes() throws IOExce
);
}

private LogIndices createIndices() throws IOException {
LogIndices indices = new LogIndices();
indices.adLdapLogsIndex = createTestIndex("ad_logs", adLdapLogMappings());
indices.s3AccessLogsIndex = createTestIndex("s3_access_logs", s3AccessLogMappings());
indices.appLogsIndex = createTestIndex("app_logs", appLogMappings());
indices.windowsIndex = createTestIndex(randomIndex(), windowsIndexMapping());
indices.vpcFlowsIndex = createTestIndex("vpc_flow", vpcFlowMappings());
return indices;
}

private String createNetworkToWindowsFieldBasedRule(LogIndices indices) throws IOException {
CorrelationQuery query1 = new CorrelationQuery(indices.vpcFlowsIndex, null, "network", "srcaddr");
CorrelationQuery query4 = new CorrelationQuery(indices.windowsIndex, null, "test_windows", "SourceIp");

CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "network to windows", List.of(query1, query4), 300000L, null);
Request request = new Request("POST", "/_plugins/_security_analytics/correlation/rules");
request.setJsonEntity(toJsonString(rule));
Response response = client().performRequest(request);

Assert.assertEquals(201, response.getStatusLine().getStatusCode());
return entityAsMap(response).get("_id").toString();
}

private String createNetworkToWindowsFilterQueryBasedRule(LogIndices indices) throws IOException {
CorrelationQuery query1 = new CorrelationQuery(indices.vpcFlowsIndex, "srcaddr:1.2.3.4", "network", null);
CorrelationQuery query4 = new CorrelationQuery(indices.windowsIndex, "SourceIp:1.2.3.4", "test_windows", null);

CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "network to windows", List.of(query1, query4), 300000L, null);
Request request = new Request("POST", "/_plugins/_security_analytics/correlation/rules");
request.setJsonEntity(toJsonString(rule));
Response response = client().performRequest(request);

Assert.assertEquals(201, response.getStatusLine().getStatusCode());
return entityAsMap(response).get("_id").toString();
}

private String createNetworkToCustomLogTypeFieldBasedRule(LogIndices indices, String customLogTypeName, String customLogTypeIndex) throws IOException {
CorrelationQuery query1 = new CorrelationQuery(indices.vpcFlowsIndex, null, "network", "srcaddr");
CorrelationQuery query4 = new CorrelationQuery(customLogTypeIndex, null, customLogTypeName, "SourceIp");

CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "network to custom log type", List.of(query1, query4), 300000L, null);
Request request = new Request("POST", "/_plugins/_security_analytics/correlation/rules");
request.setJsonEntity(toJsonString(rule));
Response response = client().performRequest(request);

Assert.assertEquals(201, response.getStatusLine().getStatusCode());
return entityAsMap(response).get("_id").toString();
}

private String createNetworkToAdLdapToWindowsRule(LogIndices indices) throws IOException {
CorrelationQuery query1 = new CorrelationQuery(indices.vpcFlowsIndex, "dstaddr:4.5.6.7", "network", null);
CorrelationQuery query2 = new CorrelationQuery(indices.adLdapLogsIndex, "ResultType:50126", "ad_ldap", null);
CorrelationQuery query4 = new CorrelationQuery(indices.windowsIndex, "Domain:NTAUTHORI*", "test_windows", null);

CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "network to ad_ldap to windows", List.of(query1, query2, query4), 300000L, null);
Request request = new Request("POST", "/_plugins/_security_analytics/correlation/rules");
request.setJsonEntity(toJsonString(rule));
Response response = client().performRequest(request);

Assert.assertEquals(201, response.getStatusLine().getStatusCode());
return entityAsMap(response).get("_id").toString();
}

private String createWindowsToAppLogsToS3LogsRule(LogIndices indices) throws IOException {
CorrelationQuery query1 = new CorrelationQuery(indices.windowsIndex, "HostName:EC2AMAZ*", "test_windows", null);
CorrelationQuery query2 = new CorrelationQuery(indices.appLogsIndex, "endpoint:\\/customer_records.txt", "others_application", null);
CorrelationQuery query4 = new CorrelationQuery(indices.s3AccessLogsIndex, "aws.cloudtrail.eventName:ReplicateObject", "s3", null);

CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "windows to app_logs to s3 logs", List.of(query1, query2, query4), 300000L, null);
Request request = new Request("POST", "/_plugins/_security_analytics/correlation/rules");
request.setJsonEntity(toJsonString(rule));
Response response = client().performRequest(request);

Assert.assertEquals(201, response.getStatusLine().getStatusCode());
return entityAsMap(response).get("_id").toString();
}

private String createCloudtrailFieldBasedRule(String index, String field, Long timeWindow) throws IOException {
CorrelationQuery query1 = new CorrelationQuery(index, "EventName:CreateUser", "cloudtrail", field);
CorrelationQuery query2 = new CorrelationQuery(index, "EventName:DeleteUser", "cloudtrail", field);

CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "cloudtrail field based", List.of(query1, query2), timeWindow, null);
Request request = new Request("POST", "/_plugins/_security_analytics/correlation/rules");
request.setJsonEntity(toJsonString(rule));
Response response = client().performRequest(request);

Assert.assertEquals(201, response.getStatusLine().getStatusCode());
return entityAsMap(response).get("_id").toString();
}

@SuppressWarnings("unchecked")
private String createVpcFlowDetector(String indexName) throws IOException {
Detector vpcFlowDetector = randomDetectorWithInputsAndTriggersAndType(List.of(new DetectorInput("vpc flow detector for security analytics", List.of(indexName), List.of(),
getPrePackagedRules("network").stream().map(DetectorRule::new).collect(Collectors.toList()))),
List.of(new DetectorTrigger(null, "test-trigger", "1", List.of("network"), List.of(), List.of(), List.of(), List.of(), List.of())), "network");

Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(vpcFlowDetector));
Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse));

Map<String, Object> responseBody = asMap(createResponse);

String createdId = responseBody.get("_id").toString();

String request = "{\n" +
" \"query\" : {\n" +
" \"match\":{\n" +
" \"_id\": \"" + createdId + "\"\n" +
" }\n" +
" }\n" +
"}";
List<SearchHit> hits = executeSearch(Detector.DETECTORS_INDEX, request);
SearchHit hit = hits.get(0);

return ((List<String>) ((Map<String, Object>) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0);
}

@SuppressWarnings("unchecked")
private String createAdLdapDetector(String indexName) throws IOException {
// Execute CreateMappingsAction to add alias mapping for index
Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI);
// both req params and req body are supported
createMappingRequest.setJsonEntity(
"{\n" +
" \"index_name\": \"" + indexName + "\",\n" +
" \"rule_topic\": \"ad_ldap\",\n" +
" \"partial\": true,\n" +
" \"alias_mappings\": {\n" +
" \"properties\": {\n" +
" \"azure.signinlogs.properties.user_id\": {\n" +
" \"path\": \"azure.signinlogs.props.user_id\",\n" +
" \"type\": \"alias\"\n" +
" },\n" +
" \"azure-platformlogs-result_type\": {\n" +
" \"path\": \"azure.platformlogs.result_type\",\n" +
" \"type\": \"alias\"\n" +
" },\n" +
" \"azure-signinlogs-result_description\": {\n" +
" \"path\": \"azure.signinlogs.result_description\",\n" +
" \"type\": \"alias\"\n" +
" },\n" +
" \"timestamp\": {\n" +
" \"path\": \"creationTime\",\n" +
" \"type\": \"alias\"\n" +
" }\n" +
" }\n" +
" }\n" +
"}"
);

Response response = client().performRequest(createMappingRequest);
assertEquals(RestStatus.OK.getStatus(), response.getStatusLine().getStatusCode());

Detector adLdapDetector = randomDetectorWithInputsAndTriggersAndType(List.of(new DetectorInput("ad_ldap logs detector for security analytics", List.of(indexName), List.of(),
getPrePackagedRules("ad_ldap").stream().map(DetectorRule::new).collect(Collectors.toList()))),
List.of(new DetectorTrigger(null, "test-trigger", "1", List.of("ad_ldap"), List.of(), List.of(), List.of(), List.of(), List.of())), "ad_ldap");

Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(adLdapDetector));
Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse));

Map<String, Object> responseBody = asMap(createResponse);

String createdId = responseBody.get("_id").toString();

String request = "{\n" +
" \"query\" : {\n" +
" \"match\":{\n" +
" \"_id\": \"" + createdId + "\"\n" +
" }\n" +
" }\n" +
"}";
List<SearchHit> hits = executeSearch(Detector.DETECTORS_INDEX, request);
SearchHit hit = hits.get(0);

return ((List<String>) ((Map<String, Object>) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0);
}

@SuppressWarnings("unchecked")
private String createTestWindowsDetector(String indexName) throws IOException {
// Execute CreateMappingsAction to add alias mapping for index
Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI);
// both req params and req body are supported
createMappingRequest.setJsonEntity(
"{ \"index_name\":\"" + indexName + "\"," +
" \"rule_topic\":\"" + randomDetectorType() + "\", " +
" \"partial\":true" +
"}"
);

Response response = client().performRequest(createMappingRequest);
assertEquals(RestStatus.OK.getStatus(), response.getStatusLine().getStatusCode());

Detector windowsDetector = randomDetectorWithInputsAndTriggers(List.of(new DetectorInput("windows detector for security analytics", List.of(indexName), List.of(),
getRandomPrePackagedRules().stream().map(DetectorRule::new).collect(Collectors.toList()))),
List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(randomDetectorType()), List.of(), List.of(), List.of(), List.of(), List.of())));

Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(windowsDetector));
Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse));

Map<String, Object> responseBody = asMap(createResponse);

String createdId = responseBody.get("_id").toString();

String request = "{\n" +
" \"query\" : {\n" +
" \"match\":{\n" +
" \"_id\": \"" + createdId + "\"\n" +
" }\n" +
" }\n" +
"}";
List<SearchHit> hits = executeSearch(Detector.DETECTORS_INDEX, request);
SearchHit hit = hits.get(0);

return ((List<String>) ((Map<String, Object>) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0);
}

@SuppressWarnings("unchecked")
private String createAppLogsDetector(String indexName) throws IOException {
Detector appLogsDetector = randomDetectorWithInputsAndTriggersAndType(List.of(new DetectorInput("app logs detector for security analytics", List.of(indexName), List.of(),
getPrePackagedRules("others_application").stream().map(DetectorRule::new).collect(Collectors.toList()))),
List.of(new DetectorTrigger(null, "test-trigger", "1", List.of("others_application"), List.of(), List.of(), List.of(), List.of(), List.of())), "others_application");

Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(appLogsDetector));
Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse));

Map<String, Object> responseBody = asMap(createResponse);

String createdId = responseBody.get("_id").toString();

String request = "{\n" +
" \"query\" : {\n" +
" \"match\":{\n" +
" \"_id\": \"" + createdId + "\"\n" +
" }\n" +
" }\n" +
"}";
List<SearchHit> hits = executeSearch(Detector.DETECTORS_INDEX, request);
SearchHit hit = hits.get(0);

return ((List<String>) ((Map<String, Object>) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0);
}

@SuppressWarnings("unchecked")
private String createS3Detector(String indexName) throws IOException {
// Execute CreateMappingsAction to add alias mapping for index
Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI);
// both req params and req body are supported
createMappingRequest.setJsonEntity(
"{\n" +
" \"index_name\": \"s3_access_logs\",\n" +
" \"rule_topic\": \"s3\",\n" +
" \"partial\": true,\n" +
" \"alias_mappings\": {\n" +
" \"properties\": {\n" +
" \"aws-cloudtrail-event_source\": {\n" +
" \"type\": \"alias\",\n" +
" \"path\": \"aws.cloudtrail.event_source\"\n" +
" },\n" +
" \"aws.cloudtrail.event_name\": {\n" +
" \"type\": \"alias\",\n" +
" \"path\": \"aws.cloudtrail.event_name\"\n" +
" }\n" +
" }\n" +
" }\n" +
"}"
);

Response response = client().performRequest(createMappingRequest);
assertEquals(RestStatus.OK.getStatus(), response.getStatusLine().getStatusCode());

Detector s3AccessLogsDetector = randomDetectorWithInputsAndTriggersAndType(List.of(new DetectorInput("s3 access logs detector for security analytics", List.of(indexName), List.of(),
getPrePackagedRules("s3").stream().map(DetectorRule::new).collect(Collectors.toList()))),
List.of(new DetectorTrigger(null, "test-trigger", "1", List.of("s3"), List.of(), List.of(), List.of(), List.of(), List.of())), "s3");

Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(s3AccessLogsDetector));
Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse));

Map<String, Object> responseBody = asMap(createResponse);

String createdId = responseBody.get("_id").toString();

String request = "{\n" +
" \"query\" : {\n" +
" \"match\":{\n" +
" \"_id\": \"" + createdId + "\"\n" +
" }\n" +
" }\n" +
"}";
List<SearchHit> hits = executeSearch(Detector.DETECTORS_INDEX, request);
SearchHit hit = hits.get(0);

return ((List<String>) ((Map<String, Object>) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0);
}

static class LogIndices {
String vpcFlowsIndex;
String adLdapLogsIndex;
String windowsIndex;
String appLogsIndex;
String s3AccessLogsIndex;
}
}
Loading

0 comments on commit e8d7879

Please sign in to comment.