Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Backport 2.9 fix #937

Merged
merged 7 commits into from
Mar 18, 2024
Merged

Backport 2.9 fix #937

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
69cb432
Fix duplicate ecs mappings which returns incorrect log index field in…
jowg-amazon Mar 7, 2024
9e507a1
Fix get mappings view API incorrectly returning ecs path (#867)
jowg-amazon Mar 11, 2024
cc0c56a
fix integ test (#918)
jowg-amazon Mar 13, 2024
bce28fd
fix detector writeTo() method missing fields (#695)
eirsep Oct 26, 2023
23e989e
removing threatIntel related code
riysaxen-amzn Mar 18, 2024
f5384cf
removing threatIntel related code
riysaxen-amzn Mar 18, 2024
f6bad2f
ignore flaky tests
riysaxen-amzn Mar 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 17 additions & 6 deletions src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -344,15 +344,26 @@
List<FieldMappingDoc> newFieldMappings = new ArrayList<>();
fieldMappingDocs.forEach( newFieldMapping -> {
Optional<FieldMappingDoc> foundFieldMappingDoc = Optional.empty();
for (FieldMappingDoc e: existingFieldMappings) {
if (e.getRawField().equals(newFieldMapping.getRawField())) {
for (FieldMappingDoc existingFieldMapping: existingFieldMappings) {
if (existingFieldMapping.getRawField().equals(newFieldMapping.getRawField())) {
if ((
e.get(defaultSchemaField) != null && newFieldMapping.get(defaultSchemaField) != null &&
e.get(defaultSchemaField).equals(newFieldMapping.get(defaultSchemaField))
existingFieldMapping.get(defaultSchemaField) != null && newFieldMapping.get(defaultSchemaField) != null &&
existingFieldMapping.get(defaultSchemaField).equals(newFieldMapping.get(defaultSchemaField))
) || (
e.get(defaultSchemaField) == null && newFieldMapping.get(defaultSchemaField) == null
existingFieldMapping.get(defaultSchemaField) == null && newFieldMapping.get(defaultSchemaField) == null
)) {
foundFieldMappingDoc = Optional.of(e);
foundFieldMappingDoc = Optional.of(existingFieldMapping);
}
// Grabs the right side of the ID with "|" as the delimiter if present representing the ecs field from predefined mappings
// Additional check to see if raw field path + log type combination is already in existingFieldMappings so a new one is not indexed
} else {
String id = existingFieldMapping.getId();
int indexOfPipe = id.indexOf("|");
if (indexOfPipe != -1) {
String ecsIdField = id.substring(indexOfPipe + 1);
if (ecsIdField.equals(newFieldMapping.getRawField()) && existingFieldMapping.getLogTypes().containsAll(newFieldMapping.getLogTypes())) {
foundFieldMappingDoc = Optional.of(existingFieldMapping);

Check warning on line 365 in src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java

View check run for this annotation

Codecov / codecov/patch

src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java#L365 

Added line #L365 was not covered by tests
}
}
}
}
Expand Down
20 changes: 12 additions & 8 deletions src/main/java/org/opensearch/securityanalytics/model/Detector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -159,12 +159,12 @@ public Detector(StreamInput sin) throws IOException {
sin.readList(DetectorInput::readFrom),
sin.readList(DetectorTrigger::readFrom),
sin.readStringList(),
sin.readString(),
sin.readString(),
sin.readString(),
sin.readString(),
sin.readString(),
sin.readString(),
sin.readOptionalString(),
sin.readOptionalString(),
sin.readOptionalString(),
sin.readOptionalString(),
sin.readOptionalString(),
sin.readOptionalString(),
sin.readMap(StreamInput::readString, StreamInput::readString)
);
}
Expand Down Expand Up @@ -197,8 +197,12 @@ public void writeTo(StreamOutput out) throws IOException {
it.writeTo(out);
}
out.writeStringCollection(monitorIds);
out.writeString(ruleIndex);

out.writeOptionalString(ruleIndex);
out.writeOptionalString(alertsIndex);
out.writeOptionalString(alertsHistoryIndex);
out.writeOptionalString(alertsHistoryIndexPattern);
out.writeOptionalString(findingsIndex);
out.writeOptionalString(findingsIndexPattern);
out.writeMap(ruleIdMonitorIdMap, StreamOutput::writeString, StreamOutput::writeString);
}

Expand Down
94 changes: 94 additions & 0 deletions src/test/java/org/opensearch/securityanalytics/TestHelpers.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,100 @@ public static String randomRule() {
" - Legitimate usage of remote file encryption\n" +
"level: high";
}
public static String randomRuleWithRawField() {
return "title: Remote Encrypting File System Abuse\n" +
"id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" +
"description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" +
"references:\n" +
" - https://attack.mitre.org/tactics/TA0008/\n" +
" - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" +
" - https://github.com/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" +
" - https://github.com/zeronetworks/rpcfirewall\n" +
" - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" +
"tags:\n" +
" - attack.defense_evasion\n" +
"status: experimental\n" +
"author: Sagie Dulce, Dekel Paz\n" +
"date: 2022/01/01\n" +
"modified: 2022/01/01\n" +
"logsource:\n" +
" product: rpc_firewall\n" +
" category: application\n" +
" definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" +
"detection:\n" +
" selection:\n" +
" eventName: testinghere\n" +
" condition: selection\n" +
"falsepositives:\n" +
" - Legitimate usage of remote file encryption\n" +
"level: high";
}

public static String randomRuleWithNotCondition() {
return "title: Remote Encrypting File System Abuse\n" +
"id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" +
"description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" +
"references:\n" +
" - https://attack.mitre.org/tactics/TA0008/\n" +
" - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" +
" - https://github.com/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" +
" - https://github.com/zeronetworks/rpcfirewall\n" +
" - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" +
"tags:\n" +
" - attack.defense_evasion\n" +
"status: experimental\n" +
"author: Sagie Dulce, Dekel Paz\n" +
"date: 2022/01/01\n" +
"modified: 2022/01/01\n" +
"logsource:\n" +
" product: rpc_firewall\n" +
" category: application\n" +
" definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" +
"detection:\n" +
" selection1:\n" +
" AccountType: TestAccountType\n" +
" selection2:\n" +
" AccountName: TestAccountName\n" +
" selection3:\n" +
" EventID: 22\n" +
" condition: (not selection1 and not selection2) and selection3\n" +
"falsepositives:\n" +
" - Legitimate usage of remote file encryption\n" +
"level: high";
}

public static String randomRuleWithNotConditionBoolAndNum() {
return "title: Remote Encrypting File System Abuse\n" +
"id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" +
"description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" +
"references:\n" +
" - https://attack.mitre.org/tactics/TA0008/\n" +
" - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" +
" - https://github.com/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" +
" - https://github.com/zeronetworks/rpcfirewall\n" +
" - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" +
"tags:\n" +
" - attack.defense_evasion\n" +
"status: experimental\n" +
"author: Sagie Dulce, Dekel Paz\n" +
"date: 2022/01/01\n" +
"modified: 2022/01/01\n" +
"logsource:\n" +
" product: rpc_firewall\n" +
" category: application\n" +
" definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" +
"detection:\n" +
" selection1:\n" +
" Initiated: \"false\"\n" +
" selection2:\n" +
" AccountName: TestAccountName\n" +
" selection3:\n" +
" EventID: 21\n" +
" condition: not selection1 and not selection3\n" +
"falsepositives:\n" +
" - Legitimate usage of remote file encryption\n" +
"level: high";
}

public static String randomNullRule() {
return "title: null field\n" +
Expand Down
3 changes: 2 additions & 1 deletion src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import java.util.stream.Collectors;
import org.apache.http.HttpStatus;
import org.junit.Assert;
import org.junit.Ignore;
import org.opensearch.client.Request;
import org.opensearch.client.Response;
import org.opensearch.client.ResponseException;
Expand Down Expand Up @@ -262,7 +263,7 @@ public void testGetFindings_byDetectorType_success() throws IOException {
getFindingsBody = entityAsMap(getFindingsResponse);
Assert.assertEquals(1, getFindingsBody.get("total_findings"));
}

@Ignore("Test is currently ignored because of flakiness, need to fix it")
public void testGetFindings_rolloverByMaxAge_success() throws IOException, InterruptedException {

updateClusterSetting(FINDING_HISTORY_ROLLOVER_PERIOD.getKey(), "1s");
Expand Down
169 changes: 169 additions & 0 deletions src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -389,6 +389,112 @@ public void testGetMappingsViewLinuxSuccess() throws IOException {
assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode());
}

// Tests mappings where multiple raw fields correspond to one ecs value
public void testGetMappingsViewWindowsSuccess() throws IOException {

String testIndexName = "get_mappings_view_index";

createSampleWindex(testIndexName);

// Execute GetMappingsViewAction to add alias mapping for index
Request request = new Request("GET", SecurityAnalyticsPlugin.MAPPINGS_VIEW_BASE_URI);
// both req params and req body are supported
request.addParameter("index_name", testIndexName);
request.addParameter("rule_topic", "windows");
Response response = client().performRequest(request);
assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode());
Map<String, Object> respMap = responseAsMap(response);

// Verify alias mappings
Map<String, Object> props = (Map<String, Object>) respMap.get("properties");
assertEquals(3, props.size());
assertTrue(props.containsKey("winlog.event_data.LogonType"));
assertTrue(props.containsKey("winlog.provider_name"));
assertTrue(props.containsKey("host.hostname"));

// Verify unmapped index fields
List<String> unmappedIndexFields = (List<String>) respMap.get("unmapped_index_fields");
assertEquals(3, unmappedIndexFields.size());
assert(unmappedIndexFields.contains("plain1"));
assert(unmappedIndexFields.contains("ParentUser.first"));
assert(unmappedIndexFields.contains("ParentUser.last"));

// Verify unmapped field aliases
List<String> filteredUnmappedFieldAliases = (List<String>) respMap.get("unmapped_field_aliases");
assertEquals(191, filteredUnmappedFieldAliases.size());
assert(!filteredUnmappedFieldAliases.contains("winlog.event_data.LogonType"));
assert(!filteredUnmappedFieldAliases.contains("winlog.provider_name"));
assert(!filteredUnmappedFieldAliases.contains("host.hostname"));

// Index a doc for a field with multiple raw fields corresponding to one ecs field
indexDoc(testIndexName, "1", "{ \"EventID\": 1 }");
// Execute GetMappingsViewAction to add alias mapping for index
request = new Request("GET", SecurityAnalyticsPlugin.MAPPINGS_VIEW_BASE_URI);
// both req params and req body are supported
request.addParameter("index_name", testIndexName);
request.addParameter("rule_topic", "windows");
response = client().performRequest(request);
assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode());
respMap = responseAsMap(response);

// Verify alias mappings
props = (Map<String, Object>) respMap.get("properties");
assertEquals(4, props.size());
assertTrue(props.containsKey("winlog.event_id"));

// verify unmapped index fields
unmappedIndexFields = (List<String>) respMap.get("unmapped_index_fields");
assertEquals(3, unmappedIndexFields.size());

// verify unmapped field aliases
filteredUnmappedFieldAliases = (List<String>) respMap.get("unmapped_field_aliases");
assertEquals(190, filteredUnmappedFieldAliases.size());
assert(!filteredUnmappedFieldAliases.contains("winlog.event_id"));
}

// Tests mappings where multiple raw fields correspond to one ecs value and all fields are present in the index
public void testGetMappingsViewMulitpleRawFieldsSuccess() throws IOException {

String testIndexName = "get_mappings_view_index";

createSampleWindex(testIndexName);
String sampleDoc = "{" +
" \"EventID\": 1," +
" \"EventId\": 2," +
" \"event_uid\": 3" +
"}";
indexDoc(testIndexName, "1", sampleDoc);

// Execute GetMappingsViewAction to add alias mapping for index
Request request = new Request("GET", SecurityAnalyticsPlugin.MAPPINGS_VIEW_BASE_URI);
// both req params and req body are supported
request.addParameter("index_name", testIndexName);
request.addParameter("rule_topic", "windows");
Response response = client().performRequest(request);
assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode());
Map<String, Object> respMap = responseAsMap(response);

// Verify alias mappings
Map<String, Object> props = (Map<String, Object>) respMap.get("properties");
assertEquals(4, props.size());
assertTrue(props.containsKey("winlog.event_data.LogonType"));
assertTrue(props.containsKey("winlog.provider_name"));
assertTrue(props.containsKey("host.hostname"));
assertTrue(props.containsKey("winlog.event_id"));

// Verify unmapped index fields
List<String> unmappedIndexFields = (List<String>) respMap.get("unmapped_index_fields");
assertEquals(5, unmappedIndexFields.size());

// Verify unmapped field aliases
List<String> filteredUnmappedFieldAliases = (List<String>) respMap.get("unmapped_field_aliases");
assertEquals(190, filteredUnmappedFieldAliases.size());
assert(!filteredUnmappedFieldAliases.contains("winlog.event_data.LogonType"));
assert(!filteredUnmappedFieldAliases.contains("winlog.provider_name"));
assert(!filteredUnmappedFieldAliases.contains("host.hostname"));
assert(!filteredUnmappedFieldAliases.contains("winlog.event_id"));
}

public void testCreateMappings_withDatastream_success() throws IOException {
String datastream = "test_datastream";

Expand Down Expand Up @@ -1272,6 +1378,69 @@ private void createSampleIndex(String indexName, Settings settings, String alias
assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode());
}

private void createSampleWindex(String indexName) throws IOException {
createSampleWindex(indexName, Settings.EMPTY, null);
}

private void createSampleWindex(String indexName, Settings settings, String aliases) throws IOException {
String indexMapping =
" \"properties\": {" +
" \"LogonType\": {" +
" \"type\": \"integer\"" +
" }," +
" \"Provider\": {" +
" \"type\": \"text\"" +
" }," +
" \"hostname\": {" +
" \"type\": \"text\"" +
" }," +
" \"plain1\": {" +
" \"type\": \"integer\"" +
" }," +
" \"ParentUser\":{" +
" \"type\":\"nested\"," +
" \"properties\":{" +
" \"first\":{" +
" \"type\":\"text\"," +
" \"fields\":{" +
" \"keyword\":{" +
" \"type\":\"keyword\"," +
" \"ignore_above\":256" +
"}" +
"}" +
"}," +
" \"last\":{" +
"\"type\":\"text\"," +
"\"fields\":{" +
" \"keyword\":{" +
" \"type\":\"keyword\"," +
" \"ignore_above\":256" +
"}" +
"}" +
"}" +
"}" +
"}" +
" }";

createIndex(indexName, settings, indexMapping, aliases);

// Insert sample doc with event_uid not explicitly mapped
String sampleDoc = "{" +
" \"LogonType\":1," +
" \"Provider\":\"Microsoft-Windows-Security-Auditing\"," +
" \"hostname\":\"FLUXCAPACITOR\"" +
"}";

// Index doc
Request indexRequest = new Request("POST", indexName + "/_doc?refresh=wait_for");
indexRequest.setJsonEntity(sampleDoc);
Response response = client().performRequest(indexRequest);
assertEquals(HttpStatus.SC_CREATED, response.getStatusLine().getStatusCode());
// Refresh everything
response = client().performRequest(new Request("POST", "_refresh"));
assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode());
}

private void createSampleDatastream(String datastreamName) throws IOException {
String indexMapping =
" \"properties\": {" +
Expand Down
Loading
Loading