From 63d266adeb2970d8b39b4a9749359d5905397f98 Mon Sep 17 00:00:00 2001 From: Joanne Wang Date: Thu, 7 Mar 2024 09:40:44 -0800 Subject: [PATCH] Fix duplicate ecs mappings which returns incorrect log index field in mapping view API (#786) (#788) * field mapping changes Signed-off-by: Joanne Wang * add integ test Signed-off-by: Joanne Wang * turn unmappedfieldaliases as set and add integ test Signed-off-by: Joanne Wang * add comments Signed-off-by: Joanne Wang * fix integ tests Signed-off-by: Joanne Wang * moved logic to method for better readability Signed-off-by: Joanne Wang --------- Signed-off-by: Joanne Wang --- .../mapper/MapperService.java | 3 +- .../mapper/MapperRestApiIT.java | 171 ++++++++++++++++++ 2 files changed, 172 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java b/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java index 77b5dca80..880e32a35 100644 --- a/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java +++ b/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java @@ -8,7 +8,6 @@ import org.apache.commons.lang3.tuple.Pair; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.opensearch.action.ActionListener; import org.opensearch.action.admin.indices.get.GetIndexRequest; import org.opensearch.action.admin.indices.get.GetIndexResponse; import org.opensearch.action.admin.indices.mapping.get.GetMappingsRequest; @@ -535,7 +534,7 @@ public void onResponse(GetMappingsResponse getMappingsResponse) { .collect(Collectors.toList()); actionListener.onResponse( - new GetMappingsViewResponse(aliasMappings, unmappedIndexFields, filteredUnmappedFieldAliases) + new GetMappingsViewResponse(aliasMappings, unmappedIndexFields, filteredUnmappedFieldAliases, logTypeService.getIocFieldsList(logType)) ); } catch (Exception e) { actionListener.onFailure(e); diff --git a/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java index 10fa189ac..d7566a1da 100644 --- a/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java @@ -389,6 +389,114 @@ public void testGetMappingsViewLinuxSuccess() throws IOException { assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); } + // Tests mappings where multiple raw fields correspond to one ecs value + public void testGetMappingsViewWindowsSuccess() throws IOException { + + String testIndexName = "get_mappings_view_index"; + + createSampleWindex(testIndexName); + + // Execute GetMappingsViewAction to add alias mapping for index + Request request = new Request("GET", SecurityAnalyticsPlugin.MAPPINGS_VIEW_BASE_URI); + // both req params and req body are supported + request.addParameter("index_name", testIndexName); + request.addParameter("rule_topic", "windows"); + Response response = client().performRequest(request); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + Map respMap = responseAsMap(response); + + // Verify alias mappings + Map props = (Map) respMap.get("properties"); + assertEquals(3, props.size()); + assertTrue(props.containsKey("winlog.event_data.LogonType")); + assertTrue(props.containsKey("winlog.provider_name")); + assertTrue(props.containsKey("host.hostname")); + + // Verify unmapped index fields + List unmappedIndexFields = (List) respMap.get("unmapped_index_fields"); + assertEquals(3, unmappedIndexFields.size()); + assert(unmappedIndexFields.contains("plain1")); + assert(unmappedIndexFields.contains("ParentUser.first")); + assert(unmappedIndexFields.contains("ParentUser.last")); + + // Verify unmapped field aliases + List filteredUnmappedFieldAliases = (List) respMap.get("unmapped_field_aliases"); + assertEquals(191, filteredUnmappedFieldAliases.size()); + assert(!filteredUnmappedFieldAliases.contains("winlog.event_data.LogonType")); + assert(!filteredUnmappedFieldAliases.contains("winlog.provider_name")); + assert(!filteredUnmappedFieldAliases.contains("host.hostname")); + List> iocFieldsList = (List>) respMap.get(GetMappingsViewResponse.THREAT_INTEL_FIELD_ALIASES); + assertEquals(iocFieldsList.size(), 1); + + // Index a doc for a field with multiple raw fields corresponding to one ecs field + indexDoc(testIndexName, "1", "{ \"EventID\": 1 }"); + // Execute GetMappingsViewAction to add alias mapping for index + request = new Request("GET", SecurityAnalyticsPlugin.MAPPINGS_VIEW_BASE_URI); + // both req params and req body are supported + request.addParameter("index_name", testIndexName); + request.addParameter("rule_topic", "windows"); + response = client().performRequest(request); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + respMap = responseAsMap(response); + + // Verify alias mappings + props = (Map) respMap.get("properties"); + assertEquals(4, props.size()); + assertTrue(props.containsKey("winlog.event_id")); + + // verify unmapped index fields + unmappedIndexFields = (List) respMap.get("unmapped_index_fields"); + assertEquals(3, unmappedIndexFields.size()); + + // verify unmapped field aliases + filteredUnmappedFieldAliases = (List) respMap.get("unmapped_field_aliases"); + assertEquals(190, filteredUnmappedFieldAliases.size()); + assert(!filteredUnmappedFieldAliases.contains("winlog.event_id")); + } + + // Tests mappings where multiple raw fields correspond to one ecs value and all fields are present in the index + public void testGetMappingsViewMulitpleRawFieldsSuccess() throws IOException { + + String testIndexName = "get_mappings_view_index"; + + createSampleWindex(testIndexName); + String sampleDoc = "{" + + " \"EventID\": 1," + + " \"EventId\": 2," + + " \"event_uid\": 3" + + "}"; + indexDoc(testIndexName, "1", sampleDoc); + + // Execute GetMappingsViewAction to add alias mapping for index + Request request = new Request("GET", SecurityAnalyticsPlugin.MAPPINGS_VIEW_BASE_URI); + // both req params and req body are supported + request.addParameter("index_name", testIndexName); + request.addParameter("rule_topic", "windows"); + Response response = client().performRequest(request); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + Map respMap = responseAsMap(response); + + // Verify alias mappings + Map props = (Map) respMap.get("properties"); + assertEquals(4, props.size()); + assertTrue(props.containsKey("winlog.event_data.LogonType")); + assertTrue(props.containsKey("winlog.provider_name")); + assertTrue(props.containsKey("host.hostname")); + assertTrue(props.containsKey("winlog.event_id")); + + // Verify unmapped index fields + List unmappedIndexFields = (List) respMap.get("unmapped_index_fields"); + assertEquals(5, unmappedIndexFields.size()); + + // Verify unmapped field aliases + List filteredUnmappedFieldAliases = (List) respMap.get("unmapped_field_aliases"); + assertEquals(190, filteredUnmappedFieldAliases.size()); + assert(!filteredUnmappedFieldAliases.contains("winlog.event_data.LogonType")); + assert(!filteredUnmappedFieldAliases.contains("winlog.provider_name")); + assert(!filteredUnmappedFieldAliases.contains("host.hostname")); + assert(!filteredUnmappedFieldAliases.contains("winlog.event_id")); + } + public void testCreateMappings_withDatastream_success() throws IOException { String datastream = "test_datastream"; @@ -1272,6 +1380,69 @@ private void createSampleIndex(String indexName, Settings settings, String alias assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); } + private void createSampleWindex(String indexName) throws IOException { + createSampleWindex(indexName, Settings.EMPTY, null); + } + + private void createSampleWindex(String indexName, Settings settings, String aliases) throws IOException { + String indexMapping = + " \"properties\": {" + + " \"LogonType\": {" + + " \"type\": \"integer\"" + + " }," + + " \"Provider\": {" + + " \"type\": \"text\"" + + " }," + + " \"hostname\": {" + + " \"type\": \"text\"" + + " }," + + " \"plain1\": {" + + " \"type\": \"integer\"" + + " }," + + " \"ParentUser\":{" + + " \"type\":\"nested\"," + + " \"properties\":{" + + " \"first\":{" + + " \"type\":\"text\"," + + " \"fields\":{" + + " \"keyword\":{" + + " \"type\":\"keyword\"," + + " \"ignore_above\":256" + + "}" + + "}" + + "}," + + " \"last\":{" + + "\"type\":\"text\"," + + "\"fields\":{" + + " \"keyword\":{" + + " \"type\":\"keyword\"," + + " \"ignore_above\":256" + + "}" + + "}" + + "}" + + "}" + + "}" + + " }"; + + createIndex(indexName, settings, indexMapping, aliases); + + // Insert sample doc with event_uid not explicitly mapped + String sampleDoc = "{" + + " \"LogonType\":1," + + " \"Provider\":\"Microsoft-Windows-Security-Auditing\"," + + " \"hostname\":\"FLUXCAPACITOR\"" + + "}"; + + // Index doc + Request indexRequest = new Request("POST", indexName + "/_doc?refresh=wait_for"); + indexRequest.setJsonEntity(sampleDoc); + Response response = client().performRequest(indexRequest); + assertEquals(HttpStatus.SC_CREATED, response.getStatusLine().getStatusCode()); + // Refresh everything + response = client().performRequest(new Request("POST", "_refresh")); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + } + private void createSampleDatastream(String datastreamName) throws IOException { String indexMapping = " \"properties\": {" +