From 3f24af9e2d67695793b0d85c13b49b64320222c8 Mon Sep 17 00:00:00 2001 From: Petar Dzepina Date: Tue, 14 Mar 2023 01:13:03 +0100 Subject: [PATCH 1/3] end to end tests for aliases and datastreams Signed-off-by: Petar Dzepina --- .../securityanalytics/TestHelpers.java | 2 + .../resthandler/DetectorRestApiIT.java | 185 ++++++++++++++++++ 2 files changed, 187 insertions(+) diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index e3eb9dcf8..d7ee3d85e 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -542,6 +542,7 @@ public static String randomAggregationRule(String aggFunction, String signAndVa public static String windowsIndexMapping() { return "\"properties\": {\n" + + " \"@timestamp\": {\"type\":\"date\"},\n" + " \"AccessList\": {\n" + " \"type\": \"text\"\n" + " },\n" + @@ -1204,6 +1205,7 @@ public static String randomDoc(int severity, int version, String opCode) { public static String randomDoc() { return "{\n" + + "\"@timestamp\":\"2020-02-04T14:59:39.343541+00:00\",\n" + "\"EventTime\":\"2020-02-04T14:59:39.343541+00:00\",\n" + "\"HostName\":\"EC2AMAZ-EPO7HKA\",\n" + "\"Keywords\":\"9223372036854775808\",\n" + diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java index 61d025242..3173bd035 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java @@ -1051,4 +1051,189 @@ public void testCreatingADetectorWithTimestampFieldAliasMapping_verifyTimeRangeI List findings = (List) getFindingsBody.get("findings"); Assert.assertEquals(findings.size(), 0); //there should be no findings as doc is not in time range of current run } + + public void testDetector_withDatastream_withTemplateField_endToEnd_success() throws IOException { + String datastream = "test_datastream"; + + createSampleDatastream(datastream, windowsIndexMapping(), false); + // Execute CreateMappingsAction to add alias mapping for index + createMappingsAPI(datastream, randomDetectorType()); + + String writeIndex = getDatastreamWriteIndex(datastream); + + // Verify mappings + Map props = getIndexMappingsAPIFlat(writeIndex); + assertTrue(props.containsKey("windows-event_data-CommandLine")); + assertTrue(props.containsKey("event_uid")); + assertTrue(props.containsKey("windows-hostname")); + assertTrue(props.containsKey("windows-message")); + assertTrue(props.containsKey("windows-provider-name")); + assertTrue(props.containsKey("windows-servicename")); + + + // Get applied mappings + props = getIndexMappingsSAFlat(datastream); + assertEquals(6, props.size()); + assertTrue(props.containsKey("windows-event_data-CommandLine")); + assertTrue(props.containsKey("event_uid")); + assertTrue(props.containsKey("windows-hostname")); + assertTrue(props.containsKey("windows-message")); + assertTrue(props.containsKey("windows-provider-name")); + assertTrue(props.containsKey("windows-servicename")); + + // Create detector + Detector detector = randomDetectorWithInputsAndTriggers(List.of(new DetectorInput("windows detector for security analytics", List.of(datastream), List.of(), + getRandomPrePackagedRules().stream().map(DetectorRule::new).collect(Collectors.toList()))), + List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(), List.of(), List.of(), List.of("attack.defense_evasion"), List.of()))); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + + String createdId = responseBody.get("_id").toString(); + int createdVersion = Integer.parseInt(responseBody.get("_version").toString()); + Assert.assertNotEquals("response is missing Id", Detector.NO_ID, createdId); + Assert.assertTrue("incorrect version", createdVersion > 0); + Assert.assertEquals("Incorrect Location header", String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, createdId), createResponse.getHeader("Location")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("rule_topic_index")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("findings_index")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("alert_index")); + + String detectorTypeInResponse = (String) ((Map)responseBody.get("detector")).get("detector_type"); + Assert.assertEquals("Detector type incorrect", randomDetectorType().toLowerCase(Locale.ROOT), detectorTypeInResponse); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + createdId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + String monitorId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); + + indexDoc(datastream, "1", randomDoc()); + + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + + int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(5, noOfSigmaRuleMatches); + + refreshAllIndices(); + + // Call GetAlerts API + Map params = new HashMap<>(); + params.put("detectorType", randomDetectorType()); + Response getAlertsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.ALERTS_BASE_URI, params, null); + Map getAlertsBody = asMap(getAlertsResponse); + // TODO enable asserts here when able + Assert.assertEquals(1, getAlertsBody.get("total_alerts")); + + // Call GetFindings API + params = new HashMap<>(); + params.put("detector_id", createdId); + Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); + Map getFindingsBody = entityAsMap(getFindingsResponse); + assertNotNull(getFindingsBody); + Assert.assertEquals(1, getFindingsBody.get("total_findings")); + List findings = (List) getFindingsBody.get("findings"); + Assert.assertEquals(findings.size(), 1); + + deleteDatastreamAPI(datastream); + } + + public void testDetector_withAlias_endToEnd_success() throws IOException { + String writeIndex = "my_windows_log-1"; + String indexAlias = "test_alias"; + + createIndex(writeIndex, Settings.EMPTY, windowsIndexMapping(), "\"" + indexAlias + "\":{}"); + // Execute CreateMappingsAction to add alias mapping for index + createMappingsAPI(indexAlias, randomDetectorType()); + + // Verify mappings + Map props = getIndexMappingsAPIFlat(writeIndex); + assertTrue(props.containsKey("windows-event_data-CommandLine")); + assertTrue(props.containsKey("event_uid")); + assertTrue(props.containsKey("windows-hostname")); + assertTrue(props.containsKey("windows-message")); + assertTrue(props.containsKey("windows-provider-name")); + assertTrue(props.containsKey("windows-servicename")); + + + // Get applied mappings + props = getIndexMappingsSAFlat(indexAlias); + assertEquals(6, props.size()); + assertTrue(props.containsKey("windows-event_data-CommandLine")); + assertTrue(props.containsKey("event_uid")); + assertTrue(props.containsKey("windows-hostname")); + assertTrue(props.containsKey("windows-message")); + assertTrue(props.containsKey("windows-provider-name")); + assertTrue(props.containsKey("windows-servicename")); + + // Create detector + Detector detector = randomDetectorWithInputsAndTriggers(List.of(new DetectorInput("windows detector for security analytics", List.of(indexAlias), List.of(), + getRandomPrePackagedRules().stream().map(DetectorRule::new).collect(Collectors.toList()))), + List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(), List.of(), List.of(), List.of("attack.defense_evasion"), List.of()))); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + + String createdId = responseBody.get("_id").toString(); + int createdVersion = Integer.parseInt(responseBody.get("_version").toString()); + Assert.assertNotEquals("response is missing Id", Detector.NO_ID, createdId); + Assert.assertTrue("incorrect version", createdVersion > 0); + Assert.assertEquals("Incorrect Location header", String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, createdId), createResponse.getHeader("Location")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("rule_topic_index")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("findings_index")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("alert_index")); + + String detectorTypeInResponse = (String) ((Map)responseBody.get("detector")).get("detector_type"); + Assert.assertEquals("Detector type incorrect", randomDetectorType().toLowerCase(Locale.ROOT), detectorTypeInResponse); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + createdId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + String monitorId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); + + indexDoc(indexAlias, "1", randomDoc()); + + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + + int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(5, noOfSigmaRuleMatches); + + refreshAllIndices(); + + // Call GetAlerts API + Map params = new HashMap<>(); + params.put("detectorType", randomDetectorType()); + Response getAlertsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.ALERTS_BASE_URI, params, null); + Map getAlertsBody = asMap(getAlertsResponse); + // TODO enable asserts here when able + Assert.assertEquals(1, getAlertsBody.get("total_alerts")); + + // Call GetFindings API + params = new HashMap<>(); + params.put("detector_id", createdId); + Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); + Map getFindingsBody = entityAsMap(getFindingsResponse); + assertNotNull(getFindingsBody); + Assert.assertEquals(1, getFindingsBody.get("total_findings")); + List findings = (List) getFindingsBody.get("findings"); + Assert.assertEquals(findings.size(), 1); + } } \ No newline at end of file From b4f2282400acb43b413ce7d3e75119401db19656 Mon Sep 17 00:00:00 2001 From: Petar Dzepina Date: Wed, 11 Jan 2023 22:57:12 +0100 Subject: [PATCH 2/3] bugfix Signed-off-by: Petar Dzepina --- .../opensearch/securityanalytics/mapper/MapperService.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java b/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java index 15112e8a5..729c8b353 100644 --- a/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java +++ b/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java @@ -398,7 +398,7 @@ private void doGetMappingsView(String mapperTopic, ActionListener allFieldsFromIndex = MapperUtils.getAllNonAliasFieldsFromIndex(mappingMetadata); // Get stored Alias Mappings as JSON string @@ -419,8 +419,7 @@ public void onResponse(GetMappingsResponse getMappingsResponse) { // Maintain list of found paths in index applyableAliases.add(alias); pathsOfApplyableAliases.add(path); - } else if (allFieldsFromIndex.contains(alias) == false) { - // we don't want to send back aliases which have same name as existing field in index + } else { unmappedFieldAliases.add(alias); } } From 1855ebfceb9a773647c4e733aea4ae65243ac1b1 Mon Sep 17 00:00:00 2001 From: Petar Dzepina Date: Tue, 2 May 2023 20:03:07 +0200 Subject: [PATCH 3/3] merge fix Signed-off-by: Petar Dzepina --- .../opensearch/securityanalytics/mapper/MapperService.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java b/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java index 729c8b353..15112e8a5 100644 --- a/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java +++ b/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java @@ -398,7 +398,7 @@ private void doGetMappingsView(String mapperTopic, ActionListener allFieldsFromIndex = MapperUtils.getAllNonAliasFieldsFromIndex(mappingMetadata); // Get stored Alias Mappings as JSON string @@ -419,7 +419,8 @@ public void onResponse(GetMappingsResponse getMappingsResponse) { // Maintain list of found paths in index applyableAliases.add(alias); pathsOfApplyableAliases.add(path); - } else { + } else if (allFieldsFromIndex.contains(alias) == false) { + // we don't want to send back aliases which have same name as existing field in index unmappedFieldAliases.add(alias); } }