From 8990aed94d24bde07de6a66e9bc162f4f0808455 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 17 Jan 2024 10:48:36 -0800 Subject: [PATCH] #683 Sigma keywords field not handled correctly (#725) (#739) * changed windows sample rule and query construction Signed-off-by: Joanne Wang * remove wildcard Signed-off-by: Joanne Wang * changed wildcardtest Signed-off-by: Joanne Wang * fixed wildcards Signed-off-by: Joanne Wang * fixed wildcard query test Signed-off-by: Joanne Wang * fixed correlation engine tests Signed-off-by: Joanne Wang * fixed query backend tests Signed-off-by: Joanne Wang * clean up Signed-off-by: Joanne Wang * added two integration tests Signed-off-by: Joanne Wang --------- Signed-off-by: Joanne Wang (cherry picked from commit 43040d6c8c6685c5bd484f736d76ed3367f4e8c4) Co-authored-by: Joanne Wang <109310487+jowg-amazon@users.noreply.github.com> --- .../rules/backend/OSQueryBackend.java | 24 +- .../rules/test_windows/win_sample_rule.yml | 4 +- .../securityanalytics/TestHelpers.java | 160 ++++++++++- .../resthandler/DetectorMonitorRestApiIT.java | 253 ++++++++++++++++++ .../rules/backend/QueryBackendTests.java | 33 ++- 5 files changed, 451 insertions(+), 23 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java b/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java index 021b3de3b..d94db371d 100644 --- a/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java +++ b/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java @@ -132,10 +132,10 @@ public OSQueryBackend(Map fieldMappings, boolean collectErrors, this.reExpression = "%s: /%s/"; this.cidrExpression = "%s: \"%s\""; this.fieldNullExpression = "%s: null"; - this.unboundValueStrExpression = "%s: \"%s\""; - this.unboundValueNumExpression = "%s: %s"; - this.unboundWildcardExpression = "%s: %s"; - this.unboundReExpression = "%s: /%s/"; + this.unboundValueStrExpression = "\"%s\""; + this.unboundValueNumExpression = "\"%s\""; + this.unboundWildcardExpression = "%s"; + this.unboundReExpression = "/%s/"; this.compareOpExpression = "\"%s\" \"%s\" %s"; this.valExpCount = 0; this.aggQuery = "{\"%s\":{\"terms\":{\"field\":\"%s\"},\"aggs\":{\"%s\":{\"%s\":{\"field\":\"%s\"}}}}}"; @@ -332,28 +332,18 @@ public Object convertConditionFieldEqValQueryExpr(ConditionFieldEqualsValueExpre @Override public Object convertConditionValStr(ConditionValueExpression condition) throws SigmaValueError { SigmaString value = (SigmaString) condition.getValue(); - - String field = getFinalValueField(); - ruleQueryFields.put(field, Map.of("type", "text", "analyzer", "rule_analyzer")); boolean containsWildcard = value.containsWildcard(); - return String.format(Locale.getDefault(), (containsWildcard? this.unboundWildcardExpression: this.unboundValueStrExpression), field, this.convertValueStr((SigmaString) condition.getValue())); + return String.format(Locale.getDefault(), (containsWildcard? this.unboundWildcardExpression: this.unboundValueStrExpression), this.convertValueStr((SigmaString) condition.getValue())); } @Override public Object convertConditionValNum(ConditionValueExpression condition) { - String field = getFinalValueField(); - - SigmaNumber number = (SigmaNumber) condition.getValue(); - ruleQueryFields.put(field, number.getNumOpt().isLeft()? Collections.singletonMap("type", "integer"): Collections.singletonMap("type", "float")); - - return String.format(Locale.getDefault(), this.unboundValueNumExpression, field, condition.getValue().toString()); + return String.format(Locale.getDefault(), this.unboundValueNumExpression, condition.getValue().toString()); } @Override public Object convertConditionValRe(ConditionValueExpression condition) { - String field = getFinalValueField(); - ruleQueryFields.put(field, Map.of("type", "text", "analyzer", "rule_analyzer")); - return String.format(Locale.getDefault(), this.unboundReExpression, field, convertValueRe((SigmaRegularExpression) condition.getValue())); + return String.format(Locale.getDefault(), this.unboundReExpression, convertValueRe((SigmaRegularExpression) condition.getValue())); } // TODO: below methods will be supported when Sigma Expand Modifier is supported. diff --git a/src/main/resources/rules/test_windows/win_sample_rule.yml b/src/main/resources/rules/test_windows/win_sample_rule.yml index b55e9c9b7..8720614a0 100644 --- a/src/main/resources/rules/test_windows/win_sample_rule.yml +++ b/src/main/resources/rules/test_windows/win_sample_rule.yml @@ -19,6 +19,8 @@ detection: EventID: 22 Message|contains: 'C:\\Program Files\\nxlog\\nxlog.exe' HostName|startswith: 'EC2AMAZ' - condition: selection + keywords: + - "NT AUTHORITY" + condition: selection or keywords falsepositives: - Unknown diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index bca37dec6..6f87900c5 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -347,6 +347,100 @@ public static String randomRuleWithAlias() { "level: high"; } + public static String randomRuleWithKeywords() { + return "title: Remote Encrypting File System Abuse\n" + + "id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" + + "description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" + + "references:\n" + + " - https://attack.mitre.org/tactics/TA0008/\n" + + " - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" + + " - https://github.com/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" + + " - https://github.com/zeronetworks/rpcfirewall\n" + + " - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" + + "tags:\n" + + " - attack.defense_evasion\n" + + "status: experimental\n" + + "author: Sagie Dulce, Dekel Paz\n" + + "date: 2022/01/01\n" + + "modified: 2022/01/01\n" + + "logsource:\n" + + " product: rpc_firewall\n" + + " category: application\n" + + " definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" + + "detection:\n" + + " selection:\n" + + " EventID: 21\n" + + " keywords:\n" + + " - 1996\n" + + " - EC2AMAZ*\n" + + " condition: selection or keywords\n" + + "falsepositives:\n" + + " - Legitimate usage of remote file encryption\n" + + "level: high"; + } + + public static String randomRuleWithStringKeywords() { + return "title: Remote Encrypting File System Abuse\n" + + "id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" + + "description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" + + "references:\n" + + " - https://attack.mitre.org/tactics/TA0008/\n" + + " - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" + + " - https://github.com/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" + + " - https://github.com/zeronetworks/rpcfirewall\n" + + " - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" + + "tags:\n" + + " - attack.defense_evasion\n" + + "status: experimental\n" + + "author: Sagie Dulce, Dekel Paz\n" + + "date: 2022/01/01\n" + + "modified: 2022/01/01\n" + + "logsource:\n" + + " product: rpc_firewall\n" + + " category: application\n" + + " definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" + + "detection:\n" + + " selection:\n" + + " EventID: 21\n" + + " keywords:\n" + + " - \"INFO\"\n" + + " condition: selection or keywords\n" + + "falsepositives:\n" + + " - Legitimate usage of remote file encryption\n" + + "level: high"; + } + + public static String randomRuleWithDateKeywords() { + return "title: Remote Encrypting File System Abuse\n" + + "id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" + + "description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" + + "references:\n" + + " - https://attack.mitre.org/tactics/TA0008/\n" + + " - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" + + " - https://github.com/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" + + " - https://github.com/zeronetworks/rpcfirewall\n" + + " - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" + + "tags:\n" + + " - attack.defense_evasion\n" + + "status: experimental\n" + + "author: Sagie Dulce, Dekel Paz\n" + + "date: 2022/01/01\n" + + "modified: 2022/01/01\n" + + "logsource:\n" + + " product: rpc_firewall\n" + + " category: application\n" + + " definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" + + "detection:\n" + + " selection:\n" + + " EventID: 21\n" + + " keywords:\n" + + " - \"2020-02-04T14:59:39.343541+00:00\"\n" + + " condition: selection or keywords\n" + + "falsepositives:\n" + + " - Legitimate usage of remote file encryption\n" + + "level: high"; + } + public static String countAggregationTestRule() { return " title: Test\n" + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + @@ -1367,6 +1461,48 @@ public static String windowsIndexMapping() { " }"; } + public static String windowsIndexMappingOnlyNumericAndDate() { + return "\"properties\": {\n" + + " \"@timestamp\": {\"type\":\"date\"},\n" + + " \"EventTime\": {\n" + + " \"type\": \"date\"\n" + + " },\n" + + " \"ExecutionProcessID\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"ExecutionThreadID\": {\n" + + " \"type\": \"integer\"\n" + + " },\n" + + " \"EventID\": {\n" + + " \"type\": \"integer\"\n" + + " },\n" + + " \"TaskValue\": {\n" + + " \"type\": \"integer\"\n" + + " }\n" + + " }"; + } + + public static String windowsIndexMappingOnlyNumericAndText() { + return "\"properties\": {\n" + + " \"TaskName\": {\n" + + " \"type\": \"text\"\n" + + " },\n" + + " \"ExecutionProcessID\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"ExecutionThreadID\": {\n" + + " \"type\": \"integer\"\n" + + " },\n" + + " \"EventID\": {\n" + + " \"type\": \"integer\"\n" + + " },\n" + + " \"TaskValue\": {\n" + + " \"type\": \"integer\"\n" + + " }\n" + + " }"; + } + + public static String randomDoc(int severity, int version, String opCode) { String doc = "{\n" + "\"EventTime\":\"2020-02-04T14:59:39.343541+00:00\",\n" + @@ -1406,6 +1542,28 @@ public static String randomDoc(int severity, int version, String opCode) { } + public static String randomDocOnlyNumericAndDate(int severity, int version, String opCode) { + String doc = "{\n" + + "\"EventTime\":\"2020-02-04T14:59:39.343541+00:00\",\n" + + "\"ExecutionProcessID\":2001,\n" + + "\"ExecutionThreadID\":2616,\n" + + "\"EventID\": 1234,\n" + + "\"TaskValue\":22\n" + + "}"; + return String.format(Locale.ROOT, doc, severity, version, opCode); + } + + public static String randomDocOnlyNumericAndText(int severity, int version, String opCode) { + String doc = "{\n" + + "\"TaskName\":\"SYSTEM\",\n" + + "\"ExecutionProcessID\":2001,\n" + + "\"ExecutionThreadID\":2616,\n" + + "\"EventID\": 1234,\n" + + "\"TaskValue\":22\n" + + "}"; + return String.format(Locale.ROOT, doc, severity, version, opCode); + } + //Add IPs in HostName field. public static String randomDocWithIpIoc(int severity, int version, String ioc) { String doc = "{\n" + @@ -1510,7 +1668,7 @@ public static String randomAppLogDoc() { return "{\n" + " \"endpoint\": \"/customer_records.txt\",\n" + " \"http_method\": \"POST\",\n" + - " \"keywords\": \"PermissionDenied\"\n" + + " \"keywords\": \"INVALID\"\n" + "}"; } diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java index 5f159c062..b91317da6 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java @@ -41,7 +41,15 @@ import static org.opensearch.securityanalytics.TestHelpers.randomDoc; import static org.opensearch.securityanalytics.TestHelpers.randomIndex; import static org.opensearch.securityanalytics.TestHelpers.randomRule; +import static org.opensearch.securityanalytics.TestHelpers.randomRuleWithKeywords; import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping; +import static org.opensearch.securityanalytics.TestHelpers.randomRuleWithStringKeywords; +import static org.opensearch.securityanalytics.TestHelpers.randomDocOnlyNumericAndDate; +import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMappingOnlyNumericAndDate; +import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMappingOnlyNumericAndText; +import static org.opensearch.securityanalytics.TestHelpers.randomRuleWithDateKeywords; +import static org.opensearch.securityanalytics.TestHelpers.randomDocOnlyNumericAndText; + import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ENABLE_WORKFLOW_USAGE; public class DetectorMonitorRestApiIT extends SecurityAnalyticsRestTestCase { @@ -1648,6 +1656,251 @@ public void testCreateDetector_verifyWorkflowExecutionMultipleBucketLevelDocLeve assertEquals(19, getFindingsBody.get("total_findings")); } + public void testCreateDetectorWithKeywordsRule_verifyFindings_success() throws IOException { + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" + ); + + Response createMappingResponse = client().performRequest(createMappingRequest); + + assertEquals(HttpStatus.SC_OK, createMappingResponse.getStatusLine().getStatusCode()); + + // Create random doc rule + String randomDocRuleId = createRule(randomRuleWithKeywords()); + List prepackagedRules = getRandomPrePackagedRules(); + DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(new DetectorRule(randomDocRuleId)), + prepackagedRules.stream().map(DetectorRule::new).collect(Collectors.toList())); + Detector detector = randomDetectorWithInputs(List.of(input)); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + + assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + Map updateResponseBody = asMap(createResponse); + String detectorId = updateResponseBody.get("_id").toString(); + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId + "\"\n" + + " }\n" + + " }\n" + + "}"; + + // Verify newly created doc level monitor + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + Map detectorAsMap = (Map) hit.getSourceAsMap().get("detector"); + List monitorIds = ((List) (detectorAsMap).get("monitor_id")); + + assertEquals(1, monitorIds.size()); + + String monitorId = monitorIds.get(0); + String monitorType = ((Map) entityAsMap(client().performRequest(new Request("GET", "/_plugins/_alerting/monitors/" + monitorId))).get("monitor")).get("monitor_type"); + + assertEquals(MonitorType.DOC_LEVEL_MONITOR.getValue(), monitorType); + + // Verify rules + request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + + assertEquals(6, response.getHits().getTotalHits().value); + + // Verify findings + indexDoc(index, "1", randomDoc(2, 5, "Test")); + indexDoc(index, "2", randomDoc(3, 5, "Test")); + + + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + // Verify 5 prepackaged rules and 1 custom rule + assertEquals(6, noOfSigmaRuleMatches); + + Map params = new HashMap<>(); + params.put("detector_id", detectorId); + Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); + Map getFindingsBody = entityAsMap(getFindingsResponse); + + assertNotNull(getFindingsBody); + // When doc level monitor is being applied one finding is generated per document + assertEquals(2, getFindingsBody.get("total_findings")); + + Set docRuleIds = new HashSet<>(prepackagedRules); + docRuleIds.add(randomDocRuleId); + + List> findings = (List) getFindingsBody.get("findings"); + List foundDocIds = new ArrayList<>(); + for (Map finding : findings) { + Set aggRulesFinding = ((List>) finding.get("queries")).stream().map(it -> it.get("id").toString()).collect( + Collectors.toSet()); + + assertTrue(docRuleIds.containsAll(aggRulesFinding)); + + List findingDocs = (List) finding.get("related_doc_ids"); + Assert.assertEquals(1, findingDocs.size()); + foundDocIds.addAll(findingDocs); + } + assertTrue(Arrays.asList("1", "2").containsAll(foundDocIds)); + } + + public void testCreateDetectorWithKeywordsRule_ensureNoFindingsWithoutTextMapping_success() throws IOException { + String index = createTestIndex(randomIndex(), windowsIndexMappingOnlyNumericAndDate()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" + ); + + Response createMappingResponse = client().performRequest(createMappingRequest); + + assertEquals(HttpStatus.SC_OK, createMappingResponse.getStatusLine().getStatusCode()); + + // Create random doc rule + String randomDocRuleId = createRule(randomRuleWithStringKeywords()); + DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(new DetectorRule(randomDocRuleId)), + emptyList()); + Detector detector = randomDetectorWithInputs(List.of(input)); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + + assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + Map updateResponseBody = asMap(createResponse); + String detectorId = updateResponseBody.get("_id").toString(); + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId + "\"\n" + + " }\n" + + " }\n" + + "}"; + + // Verify newly created doc level monitor + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + Map detectorAsMap = (Map) hit.getSourceAsMap().get("detector"); + List monitorIds = ((List) (detectorAsMap).get("monitor_id")); + + assertEquals(1, monitorIds.size()); + + String monitorId = monitorIds.get(0); + String monitorType = ((Map) entityAsMap(client().performRequest(new Request("GET", "/_plugins/_alerting/monitors/" + monitorId))).get("monitor")).get("monitor_type"); + + assertEquals(MonitorType.DOC_LEVEL_MONITOR.getValue(), monitorType); + + // Verify rules created + request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + + assertEquals(1, response.getHits().getTotalHits().value); + + // Insert test document + indexDoc(index, "1", randomDocOnlyNumericAndDate(2, 5, "Test")); + indexDoc(index, "2", randomDocOnlyNumericAndDate(3, 5, "Test")); + + + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + // Verify no rules match test document + assertEquals(0, noOfSigmaRuleMatches); + } + + public void testCreateDetectorWithKeywordsRule_ensureNoFindingsWithoutDateMapping_success() throws IOException { + String index = createTestIndex(randomIndex(), windowsIndexMappingOnlyNumericAndText()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" + ); + + Response createMappingResponse = client().performRequest(createMappingRequest); + + assertEquals(HttpStatus.SC_OK, createMappingResponse.getStatusLine().getStatusCode()); + + // Create random doc rule + String randomDocRuleId = createRule(randomRuleWithDateKeywords()); + DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(new DetectorRule(randomDocRuleId)), + emptyList()); + Detector detector = randomDetectorWithInputs(List.of(input)); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + + assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + Map updateResponseBody = asMap(createResponse); + String detectorId = updateResponseBody.get("_id").toString(); + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId + "\"\n" + + " }\n" + + " }\n" + + "}"; + + // Verify newly created doc level monitor + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + Map detectorAsMap = (Map) hit.getSourceAsMap().get("detector"); + List monitorIds = ((List) (detectorAsMap).get("monitor_id")); + + assertEquals(1, monitorIds.size()); + + String monitorId = monitorIds.get(0); + String monitorType = ((Map) entityAsMap(client().performRequest(new Request("GET", "/_plugins/_alerting/monitors/" + monitorId))).get("monitor")).get("monitor_type"); + + assertEquals(MonitorType.DOC_LEVEL_MONITOR.getValue(), monitorType); + + // Verify rules created + request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + + assertEquals(1, response.getHits().getTotalHits().value); + + // Insert test document + indexDoc(index, "1", randomDocOnlyNumericAndText(2, 5, "Test")); + indexDoc(index, "2", randomDocOnlyNumericAndText(3, 5, "Test")); + + + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + // Verify no rules match test document + assertEquals(0, noOfSigmaRuleMatches); + } private static void assertRuleMonitorFinding(Map executeResults, String ruleId, int expectedDocCount, List expectedTriggerResult) { List> buckets = ((List>) (((Map) ((Map) ((Map) ((List) ((Map) executeResults.get("input_results")).get("results")).get(0)).get("aggregations")).get("result_agg")).get("buckets"))); diff --git a/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java b/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java index 5dc4c7a9b..81a149311 100644 --- a/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java +++ b/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java @@ -328,7 +328,7 @@ public void testConvertValueRegexUnbound() throws IOException, SigmaError { " sel:\n" + " \"|re\": pat.*tern\"foo\"bar\n" + " condition: sel", false)); - Assert.assertEquals("_0: /pat.*tern\\\"foo\\\"bar/", queries.get(0).toString()); + Assert.assertEquals("/pat.*tern\\\"foo\\\"bar/", queries.get(0).toString()); } public void testConvertValueCidrWildcardNone() throws IOException, SigmaError { @@ -484,7 +484,7 @@ public void testConvertOrInMixedKeywordField() throws IOException, SigmaError { " fieldB: value2\n" + " sel3: value3\n" + " condition: sel1 or sel2 or sel3", false)); - Assert.assertEquals("((fieldA: \"value1\") OR (mappedB: \"value2\")) OR (_0: \"value3\")", queries.get(0).toString()); + Assert.assertEquals("((fieldA: \"value1\") OR (mappedB: \"value2\")) OR (\"value3\")", queries.get(0).toString()); } public void testConvertOrInMixedFields() throws IOException, SigmaError { @@ -597,9 +597,9 @@ public void testConvertUnboundValues() throws IOException, SigmaError { " sel:\n" + " - value1\n" + " - value2\n" + - " - 4\n" + + " - 123\n" + " condition: sel", false)); - Assert.assertEquals("(_0: \"value1\") OR (_1: \"value2\") OR (_2: 4)", queries.get(0).toString()); + Assert.assertEquals("(\"value1\") OR (\"value2\") OR (\"123\")", queries.get(0).toString()); } public void testConvertInvalidUnboundBool() throws IOException { @@ -882,6 +882,31 @@ public void testConvertProxyRule() throws IOException, SigmaError { Assert.assertEquals(true, true); } + public void testConvertUnboundValuesAsWildcard() throws IOException, SigmaError { + OSQueryBackend queryBackend = testBackend(); + List queries = queryBackend.convertRule(SigmaRule.fromYaml( + " title: Test\n" + + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " sel:\n" + + " fieldA1: \n" + + " - value1\n" + + " - value2\n" + + " - value3\n" + + " keywords:\n" + + " - test*\n" + + " condition: sel or keywords", false)); + Assert.assertEquals("((mappedA: \"value1\") OR (mappedA: \"value2\") OR (mappedA: \"value3\")) OR (test*)", queries.get(0).toString()); + } + private OSQueryBackend testBackend() throws IOException { return new OSQueryBackend(testFieldMapping, false, true); }