From c5882dcb471b4c7d316e13535fc7e2dc4004debb Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 4 Oct 2023 14:38:43 -0700 Subject: [PATCH] Change ruleId if it exists (#628) (#636) Signed-off-by: Ashish Agrawal (cherry picked from commit 03573e9cd31551851896aa795e7afd713833de78) Co-authored-by: Ashish Agrawal --- .../securityanalytics/util/RuleIndices.java | 12 +++- .../resthandler/RuleRestApiIT.java | 56 +++++++++++++++++++ 2 files changed, 65 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java b/src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java index baaa8da1c..be5b9f1db 100644 --- a/src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java +++ b/src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java @@ -271,10 +271,16 @@ private String getRuleCategory(Path folderPath) { private void ingestQueries(Map> logIndexToRules, WriteRequest.RefreshPolicy refreshPolicy, TimeValue indexTimeout, ActionListener listener) throws SigmaError, IOException { List queries = new ArrayList<>(); - for (Map.Entry> logIndexToRule: logIndexToRules.entrySet()) { - Map fieldMappings = logTypeService.getRuleFieldMappingsForBuiltinLogType(logIndexToRule.getKey()); + // Moving others_cloud to the top so those queries are indexed first and can be overwritten if other categories + // contain the same rules. Tracking issue: https://github.com/opensearch-project/security-analytics/issues/630 + List categories = new ArrayList<>(logIndexToRules.keySet()); + if (categories.remove("others_cloud")) { + categories.add(0, "others_cloud"); + } + for (String category: categories) { + Map fieldMappings = logTypeService.getRuleFieldMappingsForBuiltinLogType(category); final QueryBackend backend = new OSQueryBackend(fieldMappings, true, true); - queries.addAll(getQueries(backend, logIndexToRule.getKey(), logIndexToRule.getValue())); + queries.addAll(getQueries(backend, category, logIndexToRules.get(category))); } loadRules(queries, refreshPolicy, indexTimeout, listener, true); } diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/RuleRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/RuleRestApiIT.java index a230d54c5..439c7c9cb 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/RuleRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/RuleRestApiIT.java @@ -21,12 +21,17 @@ import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; import org.opensearch.securityanalytics.SecurityAnalyticsRestTestCase; import org.opensearch.securityanalytics.config.monitors.DetectorMonitorConfig; +import org.opensearch.securityanalytics.logtype.BuiltinLogTypeLoader; import org.opensearch.securityanalytics.model.Detector; import org.opensearch.securityanalytics.model.DetectorInput; import org.opensearch.securityanalytics.model.DetectorRule; import org.opensearch.securityanalytics.model.Rule; +import java.io.File; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Collections; import java.util.List; import java.util.Locale; @@ -191,6 +196,57 @@ public void testSearchingPrepackagedRules() throws IOException { Assert.assertEquals(5, ((Map) ((Map) responseBody.get("hits")).get("total")).get("value")); } + public void testSearchingForDuplicatedPrepackagedRules() throws IOException { + String gworkspaceRequest = "{\n" + + " \"query\": {\n" + + " \"nested\": {\n" + + " \"path\": \"rule\",\n" + + " \"query\": {\n" + + " \"bool\": {\n" + + " \"must\": [\n" + + " { \"match\": {\"rule.category\": \"gworkspace\"}}\n" + + " ]\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}"; + + Response gworkSpaceSearchResponse = makeRequest(client(), "POST", String.format(Locale.getDefault(), "%s/_search", SecurityAnalyticsPlugin.RULE_BASE_URI), Collections.singletonMap("pre_packaged", "true"), + new StringEntity(gworkspaceRequest), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Searching rules failed", RestStatus.OK, restStatus(gworkSpaceSearchResponse)); + + String azureRequest = "{\n" + + " \"query\": {\n" + + " \"nested\": {\n" + + " \"path\": \"rule\",\n" + + " \"query\": {\n" + + " \"bool\": {\n" + + " \"must\": [\n" + + " { \"match\": {\"rule.category\": \"azure\"}}\n" + + " ]\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}"; + + Response azureSearchResponse = makeRequest(client(), "POST", String.format(Locale.getDefault(), "%s/_search", SecurityAnalyticsPlugin.RULE_BASE_URI), Collections.singletonMap("pre_packaged", "true"), + new StringEntity(azureRequest), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Searching rules failed", RestStatus.OK, restStatus(azureSearchResponse)); + + ClassLoader classLoader = getClass().getClassLoader(); + int gworkspaceFileCount = new File(classLoader.getResource("rules/gworkspace").getFile()).listFiles().length; + int azureFileCount = new File(classLoader.getResource("rules/azure").getFile()).listFiles().length; + + // Verify azure and gworkspace categories have the right number of rules even though they + // conflict with others_cloud category + Map gworkspaceResponseBody = asMap(gworkSpaceSearchResponse); + Assert.assertEquals(gworkspaceFileCount, ((Map) ((Map) gworkspaceResponseBody.get("hits")).get("total")).get("value")); + Map azureResponseBody = asMap(azureSearchResponse); + Assert.assertEquals(azureFileCount, ((Map) ((Map) azureResponseBody.get("hits")).get("total")).get("value")); + } + @SuppressWarnings("unchecked") public void testSearchingPrepackagedRulesByMitreAttackID() throws IOException { String request = "{\n" +