diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java index f2a43ac50..5c89f727d 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java @@ -233,10 +233,10 @@ private void createMonitorFromQueries(String index, List> rul List monitorRequests = new ArrayList<>(); if (!docLevelRules.isEmpty()) { - monitorRequests.add(createDocLevelMonitorRequest(Pair.of(index, docLevelRules), detector, refreshPolicy, Monitor.NO_ID, Method.POST)); + monitorRequests.add(createDocLevelMonitorRequest(docLevelRules, detector, refreshPolicy, Monitor.NO_ID, Method.POST)); } if (!bucketLevelRules.isEmpty()) { - monitorRequests.addAll(buildBucketLevelMonitorRequests(Pair.of(index, bucketLevelRules), detector, refreshPolicy, Monitor.NO_ID, Method.POST)); + monitorRequests.addAll(buildBucketLevelMonitorRequests(bucketLevelRules, detector, refreshPolicy, Monitor.NO_ID, Method.POST)); } // Do nothing if detector doesn't have any monitor if (monitorRequests.isEmpty()){ @@ -303,7 +303,6 @@ private void updateMonitorFromQueries(String index, List> rul if (monitorPerRule.containsKey(rule.getId())) { String monitorId = monitorPerRule.get(rule.getId()); monitorsToBeUpdated.add(createBucketLevelMonitorRequest(query.getRight(), - index, detector, refreshPolicy, monitorId, @@ -311,7 +310,6 @@ private void updateMonitorFromQueries(String index, List> rul queryBackendMap.get(rule.getCategory()))); } else { monitorsToBeAdded.add(createBucketLevelMonitorRequest(query.getRight(), - index, detector, refreshPolicy, Monitor.NO_ID, @@ -328,9 +326,9 @@ private void updateMonitorFromQueries(String index, List> rul // Process doc level monitors if (!docLevelRules.isEmpty()) { if (detector.getDocLevelMonitorId() == null) { - monitorsToBeAdded.add(createDocLevelMonitorRequest(Pair.of(index, docLevelRules), detector, refreshPolicy, Monitor.NO_ID, Method.POST)); + monitorsToBeAdded.add(createDocLevelMonitorRequest(docLevelRules, detector, refreshPolicy, Monitor.NO_ID, Method.POST)); } else { - monitorsToBeUpdated.add(createDocLevelMonitorRequest(Pair.of(index, docLevelRules), detector, refreshPolicy, detector.getDocLevelMonitorId(), Method.PUT)); + monitorsToBeUpdated.add(createDocLevelMonitorRequest(docLevelRules, detector, refreshPolicy, detector.getDocLevelMonitorId(), Method.PUT)); } } @@ -394,12 +392,12 @@ private void updateAlertingMonitors( }, listener::onFailure); } - private IndexMonitorRequest createDocLevelMonitorRequest(Pair>> logIndexToQueries, Detector detector, WriteRequest.RefreshPolicy refreshPolicy, String monitorId, RestRequest.Method restMethod) { + private IndexMonitorRequest createDocLevelMonitorRequest(List> queries, Detector detector, WriteRequest.RefreshPolicy refreshPolicy, String monitorId, RestRequest.Method restMethod) { List docLevelMonitorInputs = new ArrayList<>(); List docLevelQueries = new ArrayList<>(); - for (Pair query: logIndexToQueries.getRight()) { + for (Pair query: queries) { String id = query.getLeft(); Rule rule = query.getRight(); @@ -415,7 +413,7 @@ private IndexMonitorRequest createDocLevelMonitorRequest(Pair triggers = new ArrayList<>(); @@ -445,8 +443,8 @@ private IndexMonitorRequest createDocLevelMonitorRequest(Pair buildBucketLevelMonitorRequests(Pair>> logIndexToQueries, Detector detector, WriteRequest.RefreshPolicy refreshPolicy, String monitorId, RestRequest.Method restMethod) throws IOException, SigmaError { - List ruleCategories = logIndexToQueries.getRight().stream().map(Pair::getRight).map(Rule::getCategory).distinct().collect( + private List buildBucketLevelMonitorRequests(List> queries, Detector detector, WriteRequest.RefreshPolicy refreshPolicy, String monitorId, RestRequest.Method restMethod) throws IOException, SigmaError { + List ruleCategories = queries.stream().map(Pair::getRight).map(Rule::getCategory).distinct().collect( Collectors.toList()); Map queryBackendMap = new HashMap<>(); @@ -456,14 +454,13 @@ private List buildBucketLevelMonitorRequests(Pair monitorRequests = new ArrayList<>(); - for (Pair query: logIndexToQueries.getRight()) { + for (Pair query: queries) { Rule rule = query.getRight(); // Creating bucket level monitor per each aggregation rule if (rule.getAggregationQueries() != null){ monitorRequests.add(createBucketLevelMonitorRequest( query.getRight(), - logIndexToQueries.getLeft(), detector, refreshPolicy, Monitor.NO_ID, @@ -476,13 +473,15 @@ private List buildBucketLevelMonitorRequests(Pair indices = detector.getInputs().get(0).getIndices(); + AggregationQueries aggregationQueries = queryBackend.convertAggregation(rule.getAggregationItemsFromRule().get(0)); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder() @@ -491,10 +490,11 @@ private IndexMonitorRequest createBucketLevelMonitorRequest( // Build query string filter .query(QueryBuilders.queryStringQuery(rule.getQueries().get(0).getValue())) .aggregation(aggregationQueries.getAggBuilder()); - String concreteIndex = IndexUtils.getNewIndexByCreationDate( // index variable in method signature can also be an index pattern + // input index can also be an index pattern or alias so we have to resolve it to concrete index + String concreteIndex = IndexUtils.getNewIndexByCreationDate( clusterService.state(), indexNameExpressionResolver, - index + indices.get(0) // taking first one is fine because we expect that all indices in list share same mappings ); try { GetIndexMappingsResponse getIndexMappingsResponse = client.execute( @@ -526,7 +526,7 @@ private IndexMonitorRequest createBucketLevelMonitorRequest( } List bucketLevelMonitorInputs = new ArrayList<>(); - bucketLevelMonitorInputs.add(new SearchInput(Arrays.asList(index), searchSourceBuilder)); + bucketLevelMonitorInputs.add(new SearchInput(indices, searchSourceBuilder)); List triggers = new ArrayList<>(); BucketLevelTrigger bucketLevelTrigger = new BucketLevelTrigger(rule.getId(), rule.getTitle(), rule.getLevel(), aggregationQueries.getCondition(), diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java index 1711a6ca3..4b3cf281c 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java @@ -100,6 +100,81 @@ public void testCreatingADetector() throws IOException { Assert.assertEquals(5, noOfSigmaRuleMatches); } + public void testCreatingADetectorWithMultipleIndices() throws IOException { + String index1 = createTestIndex("windows-1", windowsIndexMapping()); + String index2 = createTestIndex("windows-2", windowsIndexMapping()); + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"windows*\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + Detector detector = randomDetectorWithTriggers( + getRandomPrePackagedRules(), + List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(randomDetectorType()), List.of(), List.of(), List.of(), List.of())), + List.of(index1, index2) + ); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + + String createdId = responseBody.get("_id").toString(); + int createdVersion = Integer.parseInt(responseBody.get("_version").toString()); + Assert.assertNotEquals("response is missing Id", Detector.NO_ID, createdId); + Assert.assertTrue("incorrect version", createdVersion > 0); + Assert.assertEquals("Incorrect Location header", String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, createdId), createResponse.getHeader("Location")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("rule_topic_index")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("findings_index")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("alert_index")); + + String detectorTypeInResponse = (String) ((Map)responseBody.get("detector")).get("detector_type"); + Assert.assertEquals("Detector type incorrect", randomDetectorType().toLowerCase(Locale.ROOT), detectorTypeInResponse); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + createdId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + String monitorId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); + + indexDoc(index1, "1", randomDoc()); + indexDoc(index2, "1", randomDoc()); + + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + + int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(5, noOfSigmaRuleMatches); + List> results = ((List>) ((Map) executeResults.get("input_results")).get("results")); + List matchedDocs = (List) (results.get(0)).values().iterator().next(); + assertTrue(matchedDocs.get(0).equals("1|windows-1")); + assertTrue(matchedDocs.get(1).equals("1|windows-2")); + + // Check findings + Map params = new HashMap<>(); + params.put("detector_id", createdId); + Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); + Map getFindingsBody = entityAsMap(getFindingsResponse); + assertNotNull(getFindingsBody); + Assert.assertEquals(2, getFindingsBody.get("total_findings")); + List findings = (List) getFindingsBody.get("findings"); + Assert.assertEquals(findings.size(), 2); + } + public void testCreatingADetectorWithIndexNotExists() throws IOException { Detector detector = randomDetectorWithTriggers(getRandomPrePackagedRules(), List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(randomDetectorType()), List.of(), List.of(), List.of(), List.of())));