From 6c4e49c19048b7e5228f06ce3697a4b0d6b33e52 Mon Sep 17 00:00:00 2001 From: AWSHurneyt Date: Tue, 18 Jun 2024 10:49:39 -0700 Subject: [PATCH 1/3] Implemented logic to update the IocStoreConfig with the saTifSourceConfig ID and IOC index names. Signed-off-by: AWSHurneyt --- .../services/STIX2IOCFeedStore.java | 16 ++++++++++------ .../services/STIX2IOCFetchService.java | 18 +++++++++++++++--- .../model/DefaultIocStoreConfig.java | 2 ++ .../SATIFSourceConfigManagementService.java | 2 +- .../SATIFSourceConfigRestApiIT.java | 7 +------ 5 files changed, 29 insertions(+), 16 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFeedStore.java b/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFeedStore.java index 5eaf6a164..4b4ab8fea 100644 --- a/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFeedStore.java +++ b/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFeedStore.java @@ -30,6 +30,7 @@ import org.opensearch.securityanalytics.model.STIX2IOC; import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; import org.opensearch.securityanalytics.threatIntel.common.StashedThreadContext; +import org.opensearch.securityanalytics.threatIntel.model.DefaultIocStoreConfig; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; import java.io.ByteArrayOutputStream; @@ -111,10 +112,11 @@ public void storeIOCs(Map actionToIOCs) { } public void indexIocs(List iocs) throws IOException { - // TODO @jowg, there seems to be a bug in SATIFSourceConfigManagementService. - // downloadAndSaveIOCs is called before indexTIFSourceConfig, which means the config doesn't have an ID to use when creating the system index to store IOCs. - // Testing using SaTifSourceConfigDto.getName() instead of .getId() for now. - String feedIndexName = initFeedIndex(saTifSourceConfig.getName()); + String feedIndexName = initFeedIndex(saTifSourceConfig.getId()); + + // Add the created index to the IocStoreConfig + ((DefaultIocStoreConfig) saTifSourceConfig.getIocStoreConfig()).getIocMapStore().putIfAbsent(saTifSourceConfig.getId(), new ArrayList<>()); + ((DefaultIocStoreConfig) saTifSourceConfig.getIocStoreConfig()).getIocMapStore().get(saTifSourceConfig.getId()).add(feedIndexName); List bulkRequestList = new ArrayList<>(); BulkRequest bulkRequest = new BulkRequest(); @@ -150,7 +152,10 @@ public void indexIocs(List iocs) throws IOException { long duration = Duration.between(startTime, Instant.now()).toMillis(); STIX2IOCFetchService.STIX2IOCFetchResponse output = new STIX2IOCFetchService.STIX2IOCFetchResponse(iocs, duration); baseListener.onResponse(output); - }, baseListener::onFailure), bulkRequestList.size()); + }, e -> { + log.error("Failed to index IOCs.", e); + baseListener.onFailure(e); + }), bulkRequestList.size()); for (BulkRequest req : bulkRequestList) { try { @@ -175,7 +180,6 @@ public static String getFeedConfigIndexName(String feedSourceConfigId) { return IOC_INDEX_NAME_TEMPLATE.replace(IOC_FEED_ID_PLACEHOLDER, feedSourceConfigId.toLowerCase(Locale.ROOT)); } - // TODO hurneyt change ActionResponse to more specific response once it's available public String initFeedIndex(String feedSourceConfigId) { String feedIndexName = getFeedConfigIndexName(feedSourceConfigId); if (!feedIndexExists(feedIndexName)) { diff --git a/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFetchService.java b/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFetchService.java index 76c672f38..7861a86b6 100644 --- a/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFetchService.java +++ b/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFetchService.java @@ -62,7 +62,7 @@ public STIX2IOCFetchService(Client client, ClusterService clusterService) { batchSize = clusterService.getClusterSettings().get(SecurityAnalyticsSettings.BATCH_SIZE); } - public void fetchIocs(SATIFSourceConfig saTifSourceConfig, ActionListener listener) { + public void downloadAndIndexIOCs(SATIFSourceConfig saTifSourceConfig, ActionListener listener) { S3ConnectorConfig s3ConnectorConfig = new S3ConnectorConfig( ((S3Source) saTifSourceConfig.getSource()).getBucketName(), ((S3Source) saTifSourceConfig.getSource()).getObjectKey(), @@ -76,8 +76,20 @@ public void fetchIocs(SATIFSourceConfig saTifSourceConfig, ActionListener> iocMapStore; public DefaultIocStoreConfig(Map> iocMapStore) { diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java index 808199905..605789b62 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java @@ -122,7 +122,7 @@ public void downloadAndSaveIOCs(SATIFSourceConfig SaTifSourceConfig, ActionListe // call to update or create IOCs - state can be either creating or refreshing here // on success, change state back to available // on failure, change state to refresh failed and mark source config as refresh failed - stix2IOCFetchService.fetchIocs(SaTifSourceConfig, actionListener); + stix2IOCFetchService.downloadAndIndexIOCs(SaTifSourceConfig, actionListener); } public void getTIFSourceConfig( diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/SATIFSourceConfigRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/SATIFSourceConfigRestApiIT.java index ade34d5a6..03994b2df 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/SATIFSourceConfigRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/SATIFSourceConfigRestApiIT.java @@ -364,8 +364,6 @@ public void testRetrieveIOCsSuccessfully() throws IOException, InterruptedExcept // Wait for feed to execute - // TODO @jowg, last_updated_time is null in responseBody, but last_refreshed_time is present. - // Can you clarify which should be used here? String firstUpdatedTime = (String) ((Map)responseBody.get("tif_config")).get("last_refreshed_time"); waitUntil(() -> { try { @@ -376,10 +374,7 @@ public void testRetrieveIOCsSuccessfully() throws IOException, InterruptedExcept }, 240, TimeUnit.SECONDS); // Confirm IOCs were ingested to system index for the feed - // TODO @jowg, there seems to be a bug in SATIFSourceConfigManagementService. - // downloadAndSaveIOCs is called before indexTIFSourceConfig, which means the config doesn't have an ID to use when creating the system index to store IOCs. - // Testing using SaTifSourceConfigDto.getName() instead of .getId() for now. - String indexName = STIX2IOCFeedStore.getFeedConfigIndexName(SaTifSourceConfigDto.getName()); + String indexName = STIX2IOCFeedStore.getFeedConfigIndexName(SaTifSourceConfigDto.getId()); String request = "{\n" + " \"query\" : {\n" + " \"match_all\":{\n" + From 6d6e32a1d542abe4b9a4b3d1b10323e9a1bf7dfc Mon Sep 17 00:00:00 2001 From: AWSHurneyt Date: Tue, 18 Jun 2024 11:12:38 -0700 Subject: [PATCH 2/3] Removed unused test suite. Signed-off-by: AWSHurneyt --- .../services/STIX2IOCFetchServiceIT.java | 210 ------------------ 1 file changed, 210 deletions(-) delete mode 100644 src/test/java/org/opensearch/securityanalytics/services/STIX2IOCFetchServiceIT.java diff --git a/src/test/java/org/opensearch/securityanalytics/services/STIX2IOCFetchServiceIT.java b/src/test/java/org/opensearch/securityanalytics/services/STIX2IOCFetchServiceIT.java deleted file mode 100644 index d2b41f21b..000000000 --- a/src/test/java/org/opensearch/securityanalytics/services/STIX2IOCFetchServiceIT.java +++ /dev/null @@ -1,210 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.securityanalytics.services; - -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.opensearch.action.delete.DeleteRequest; -import org.opensearch.core.action.ActionListener; -import org.opensearch.securityanalytics.commons.connector.model.S3ConnectorConfig; -import org.opensearch.securityanalytics.commons.utils.testUtils.S3ObjectGenerator; -import org.opensearch.securityanalytics.model.STIX2IOC; -import org.opensearch.securityanalytics.model.STIX2IOCDto; -import org.opensearch.securityanalytics.util.STIX2IOCGenerator; -import org.opensearch.test.OpenSearchIntegTestCase; -import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.services.s3.S3Client; - -import java.io.IOException; -import java.util.Locale; -import java.util.UUID; - -public class STIX2IOCFetchServiceIT extends OpenSearchIntegTestCase { - private String bucket; - private String region; - private String roleArn; - - private S3Client s3Client; - private S3ObjectGenerator s3ObjectGenerator; - private STIX2IOCFetchService service; - - private String testFeedSourceConfigId; - private String testIndex; - private S3ConnectorConfig s3ConnectorConfig; - - @Before - public void beforeTest() { - if (service == null) { - region = System.getProperty("tests.STIX2IOCFetchServiceIT.region"); - roleArn = System.getProperty("tests.STIX2IOCFetchServiceIT.roleArn"); - bucket = System.getProperty("tests.STIX2IOCFetchServiceIT.bucket"); - - s3Client = S3Client.builder() - .region(Region.of(region)) - .build(); - s3ObjectGenerator = new S3ObjectGenerator(s3Client, bucket); - - service = new STIX2IOCFetchService(); - } - testFeedSourceConfigId = UUID.randomUUID().toString(); - testIndex = null; - s3ConnectorConfig = new S3ConnectorConfig(bucket, testFeedSourceConfigId, region, roleArn); - } - - @After - private void afterTest() { - if (testIndex != null && !testIndex.isBlank()) { - client().delete(new DeleteRequest(testIndex)); - } - } - - @Test - public void test_fetchIocs_fetchesIocsCorrectly() throws IOException { - int numOfIOCs = 5; - s3ObjectGenerator.write(numOfIOCs, testFeedSourceConfigId, new STIX2IOCGenerator()); - - ActionListener listener = new ActionListener<>() { - @Override - public void onResponse(STIX2IOCFetchService.STIX2IOCFetchResponse stix2IOCFetchResponse) { - assertEquals(numOfIOCs, stix2IOCFetchResponse.getIocs().size()); - //TODO hurneyt need to retrieve the test IOCs from s3ObjectGenerator.write, and compare to output - } - - @Override - public void onFailure(Exception e) { - fail("STIX2IOCFetchService.fetchIocs failed with error: " + e); - } - }; - - service.fetchIocs(s3ConnectorConfig, listener); - } - - - // TODO hurneyt extract feedIndexExists and initFeedIndex to helper function, or expose for testing -// @Test -// public void test_hasIocSystemIndex_returnsFalse_whenIndexNotCreated() throws ExecutionException, InterruptedException { -// // Confirm index doesn't exist before running test case -// testIndex = STIX2IOCFeedStore.getFeedConfigIndexName(testFeedSourceConfigId); -// ClusterHealthResponse clusterHealthResponse = client().admin().cluster().health(new ClusterHealthRequest()).get(); -// assertFalse(clusterHealthResponse.getIndices().containsKey(testIndex)); -// -// // Run test case -// assertFalse(service.feedIndexExists(testIndex)); -// } -// -// @Test -// public void test_hasIocSystemIndex_returnsFalse_withInvalidIndex() throws ExecutionException, InterruptedException { -// // Create test index -// testIndex = STIX2IOCFeedStore.getFeedConfigIndexName(testFeedSourceConfigId); -// client().admin().indices().create(new CreateIndexRequest(testIndex)).get(); -// -// // Run test case -// assertFalse(service.feedIndexExists(testIndex)); -// } -// -// @Test -// public void test_hasIocSystemIndex_returnsTrue_whenIndexExists() throws ExecutionException, InterruptedException { -// // Create test index -// testIndex = STIX2IOCFeedStore.getFeedConfigIndexName(testFeedSourceConfigId); -// client().admin().indices().create(new CreateIndexRequest(testIndex)).get(); -// -// // Run test case -// assertTrue(service.feedIndexExists(testIndex)); -// } -// -// @Test -// public void test_initSystemIndexes_createsIndexes() { -// // Confirm index doesn't exist -// testIndex = IocService.getFeedConfigIndexName(testFeedSourceConfigId); -// assertFalse(service.feedIndexExists(testIndex)); -// -// // Run test case -// service.initFeedIndex(testIndex, new ActionListener<>() { -// @Override -// public void onResponse(FetchIocsActionResponse fetchIocsActionResponse) {} -// -// @Override -// public void onFailure(Exception e) { -// fail(String.format("Creation of %s should not fail: %s", testIndex, e)); -// } -// }); -// assertTrue(service.feedIndexExists(testIndex)); -// } -// -// @Test -// public void test_indexIocs_ingestsIocsCorrectly() throws IOException { -// // Prepare test IOCs -// List iocs = IntStream.range(0, randomInt()) -// .mapToObj(i -> STIX2IOCGenerator.randomIOC()) -// .collect(Collectors.toList()); -// -// // Run test case -// service.indexIocs(testFeedSourceConfigId, iocs, new ActionListener<>() { -// @Override -// public void onResponse(FetchIocsActionResponse fetchIocsActionResponse) { -// // Confirm expected number of IOCs in response -// assertEquals(iocs.size(), fetchIocsActionResponse.getIocs().size()); -// -// try { -// // Search system indexes directly -// SearchRequest searchRequest = new SearchRequest() -// .indices(IOC_ALL_INDEX_PATTERN) -// .source(new SearchSourceBuilder().query(QueryBuilders.matchAllQuery())); -// SearchResponse searchResponse = client().search(searchRequest).get(); -// -// // Confirm expected number of hits -// assertEquals(iocs.size(), searchResponse.getHits().getHits().length); -// -// // Parse hits to IOCs -// List iocHits = Collections.emptyList(); -// for (SearchHit ioc : searchResponse.getHits()) { -// try { -// iocHits.add(IocModel.parse(TestHelpers.parser(ioc.getSourceAsString()), null)); -// } catch (IOException e) { -// fail(String.format("Failed to parse IOC hit: %s", e)); -// } -// } -// -// // Confirm expected number of IOCs -// assertEquals(iocs.size(), iocHits.size()); -// -// // Sort IOCs for comparison -// iocs.sort(Comparator.comparing(IocModel::getId)); -// fetchIocsActionResponse.getIocs().sort(Comparator.comparing(IocDto::getId)); -// iocHits.sort(Comparator.comparing(IocModel::getId)); -// -// // Confirm IOCs are equal -// for (int i = 0; i < iocs.size(); i++) { -// assertEqualIocs(iocs.get(i), fetchIocsActionResponse.getIocs().get(i)); -// IocModelTests.assertEqualIOCs(iocs.get(i), iocHits.get(i)); -// } -// } catch (InterruptedException | ExecutionException e) { -// fail(String.format("IOC_ALL_INDEX_PATTERN search failed: %s", e)); -// } -// } -// -// @Override -// public void onFailure(Exception e) { -// fail(String.format("Ingestion of IOCs should not fail: %s", e)); -// } -// }); -// } - - private String createEndpointString() { - return STIX2IOCServiceTestAPI.RestSTIX2IOCServiceTestAPIAction.ROUTE + String.format(Locale.getDefault(), - "?%s=%s&%s=%s&%s=%s&%s=%s", - STIX2IOCServiceTestAPI.STIX2IOCServiceTestAPIRequest.BUCKET_FIELD, - bucket, - STIX2IOCServiceTestAPI.STIX2IOCServiceTestAPIRequest.REGION_FIELD, - region, - STIX2IOCServiceTestAPI.STIX2IOCServiceTestAPIRequest.ROLE_ARN_FIELD, - roleArn, - STIX2IOCServiceTestAPI.STIX2IOCServiceTestAPIRequest.OBJECT_KEY_FIELD, - testFeedSourceConfigId - ); - } -} From 26351ea8fe520625fbf7059cd3273aee0f9f7bc5 Mon Sep 17 00:00:00 2001 From: AWSHurneyt Date: Wed, 19 Jun 2024 12:41:05 -0700 Subject: [PATCH 3/3] Added configId to error logs. Signed-off-by: AWSHurneyt --- .../securityanalytics/services/STIX2IOCFeedStore.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFeedStore.java b/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFeedStore.java index 4b4ab8fea..06a5d93b7 100644 --- a/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFeedStore.java +++ b/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFeedStore.java @@ -153,7 +153,7 @@ public void indexIocs(List iocs) throws IOException { STIX2IOCFetchService.STIX2IOCFetchResponse output = new STIX2IOCFetchService.STIX2IOCFetchResponse(iocs, duration); baseListener.onResponse(output); }, e -> { - log.error("Failed to index IOCs.", e); + log.error("Failed to index IOCs for config {}", saTifSourceConfig.getId(), e); baseListener.onFailure(e); }), bulkRequestList.size()); @@ -161,7 +161,7 @@ public void indexIocs(List iocs) throws IOException { try { StashedThreadContext.run(client, () -> client.bulk(req, bulkResponseListener)); } catch (OpenSearchException e) { - log.error("Failed to save IOCs.", e); + log.error("Failed to save IOCs for config {}", saTifSourceConfig.getId(), e); baseListener.onFailure(e); } }