diff --git a/release-notes/opensearch-security-analytics.release-notes-2.15.0.0.md b/release-notes/opensearch-security-analytics.release-notes-2.15.0.0.md new file mode 100644 index 000000000..9bd0879dd --- /dev/null +++ b/release-notes/opensearch-security-analytics.release-notes-2.15.0.0.md @@ -0,0 +1,19 @@ +## Version 2.15.0.0 2024-06-10 + +Compatible with OpenSearch 2.15.0 + +### Features +* Alerts in correlations [Experminental] ([#1040](https://github.com/opensearch-project/security-analytics/pull/1040)) +* Alerts in Correlations Part 2 ([#1062](https://github.com/opensearch-project/security-analytics/pull/1062)) + +### Maintenance +* Increment version to 2.15.0-SNAPSHOT. ([#1055](https://github.com/opensearch-project/security-analytics/pull/1055)) +* Fix codecov calculation ([#1021](https://github.com/opensearch-project/security-analytics/pull/1021)) +* Stabilize integ tests ([#1014](https://github.com/opensearch-project/security-analytics/pull/1014)) + +### Bug Fixes +* Fix chained findings monitor logic in update detector flow ([#1019](https://github.com/opensearch-project/security-analytics/pull/1019)) +* Change default filter to time based fields ([#1030](https://github.com/opensearch-project/security-analytics/pull/1030)) + +### Documentation +* Added 2.15.0 release notes. ([#1061](https://github.com/opensearch-project/security-analytics/pull/1061)) diff --git a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java index e458afce4..98c041e4b 100644 --- a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java +++ b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java @@ -10,6 +10,7 @@ import org.opensearch.alerting.spi.RemoteMonitorRunner; import org.opensearch.alerting.spi.RemoteMonitorRunnerExtension; import org.opensearch.client.Client; +import org.opensearch.client.node.NodeClient; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.node.DiscoveryNode; @@ -50,6 +51,7 @@ import org.opensearch.rest.RestHandler; import org.opensearch.script.ScriptService; import org.opensearch.securityanalytics.action.AckAlertsAction; +import org.opensearch.securityanalytics.action.AckCorrelationAlertsAction; import org.opensearch.securityanalytics.action.CorrelatedFindingAction; import org.opensearch.securityanalytics.action.CreateIndexMappingsAction; import org.opensearch.securityanalytics.action.DeleteCorrelationRuleAction; @@ -58,6 +60,7 @@ import org.opensearch.securityanalytics.action.DeleteRuleAction; import org.opensearch.securityanalytics.action.GetAlertsAction; import org.opensearch.securityanalytics.action.GetAllRuleCategoriesAction; +import org.opensearch.securityanalytics.action.GetCorrelationAlertsAction; import org.opensearch.securityanalytics.action.GetDetectorAction; import org.opensearch.securityanalytics.action.GetFindingsAction; import org.opensearch.securityanalytics.action.GetIndexMappingsAction; @@ -75,6 +78,8 @@ import org.opensearch.securityanalytics.action.TestS3ConnectionAction; import org.opensearch.securityanalytics.action.UpdateIndexMappingsAction; import org.opensearch.securityanalytics.action.ValidateRulesAction; +import org.opensearch.securityanalytics.correlation.alert.CorrelationAlertService; +import org.opensearch.securityanalytics.correlation.alert.notifications.NotificationService; import org.opensearch.securityanalytics.correlation.index.codec.CorrelationCodecService; import org.opensearch.securityanalytics.correlation.index.mapper.CorrelationVectorFieldMapper; import org.opensearch.securityanalytics.correlation.index.query.CorrelationQueryBuilder; @@ -90,6 +95,7 @@ import org.opensearch.securityanalytics.model.Rule; import org.opensearch.securityanalytics.model.ThreatIntelFeedData; import org.opensearch.securityanalytics.resthandler.RestAcknowledgeAlertsAction; +import org.opensearch.securityanalytics.resthandler.RestAcknowledgeCorrelationAlertsAction; import org.opensearch.securityanalytics.resthandler.RestCreateIndexMappingsAction; import org.opensearch.securityanalytics.resthandler.RestDeleteCorrelationRuleAction; import org.opensearch.securityanalytics.resthandler.RestDeleteCustomLogTypeAction; @@ -97,6 +103,7 @@ import org.opensearch.securityanalytics.resthandler.RestDeleteRuleAction; import org.opensearch.securityanalytics.resthandler.RestGetAlertsAction; import org.opensearch.securityanalytics.resthandler.RestGetAllRuleCategoriesAction; +import org.opensearch.securityanalytics.resthandler.RestGetCorrelationsAlertsAction; import org.opensearch.securityanalytics.resthandler.RestGetDetectorAction; import org.opensearch.securityanalytics.resthandler.RestGetFindingsAction; import org.opensearch.securityanalytics.resthandler.RestGetIndexMappingsAction; @@ -117,30 +124,35 @@ import org.opensearch.securityanalytics.resthandler.RestValidateRulesAction; import org.opensearch.securityanalytics.services.STIX2IOCFetchService; import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.securityanalytics.threatIntel.action.GetIocFindingsAction; import org.opensearch.securityanalytics.threatIntel.action.PutTIFJobAction; import org.opensearch.securityanalytics.threatIntel.action.SADeleteTIFSourceConfigAction; -import org.opensearch.securityanalytics.threatIntel.action.GetIocFindingsAction; import org.opensearch.securityanalytics.threatIntel.action.SAGetTIFSourceConfigAction; import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigAction; import org.opensearch.securityanalytics.threatIntel.action.SARefreshTIFSourceConfigAction; import org.opensearch.securityanalytics.threatIntel.action.SASearchTIFSourceConfigsAction; import org.opensearch.securityanalytics.threatIntel.action.monitor.DeleteThreatIntelMonitorAction; +import org.opensearch.securityanalytics.threatIntel.action.monitor.GetThreatIntelAlertsAction; import org.opensearch.securityanalytics.threatIntel.action.monitor.IndexThreatIntelMonitorAction; import org.opensearch.securityanalytics.threatIntel.action.monitor.SearchThreatIntelMonitorAction; import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; import org.opensearch.securityanalytics.threatIntel.feedMetadata.BuiltInTIFMetadataLoader; +import org.opensearch.securityanalytics.threatIntel.iocscan.dao.IocFindingService; +import org.opensearch.securityanalytics.threatIntel.iocscan.dao.ThreatIntelAlertService; +import org.opensearch.securityanalytics.threatIntel.iocscan.service.SaIoCScanService; +import org.opensearch.securityanalytics.threatIntel.iocscan.service.ThreatIntelMonitorRunner; import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobRunner; import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFSourceConfigRunner; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; +import org.opensearch.securityanalytics.threatIntel.model.monitor.TransportThreatIntelMonitorFanOutAction; import org.opensearch.securityanalytics.threatIntel.resthandler.RestDeleteTIFSourceConfigAction; -import org.opensearch.securityanalytics.threatIntel.model.monitor.SampleRemoteDocLevelMonitorRunner; -import org.opensearch.securityanalytics.threatIntel.model.monitor.TransportRemoteDocLevelMonitorFanOutAction; import org.opensearch.securityanalytics.threatIntel.resthandler.RestGetIocFindingsAction; import org.opensearch.securityanalytics.threatIntel.resthandler.RestGetTIFSourceConfigAction; import org.opensearch.securityanalytics.threatIntel.resthandler.RestIndexTIFSourceConfigAction; import org.opensearch.securityanalytics.threatIntel.resthandler.RestRefreshTIFSourceConfigAction; import org.opensearch.securityanalytics.threatIntel.resthandler.RestSearchTIFSourceConfigsAction; import org.opensearch.securityanalytics.threatIntel.resthandler.monitor.RestDeleteThreatIntelMonitorAction; +import org.opensearch.securityanalytics.threatIntel.resthandler.monitor.RestGetThreatIntelAlertsAction; import org.opensearch.securityanalytics.threatIntel.resthandler.monitor.RestIndexThreatIntelMonitorAction; import org.opensearch.securityanalytics.threatIntel.resthandler.monitor.RestSearchThreatIntelMonitorAction; import org.opensearch.securityanalytics.threatIntel.service.DetectorThreatIntelService; @@ -150,14 +162,17 @@ import org.opensearch.securityanalytics.threatIntel.service.TIFJobUpdateService; import org.opensearch.securityanalytics.threatIntel.service.ThreatIntelFeedDataService; import org.opensearch.securityanalytics.threatIntel.transport.TransportDeleteTIFSourceConfigAction; +import org.opensearch.securityanalytics.threatIntel.transport.TransportGetIocFindingsAction; import org.opensearch.securityanalytics.threatIntel.transport.TransportGetTIFSourceConfigAction; import org.opensearch.securityanalytics.threatIntel.transport.TransportIndexTIFSourceConfigAction; import org.opensearch.securityanalytics.threatIntel.transport.TransportPutTIFJobAction; import org.opensearch.securityanalytics.threatIntel.transport.TransportRefreshTIFSourceConfigAction; import org.opensearch.securityanalytics.threatIntel.transport.TransportSearchTIFSourceConfigsAction; import org.opensearch.securityanalytics.threatIntel.transport.monitor.TransportDeleteThreatIntelMonitorAction; +import org.opensearch.securityanalytics.threatIntel.transport.monitor.TransportGetThreatIntelAlertsAction; import org.opensearch.securityanalytics.threatIntel.transport.monitor.TransportIndexThreatIntelMonitorAction; import org.opensearch.securityanalytics.threatIntel.transport.monitor.TransportSearchThreatIntelMonitorAction; +import org.opensearch.securityanalytics.transport.TransportAckCorrelationAlertsAction; import org.opensearch.securityanalytics.transport.TransportAcknowledgeAlertsAction; import org.opensearch.securityanalytics.transport.TransportCorrelateFindingAction; import org.opensearch.securityanalytics.transport.TransportCreateIndexMappingsAction; @@ -167,6 +182,7 @@ import org.opensearch.securityanalytics.transport.TransportDeleteRuleAction; import org.opensearch.securityanalytics.transport.TransportGetAlertsAction; import org.opensearch.securityanalytics.transport.TransportGetAllRuleCategoriesAction; +import org.opensearch.securityanalytics.transport.TransportGetCorrelationAlertsAction; import org.opensearch.securityanalytics.transport.TransportGetDetectorAction; import org.opensearch.securityanalytics.transport.TransportGetFindingsAction; import org.opensearch.securityanalytics.transport.TransportGetIndexMappingsAction; @@ -185,7 +201,6 @@ import org.opensearch.securityanalytics.transport.TransportTestS3ConnectionAction; import org.opensearch.securityanalytics.transport.TransportUpdateIndexMappingsAction; import org.opensearch.securityanalytics.transport.TransportValidateRulesAction; -import org.opensearch.securityanalytics.threatIntel.transport.TransportGetIocFindingsAction; import org.opensearch.securityanalytics.util.CorrelationIndices; import org.opensearch.securityanalytics.util.CorrelationRuleIndices; import org.opensearch.securityanalytics.util.CustomLogTypeIndices; @@ -203,9 +218,9 @@ import java.util.Optional; import java.util.function.Supplier; +import static org.opensearch.securityanalytics.threatIntel.iocscan.service.ThreatIntelMonitorRunner.THREAT_INTEL_MONITOR_TYPE; import static org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig.SOURCE_CONFIG_FIELD; import static org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter.THREAT_INTEL_DATA_INDEX_NAME_PREFIX; -import static org.opensearch.securityanalytics.threatIntel.model.monitor.SampleRemoteDocLevelMonitorRunner.THREAT_INTEL_MONITOR_TYPE; public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, MapperPlugin, SearchPlugin, EnginePlugin, ClusterPlugin, SystemIndexPlugin, JobSchedulerExtension, RemoteMonitorRunnerExtension { @@ -222,14 +237,16 @@ public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, Map public static final String LIST_CORRELATIONS_URI = PLUGINS_BASE_URI + "/correlations"; public static final String CORRELATION_RULES_BASE_URI = PLUGINS_BASE_URI + "/correlation/rules"; public static final String THREAT_INTEL_BASE_URI = PLUGINS_BASE_URI + "/threat_intel"; - public static final String THREAT_INTEL_SOURCE_URI = PLUGINS_BASE_URI + "/threat_intel/source"; - public static final String THREAT_INTEL_MONITOR_URI = PLUGINS_BASE_URI + "/threat_intel/monitor"; - public static final String IOCS_URI = PLUGINS_BASE_URI + "/iocs"; - public static final String LIST_IOCS_URI = IOCS_URI + "/list"; + public static final String THREAT_INTEL_SOURCE_URI = PLUGINS_BASE_URI + "/threat_intel/sources"; + public static final String THREAT_INTEL_MONITOR_URI = PLUGINS_BASE_URI + "/threat_intel/monitors"; + public static final String LIST_IOCS_URI = PLUGINS_BASE_URI + "/threat_intel/iocs"; + public static final String THREAT_INTEL_ALERTS_URI = PLUGINS_BASE_URI + "/threat_intel/alerts"; public static final String TEST_CONNECTION_BASE_URI = PLUGINS_BASE_URI + "/connections/%s/test"; public static final String TEST_S3_CONNECTION_URI = String.format(TEST_CONNECTION_BASE_URI, "s3"); public static final String CUSTOM_LOG_TYPE_URI = PLUGINS_BASE_URI + "/logtype"; + + public static final String CORRELATIONS_ALERTS_BASE_URI = PLUGINS_BASE_URI + "/correlationAlerts"; public static final String JOB_INDEX_NAME = ".opensearch-sap--job"; public static final String JOB_TYPE = "opensearch_sap_job"; @@ -260,12 +277,11 @@ public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, Map private SATIFSourceConfigService saTifSourceConfigService; @Override - public Collection getSystemIndexDescriptors(Settings settings){ + public Collection getSystemIndexDescriptors(Settings settings) { return Collections.singletonList(new SystemIndexDescriptor(THREAT_INTEL_DATA_INDEX_NAME_PREFIX, "System index used for threat intel data")); } - @Override public Collection createComponents(Client client, ClusterService clusterService, @@ -300,12 +316,18 @@ public Collection createComponents(Client client, SATIFSourceConfigManagementService saTifSourceConfigManagementService = new SATIFSourceConfigManagementService(saTifSourceConfigService, threatIntelLockService, stix2IOCFetchService, xContentRegistry, clusterService); SecurityAnalyticsRunner.getJobRunnerInstance(); TIFSourceConfigRunner.getJobRunnerInstance().initialize(clusterService, threatIntelLockService, threadPool, saTifSourceConfigManagementService, saTifSourceConfigService); + CorrelationAlertService correlationAlertService = new CorrelationAlertService(client, xContentRegistry); + NotificationService notificationService = new NotificationService((NodeClient) client, scriptService); TIFJobRunner.getJobRunnerInstance().initialize(clusterService, tifJobUpdateService, tifJobParameterService, threatIntelLockService, threadPool, detectorThreatIntelService); - + IocFindingService iocFindingService = new IocFindingService(client, clusterService, xContentRegistry); + ThreatIntelAlertService threatIntelAlertService = new ThreatIntelAlertService(client, clusterService, xContentRegistry); + SaIoCScanService ioCScanService = new SaIoCScanService(client, xContentRegistry, iocFindingService, threatIntelAlertService, notificationService); return List.of( - detectorIndices, correlationIndices, correlationRuleIndices, ruleTopicIndices, customLogTypeIndices, ruleIndices, + detectorIndices, correlationIndices, correlationRuleIndices, ruleTopicIndices, customLogTypeIndices, ruleIndices,threatIntelAlertService, mapperService, indexTemplateManager, builtinLogTypeLoader, builtInTIFMetadataLoader, threatIntelFeedDataService, detectorThreatIntelService, - tifJobUpdateService, tifJobParameterService, threatIntelLockService, saTifSourceConfigService, saTifSourceConfigManagementService, stix2IOCFetchService); + correlationAlertService, notificationService, + tifJobUpdateService, tifJobParameterService, threatIntelLockService, saTifSourceConfigService, saTifSourceConfigManagementService, stix2IOCFetchService, + ioCScanService); } @Override @@ -333,6 +355,7 @@ public List getRestHandlers(Settings settings, new RestGetFindingsAction(), new RestGetMappingsViewAction(), new RestGetAlertsAction(), + new RestGetThreatIntelAlertsAction(), new RestIndexRuleAction(), new RestSearchRuleAction(), new RestDeleteRuleAction(), @@ -356,7 +379,9 @@ public List getRestHandlers(Settings settings, new RestRefreshTIFSourceConfigAction(), new RestListIOCsAction(), new RestGetIocFindingsAction(), - new RestTestS3ConnectionAction() + new RestTestS3ConnectionAction(), + new RestGetCorrelationsAlertsAction(), + new RestAcknowledgeCorrelationAlertsAction() ); } @@ -385,7 +410,7 @@ public ScheduledJobParser getJobParser() { xcp.nextToken(); switch (fieldName) { case SOURCE_CONFIG_FIELD: - return SATIFSourceConfig.parse(xcp, id, null); + return SATIFSourceConfig.parse(xcp, id, jobDocVersion.getVersion()); default: log.error("Job parser failed for [{}] in security analytics job registration", fieldName); xcp.skipChildren(); @@ -496,10 +521,10 @@ public List> getSettings() { new ActionPlugin.ActionHandler<>(AlertingActions.SUBSCRIBE_FINDINGS_ACTION_TYPE, TransportCorrelateFindingAction.class), new ActionPlugin.ActionHandler<>(ListCorrelationsAction.INSTANCE, TransportListCorrelationAction.class), new ActionPlugin.ActionHandler<>(SearchCorrelationRuleAction.INSTANCE, TransportSearchCorrelationRuleAction.class), + new ActionPlugin.ActionHandler<>(GetThreatIntelAlertsAction.INSTANCE, TransportGetThreatIntelAlertsAction.class), new ActionHandler<>(IndexCustomLogTypeAction.INSTANCE, TransportIndexCustomLogTypeAction.class), new ActionHandler<>(SearchCustomLogTypeAction.INSTANCE, TransportSearchCustomLogTypeAction.class), new ActionHandler<>(DeleteCustomLogTypeAction.INSTANCE, TransportDeleteCustomLogTypeAction.class), - new ActionHandler<>(PutTIFJobAction.INSTANCE, TransportPutTIFJobAction.class), new ActionHandler<>(IndexThreatIntelMonitorAction.INSTANCE, TransportIndexThreatIntelMonitorAction.class), new ActionHandler<>(DeleteThreatIntelMonitorAction.INSTANCE, TransportDeleteThreatIntelMonitorAction.class), new ActionHandler<>(SearchThreatIntelMonitorAction.INSTANCE, TransportSearchThreatIntelMonitorAction.class), @@ -508,10 +533,13 @@ public List> getSettings() { new ActionHandler<>(SADeleteTIFSourceConfigAction.INSTANCE, TransportDeleteTIFSourceConfigAction.class), new ActionHandler<>(SASearchTIFSourceConfigsAction.INSTANCE, TransportSearchTIFSourceConfigsAction.class), new ActionHandler<>(SARefreshTIFSourceConfigAction.INSTANCE, TransportRefreshTIFSourceConfigAction.class), - new ActionHandler<>(SampleRemoteDocLevelMonitorRunner.REMOTE_DOC_LEVEL_MONITOR_ACTION_INSTANCE, TransportRemoteDocLevelMonitorFanOutAction.class), + new ActionHandler<>(ThreatIntelMonitorRunner.REMOTE_DOC_LEVEL_MONITOR_ACTION_INSTANCE, TransportThreatIntelMonitorFanOutAction.class), new ActionHandler<>(ListIOCsAction.INSTANCE, TransportListIOCsAction.class), + new ActionHandler<>(TestS3ConnectionAction.INSTANCE, TransportTestS3ConnectionAction.class), new ActionHandler<>(GetIocFindingsAction.INSTANCE, TransportGetIocFindingsAction.class), - new ActionHandler<>(TestS3ConnectionAction.INSTANCE, TransportTestS3ConnectionAction.class) + new ActionHandler<>(PutTIFJobAction.INSTANCE, TransportPutTIFJobAction.class), + new ActionPlugin.ActionHandler<>(GetCorrelationAlertsAction.INSTANCE, TransportGetCorrelationAlertsAction.class), + new ActionPlugin.ActionHandler<>(AckCorrelationAlertsAction.INSTANCE, TransportAckCorrelationAlertsAction.class) ); } @@ -535,7 +563,7 @@ public void onFailure(Exception e) { @Override public Map getMonitorTypesToMonitorRunners() { return Map.of( - THREAT_INTEL_MONITOR_TYPE, SampleRemoteDocLevelMonitorRunner.getMonitorRunner() + THREAT_INTEL_MONITOR_TYPE, ThreatIntelMonitorRunner.getMonitorRunner() ); } } \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/action/AckCorrelationAlertsAction.java b/src/main/java/org/opensearch/securityanalytics/action/AckCorrelationAlertsAction.java new file mode 100644 index 000000000..d85d23f1c --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/action/AckCorrelationAlertsAction.java @@ -0,0 +1,20 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.action; + +import org.opensearch.action.ActionType; + +/** + * Acknowledge Correlation Alert Action + */ +public class AckCorrelationAlertsAction extends ActionType { + public static final String NAME = "cluster:admin/opensearch/securityanalytics/correlationAlerts/ack"; + public static final AckCorrelationAlertsAction INSTANCE = new AckCorrelationAlertsAction(); + + public AckCorrelationAlertsAction() { + super(NAME, AckCorrelationAlertsResponse::new); + } +} + diff --git a/src/main/java/org/opensearch/securityanalytics/action/AckCorrelationAlertsRequest.java b/src/main/java/org/opensearch/securityanalytics/action/AckCorrelationAlertsRequest.java new file mode 100644 index 000000000..2183b4658 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/action/AckCorrelationAlertsRequest.java @@ -0,0 +1,56 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.action; + +import org.opensearch.action.ActionRequest; +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.action.ValidateActions; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +public class AckCorrelationAlertsRequest extends ActionRequest { + private final List correlationAlertIds; + + public AckCorrelationAlertsRequest(List correlationAlertIds) { + this.correlationAlertIds = correlationAlertIds; + } + + public AckCorrelationAlertsRequest(StreamInput in) throws IOException { + correlationAlertIds = Collections.unmodifiableList(in.readStringList()); + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + if(correlationAlertIds == null || correlationAlertIds.isEmpty()) { + validationException = ValidateActions.addValidationError("alert ids list cannot be empty", validationException); + } + return validationException; + } + + public void writeTo(StreamOutput out) throws IOException { + out.writeStringCollection(this.correlationAlertIds); + } + + public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { + return builder.startObject() + .field("correlation_alert_ids", correlationAlertIds) + .endObject(); + } + + public static AckAlertsRequest readFrom(StreamInput sin) throws IOException { + return new AckAlertsRequest(sin); + } + + public List getCorrelationAlertIds() { + return correlationAlertIds; + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/action/AckCorrelationAlertsResponse.java b/src/main/java/org/opensearch/securityanalytics/action/AckCorrelationAlertsResponse.java new file mode 100644 index 000000000..a34ae6b74 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/action/AckCorrelationAlertsResponse.java @@ -0,0 +1,56 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.action; + +import org.opensearch.commons.alerting.model.CorrelationAlert; +import org.opensearch.core.action.ActionResponse; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +public class AckCorrelationAlertsResponse extends ActionResponse implements ToXContentObject { + + private final List acknowledged; + private final List failed; + + public AckCorrelationAlertsResponse(List acknowledged, List failed) { + this.acknowledged = acknowledged; + this.failed = failed; + } + + public AckCorrelationAlertsResponse(StreamInput sin) throws IOException { + this( + Collections.unmodifiableList(sin.readList(CorrelationAlert::new)), + Collections.unmodifiableList(sin.readList(CorrelationAlert::new)) + ); + } + + @Override + public void writeTo(StreamOutput streamOutput) throws IOException { + streamOutput.writeList(this.acknowledged); + streamOutput.writeList(this.failed); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject() + .field("acknowledged",this.acknowledged) + .field("failed",this.failed); + return builder.endObject(); + } + + public List getAcknowledged() { + return acknowledged; + } + + public List getFailed() { + return failed; + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/action/GetAlertsAction.java b/src/main/java/org/opensearch/securityanalytics/action/GetAlertsAction.java index df9422a77..39f415e90 100644 --- a/src/main/java/org/opensearch/securityanalytics/action/GetAlertsAction.java +++ b/src/main/java/org/opensearch/securityanalytics/action/GetAlertsAction.java @@ -14,4 +14,4 @@ public class GetAlertsAction extends ActionType { public GetAlertsAction() { super(NAME, GetAlertsResponse::new); } -} \ No newline at end of file +} diff --git a/src/main/java/org/opensearch/securityanalytics/action/GetCorrelationAlertsAction.java b/src/main/java/org/opensearch/securityanalytics/action/GetCorrelationAlertsAction.java new file mode 100644 index 000000000..d07fc7bfc --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/action/GetCorrelationAlertsAction.java @@ -0,0 +1,17 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.action; + +import org.opensearch.action.ActionType; + +public class GetCorrelationAlertsAction extends ActionType { + + public static final GetCorrelationAlertsAction INSTANCE = new GetCorrelationAlertsAction(); + public static final String NAME = "cluster:admin/opensearch/securityanalytics/correlationAlerts/get"; + + public GetCorrelationAlertsAction() { + super(NAME, GetCorrelationAlertsResponse::new); + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/action/GetCorrelationAlertsRequest.java b/src/main/java/org/opensearch/securityanalytics/action/GetCorrelationAlertsRequest.java new file mode 100644 index 000000000..41213b3cd --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/action/GetCorrelationAlertsRequest.java @@ -0,0 +1,112 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.action; + +import org.opensearch.action.ActionRequest; +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.commons.alerting.model.Table; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; + +import java.io.IOException; +import java.time.Instant; +import java.util.Locale; + +import static org.opensearch.action.ValidateActions.addValidationError; + +public class GetCorrelationAlertsRequest extends ActionRequest { + private String correlationRuleId; + private String correlationRuleName; + private Table table; + private String severityLevel; + private String alertState; + + private Instant startTime; + + private Instant endTime; + + public static final String CORRELATION_RULE_ID = "correlation_rule_id"; + + public GetCorrelationAlertsRequest( + String correlationRuleId, + String correlationRuleName, + Table table, + String severityLevel, + String alertState, + Instant startTime, + Instant endTime + ) { + super(); + this.correlationRuleId = correlationRuleId; + this.correlationRuleName = correlationRuleName; + this.table = table; + this.severityLevel = severityLevel; + this.alertState = alertState; + this.startTime = startTime; + this.endTime = endTime; + } + public GetCorrelationAlertsRequest(StreamInput sin) throws IOException { + this( + sin.readOptionalString(), + sin.readOptionalString(), + Table.readFrom(sin), + sin.readString(), + sin.readString(), + sin.readOptionalInstant(), + sin.readOptionalInstant() + ); + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + if ((correlationRuleId != null && correlationRuleId.isEmpty())) { + validationException = addValidationError(String.format(Locale.getDefault(), + "Correlation ruleId is empty or not valid", CORRELATION_RULE_ID), + validationException); + } + return validationException; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeOptionalString(correlationRuleId); + out.writeOptionalString(correlationRuleName); + table.writeTo(out); + out.writeString(severityLevel); + out.writeString(alertState); + out.writeOptionalInstant(startTime); + out.writeOptionalInstant(endTime); + } + + public String getCorrelationRuleId() { + return correlationRuleId; + } + + public Table getTable() { + return table; + } + + public String getSeverityLevel() { + return severityLevel; + } + + public String getAlertState() { + return alertState; + } + + public String getCorrelationRuleName() { + return correlationRuleName; + } + + public Instant getStartTime() { + return startTime; + } + + public Instant getEndTime() { + return endTime; + } +} + diff --git a/src/main/java/org/opensearch/securityanalytics/action/GetCorrelationAlertsResponse.java b/src/main/java/org/opensearch/securityanalytics/action/GetCorrelationAlertsResponse.java new file mode 100644 index 000000000..33ffc1e93 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/action/GetCorrelationAlertsResponse.java @@ -0,0 +1,54 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.action; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.commons.alerting.model.CorrelationAlert; +import org.opensearch.core.action.ActionResponse; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +public class GetCorrelationAlertsResponse extends ActionResponse implements ToXContentObject { + + private static final Logger log = LogManager.getLogger(GetCorrelationAlertsResponse.class); + private static final String CORRELATION_ALERTS_FIELD = "correlationAlerts"; + private static final String TOTAL_ALERTS_FIELD = "total_alerts"; + + private List alerts; + private Integer totalAlerts; + + public GetCorrelationAlertsResponse(List alerts, Integer totalAlerts) { + super(); + this.alerts = alerts; + this.totalAlerts = totalAlerts; + } + + public GetCorrelationAlertsResponse(StreamInput sin) throws IOException { + this( + Collections.unmodifiableList(sin.readList(CorrelationAlert::new)), + sin.readInt() + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeCollection(this.alerts); + out.writeInt(this.totalAlerts); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject() + .field(CORRELATION_ALERTS_FIELD, this.alerts) + .field(TOTAL_ALERTS_FIELD, this.totalAlerts); + return builder.endObject(); + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/correlation/JoinEngine.java b/src/main/java/org/opensearch/securityanalytics/correlation/JoinEngine.java index 3b4314e12..20cff273a 100644 --- a/src/main/java/org/opensearch/securityanalytics/correlation/JoinEngine.java +++ b/src/main/java/org/opensearch/securityanalytics/correlation/JoinEngine.java @@ -21,6 +21,7 @@ import org.opensearch.common.xcontent.XContentType; import org.opensearch.commons.alerting.action.PublishFindingsRequest; import org.opensearch.commons.alerting.model.Finding; +import org.opensearch.commons.authuser.User; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.query.BoolQueryBuilder; @@ -32,9 +33,13 @@ import org.opensearch.search.SearchHit; import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.securityanalytics.config.monitors.DetectorMonitorConfig; +import org.opensearch.securityanalytics.correlation.alert.CorrelationAlertService; +import org.opensearch.securityanalytics.correlation.alert.CorrelationRuleScheduler; +import org.opensearch.securityanalytics.correlation.alert.notifications.NotificationService; import org.opensearch.securityanalytics.logtype.LogTypeService; import org.opensearch.securityanalytics.model.CorrelationQuery; import org.opensearch.securityanalytics.model.CorrelationRule; +import org.opensearch.securityanalytics.model.CorrelationRuleTrigger; import org.opensearch.securityanalytics.model.Detector; import org.opensearch.securityanalytics.transport.TransportCorrelateFindingAction; import org.opensearch.securityanalytics.util.AutoCorrelationsRepo; @@ -68,18 +73,30 @@ public class JoinEngine { private final LogTypeService logTypeService; + private final CorrelationAlertService correlationAlertService; + + private final NotificationService notificationService; + + private volatile TimeValue indexTimeout; + private static final Logger log = LogManager.getLogger(JoinEngine.class); + private final User user; + public JoinEngine(Client client, PublishFindingsRequest request, NamedXContentRegistry xContentRegistry, - long corrTimeWindow, TransportCorrelateFindingAction.AsyncCorrelateFindingAction correlateFindingAction, - LogTypeService logTypeService, boolean enableAutoCorrelations) { + long corrTimeWindow, TimeValue indexTimeout, TransportCorrelateFindingAction.AsyncCorrelateFindingAction correlateFindingAction, + LogTypeService logTypeService, boolean enableAutoCorrelations, CorrelationAlertService correlationAlertService, NotificationService notificationService, User user) { this.client = client; this.request = request; this.xContentRegistry = xContentRegistry; this.corrTimeWindow = corrTimeWindow; + this.indexTimeout = indexTimeout; this.correlateFindingAction = correlateFindingAction; this.logTypeService = logTypeService; this.enableAutoCorrelations = enableAutoCorrelations; + this.correlationAlertService = correlationAlertService; + this.notificationService = notificationService; + this.user = user; } public void onSearchDetectorResponse(Detector detector, Finding finding) { @@ -349,7 +366,7 @@ private void getValidDocuments(String detectorType, List indices, List it.correlationRule).map(CorrelationRule::getId).collect(Collectors.toList()), + filteredCorrelationRules.stream().map(it -> it.correlationRule).collect(Collectors.toList()), autoCorrelations ); }, this::onFailure)); @@ -362,7 +379,7 @@ private void getValidDocuments(String detectorType, List indices, List> categoryToQueriesMap, Map categoryToTimeWindowMap, List correlationRules, Map> autoCorrelations) { + private void searchFindingsByTimestamp(String detectorType, Map> categoryToQueriesMap, Map categoryToTimeWindowMap, List correlationRules, Map> autoCorrelations) { long findingTimestamp = request.getFinding().getTimestamp().toEpochMilli(); MultiSearchRequest mSearchRequest = new MultiSearchRequest(); List>> categoryToQueriesPairs = new ArrayList<>(); @@ -418,14 +435,14 @@ private void searchFindingsByTimestamp(String detectorType, Map relatedDocsMap, Map categoryToTimeWindowMap, List correlationRules, Map> autoCorrelations) { + private void searchDocsWithFilterKeys(String detectorType, Map relatedDocsMap, Map categoryToTimeWindowMap, List correlationRules, Map> autoCorrelations) { MultiSearchRequest mSearchRequest = new MultiSearchRequest(); List categories = new ArrayList<>(); @@ -476,7 +493,7 @@ private void searchDocsWithFilterKeys(String detectorType, Map> filteredRelatedDocIds, Map categoryToTimeWindowMap, List correlationRules, Map> autoCorrelations) { + private void getCorrelatedFindings(String detectorType, Map> filteredRelatedDocIds, Map categoryToTimeWindowMap, List correlationRules, Map> autoCorrelations) { long findingTimestamp = request.getFinding().getTimestamp().toEpochMilli(); MultiSearchRequest mSearchRequest = new MultiSearchRequest(); List categories = new ArrayList<>(); @@ -540,6 +557,11 @@ private void getCorrelatedFindings(String detectorType, Map ++idx; } + if (!correlatedFindings.isEmpty()) { + CorrelationRuleScheduler correlationRuleScheduler = new CorrelationRuleScheduler(client, correlationAlertService, notificationService); + correlationRuleScheduler.schedule(correlationRules, correlatedFindings, request.getFinding().getId(), indexTimeout, user); + } + for (Map.Entry> autoCorrelation: autoCorrelations.entrySet()) { if (correlatedFindings.containsKey(autoCorrelation.getKey())) { Set alreadyCorrelatedFindings = new HashSet<>(correlatedFindings.get(autoCorrelation.getKey())); @@ -549,10 +571,10 @@ private void getCorrelatedFindings(String detectorType, Map correlatedFindings.put(autoCorrelation.getKey(), autoCorrelation.getValue()); } } - correlateFindingAction.initCorrelationIndex(detectorType, correlatedFindings, correlationRules); + correlateFindingAction.initCorrelationIndex(detectorType, correlatedFindings, correlationRules.stream().map(CorrelationRule::getId).collect(Collectors.toList())); }, this::onFailure)); } else { - getTimestampFeature(detectorType, correlationRules, autoCorrelations); + getTimestampFeature(detectorType, correlationRules.stream().map(CorrelationRule::getId).collect(Collectors.toList()), autoCorrelations); } } diff --git a/src/main/java/org/opensearch/securityanalytics/correlation/alert/CorrelationAlertService.java b/src/main/java/org/opensearch/securityanalytics/correlation/alert/CorrelationAlertService.java new file mode 100644 index 000000000..54c09d29a --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/correlation/alert/CorrelationAlertService.java @@ -0,0 +1,327 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.correlation.alert; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.bulk.BulkItemResponse; +import org.opensearch.action.bulk.BulkRequest; +import org.opensearch.action.bulk.BulkResponse; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.action.index.IndexResponse; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.update.UpdateRequest; +import org.opensearch.client.Client; +import org.opensearch.common.lucene.uid.Versions; +import org.opensearch.commons.alerting.model.Alert; +import org.opensearch.commons.alerting.model.Table; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.index.query.TermsQueryBuilder; +import org.opensearch.script.Script; +import org.opensearch.script.ScriptType; +import org.opensearch.search.SearchHit; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.commons.alerting.model.CorrelationAlert; +import org.opensearch.search.sort.FieldSortBuilder; +import org.opensearch.search.sort.SortBuilders; +import org.opensearch.search.sort.SortOrder; +import org.opensearch.securityanalytics.action.AckCorrelationAlertsResponse; +import org.opensearch.securityanalytics.action.GetCorrelationAlertsResponse; +import org.opensearch.securityanalytics.util.CorrelationIndices; +import java.io.IOException; +import java.time.Instant; +import java.util.List; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Map; + +public class CorrelationAlertService { + private static final Logger log = LogManager.getLogger(CorrelationAlertService.class); + + private final NamedXContentRegistry xContentRegistry; + private final Client client; + + protected static final String CORRELATED_FINDING_IDS = "correlated_finding_ids"; + protected static final String CORRELATION_RULE_ID = "correlation_rule_id"; + protected static final String CORRELATION_RULE_NAME = "correlation_rule_name"; + protected static final String ALERT_ID_FIELD = "id"; + protected static final String SCHEMA_VERSION_FIELD = "schema_version"; + protected static final String ALERT_VERSION_FIELD = "version"; + protected static final String USER_FIELD = "user"; + protected static final String TRIGGER_NAME_FIELD = "trigger_name"; + protected static final String STATE_FIELD = "state"; + protected static final String START_TIME_FIELD = "start_time"; + protected static final String END_TIME_FIELD = "end_time"; + protected static final String ACKNOWLEDGED_TIME_FIELD = "acknowledged_time"; + protected static final String ERROR_MESSAGE_FIELD = "error_message"; + protected static final String SEVERITY_FIELD = "severity"; + protected static final String ACTION_EXECUTION_RESULTS_FIELD = "action_execution_results"; + protected static final String NO_ID = ""; + protected static final long NO_VERSION = Versions.NOT_FOUND; + + public CorrelationAlertService(Client client, NamedXContentRegistry xContentRegistry) { + this.client = client; + this.xContentRegistry = xContentRegistry; + } + + /** + * Searches for active Alerts in the correlation alerts index within a specified time range. + * + * @param ruleId The correlation rule ID to filter the alerts + * @param currentTime The current time of the search range + */ + public void getActiveAlerts(String ruleId, long currentTime, ActionListener listener) { + Instant currentTimeDate = Instant.ofEpochMilli(currentTime); + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery() + .must(QueryBuilders.termQuery("correlation_rule_id", ruleId)) + .must(QueryBuilders.rangeQuery("start_time").lte(currentTimeDate)) + .must(QueryBuilders.rangeQuery("end_time").gte(currentTimeDate)) + .must(QueryBuilders.termQuery("state", "ACTIVE")); + + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder() + .seqNoAndPrimaryTerm(true) + .version(true) + .size(10000) // set the size to 10,000 + .query(queryBuilder); + + SearchRequest searchRequest = new SearchRequest(CorrelationIndices.CORRELATION_ALERT_INDEX) + .source(searchSourceBuilder); + + client.search(searchRequest, ActionListener.wrap( + searchResponse -> { + if (searchResponse.getHits().getTotalHits().equals(0)) { + listener.onResponse(new CorrelationAlertsList(Collections.emptyList(), 0)); + } else { + listener.onResponse(new CorrelationAlertsList( + parseCorrelationAlerts(searchResponse), + searchResponse.getHits() != null && searchResponse.getHits().getTotalHits() != null ? + (int) searchResponse.getHits().getTotalHits().value : 0) + ); + } + }, + e -> { + log.error("Search request to fetch correlation alerts failed", e); + listener.onFailure(e); + } + )); + } + + public void indexCorrelationAlert(CorrelationAlert correlationAlert, TimeValue indexTimeout, ActionListener listener) { + // Convert CorrelationAlert to a map + try { + XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); + builder.field("correlated_finding_ids", correlationAlert.getCorrelatedFindingIds()); + builder.field("correlation_rule_id", correlationAlert.getCorrelationRuleId()); + builder.field("correlation_rule_name", correlationAlert.getCorrelationRuleName()); + builder.field("id", correlationAlert.getId()); + builder.field("user", correlationAlert.getUser()); // Convert User object to map + builder.field("schema_version", correlationAlert.getSchemaVersion()); + builder.field("severity", correlationAlert.getSeverity()); + builder.field("state", correlationAlert.getState()); + builder.field("trigger_name", correlationAlert.getTriggerName()); + builder.field("version", correlationAlert.getVersion()); + builder.field("start_time", correlationAlert.getStartTime()); + builder.field("end_time", correlationAlert.getEndTime()); + builder.field("action_execution_results", correlationAlert.getActionExecutionResults()); + builder.field("error_message", correlationAlert.getErrorMessage()); + builder.field("acknowledged_time", correlationAlert.getAcknowledgedTime()); + builder.endObject(); + IndexRequest indexRequest = new IndexRequest(CorrelationIndices.CORRELATION_ALERT_INDEX) + .id(correlationAlert.getId()) + .source(builder) + .timeout(indexTimeout); + + client.index(indexRequest, listener); + } catch (IOException ex) { + log.error("Exception while adding alerts in .opensearch-sap-correlation-alerts index", ex); + } + } + + public void getCorrelationAlerts(String ruleId, Table tableProp, ActionListener listener) { + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + if (ruleId != null) { + queryBuilder = QueryBuilders.boolQuery() + .must(QueryBuilders.termQuery("correlation_rule_id", ruleId)); + } + + FieldSortBuilder sortBuilder = SortBuilders + .fieldSort(tableProp.getSortString()) + .order(SortOrder.fromString(tableProp.getSortOrder())); + if (tableProp.getMissing() != null && !tableProp.getMissing().isEmpty()) { + sortBuilder.missing(tableProp.getMissing()); + } + + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder() + .version(true) + .seqNoAndPrimaryTerm(true) + .query(queryBuilder) + .sort(sortBuilder) + .size(tableProp.getSize()) + .from(tableProp.getStartIndex()); + + SearchRequest searchRequest = new SearchRequest(CorrelationIndices.CORRELATION_ALERT_INDEX) + .source(searchSourceBuilder); + + client.search(searchRequest, ActionListener.wrap( + searchResponse -> { + if (searchResponse.getHits().getTotalHits().equals(0)) { + listener.onResponse(new GetCorrelationAlertsResponse(Collections.emptyList(), 0)); + } else { + listener.onResponse(new GetCorrelationAlertsResponse( + parseCorrelationAlerts(searchResponse), + searchResponse.getHits() != null && searchResponse.getHits().getTotalHits() != null ? + (int) searchResponse.getHits().getTotalHits().value : 0) + ); + } + }, + e -> { + log.error("Search request to fetch correlation alerts failed", e); + listener.onFailure(e); + } + )); + } + + public void acknowledgeAlerts(List alertIds, ActionListener listener) { + BulkRequest bulkRequest = new BulkRequest(); + List acknowledgedAlerts = new ArrayList<>(); + List failedAlerts = new ArrayList<>(); + + TermsQueryBuilder termsQueryBuilder = QueryBuilders.termsQuery("id", alertIds); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().query(termsQueryBuilder); + SearchRequest searchRequest = new SearchRequest(CorrelationIndices.CORRELATION_ALERT_INDEX) + .source(searchSourceBuilder); + + // Execute the search request + client.search(searchRequest, new ActionListener() { + @Override + public void onResponse(SearchResponse searchResponse) { + // Iterate through the search hits + for (SearchHit hit : searchResponse.getHits().getHits()) { + // Construct a script to update the document with the new state and acknowledgedTime + // Construct a script to update the document with the new state and acknowledgedTime + Script script = new Script(ScriptType.INLINE, "painless", + "ctx._source.state = params.state; ctx._source.acknowledged_time = params.time", + Map.of("state", Alert.State.ACKNOWLEDGED, "time", Instant.now())); + // Create an update request with the script + UpdateRequest updateRequest = new UpdateRequest(CorrelationIndices.CORRELATION_ALERT_INDEX, hit.getId()) + .script(script); + + // Add the update request to the bulk request + bulkRequest.add(updateRequest); + + // Add the current alert to the acknowledged alerts list + try { + acknowledgedAlerts.add(getParsedCorrelationAlert(hit)); + } catch (IOException e) { + log.error("Exception while acknowledging alerts: {}", e.toString()); + } + } + + // Check if there are any update requests in the bulk request + if (!bulkRequest.requests().isEmpty()) { + // Execute the bulk request asynchronously + client.bulk(bulkRequest, new ActionListener() { + @Override + public void onResponse(BulkResponse bulkResponse) { + // Iterate through the bulk response to identify failed updates + for (BulkItemResponse itemResponse : bulkResponse.getItems()) { + if (itemResponse.isFailed()) { + // If an update failed, add the corresponding alert to the failed alerts list + failedAlerts.add(acknowledgedAlerts.get(itemResponse.getItemId())); + } + } + // Create and pass the CorrelationAckAlertsResponse to the listener + listener.onResponse(new AckCorrelationAlertsResponse(acknowledgedAlerts, failedAlerts)); + } + + @Override + public void onFailure(Exception e) { + // Handle failure + listener.onFailure(e); + } + }); + } else { + // If there are no update requests, return an empty response + listener.onResponse(new AckCorrelationAlertsResponse(acknowledgedAlerts, failedAlerts)); + } + } + + @Override + public void onFailure(Exception e) { + // Handle failure + listener.onFailure(e); + } + }); + } + + public void updateCorrelationAlertsWithError(String correlationRuleId) { + BulkRequest bulkRequest = new BulkRequest(); + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery() + .must(QueryBuilders.termQuery("correlation_rule_id", correlationRuleId)); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().query(queryBuilder); + SearchRequest searchRequest = new SearchRequest(CorrelationIndices.CORRELATION_ALERT_INDEX) + .source(searchSourceBuilder); + + // Execute the search request + client.search(searchRequest, new ActionListener() { + @Override + public void onResponse(SearchResponse searchResponse) { + // Iterate through the search hits + for (SearchHit hit : searchResponse.getHits().getHits()) { + // Construct a script to update the document with the new state and error_message + Script script = new Script(ScriptType.INLINE, "painless", + "ctx._source.state = params.state; ctx._source.error_message = params.error_message", + Map.of("state", Alert.State.ERROR, "error_message", "The rule associated to this Alert is deleted")); + // Create an update request with the script + UpdateRequest updateRequest = new UpdateRequest(CorrelationIndices.CORRELATION_ALERT_INDEX, hit.getId()) + .script(script); + // Add the update request to the bulk request + bulkRequest.add(updateRequest); + client.bulk(bulkRequest); + } + } + @Override + public void onFailure(Exception e) { + log.error("Error updating the alerts with Error message for correlation ruleId: {}", correlationRuleId); + } + }); + } + + + public List parseCorrelationAlerts(final SearchResponse response) throws IOException { + List alerts = new ArrayList<>(); + for (SearchHit hit : response.getHits()) { + CorrelationAlert correlationAlert = getParsedCorrelationAlert(hit); + alerts.add(correlationAlert); + } + return alerts; + } + + private CorrelationAlert getParsedCorrelationAlert(SearchHit hit) throws IOException { + XContentParser xcp = XContentType.JSON.xContent().createParser( + xContentRegistry, + LoggingDeprecationHandler.INSTANCE, + hit.getSourceAsString() + ); + xcp.nextToken(); + CorrelationAlert correlationAlert = CorrelationAlertsList.parse(xcp, hit.getId(), hit.getVersion()); + return correlationAlert; + } + +} + + + + diff --git a/src/main/java/org/opensearch/securityanalytics/correlation/alert/CorrelationAlertsList.java b/src/main/java/org/opensearch/securityanalytics/correlation/alert/CorrelationAlertsList.java new file mode 100644 index 000000000..2770f3eaa --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/correlation/alert/CorrelationAlertsList.java @@ -0,0 +1,142 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.correlation.alert; + + +import org.opensearch.commons.alerting.model.ActionExecutionResult; +import org.opensearch.commons.alerting.model.Alert; +import org.opensearch.commons.alerting.model.CorrelationAlert; +import org.opensearch.commons.authuser.User; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; + +import java.io.IOException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; + +/** + * Wrapper class that holds list of correlation alerts and total number of alerts available. + * Useful for pagination. + */ +public class CorrelationAlertsList { + + private final List correlationAlertList; + private final Integer totalAlerts; + + public CorrelationAlertsList(List correlationAlertList, Integer totalAlerts) { + this.correlationAlertList = correlationAlertList; + this.totalAlerts = totalAlerts; + } + + // logic will be moved to common-utils, once the parsing logic in common-utils is fixed + public static CorrelationAlert parse(XContentParser xcp, String id, long version) throws IOException { + // Parse additional CorrelationAlert-specific fields + List correlatedFindingIds = new ArrayList<>(); + String correlationRuleId = null; + String correlationRuleName = null; + User user = null; + int schemaVersion = 0; + String triggerName = null; + Alert.State state = null; + String errorMessage = null; + String severity = null; + List actionExecutionResults = new ArrayList<>(); + Instant startTime = null; + Instant endTime = null; + Instant acknowledgedTime = null; + + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = xcp.currentName(); + xcp.nextToken(); + switch (fieldName) { + case CorrelationAlertService.CORRELATED_FINDING_IDS: + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { + correlatedFindingIds.add(xcp.text()); + } + break; + case CorrelationAlertService.CORRELATION_RULE_ID: + correlationRuleId = xcp.text(); + break; + case CorrelationAlertService.CORRELATION_RULE_NAME: + correlationRuleName = xcp.text(); + break; + case CorrelationAlertService.USER_FIELD: + user = (xcp.currentToken() == XContentParser.Token.VALUE_NULL) ? null : User.parse(xcp); + break; + case CorrelationAlertService.ALERT_ID_FIELD: + id = xcp.text(); + break; + case CorrelationAlertService.ALERT_VERSION_FIELD: + version = xcp.longValue(); + break; + case CorrelationAlertService.SCHEMA_VERSION_FIELD: + schemaVersion = xcp.intValue(); + break; + case CorrelationAlertService.TRIGGER_NAME_FIELD: + triggerName = xcp.text(); + break; + case CorrelationAlertService.STATE_FIELD: + state = Alert.State.valueOf(xcp.text()); + break; + case CorrelationAlertService.ERROR_MESSAGE_FIELD: + errorMessage = xcp.textOrNull(); + break; + case CorrelationAlertService.SEVERITY_FIELD: + severity = xcp.text(); + break; + case CorrelationAlertService.ACTION_EXECUTION_RESULTS_FIELD: + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { + actionExecutionResults.add(ActionExecutionResult.parse(xcp)); + } + break; + case CorrelationAlertService.START_TIME_FIELD: + startTime = Instant.parse(xcp.text()); + break; + case CorrelationAlertService.END_TIME_FIELD: + endTime = Instant.parse(xcp.text()); + break; + case CorrelationAlertService.ACKNOWLEDGED_TIME_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + acknowledgedTime = null; + } else { + acknowledgedTime = Instant.parse(xcp.text()); + } + break; + } + } + + // Create and return CorrelationAlert object + return new CorrelationAlert( + correlatedFindingIds, + correlationRuleId, + correlationRuleName, + id, + version, + schemaVersion, + user, + triggerName, + state, + startTime, + endTime, + acknowledgedTime, + errorMessage, + severity, + actionExecutionResults + ); + } + + + public List getCorrelationAlertList() { + return correlationAlertList; + } + + public Integer getTotalAlerts() { + return totalAlerts; + } + +} diff --git a/src/main/java/org/opensearch/securityanalytics/correlation/alert/CorrelationRuleScheduler.java b/src/main/java/org/opensearch/securityanalytics/correlation/alert/CorrelationRuleScheduler.java new file mode 100644 index 000000000..10e61857b --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/correlation/alert/CorrelationRuleScheduler.java @@ -0,0 +1,190 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.correlation.alert; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.index.IndexResponse; +import org.opensearch.client.Client; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.commons.alerting.model.Alert; +import org.opensearch.commons.alerting.model.CorrelationAlert; +import org.opensearch.commons.authuser.User; +import org.opensearch.core.action.ActionListener; +import org.opensearch.securityanalytics.model.CorrelationQuery; +import org.opensearch.securityanalytics.model.CorrelationRule; +import org.opensearch.securityanalytics.model.CorrelationRuleTrigger; +import org.opensearch.securityanalytics.correlation.alert.notifications.NotificationService; +import org.opensearch.securityanalytics.correlation.alert.notifications.CorrelationAlertContext; +import org.opensearch.commons.alerting.model.action.Action; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.securityanalytics.util.SecurityAnalyticsException; +import java.time.Instant; +import java.util.UUID; +import java.util.List; +import java.util.ArrayList; +import java.util.Map; + +public class CorrelationRuleScheduler { + + private final Logger log = LogManager.getLogger(CorrelationRuleScheduler.class); + private final Client client; + private final CorrelationAlertService correlationAlertService; + private final NotificationService notificationService; + + public CorrelationRuleScheduler(Client client, CorrelationAlertService correlationAlertService, NotificationService notificationService) { + this.client = client; + this.correlationAlertService = correlationAlertService; + this.notificationService = notificationService; + } + + public void schedule(List correlationRules, Map> correlatedFindings, String sourceFinding, TimeValue indexTimeout, User user) { + for (CorrelationRule rule : correlationRules) { + CorrelationRuleTrigger trigger = rule.getCorrelationTrigger(); + if (trigger != null) { + List findingIds = new ArrayList<>(); + for (CorrelationQuery query : rule.getCorrelationQueries()) { + List categoryFindingIds = correlatedFindings.get(query.getCategory()); + if (categoryFindingIds != null) { + findingIds.addAll(categoryFindingIds); + } + } + scheduleRule(rule, findingIds, indexTimeout, sourceFinding, user); + } + } + } + + private void scheduleRule(CorrelationRule correlationRule, List findingIds, TimeValue indexTimeout, String sourceFindingId, User user) { + long startTime = Instant.now().toEpochMilli(); + long endTime = startTime + correlationRule.getCorrTimeWindow(); + RuleTask ruleTask = new RuleTask(correlationRule, findingIds, startTime, endTime, correlationAlertService, notificationService, indexTimeout, sourceFindingId, user); + ruleTask.run(); + } + + private class RuleTask implements Runnable { + private final CorrelationRule correlationRule; + private final long startTime; + private final long endTime; + private final List correlatedFindingIds; + private final CorrelationAlertService correlationAlertService; + private final NotificationService notificationService; + private final TimeValue indexTimeout; + private final String sourceFindingId; + private final User user; + + public RuleTask(CorrelationRule correlationRule, List correlatedFindingIds, long startTime, long endTime, CorrelationAlertService correlationAlertService, NotificationService notificationService, TimeValue indexTimeout, String sourceFindingId, User user) { + this.correlationRule = correlationRule; + this.correlatedFindingIds = correlatedFindingIds; + this.startTime = startTime; + this.endTime = endTime; + this.correlationAlertService = correlationAlertService; + this.notificationService = notificationService; + this.indexTimeout = indexTimeout; + this.sourceFindingId = sourceFindingId; + this.user = user; + } + + @Override + public void run() { + long currentTime = Instant.now().toEpochMilli(); + if (currentTime >= startTime && currentTime <= endTime) { + try { + correlationAlertService.getActiveAlerts(correlationRule.getId(), currentTime, new ActionListener<>() { + @Override + public void onResponse(CorrelationAlertsList correlationAlertsList) { + if (correlationAlertsList.getTotalAlerts() == 0) { + addCorrelationAlertIntoIndex(); + List actions = correlationRule.getCorrelationTrigger().getActions(); + for (Action action : actions) { + String configId = action.getDestinationId(); + CorrelationAlertContext ctx = new CorrelationAlertContext(correlatedFindingIds, correlationRule.getName(), correlationRule.getCorrTimeWindow(), sourceFindingId); + String transformedSubject = notificationService.compileTemplate(ctx, action.getSubjectTemplate()); + String transformedMessage = notificationService.compileTemplate(ctx, action.getMessageTemplate()); + try { + notificationService.sendNotification(configId, correlationRule.getCorrelationTrigger().getSeverity(), transformedSubject, transformedMessage); + } catch (Exception e) { + log.error("Failed while sending a notification with " + configId + "for correlationRule id " + correlationRule.getId(), e); + new SecurityAnalyticsException("Failed to send notification", RestStatus.INTERNAL_SERVER_ERROR, e); + } + + } + } else { + for (CorrelationAlert correlationAlert: correlationAlertsList.getCorrelationAlertList()) { + updateCorrelationAlert(correlationAlert); + } + } + } + + @Override + public void onFailure(Exception e) { + log.error("Failed to search active correlation alert", e); + new SecurityAnalyticsException("Failed to search active correlation alert", RestStatus.INTERNAL_SERVER_ERROR, e); + } + }); + } catch (Exception e) { + log.error("Failed to fetch active alerts in the time window", e); + new SecurityAnalyticsException("Failed to get active alerts in the correlationRuletimewindow", RestStatus.INTERNAL_SERVER_ERROR, e); + } + } + } + + private void addCorrelationAlertIntoIndex() { + CorrelationAlert correlationAlert = new CorrelationAlert( + correlatedFindingIds, + correlationRule.getId(), + correlationRule.getName(), + UUID.randomUUID().toString(), + 1L, + 1, + user, + correlationRule.getCorrelationTrigger().getName(), + Alert.State.ACTIVE, + Instant.ofEpochMilli(startTime), + Instant.ofEpochMilli(endTime), + null, + null, + correlationRule.getCorrelationTrigger().getSeverity(), + new ArrayList<>() + ); + insertCorrelationAlert(correlationAlert); + } + + private void updateCorrelationAlert(CorrelationAlert correlationAlert) { + CorrelationAlert newCorrelationAlert = new CorrelationAlert( + correlatedFindingIds, + correlationAlert.getCorrelationRuleId(), + correlationAlert.getCorrelationRuleName(), + correlationAlert.getId(), + 1L, + 1, + correlationAlert.getUser(), + correlationRule.getCorrelationTrigger().getName(), + Alert.State.ACTIVE, + Instant.ofEpochMilli(startTime), + Instant.ofEpochMilli(endTime), + null, + null, + correlationRule.getCorrelationTrigger().getSeverity(), + new ArrayList<>() + ); + insertCorrelationAlert(newCorrelationAlert); + } + + private void insertCorrelationAlert(CorrelationAlert correlationAlert) { + correlationAlertService.indexCorrelationAlert(correlationAlert, indexTimeout, new ActionListener<>() { + @Override + public void onResponse(IndexResponse indexResponse) { + log.info("Successfully updated the index .opensearch-sap-correlation-alerts: {}", indexResponse); + } + + @Override + public void onFailure(Exception e) { + log.error("Failed to index correlation alert", e); + } + }); + } + } +} + diff --git a/src/main/java/org/opensearch/securityanalytics/correlation/alert/notifications/CorrelationAlertContext.java b/src/main/java/org/opensearch/securityanalytics/correlation/alert/notifications/CorrelationAlertContext.java new file mode 100644 index 000000000..148da9a50 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/correlation/alert/notifications/CorrelationAlertContext.java @@ -0,0 +1,37 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.correlation.alert.notifications; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class CorrelationAlertContext { + + private final List correlatedFindingIds; + private final String sourceFinding; + private final String correlationRuleName; + private final long timeWindow; + public CorrelationAlertContext(List correlatedFindingIds, String correlationRuleName, long timeWindow, String sourceFinding) { + this.correlatedFindingIds = correlatedFindingIds; + this.correlationRuleName = correlationRuleName; + this.timeWindow = timeWindow; + this.sourceFinding = sourceFinding; + } + + /** + * Mustache templates need special permissions to reflectively introspect field names. To avoid doing this we + * translate the context to a Map of Strings to primitive types, which can be accessed without reflection. + */ + public Map asTemplateArg() { + Map templateArg = new HashMap<>(); + templateArg.put("correlatedFindingIds", correlatedFindingIds); + templateArg.put("sourceFinding", sourceFinding); + templateArg.put("correlationRuleName", correlationRuleName); + templateArg.put("timeWindow", timeWindow); + return templateArg; + } + +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/correlation/alert/notifications/NotificationService.java b/src/main/java/org/opensearch/securityanalytics/correlation/alert/notifications/NotificationService.java new file mode 100644 index 000000000..ca55e0dd5 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/correlation/alert/notifications/NotificationService.java @@ -0,0 +1,151 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.correlation.alert.notifications; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.client.node.NodeClient; +import org.opensearch.commons.alerting.model.ActionExecutionResult; +import org.opensearch.commons.notifications.NotificationsPluginInterface; +import org.opensearch.commons.notifications.action.*; +import org.opensearch.commons.notifications.model.ChannelMessage; +import org.opensearch.commons.notifications.model.EventSource; +import org.opensearch.commons.notifications.model.SeverityType; +import org.opensearch.commons.notifications.model.NotificationConfigInfo; +import org.opensearch.commons.notifications.action.GetNotificationConfigRequest; +import org.opensearch.commons.notifications.action.GetNotificationConfigResponse; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.securityanalytics.threatIntel.iocscan.service.ThreatIntelAlertContext; +import org.opensearch.securityanalytics.util.SecurityAnalyticsException; +import org.opensearch.script.ScriptService; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.Set; +import java.util.HashSet; +import java.util.Collections; + +import org.opensearch.script.Script; +import org.opensearch.script.TemplateScript; +import org.opensearch.commons.notifications.model.SeverityType; + +public class NotificationService { + + private static final Logger logger = LogManager.getLogger(NotificationService.class); + + private static ScriptService scriptService; + private final NodeClient client; + + public NotificationService(NodeClient client, ScriptService scriptService) { + this.client = client; + this.scriptService = scriptService; + } + + /** + * Extension function for publishing a notification to a channel in the Notification plugin. + */ + public void sendNotification(String configId, String severity, String subject, String notificationMessageText) throws IOException { + ChannelMessage message = generateMessage(notificationMessageText); + List channelIds = new ArrayList<>(); + channelIds.add(configId); + SeverityType severityType = SeverityType.Companion.fromTagOrDefault(severity); + NotificationsPluginInterface.INSTANCE.sendNotification(client, new EventSource(subject, configId, severityType, Collections.emptyList()), message, channelIds, new ActionListener() { + @Override + public void onResponse(SendNotificationResponse sendNotificationResponse) { + if (sendNotificationResponse.getStatus() == RestStatus.OK) { + logger.info("Successfully sent a notification, Notification Event: " + sendNotificationResponse.getNotificationEvent()); + } else { + logger.error("Error while sending a notification, Notification Event: " + sendNotificationResponse.getNotificationEvent()); + } + } + @Override + public void onFailure(Exception e) { + logger.error("Failed while sending a notification with " + configId, e); + } + }); + } + + /** + * Extension function for publishing a notification to a channel in the Notification plugin. + */ + public void sendNotification(String configId, String severity, String subject, String notificationMessageText, + ActionListener listener) { + ChannelMessage message = generateMessage(notificationMessageText); + List channelIds = new ArrayList<>(); + channelIds.add(configId); + SeverityType severityType = SeverityType.Companion.fromTagOrDefault(severity); + NotificationsPluginInterface.INSTANCE.sendNotification(client, new EventSource(subject, configId, severityType, Collections.emptyList()), message, channelIds, ActionListener.wrap( + sendNotificationResponse -> { + if (sendNotificationResponse.getStatus() == RestStatus.OK) { + logger.info("Successfully sent a notification, Notification Event: " + sendNotificationResponse.getNotificationEvent()); + } else { + listener.onFailure(new Exception("Error while sending a notification, Notification Event: " + sendNotificationResponse.getNotificationEvent())); + } + + }, e -> { + logger.error("Failed while sending a notification with " + configId, e); + listener.onFailure(e); + } + )); + } + + /** + * Gets a NotificationConfigInfo object by ID if it exists. + */ + public GetNotificationConfigResponse getNotificationConfigInfo(String id) { + + Set idSet = new HashSet(); + idSet.add(id); + GetNotificationConfigRequest getNotificationConfigRequest = new GetNotificationConfigRequest(idSet, 0, 10, null, null, new HashMap<>()); + GetNotificationConfigResponse configResp = null; + NotificationsPluginInterface.INSTANCE.getNotificationConfig(client, getNotificationConfigRequest, new ActionListener() { + @Override + public void onResponse(GetNotificationConfigResponse getNotificationConfigResponse) { + if (getNotificationConfigResponse.getStatus() == RestStatus.OK) { + getNotificationConfigResponse = configResp; + } else { + logger.error("Successfully sent a notification, Notification Event: " + getNotificationConfigResponse); + } + } + + @Override + public void onFailure(Exception e) { + logger.error("Notification config [" + id + "] was not found"); + new SecurityAnalyticsException("Failed to fetch notification config", RestStatus.INTERNAL_SERVER_ERROR, e); + } + }); + logger.info("Notification config response is: {} ", configResp); + return configResp; + } + + public static ChannelMessage generateMessage(String message) { + return new ChannelMessage( + message, + null, + null + ); + } + + public static String compileTemplate(CorrelationAlertContext ctx, Script template) { + return compileTemplateGeneric(template, ctx.asTemplateArg()); + } + + public static String compileTemplate(ThreatIntelAlertContext ctx, Script template) { + return compileTemplateGeneric(template, ctx.asTemplateArg()); + } + + private static String compileTemplateGeneric(Script template, Map templateArg) { + TemplateScript.Factory factory = scriptService.compile(template, TemplateScript.CONTEXT); + Map params = new HashMap<>(template.getParams()); + params.put("ctx", templateArg); + TemplateScript templateScript = factory.newInstance(params); + return templateScript.execute(); + } + +} diff --git a/src/main/java/org/opensearch/securityanalytics/logtype/BuiltinLogTypeLoader.java b/src/main/java/org/opensearch/securityanalytics/logtype/BuiltinLogTypeLoader.java index 0d28bce4d..80d0ae50d 100644 --- a/src/main/java/org/opensearch/securityanalytics/logtype/BuiltinLogTypeLoader.java +++ b/src/main/java/org/opensearch/securityanalytics/logtype/BuiltinLogTypeLoader.java @@ -10,6 +10,7 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -69,8 +70,9 @@ public void ensureLogTypesLoaded() { private List loadBuiltinLogTypes() throws URISyntaxException, IOException { List logTypes = new ArrayList<>(); - final String url = Objects.requireNonNull(BuiltinLogTypeLoader.class.getClassLoader().getResource(BASE_PATH)).toURI().toString(); + String pathurl = Paths.get(BuiltinLogTypeLoader.class.getClassLoader().getResource(BASE_PATH).toURI()).toString(); + final String url = Objects.requireNonNull(BuiltinLogTypeLoader.class.getClassLoader().getResource(BASE_PATH)).toURI().toString(); Path dirPath = null; if (url.contains("!")) { final String[] paths = url.split("!"); diff --git a/src/main/java/org/opensearch/securityanalytics/model/CorrelationRule.java b/src/main/java/org/opensearch/securityanalytics/model/CorrelationRule.java index b7f5a4f70..c4a1d4e2c 100644 --- a/src/main/java/org/opensearch/securityanalytics/model/CorrelationRule.java +++ b/src/main/java/org/opensearch/securityanalytics/model/CorrelationRule.java @@ -10,6 +10,7 @@ import java.util.Objects; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.opensearch.commons.authuser.User; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; @@ -29,6 +30,7 @@ public class CorrelationRule implements Writeable, ToXContentObject { public static final Long NO_VERSION = 1L; private static final String CORRELATION_QUERIES = "correlate"; private static final String CORRELATION_TIME_WINDOW = "time_window"; + private static final String TRIGGER_FIELD = "trigger"; private String id; @@ -40,16 +42,19 @@ public class CorrelationRule implements Writeable, ToXContentObject { private Long corrTimeWindow; - public CorrelationRule(String id, Long version, String name, List correlationQueries, Long corrTimeWindow) { + private CorrelationRuleTrigger trigger; + + public CorrelationRule(String id, Long version, String name, List correlationQueries, Long corrTimeWindow, CorrelationRuleTrigger trigger) { this.id = id != null ? id : NO_ID; this.version = version != null ? version : NO_VERSION; this.name = name; this.correlationQueries = correlationQueries; this.corrTimeWindow = corrTimeWindow != null? corrTimeWindow: 300000L; + this.trigger = trigger; } public CorrelationRule(StreamInput sin) throws IOException { - this(sin.readString(), sin.readLong(), sin.readString(), sin.readList(CorrelationQuery::readFrom), sin.readLong()); + this(sin.readString(), sin.readLong(), sin.readString(), sin.readList(CorrelationQuery::readFrom), sin.readLong(), sin.readBoolean() ? new CorrelationRuleTrigger(sin) : null); } @Override @@ -62,6 +67,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws correlationQueries = this.correlationQueries.toArray(correlationQueries); builder.field(CORRELATION_QUERIES, correlationQueries); builder.field(CORRELATION_TIME_WINDOW, corrTimeWindow); + builder.field(TRIGGER_FIELD, trigger); return builder.endObject(); } @@ -74,6 +80,11 @@ public void writeTo(StreamOutput out) throws IOException { for (CorrelationQuery query : correlationQueries) { query.writeTo(out); } + + out.writeBoolean(trigger != null); + if (trigger != null) { + trigger.writeTo(out); + } out.writeLong(corrTimeWindow); } @@ -88,7 +99,7 @@ public static CorrelationRule parse(XContentParser xcp, String id, Long version) String name = null; List correlationQueries = new ArrayList<>(); Long corrTimeWindow = null; - + CorrelationRuleTrigger trigger = null; XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp); while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { String fieldName = xcp.currentName(); @@ -108,11 +119,18 @@ public static CorrelationRule parse(XContentParser xcp, String id, Long version) case CORRELATION_TIME_WINDOW: corrTimeWindow = xcp.longValue(); break; + case TRIGGER_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + trigger = null; + } else { + trigger = CorrelationRuleTrigger.parse(xcp); + } + break; default: xcp.skipChildren(); } } - return new CorrelationRule(id, version, name, correlationQueries, corrTimeWindow); + return new CorrelationRule(id, version, name, correlationQueries, corrTimeWindow, trigger); } public static CorrelationRule readFrom(StreamInput sin) throws IOException { @@ -151,6 +169,10 @@ public Long getCorrTimeWindow() { return corrTimeWindow; } + public CorrelationRuleTrigger getCorrelationTrigger() { + return trigger; + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -159,7 +181,8 @@ public boolean equals(Object o) { return id.equals(that.id) && version.equals(that.version) && name.equals(that.name) - && correlationQueries.equals(that.correlationQueries); + && correlationQueries.equals(that.correlationQueries) + && trigger.equals(that.trigger); } @Override diff --git a/src/main/java/org/opensearch/securityanalytics/model/CorrelationRuleTrigger.java b/src/main/java/org/opensearch/securityanalytics/model/CorrelationRuleTrigger.java new file mode 100644 index 000000000..3426c7eb1 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/model/CorrelationRuleTrigger.java @@ -0,0 +1,193 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.model; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.common.UUIDs; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.XContentParserUtils; +import org.opensearch.commons.alerting.model.action.Action; +import org.opensearch.core.ParseField; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.script.Script; +import org.opensearch.script.ScriptType; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +public class CorrelationRuleTrigger implements Writeable, ToXContentObject { + + private static final Logger log = LogManager.getLogger(DetectorTrigger.class); + + private String id; + + private String name; + + private String severity; + + private List actions; + + private static final String ID_FIELD = "id"; + + private static final String SEVERITY_FIELD = "severity"; + private static final String ACTIONS_FIELD = "actions"; + + private static final String NAME_FIELD = "name"; + + public static final NamedXContentRegistry.Entry XCONTENT_REGISTRY = new NamedXContentRegistry.Entry( + CorrelationRuleTrigger.class, + new ParseField(ID_FIELD), + CorrelationRuleTrigger::parse + ); + + public CorrelationRuleTrigger(String id, + String name, + String severity, + List actions) { + this.id = id == null ? UUIDs.base64UUID() : id; + this.name = name; + this.severity = severity; + this.actions = actions; + } + + public CorrelationRuleTrigger(StreamInput sin) throws IOException { + this( + sin.readString(), + sin.readString(), + sin.readString(), + sin.readList(Action::readFrom) + ); + } + + public Map asTemplateArg() { + return Map.of( + ACTIONS_FIELD, actions.stream().map(Action::asTemplateArg) + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(id); + out.writeString(name); + out.writeString(severity); + out.writeCollection(actions); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + + Action[] actionArray = new Action[]{}; + actionArray = actions.toArray(actionArray); + + return builder.startObject() + .field(ID_FIELD, id) + .field(NAME_FIELD, name) + .field(SEVERITY_FIELD, severity) + .field(ACTIONS_FIELD, actionArray) + .endObject(); + } + + public static CorrelationRuleTrigger parse(XContentParser xcp) throws IOException { + String id = null; + String name = null; + String severity = null; + List actions = new ArrayList<>(); + + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = xcp.currentName(); + xcp.nextToken(); + + switch (fieldName) { + case ID_FIELD: + id = xcp.text(); + break; + case NAME_FIELD: + name = xcp.text(); + break; + case SEVERITY_FIELD: + severity = xcp.text(); + break; + case ACTIONS_FIELD: + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { + Action action = Action.parse(xcp); + actions.add(action); + } + break; + default: + xcp.skipChildren(); + } + } + return new CorrelationRuleTrigger(id, name, severity, actions); + } + + public static CorrelationRuleTrigger readFrom(StreamInput sin) throws IOException { + return new CorrelationRuleTrigger(sin); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + CorrelationRuleTrigger that = (CorrelationRuleTrigger) o; + return Objects.equals(id, that.id) && Objects.equals(name, that.name) && Objects.equals(severity, that.severity) && Objects.equals(actions, that.actions); + } + + @Override + public int hashCode() { + return Objects.hash(id, name, severity, actions); + } + + public String getId() { + return id; + } + + public String getName() { + return name; + } + + public String getSeverity() { + return severity; + } + + public List getActions() { +// List transformedActions = new ArrayList<>(); +// +// if (actions != null) { +// for (Action action : actions) { +// String subjectTemplate = action.getSubjectTemplate() != null ? action.getSubjectTemplate().getIdOrCode() : ""; +// CorrelationContext ctx = CorrelationContext(rule, sourceFindingId); +// no +// +// action.getMessageTemplate(); +// String messageTemplate = action.getMessageTemplate().getIdOrCode(); +// messageTemplate = messageTemplate.replace("{{ctx.detector", "{{ctx.monitor"); +// +// Action transformedAction = new Action(action.getName(), action.getDestinationId(), +// new Script(ScriptType.INLINE, Script.DEFAULT_TEMPLATE_LANG, subjectTemplate, Collections.emptyMap()), +// new Script(ScriptType.INLINE, Script.DEFAULT_TEMPLATE_LANG, messageTemplate, Collections.emptyMap()), +// action.getThrottleEnabled(), action.getThrottle(), +// action.getId(), action.getActionExecutionPolicy()); +// +// transformedActions.add(transformedAction); +// } +// } + return actions; + } + +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/model/STIX2IOC.java b/src/main/java/org/opensearch/securityanalytics/model/STIX2IOC.java index 57e99a1c5..e6c361f09 100644 --- a/src/main/java/org/opensearch/securityanalytics/model/STIX2IOC.java +++ b/src/main/java/org/opensearch/securityanalytics/model/STIX2IOC.java @@ -184,7 +184,7 @@ public static STIX2IOC parse(XContentParser xcp, String id, Long version) throws name = xcp.text(); break; case TYPE_FIELD: - type = IOCType.valueOf(xcp.text().toUpperCase(Locale.ROOT)); + type = IOCType.valueOf(xcp.text().toLowerCase(Locale.ROOT)); break; case VALUE_FIELD: value = xcp.text(); diff --git a/src/main/java/org/opensearch/securityanalytics/model/threatintel/BaseEntity.java b/src/main/java/org/opensearch/securityanalytics/model/threatintel/BaseEntity.java new file mode 100644 index 000000000..e72fac958 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/model/threatintel/BaseEntity.java @@ -0,0 +1,18 @@ +package org.opensearch.securityanalytics.model.threatintel; + +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; + +import java.io.IOException; + +public abstract class BaseEntity implements Writeable, ToXContentObject { + @Override + public abstract void writeTo(StreamOutput out) throws IOException; + + @Override + public abstract XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException; + + public abstract String getId(); +} diff --git a/src/main/java/org/opensearch/securityanalytics/model/IocFinding.java b/src/main/java/org/opensearch/securityanalytics/model/threatintel/IocFinding.java similarity index 96% rename from src/main/java/org/opensearch/securityanalytics/model/IocFinding.java rename to src/main/java/org/opensearch/securityanalytics/model/threatintel/IocFinding.java index 6c34b2cb3..2beff07e6 100644 --- a/src/main/java/org/opensearch/securityanalytics/model/IocFinding.java +++ b/src/main/java/org/opensearch/securityanalytics/model/threatintel/IocFinding.java @@ -1,10 +1,8 @@ -package org.opensearch.securityanalytics.model; +package org.opensearch.securityanalytics.model.threatintel; import org.apache.commons.lang3.StringUtils; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; -import org.opensearch.core.common.io.stream.Writeable; -import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.core.xcontent.XContentParserUtils; @@ -14,6 +12,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; @@ -21,7 +20,7 @@ * IoC Match provides mapping of the IoC Value to the list of docs that contain the ioc in a given execution of IoC_Scan_job * It's the inverse of an IoC finding which maps a document to list of IoC's */ -public class IocFinding implements Writeable, ToXContent { +public class IocFinding extends BaseEntity { //TODO implement IoC_Match interface from security-analytics-commons public static final String ID_FIELD = "id"; public static final String RELATED_DOC_IDS_FIELD = "related_doc_ids"; @@ -84,9 +83,9 @@ public void writeTo(StreamOutput out) throws IOException { public Map asTemplateArg() { return Map.of( - ID_FIELD,id, + ID_FIELD, id, RELATED_DOC_IDS_FIELD, relatedDocIds, - IOC_WITH_FEED_IDS_FIELD, iocWithFeeds, + IOC_WITH_FEED_IDS_FIELD, iocWithFeeds.stream().map(IocWithFeeds::asTemplateArg).collect(Collectors.toList()), MONITOR_ID_FIELD, monitorId, MONITOR_NAME_FIELD, monitorName, IOC_VALUE_FIELD, iocValue, @@ -242,7 +241,7 @@ private static void validateIoCMatch(String id, String iocScanJobId, String iocS if (timestamp == null) { throw new IllegalArgumentException("timestamp cannot be null in IoC_Match Object"); } - if(relatedDocIds == null || relatedDocIds.isEmpty()) { + if (relatedDocIds == null || relatedDocIds.isEmpty()) { throw new IllegalArgumentException("related_doc_ids cannot be null or empty in IoC_Match Object"); } } diff --git a/src/main/java/org/opensearch/securityanalytics/model/IocWithFeeds.java b/src/main/java/org/opensearch/securityanalytics/model/threatintel/IocWithFeeds.java similarity index 77% rename from src/main/java/org/opensearch/securityanalytics/model/IocWithFeeds.java rename to src/main/java/org/opensearch/securityanalytics/model/threatintel/IocWithFeeds.java index d858619fc..dc6b3d1c8 100644 --- a/src/main/java/org/opensearch/securityanalytics/model/IocWithFeeds.java +++ b/src/main/java/org/opensearch/securityanalytics/model/threatintel/IocWithFeeds.java @@ -1,4 +1,4 @@ -package org.opensearch.securityanalytics.model; +package org.opensearch.securityanalytics.model.threatintel; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; @@ -9,6 +9,7 @@ import java.io.IOException; import java.util.Map; +import java.util.Objects; import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; @@ -108,4 +109,30 @@ public static IocWithFeeds parse(XContentParser xcp) throws IOException { public static IocWithFeeds readFrom(StreamInput sin) throws IOException { return new IocWithFeeds(sin); } + + @Override + public int hashCode() { + return Objects.hash(feedId, index, iocId); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + IocWithFeeds that = (IocWithFeeds) o; + + if (feedId != null ? !feedId.equals(that.feedId) : that.feedId != null) return false; + if (iocId != null ? !iocId.equals(that.iocId) : that.iocId != null) return false; + return index != null ? index.equals(that.index) : that.index == null; + } + + @Override + public String toString() { + return "IocWithFeeds{" + + "feedId='" + feedId + '\'' + + ", iocId='" + iocId + '\'' + + ", index='" + index + '\'' + + '}'; + } } \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/model/threatintel/ThreatIntelAlert.java b/src/main/java/org/opensearch/securityanalytics/model/threatintel/ThreatIntelAlert.java new file mode 100644 index 000000000..87bd765d1 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/model/threatintel/ThreatIntelAlert.java @@ -0,0 +1,431 @@ +package org.opensearch.securityanalytics.model.threatintel; + +import org.opensearch.commons.alerting.model.ActionExecutionResult; +import org.opensearch.commons.alerting.model.Alert; +import org.opensearch.commons.authuser.User; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.securityanalytics.util.XContentUtils; + +import java.io.IOException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.securityanalytics.util.XContentUtils.getInstant; + +public class ThreatIntelAlert extends BaseEntity { + + public static final String ALERT_ID_FIELD = "id"; + public static final String SCHEMA_VERSION_FIELD = "schema_version"; + public static final String ALERT_VERSION_FIELD = "version"; + public static final String USER_FIELD = "user"; + public static final String TRIGGER_NAME_FIELD = "trigger_name"; + public static final String TRIGGER_ID_FIELD = "trigger_id"; + public static final String MONITOR_ID_FIELD = "monitor_id"; + public static final String MONITOR_NAME_FIELD = "monitor_name"; + public static final String STATE_FIELD = "state"; + public static final String START_TIME_FIELD = "start_time"; + public static final String END_TIME_FIELD = "end_time"; + public static final String LAST_UPDATED_TIME_FIELD = "last_updated_time"; + public static final String ACKNOWLEDGED_TIME_FIELD = "acknowledged_time"; + public static final String ERROR_MESSAGE_FIELD = "error_message"; + public static final String SEVERITY_FIELD = "severity"; + public static final String ACTION_EXECUTION_RESULTS_FIELD = "action_execution_results"; + public static final String IOC_VALUE_FIELD = "ioc_value"; + public static final String IOC_TYPE_FIELD = "ioc_type"; + public static final String FINDING_IDS_FIELD = "finding_ids"; + public static final String NO_ID = ""; + public static final long NO_VERSION = 1L; + public static final long NO_SCHEMA_VERSION = 0; + + private final String id; + private final long version; + private final long schemaVersion; + private final User user; + private final String triggerName; + private final String triggerId; + private final String monitorId; + private final String monitorName; + private final Alert.State state; + private final Instant startTime; + private final Instant endTime; + private final Instant acknowledgedTime; + private final Instant lastUpdatedTime; + private final String errorMessage; + private final String severity; + private final String iocValue; + private final String iocType; + private final List actionExecutionResults; + private List findingIds; + + public ThreatIntelAlert( + String id, + long version, + long schemaVersion, + User user, + String triggerId, + String triggerName, + String monitorId, + String monitorName, + Alert.State state, + Instant startTime, + Instant endTime, + Instant lastUpdatedTime, + Instant acknowledgedTime, + String errorMessage, + String severity, + String iocValue, + String iocType, + List actionExecutionResults, + List findingIds + ) { + + this.id = id != null ? id : NO_ID; + this.version = version != 0 ? version : NO_VERSION; + this.schemaVersion = schemaVersion; + this.user = user; + this.triggerId = triggerId; + this.triggerName = triggerName; + this.monitorId = monitorId; + this.monitorName = monitorName; + this.state = state; + this.startTime = startTime; + this.endTime = endTime; + this.acknowledgedTime = acknowledgedTime; + this.errorMessage = errorMessage; + this.severity = severity; + this.iocValue = iocValue; + this.iocType = iocType; + this.actionExecutionResults = actionExecutionResults; + this.lastUpdatedTime = lastUpdatedTime; + this.findingIds = findingIds; + } + + public ThreatIntelAlert(StreamInput sin) throws IOException { + this.id = sin.readString(); + this.version = sin.readLong(); + this.schemaVersion = sin.readLong(); + this.user = sin.readBoolean() ? new User(sin) : null; + this.triggerId = sin.readString(); + this.triggerName = sin.readString(); + this.monitorId = sin.readString(); + this.monitorName = sin.readString(); + this.state = sin.readEnum(Alert.State.class); + this.startTime = sin.readInstant(); + this.endTime = sin.readOptionalInstant(); + this.acknowledgedTime = sin.readOptionalInstant(); + this.errorMessage = sin.readOptionalString(); + this.severity = sin.readString(); + this.actionExecutionResults = sin.readList(ActionExecutionResult::new); + this.lastUpdatedTime = sin.readOptionalInstant(); + this.iocType = sin.readString(); + this.iocValue = sin.readString(); + this.findingIds = sin.readStringList(); + } + + public ThreatIntelAlert(ThreatIntelAlert currentAlert, List findingIds) { + this.findingIds = findingIds; + this.id = currentAlert.id; + this.version = currentAlert.version; + this.schemaVersion = currentAlert.schemaVersion; + this.user = currentAlert.user; + this.triggerId = currentAlert.triggerId; + this.triggerName = currentAlert.triggerName; + this.monitorId = currentAlert.monitorId; + this.monitorName = currentAlert.monitorName; + this.state = currentAlert.state; + this.startTime = currentAlert.startTime; + this.endTime = currentAlert.endTime; + this.acknowledgedTime = currentAlert.acknowledgedTime; + this.errorMessage = currentAlert.errorMessage; + this.severity = currentAlert.severity; + this.iocValue = currentAlert.iocValue; + this.iocType = currentAlert.iocType; + this.actionExecutionResults = currentAlert.actionExecutionResults; + this.lastUpdatedTime = Instant.now(); + } + + public boolean isAcknowledged() { + return state == Alert.State.ACKNOWLEDGED; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(id); + out.writeLong(version); + out.writeLong(schemaVersion); + out.writeBoolean(user != null); + if (user != null) { + user.writeTo(out); + } + out.writeString(triggerId); + out.writeString(triggerName); + out.writeString(monitorId); + out.writeString(monitorName); + out.writeEnum(state); + out.writeInstant(startTime); + out.writeOptionalInstant(endTime); + out.writeOptionalInstant(acknowledgedTime); + out.writeOptionalString(errorMessage); + out.writeString(severity); + out.writeCollection(actionExecutionResults); + out.writeOptionalInstant(lastUpdatedTime); + out.writeString(iocType); + out.writeString(iocValue); + out.writeStringCollection(findingIds); + } + + public static ThreatIntelAlert parse(XContentParser xcp, long version) throws IOException { + String id = NO_ID; + long schemaVersion = NO_SCHEMA_VERSION; + User user = null; + String triggerId = null; + String triggerName = null; + String monitorId = null; + String monitorName = null; + Alert.State state = null; + Instant startTime = null; + String severity = null; + Instant endTime = null; + Instant acknowledgedTime = null; + Instant lastUpdatedTime = null; + String errorMessage = null; + List actionExecutionResults = new ArrayList<>(); + String iocValue = null; + String iocType = null; + List findingIds = new ArrayList<>(); + + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = xcp.currentName(); + xcp.nextToken(); + switch (fieldName) { + case USER_FIELD: + user = xcp.currentToken() == XContentParser.Token.VALUE_NULL ? null : User.parse(xcp); + break; + case ALERT_ID_FIELD: + id = xcp.text(); + break; + case IOC_VALUE_FIELD: + iocValue = xcp.textOrNull(); + break; + case IOC_TYPE_FIELD: + iocType = xcp.textOrNull(); + break; + case ALERT_VERSION_FIELD: + version = xcp.longValue(); + break; + case SCHEMA_VERSION_FIELD: + schemaVersion = xcp.intValue(); + break; + case TRIGGER_ID_FIELD: + triggerId = xcp.text(); + break; + case TRIGGER_NAME_FIELD: + triggerName = xcp.text(); + break; + case MONITOR_ID_FIELD: + monitorId = xcp.text(); + break; + case MONITOR_NAME_FIELD: + monitorName = xcp.text(); + break; + case STATE_FIELD: + state = Alert.State.valueOf(xcp.text()); + break; + case ERROR_MESSAGE_FIELD: + errorMessage = xcp.textOrNull(); + break; + case SEVERITY_FIELD: + severity = xcp.text(); + break; + case ACTION_EXECUTION_RESULTS_FIELD: + ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { + actionExecutionResults.add(ActionExecutionResult.parse(xcp)); + } + break; + case START_TIME_FIELD: + startTime = getInstant(xcp); + break; + case END_TIME_FIELD: + endTime = getInstant(xcp); + break; + case ACKNOWLEDGED_TIME_FIELD: + acknowledgedTime = getInstant(xcp); + break; + case LAST_UPDATED_TIME_FIELD: + lastUpdatedTime = getInstant(xcp); + break; + case FINDING_IDS_FIELD: + ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { + findingIds.add(xcp.text()); + } + default: + xcp.skipChildren(); + } + } + + return new ThreatIntelAlert(id, + version, + schemaVersion, + user, + triggerId, + triggerName, + monitorId, + monitorName, + state, + startTime, + endTime, + acknowledgedTime, + lastUpdatedTime, + errorMessage, + severity, + iocValue, iocType, actionExecutionResults, findingIds); + } + + public static Alert readFrom(StreamInput sin) throws IOException { + return new Alert(sin); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return createXContentBuilder(builder, true); + } + + @Override + public String getId() { + return id; + } + + public XContentBuilder toXContentWithUser(XContentBuilder builder) throws IOException { + return createXContentBuilder(builder, false); + } + + private XContentBuilder createXContentBuilder(XContentBuilder builder, boolean secure) throws IOException { + builder.startObject() + .field(ALERT_ID_FIELD, id) + .field(ALERT_VERSION_FIELD, version) + .field(SCHEMA_VERSION_FIELD, schemaVersion) + .field(TRIGGER_NAME_FIELD, triggerName) + .field(TRIGGER_ID_FIELD, triggerId) + .field(MONITOR_ID_FIELD, monitorId) + .field(MONITOR_NAME_FIELD, monitorName) + .field(STATE_FIELD, state) + .field(ERROR_MESSAGE_FIELD, errorMessage) + .field(IOC_VALUE_FIELD, iocValue) + .field(IOC_TYPE_FIELD, iocType) + .field(SEVERITY_FIELD, severity) + .field(ACTION_EXECUTION_RESULTS_FIELD, actionExecutionResults.toArray()) + .field(FINDING_IDS_FIELD, findingIds.toArray(new String[0])); + XContentUtils.buildInstantAsField(builder, acknowledgedTime, ACKNOWLEDGED_TIME_FIELD); + XContentUtils.buildInstantAsField(builder, lastUpdatedTime, LAST_UPDATED_TIME_FIELD); + XContentUtils.buildInstantAsField(builder, startTime, START_TIME_FIELD); + XContentUtils.buildInstantAsField(builder, endTime, END_TIME_FIELD); + if (!secure) { + if (user == null) { + builder.nullField(USER_FIELD); + } else { + builder.field(USER_FIELD, user); + } + } + return builder.endObject(); + } + + public Map asTemplateArg() { + Map map = new HashMap<>(); + map.put(ACKNOWLEDGED_TIME_FIELD, acknowledgedTime != null ? acknowledgedTime.toEpochMilli() : null); + map.put(ALERT_ID_FIELD, id); + map.put(ALERT_VERSION_FIELD, version); + map.put(END_TIME_FIELD, endTime != null ? endTime.toEpochMilli() : null); + map.put(ERROR_MESSAGE_FIELD, errorMessage); + map.put(SEVERITY_FIELD, severity); + map.put(START_TIME_FIELD, startTime.toEpochMilli()); + map.put(STATE_FIELD, state.toString()); + map.put(TRIGGER_ID_FIELD, triggerId); + map.put(TRIGGER_NAME_FIELD, triggerName); + map.put(FINDING_IDS_FIELD, findingIds); + map.put(LAST_UPDATED_TIME_FIELD, lastUpdatedTime); + map.put(IOC_TYPE_FIELD, iocType); + map.put(IOC_VALUE_FIELD, iocValue); + return map; + } + + public long getVersion() { + return version; + } + + public long getSchemaVersion() { + return schemaVersion; + } + + public User getUser() { + return user; + } + + public String getTriggerName() { + return triggerName; + } + + public Alert.State getState() { + return state; + } + + public Instant getStartTime() { + return startTime; + } + + public Instant getEndTime() { + return endTime; + } + + public Instant getAcknowledgedTime() { + return acknowledgedTime; + } + + public String getErrorMessage() { + return errorMessage; + } + + public String getSeverity() { + return severity; + } + + public List getActionExecutionResults() { + return actionExecutionResults; + } + + public String getTriggerId() { + return triggerId; + } + + public Instant getLastUpdatedTime() { + return lastUpdatedTime; + } + + public String getIocValue() { + return iocValue; + } + + public String getIocType() { + return iocType; + } + + public List getFindingIds() { + return findingIds; + } + + public String getMonitorId() { + return monitorId; + } + + public String getMonitorName() { + return monitorName; + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/resthandler/RestAcknowledgeCorrelationAlertsAction.java b/src/main/java/org/opensearch/securityanalytics/resthandler/RestAcknowledgeCorrelationAlertsAction.java new file mode 100644 index 000000000..1a1eede54 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/resthandler/RestAcknowledgeCorrelationAlertsAction.java @@ -0,0 +1,72 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.resthandler; + +import org.opensearch.client.node.NodeClient; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.action.RestToXContentListener; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.action.AckCorrelationAlertsAction; +import org.opensearch.securityanalytics.action.AckCorrelationAlertsRequest; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Locale; + +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; + + +/** + * Acknowledge list of correlation alerts generated by correlation rules. + */ +public class RestAcknowledgeCorrelationAlertsAction extends BaseRestHandler { + @Override + public String getName() { + return "ack_correlation_alerts_action"; + } + + @Override + public List routes() { + return Collections.singletonList( + new Route(RestRequest.Method.POST, String.format( + Locale.getDefault(), + "%s/_acknowledge/correlationAlerts", + SecurityAnalyticsPlugin.PLUGINS_BASE_URI) + )); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient nodeClient) throws IOException { + List alertIds = getAlertIds(request.contentParser()); + AckCorrelationAlertsRequest CorrelationAckAlertsRequest = new AckCorrelationAlertsRequest(alertIds); + return channel -> nodeClient.execute( + AckCorrelationAlertsAction.INSTANCE, + CorrelationAckAlertsRequest, + new RestToXContentListener<>(channel) + ); + } + + private List getAlertIds(XContentParser xcp) throws IOException { + List ids = new ArrayList<>(); + ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = xcp.currentName(); + xcp.nextToken(); + if (fieldName.equals("alertIds")) { + ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { + ids.add(xcp.text()); + } + } + + } + return ids; + } +} + diff --git a/src/main/java/org/opensearch/securityanalytics/resthandler/RestGetCorrelationsAlertsAction.java b/src/main/java/org/opensearch/securityanalytics/resthandler/RestGetCorrelationsAlertsAction.java new file mode 100644 index 000000000..471c26915 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/resthandler/RestGetCorrelationsAlertsAction.java @@ -0,0 +1,96 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.resthandler; + +import org.opensearch.client.node.NodeClient; +import org.opensearch.commons.alerting.model.Table; +import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.action.RestToXContentListener; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.action.GetCorrelationAlertsAction; +import org.opensearch.securityanalytics.action.GetCorrelationAlertsRequest; + +import java.io.IOException; +import java.time.DateTimeException; +import java.time.Instant; +import java.util.List; + +import static java.util.Collections.singletonList; +import static org.opensearch.rest.RestRequest.Method.GET; + +public class RestGetCorrelationsAlertsAction extends BaseRestHandler { + + @Override + public String getName() { + return "get_correlation_alerts_action_sa"; + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + + String correlationRuleId = request.param("correlation_rule_id", null); + String correlationRuleName = request.param("correlation_rule_name", null); + String severityLevel = request.param("severityLevel", "ALL"); + String alertState = request.param("alertState", "ALL"); + // Table params + String sortString = request.param("sortString", "start_time"); + String sortOrder = request.param("sortOrder", "asc"); + String missing = request.param("missing"); + int size = request.paramAsInt("size", 20); + int startIndex = request.paramAsInt("startIndex", 0); + String searchString = request.param("searchString", ""); + + Instant startTime = null; + String startTimeParam = request.param("startTime"); + if (startTimeParam != null && !startTimeParam.isEmpty()) { + try { + startTime = Instant.ofEpochMilli(Long.parseLong(startTimeParam)); + } catch (NumberFormatException | NullPointerException | DateTimeException e) { + startTime = Instant.now(); + } + } + + Instant endTime = null; + String endTimeParam = request.param("endTime"); + if (endTimeParam != null && !endTimeParam.isEmpty()) { + try { + endTime = Instant.ofEpochMilli(Long.parseLong(endTimeParam)); + } catch (NumberFormatException | NullPointerException | DateTimeException e) { + endTime = Instant.now(); + } + } + + Table table = new Table( + sortOrder, + sortString, + missing, + size, + startIndex, + searchString + ); + + GetCorrelationAlertsRequest req = new GetCorrelationAlertsRequest( + correlationRuleId, + correlationRuleName, + table, + severityLevel, + alertState, + startTime, + endTime + ); + + return channel -> client.execute( + GetCorrelationAlertsAction.INSTANCE, + req, + new RestToXContentListener<>(channel) + ); + } + + @Override + public List routes() { + return singletonList(new Route(GET, SecurityAnalyticsPlugin.CORRELATIONS_ALERTS_BASE_URI)); + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsResponse.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsResponse.java index 4c0dea477..50ae08dd4 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsResponse.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsResponse.java @@ -9,7 +9,7 @@ import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.securityanalytics.model.IocFinding; +import org.opensearch.securityanalytics.model.threatintel.IocFinding; import java.io.IOException; import java.util.Collections; diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/GetThreatIntelAlertsAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/GetThreatIntelAlertsAction.java new file mode 100644 index 000000000..16ba20543 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/GetThreatIntelAlertsAction.java @@ -0,0 +1,15 @@ +package org.opensearch.securityanalytics.threatIntel.action.monitor; + +import org.opensearch.action.ActionType; +import org.opensearch.securityanalytics.threatIntel.action.monitor.response.GetThreatIntelAlertsResponse; +import org.opensearch.securityanalytics.threatIntel.sacommons.monitor.ThreatIntelMonitorActions; + +public class GetThreatIntelAlertsAction extends ActionType { + + public static final GetThreatIntelAlertsAction INSTANCE = new GetThreatIntelAlertsAction(); + public static final String NAME = ThreatIntelMonitorActions.GET_THREAT_INTEL_ALERTS_ACTION_NAME; + + public GetThreatIntelAlertsAction() { + super(NAME, GetThreatIntelAlertsResponse::new); + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/IocScanMonitorFanOutAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/IocScanMonitorFanOutAction.java deleted file mode 100644 index eb3665992..000000000 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/IocScanMonitorFanOutAction.java +++ /dev/null @@ -1,19 +0,0 @@ -package org.opensearch.securityanalytics.threatIntel.action.monitor; - -import org.opensearch.action.ActionType; -import org.opensearch.commons.alerting.action.DocLevelMonitorFanOutResponse; -import org.opensearch.core.common.io.stream.Writeable; - -/** - * Ioc Scan Monitor fan out action that distributes the monitor runner logic to mutliple data node. - */ -public class IocScanMonitorFanOutAction extends ActionType { - /** - * @param name The name of the action, must be unique across actions. - * @param docLevelMonitorFanOutResponseReader A reader for the response type - */ - public IocScanMonitorFanOutAction(String name, Writeable.Reader docLevelMonitorFanOutResponseReader) { - super(name, docLevelMonitorFanOutResponseReader); - } - -} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/request/GetThreatIntelAlertsRequest.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/request/GetThreatIntelAlertsRequest.java new file mode 100644 index 000000000..6dbc5666c --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/request/GetThreatIntelAlertsRequest.java @@ -0,0 +1,103 @@ +package org.opensearch.securityanalytics.threatIntel.action.monitor.request; + +import org.opensearch.action.ActionRequest; +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.commons.alerting.model.Table; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; + +import java.io.IOException; +import java.time.Instant; + +public class GetThreatIntelAlertsRequest extends ActionRequest { + + private final String monitorId; + private final Table table; + private final String severityLevel; + private final String alertState; + private final Instant startTime; + private final Instant endTime; + + public GetThreatIntelAlertsRequest( + String monitorId, + Table table, + String severityLevel, + String alertState, + Instant startTime, + Instant endTime + ) { + super(); + this.monitorId = monitorId; + this.table = table; + this.severityLevel = severityLevel; + this.alertState = alertState; + this.startTime = startTime; + this.endTime = endTime; + } + + public GetThreatIntelAlertsRequest( + Table table, + String severityLevel, + String alertState, + Instant startTime, + Instant endTime + ) { + super(); + this.monitorId = null; + this.table = table; + this.severityLevel = severityLevel; + this.alertState = alertState; + this.startTime = startTime; + this.endTime = endTime; + } + + public GetThreatIntelAlertsRequest(StreamInput sin) throws IOException { + this( + sin.readOptionalString(), + Table.readFrom(sin), + sin.readString(), + sin.readString(), + sin.readOptionalInstant(), + sin.readOptionalInstant() + ); + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeOptionalString(monitorId); + table.writeTo(out); + out.writeString(severityLevel); + out.writeString(alertState); + out.writeOptionalInstant(startTime); + out.writeOptionalInstant(endTime); + } + + public String getmonitorId() { + return monitorId; + } + + public Table getTable() { + return table; + } + + public String getSeverityLevel() { + return severityLevel; + } + + public String getAlertState() { + return alertState; + } + + public Instant getStartTime() { + return startTime; + } + + public Instant getEndTime() { + return endTime; + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/request/IndexThreatIntelMonitorRequest.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/request/IndexThreatIntelMonitorRequest.java index 64d4a433b..7f7205c5f 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/request/IndexThreatIntelMonitorRequest.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/request/IndexThreatIntelMonitorRequest.java @@ -16,13 +16,13 @@ public class IndexThreatIntelMonitorRequest extends ActionRequest implements Ind private final String id; private final RestRequest.Method method; - private final ThreatIntelMonitorDto threatIntelMonitor; + private final ThreatIntelMonitorDto monitor; - public IndexThreatIntelMonitorRequest(String id, RestRequest.Method method, ThreatIntelMonitorDto threatIntelMonitor) { + public IndexThreatIntelMonitorRequest(String id, RestRequest.Method method, ThreatIntelMonitorDto monitor) { super(); this.id = id; this.method = method; - this.threatIntelMonitor = threatIntelMonitor; + this.monitor = monitor; } public IndexThreatIntelMonitorRequest(StreamInput sin) throws IOException { @@ -37,7 +37,7 @@ public IndexThreatIntelMonitorRequest(StreamInput sin) throws IOException { public void writeTo(StreamOutput out) throws IOException { out.writeString(id); out.writeEnum(method); - threatIntelMonitor.writeTo(out); + monitor.writeTo(out); } @Override @@ -53,7 +53,7 @@ public RestRequest.Method getMethod() { return method; } - public ThreatIntelMonitorDto getThreatIntelMonitor() { - return threatIntelMonitor; + public ThreatIntelMonitorDto getMonitor() { + return monitor; } } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/response/GetThreatIntelAlertsResponse.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/response/GetThreatIntelAlertsResponse.java new file mode 100644 index 000000000..8e2df4a69 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/response/GetThreatIntelAlertsResponse.java @@ -0,0 +1,56 @@ +package org.opensearch.securityanalytics.threatIntel.action.monitor.response; + +import org.opensearch.core.action.ActionResponse; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.securityanalytics.threatIntel.sacommons.monitor.ThreatIntelAlertDto; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +public class GetThreatIntelAlertsResponse extends ActionResponse implements ToXContentObject { + + private static final String ALERTS_FIELD = "alerts"; + private static final String TOTAL_ALERTS_FIELD = "total_alerts"; + + private List alerts; + private Integer totalAlerts; + + public GetThreatIntelAlertsResponse(List alerts, Integer totalAlerts) { + super(); + this.alerts = alerts; + this.totalAlerts = totalAlerts; + } + + public GetThreatIntelAlertsResponse(StreamInput sin) throws IOException { + this( + Collections.unmodifiableList(sin.readList(ThreatIntelAlertDto::new)), + sin.readInt() + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeCollection(this.alerts); + out.writeInt(this.totalAlerts); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject() + .field(ALERTS_FIELD, alerts) + .field(TOTAL_ALERTS_FIELD, totalAlerts); + return builder.endObject(); + } + + public List getAlerts() { + return this.alerts; + } + + public Integer getTotalAlerts() { + return this.totalAlerts; + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/BaseEntityCrudService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/BaseEntityCrudService.java new file mode 100644 index 000000000..e69706b94 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/BaseEntityCrudService.java @@ -0,0 +1,256 @@ +package org.opensearch.securityanalytics.threatIntel.iocscan.dao; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.opensearch.ResourceAlreadyExistsException; +import org.opensearch.action.DocWriteRequest; +import org.opensearch.action.admin.indices.create.CreateIndexRequest; +import org.opensearch.action.bulk.BulkRequest; +import org.opensearch.action.bulk.BulkResponse; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.support.GroupedActionListener; +import org.opensearch.action.support.WriteRequest; +import org.opensearch.client.Client; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.index.IndexNotFoundException; +import org.opensearch.rest.action.admin.indices.AliasesNotFoundException; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.securityanalytics.model.threatintel.BaseEntity; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.opensearch.securityanalytics.util.DetectorUtils.getEmptySearchResponse; + +/** + * Provides generic CRUD implementations for entity that is stored in system index. Provides generic implementation + * of system index too. + */ +public abstract class BaseEntityCrudService { + // todo rollover + private static final Logger log = LogManager.getLogger(BaseEntityCrudService.class); + private final Client client; + private final ClusterService clusterService; + private final NamedXContentRegistry xContentRegistry; + + public BaseEntityCrudService(Client client, ClusterService clusterService, NamedXContentRegistry xContentRegistry) { + this.client = client; + this.clusterService = clusterService; + this.xContentRegistry = xContentRegistry; + } + + + public void bulkIndexEntities(List newEntityList, List updatedEntityList, + ActionListener actionListener) { + try { + Integer batchSize = this.clusterService.getClusterSettings().get(SecurityAnalyticsSettings.BATCH_SIZE); + createIndexIfNotExists(ActionListener.wrap( + r -> { + List bulkRequestList = new ArrayList<>(); + BulkRequest bulkRequest = new BulkRequest(getEntityAliasName()); + for (int i = 0; i < newEntityList.size(); i++) { + Entity entity = newEntityList.get(i); + try { + IndexRequest indexRequest = new IndexRequest(getEntityAliasName()) + .id(entity.getId()) + .source(entity.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) + .opType(DocWriteRequest.OpType.CREATE); + bulkRequest.add(indexRequest); + if ( + bulkRequest.requests().size() == batchSize + && i != newEntityList.size() - 1 // final bulk request will be added outside for loop with refresh policy none + ) { + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.NONE); + bulkRequestList.add(bulkRequest); + bulkRequest = new BulkRequest(); + } + } catch (IOException e) { + log.error(String.format("Failed to create index request for %s moving on to next", getEntityName()), e); + } + } + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + bulkRequestList.add(bulkRequest); + for (int i = 0; i < updatedEntityList.size(); i++) { + Entity entity = updatedEntityList.get(i); + try { + IndexRequest indexRequest = new IndexRequest(getEntityAliasName()) + .id(entity.getId()) + .source(entity.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) + .opType(DocWriteRequest.OpType.INDEX); + bulkRequest.add(indexRequest); + if ( + bulkRequest.requests().size() == batchSize + && i != updatedEntityList.size() - 1 // final bulk request will be added outside for loop with refresh policy none + ) { + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.NONE); + bulkRequestList.add(bulkRequest); + bulkRequest = new BulkRequest(); + } + } catch (IOException e) { + log.error(String.format("Failed to create index request for %s moving on to next", getEntityName()), e); + } + } + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + bulkRequestList.add(bulkRequest); + GroupedActionListener groupedListener = new GroupedActionListener<>(ActionListener.wrap(bulkResponses -> { + int idx = 0; + for (BulkResponse response : bulkResponses) { + BulkRequest request = bulkRequestList.get(idx); + if (response.hasFailures()) { + log.error("Failed to bulk index {} {}s. Failure: {}", request.requests().size(), getEntityName(), response.buildFailureMessage()); + } + } + actionListener.onResponse(null); + }, actionListener::onFailure), bulkRequestList.size()); + for (BulkRequest req : bulkRequestList) { + try { + client.bulk(req, groupedListener); //todo why stash context here? + } catch (Exception e) { + log.error( + () -> new ParameterizedMessage("Failed to bulk save {} {}.", req.batchSize(), getEntityName()), + e); + } + } + }, e -> { + log.error(() -> new ParameterizedMessage("Failed to create System Index {}", getEntityAliasName()), e); + actionListener.onFailure(e); + })); + + + } catch (Exception e) { + log.error("Exception saving the threat intel source config in index", e); + actionListener.onFailure(e); + } + } + + public void bulkIndexEntities(List entityList, + ActionListener actionListener) { + try { + Integer batchSize = this.clusterService.getClusterSettings().get(SecurityAnalyticsSettings.BATCH_SIZE); + createIndexIfNotExists(ActionListener.wrap( + r -> { + List bulkRequestList = new ArrayList<>(); + BulkRequest bulkRequest = new BulkRequest(getEntityAliasName()); + for (int i = 0; i < entityList.size(); i++) { + Entity entity = entityList.get(i); + try { + IndexRequest indexRequest = new IndexRequest(getEntityAliasName()) + .id(entity.getId()) + .source(entity.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) + .opType(DocWriteRequest.OpType.CREATE); + bulkRequest.add(indexRequest); + if ( + bulkRequest.requests().size() == batchSize + && i != entityList.size() - 1 // final bulk request will be added outside for loop with refresh policy none + ) { + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.NONE); + bulkRequestList.add(bulkRequest); + bulkRequest = new BulkRequest(); + } + } catch (IOException e) { + log.error(String.format("Failed to create index request for %s moving on to next", getEntityName()), e); + } + } + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + bulkRequestList.add(bulkRequest); + GroupedActionListener groupedListener = new GroupedActionListener<>(ActionListener.wrap(bulkResponses -> { + int idx = 0; + for (BulkResponse response : bulkResponses) { + BulkRequest request = bulkRequestList.get(idx); + if (response.hasFailures()) { + log.error("Failed to bulk index {} {}s. Failure: {}", request.batchSize(), getEntityName(), response.buildFailureMessage()); + } + } + actionListener.onResponse(null); + }, actionListener::onFailure), bulkRequestList.size()); + for (BulkRequest req : bulkRequestList) { + try { + client.bulk(req, groupedListener); //todo why stash context here? + } catch (Exception e) { + log.error( + () -> new ParameterizedMessage("Failed to bulk save {} {}.", req.batchSize(), getEntityName()), + e); + } + } + }, e -> { + log.error(() -> new ParameterizedMessage("Failed to create System Index {}", getEntityIndexPattern()), e); + actionListener.onFailure(e); + })); + + + } catch (Exception e) { + log.error("Exception saving the threat intel source config in index", e); + actionListener.onFailure(e); + } + } + + public void search(SearchSourceBuilder searchSourceBuilder, final ActionListener listener) { + SearchRequest searchRequest = new SearchRequest() + .source(searchSourceBuilder) + .indices(getEntityAliasName()); + client.search(searchRequest, ActionListener.wrap( + listener::onResponse, + e -> { + if (e instanceof IndexNotFoundException || e instanceof AliasesNotFoundException) { + listener.onResponse(getEmptySearchResponse()); + return; + } + log.error( + () -> new ParameterizedMessage("Failed to search {}s from index {}.", getEntityName(), getEntityAliasName()), + e); + listener.onFailure(e); + } + )); + } + + public void createIndexIfNotExists(final ActionListener listener) { + try { + if (clusterService.state().metadata().hasAlias(getEntityAliasName())) { + listener.onResponse(null); + return; + } + final CreateIndexRequest createIndexRequest = new CreateIndexRequest(getEntityIndexPattern()).mapping(getEntityIndexMapping()) + .settings(getIndexSettings()); + client.admin().indices().create(createIndexRequest, ActionListener.wrap( + r -> { + log.debug("{} index created", getEntityName()); + listener.onResponse(null); + }, e -> { + if (e instanceof ResourceAlreadyExistsException) { + log.debug("index {} already exist", getEntityIndexMapping()); + listener.onResponse(null); + return; + } + log.error(String.format("Failed to create security analytics threat intel %s index", getEntityName()), e); + listener.onFailure(e); + } + )); + } catch (Exception e) { + log.error(String.format("Failure in creating %s index", getEntityName()), e); + listener.onFailure(e); + } + } + + protected abstract String getEntityIndexMapping(); + + public abstract String getEntityName(); + + protected Settings.Builder getIndexSettings() { + return Settings.builder().put("index.hidden", true); + } + + public abstract String getEntityAliasName(); + + public abstract String getEntityIndexPattern(); + +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocFindingService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocFindingService.java index 0e1b955b1..eaf94bdbf 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocFindingService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocFindingService.java @@ -2,35 +2,11 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.opensearch.ResourceAlreadyExistsException; -import org.opensearch.action.DocWriteRequest; -import org.opensearch.action.admin.indices.alias.Alias; -import org.opensearch.action.admin.indices.create.CreateIndexRequest; -import org.opensearch.action.bulk.BulkRequest; -import org.opensearch.action.bulk.BulkResponse; -import org.opensearch.action.index.IndexRequest; -import org.opensearch.action.search.SearchRequest; -import org.opensearch.action.search.SearchResponse; -import org.opensearch.action.support.GroupedActionListener; -import org.opensearch.action.support.WriteRequest; import org.opensearch.client.Client; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.xcontent.LoggingDeprecationHandler; -import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.common.xcontent.XContentType; -import org.opensearch.core.action.ActionListener; import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.core.xcontent.ToXContent; -import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.core.xcontent.XContentParserUtils; -import org.opensearch.index.IndexNotFoundException; -import org.opensearch.search.SearchHit; -import org.opensearch.search.builder.SearchSourceBuilder; -import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; -import org.opensearch.securityanalytics.model.IocFinding; -import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; -import org.opensearch.securityanalytics.threatIntel.action.GetIocFindingsResponse; +import org.opensearch.securityanalytics.model.threatintel.IocFinding; import org.opensearch.securityanalytics.util.SecurityAnalyticsException; import java.io.BufferedReader; @@ -38,15 +14,13 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; import java.util.stream.Collectors; /** - * Data layer to perform CRUD operations for threat intel ioc match : store in system index. + * Data layer to perform CRUD operations for threat intel ioc finding : store in system index. */ -public class IocFindingService { - //TODO manage index rollover +public class IocFindingService extends BaseEntityCrudService { + public static final String IOC_FINDING_ALIAS_NAME = ".opensearch-sap-ioc-findings"; public static final String IOC_FINDING_INDEX_PATTERN = "<.opensearch-sap-ioc-findings-history-{now/d}-1>"; @@ -60,67 +34,15 @@ public class IocFindingService { private final NamedXContentRegistry xContentRegistry; public IocFindingService(final Client client, final ClusterService clusterService, final NamedXContentRegistry xContentRegistry) { + super(client, clusterService, xContentRegistry); this.client = client; this.clusterService = clusterService; this.xContentRegistry = xContentRegistry; } - public void indexIocMatches(List iocFindings, - final ActionListener actionListener) { - try { - Integer batchSize = this.clusterService.getClusterSettings().get(SecurityAnalyticsSettings.BATCH_SIZE); - createIndexIfNotExists(ActionListener.wrap( - r -> { - List bulkRequestList = new ArrayList<>(); - BulkRequest bulkRequest = new BulkRequest(IOC_FINDING_ALIAS_NAME); - for (int i = 0; i < iocFindings.size(); i++) { - IocFinding iocFinding = iocFindings.get(i); - try { - IndexRequest indexRequest = new IndexRequest(IOC_FINDING_ALIAS_NAME) - .source(iocFinding.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) - .opType(DocWriteRequest.OpType.CREATE); - bulkRequest.add(indexRequest); - if ( - bulkRequest.requests().size() == batchSize - && i != iocFindings.size() - 1 // final bulk request will be added outside for loop with refresh policy none - ) { - bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.NONE); - bulkRequestList.add(bulkRequest); - bulkRequest = new BulkRequest(); - } - } catch (IOException e) { - log.error(String.format("Failed to create index request for ioc match %s moving on to next"), e); - } - } - bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - bulkRequestList.add(bulkRequest); - GroupedActionListener groupedListener = new GroupedActionListener<>(ActionListener.wrap(bulkResponses -> { - int idx = 0; - for (BulkResponse response : bulkResponses) { - BulkRequest request = bulkRequestList.get(idx); - if (response.hasFailures()) { - log.error("Failed to bulk index {} Ioc Matches. Failure: {}", request.batchSize(), response.buildFailureMessage()); - } - } - actionListener.onResponse(null); - }, actionListener::onFailure), bulkRequestList.size()); - for (BulkRequest req : bulkRequestList) { - try { - client.bulk(req, groupedListener); //todo why stash context here? - } catch (Exception e) { - log.error("Failed to save ioc matches.", e); - } - } - }, e -> { - log.error("Failed to create System Index"); - actionListener.onFailure(e); - })); - - - } catch (Exception e) { - log.error("Exception saving the threat intel source config in index", e); - actionListener.onFailure(e); - } + @Override + public String getEntityIndexMapping() { + return getIndexMapping(); } public static String getIndexMapping() { @@ -131,85 +53,22 @@ public static String getIndexMapping() { } } } catch (IOException e) { - log.error("Failed to get the threat intel ioc match index mapping", e); - throw new SecurityAnalyticsException("Failed to get the threat intel ioc match index mapping", RestStatus.INTERNAL_SERVER_ERROR, e); + log.error("Failed to get the threat intel ioc finding index mapping", e); + throw new SecurityAnalyticsException("Failed to get the threat intel ioc finding index mapping", RestStatus.INTERNAL_SERVER_ERROR, e); } } - - /** - * Index name: .opensearch-sap-iocmatch - * Mapping: /mappings/ioc_finding_mapping.json - * - * @param listener setup listener - */ - public void createIndexIfNotExists(final ActionListener listener) { - // check if job index exists - try { - if (clusterService.state().metadata().hasAlias(IOC_FINDING_ALIAS_NAME) == true) { - listener.onResponse(null); - return; - } - final CreateIndexRequest createIndexRequest = new CreateIndexRequest(IOC_FINDING_INDEX_PATTERN).mapping(getIndexMapping()) - .settings(SecurityAnalyticsPlugin.TIF_JOB_INDEX_SETTING).alias(new Alias(IOC_FINDING_ALIAS_NAME)); - client.admin().indices().create(createIndexRequest, ActionListener.wrap( - r -> { - log.debug("Ioc match index created"); - listener.onResponse(null); - }, e -> { - if (e instanceof ResourceAlreadyExistsException) { - log.debug("index {} already exist", IOC_FINDING_INDEX_PATTERN); - listener.onResponse(null); - return; - } - log.error("Failed to create security analytics threat intel job index", e); - listener.onFailure(e); - } - )); - } catch (Exception e) { - log.error("Failure in creating ioc_match index", e); - listener.onFailure(e); - } + @Override + public String getEntityAliasName() { + return IOC_FINDING_ALIAS_NAME; } - public void searchIocMatches(SearchSourceBuilder searchSourceBuilder, final ActionListener actionListener) { - createIndexIfNotExists(ActionListener.wrap( - r -> { - SearchRequest searchRequest = new SearchRequest() - .source(searchSourceBuilder) - .indices(IOC_FINDING_ALIAS_NAME); - - client.search(searchRequest, new ActionListener<>() { - @Override - public void onResponse(SearchResponse searchResponse) { - try { - long totalIocFindingsCount = searchResponse.getHits().getTotalHits().value; - List iocFindings = new ArrayList<>(); - - for (SearchHit hit: searchResponse.getHits()) { - XContentParser xcp = XContentType.JSON.xContent() - .createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, hit.getSourceAsString()); - XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp); - IocFinding iocFinding = IocFinding.parse(xcp); - iocFindings.add(iocFinding); - } - actionListener.onResponse(new GetIocFindingsResponse((int) totalIocFindingsCount, iocFindings)); - } catch (Exception ex) { - this.onFailure(ex); - } - } + @Override + public String getEntityIndexPattern() { + return IOC_FINDING_INDEX_PATTERN; + } - @Override - public void onFailure(Exception e) { - if (e instanceof IndexNotFoundException) { - actionListener.onResponse(new GetIocFindingsResponse(0, List.of())); - return; - } - actionListener.onFailure(e); - } - }); - }, e -> { - log.error("Failed to create System Index"); - actionListener.onFailure(e); - })); + @Override + public String getEntityName() { + return "ioc_finding"; } } \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/ThreatIntelAlertService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/ThreatIntelAlertService.java new file mode 100644 index 000000000..987203cda --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/ThreatIntelAlertService.java @@ -0,0 +1,66 @@ +package org.opensearch.securityanalytics.threatIntel.iocscan.dao; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.client.Client; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.securityanalytics.model.threatintel.ThreatIntelAlert; +import org.opensearch.securityanalytics.util.SecurityAnalyticsException; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.stream.Collectors; + +public class ThreatIntelAlertService extends BaseEntityCrudService { + + public static final String THREAT_INTEL_ALERT_ALIAS_NAME = ".opensearch-sap-threat-intel-alerts"; + + public static final String THREAT_INTEL_ALERT_INDEX_PATTERN = "<.opensearch-sap-threat-intel-alerts-history-{now/d}-1>"; + + public static final String THREAT_INTEL_ALERT_INDEX_PATTERN_REGEXP = ".opensearch-sap-threat-intel-alerts*"; + + private static final Logger log = LogManager.getLogger(ThreatIntelAlertService.class); + + public ThreatIntelAlertService(Client client, ClusterService clusterService, NamedXContentRegistry xContentRegistry) { + super(client, clusterService, xContentRegistry); + } + + @Override + protected String getEntityIndexMapping() { + return getIndexMapping(); + } + + public static String getIndexMapping() { + try { + try (InputStream is = IocFindingService.class.getResourceAsStream("/mappings/threat_intel_alert_mapping.json")) { + try (BufferedReader reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) { + return reader.lines().map(String::trim).collect(Collectors.joining()); + } + } + } catch (IOException e) { + log.error("Failed to get the threat intel alert index mapping", e); + throw new SecurityAnalyticsException("Failed to get the threat intel alert index mapping", RestStatus.INTERNAL_SERVER_ERROR, e); + } + } + + @Override + public String getEntityName() { + return "threat_intel_alert"; + } + + @Override + public String getEntityAliasName() { + return THREAT_INTEL_ALERT_ALIAS_NAME; + } + + @Override + public String getEntityIndexPattern() { + return THREAT_INTEL_ALERT_INDEX_PATTERN; + } + +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dto/IocScanContext.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dto/IocScanContext.java new file mode 100644 index 000000000..d04a85bc5 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dto/IocScanContext.java @@ -0,0 +1,56 @@ +package org.opensearch.securityanalytics.threatIntel.iocscan.dto; + +import org.opensearch.commons.alerting.model.Monitor; +import org.opensearch.commons.alerting.model.MonitorMetadata; +import org.opensearch.securityanalytics.threatIntel.model.monitor.ThreatIntelInput; +import org.opensearch.securityanalytics.threatIntel.model.monitor.ThreatIntelTrigger; + +import java.util.List; +import java.util.Map; + +public class IocScanContext { + private final Monitor monitor; + private final MonitorMetadata monitorMetadata; + private final boolean dryRun; + private final List data; + private final ThreatIntelInput threatIntelInput; // deserialize threat intel input + private final List indices; // user's log data indices + private final Map> iocTypeToIndices; + public IocScanContext(Monitor monitor, MonitorMetadata monitorMetadata, boolean dryRun, List data, ThreatIntelInput threatIntelInput, List indices, Map> iocTypeToIndices) { + this.monitor = monitor; + this.monitorMetadata = monitorMetadata; + this.dryRun = dryRun; + this.data = data; + this.threatIntelInput = threatIntelInput; + this.indices = indices; + this.iocTypeToIndices = iocTypeToIndices; + } + + public Monitor getMonitor() { + return monitor; + } + + public boolean isDryRun() { + return dryRun; + } + + public List getData() { + return data; + } + + public MonitorMetadata getMonitorMetadata() { + return monitorMetadata; + } + + public ThreatIntelInput getThreatIntelInput() { + return threatIntelInput; + } + + public List getIndices() { + return indices; + } + + public Map> getIocTypeToIndices() { + return iocTypeToIndices; + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dto/PerIocTypeScanInputDto.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dto/PerIocTypeScanInputDto.java index 3be4b5542..36f34eebb 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dto/PerIocTypeScanInputDto.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dto/PerIocTypeScanInputDto.java @@ -10,6 +10,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -28,7 +29,7 @@ public class PerIocTypeScanInputDto implements Writeable, ToXContentObject { public PerIocTypeScanInputDto(String iocType, Map> indexToFieldsMap) { this.iocType = iocType; - this.indexToFieldsMap = indexToFieldsMap; + this.indexToFieldsMap = indexToFieldsMap == null ? Collections.emptyMap() : indexToFieldsMap; } public PerIocTypeScanInputDto(StreamInput sin) throws IOException { diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/IoCScanService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/IoCScanService.java new file mode 100644 index 000000000..fdc4d3f1c --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/IoCScanService.java @@ -0,0 +1,233 @@ +package org.opensearch.securityanalytics.threatIntel.iocscan.service; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.opensearch.commons.alerting.model.Monitor; +import org.opensearch.securityanalytics.model.STIX2IOC; +import org.opensearch.securityanalytics.model.threatintel.IocFinding; +import org.opensearch.securityanalytics.model.threatintel.IocWithFeeds; +import org.opensearch.securityanalytics.model.threatintel.ThreatIntelAlert; +import org.opensearch.securityanalytics.threatIntel.iocscan.dto.IocScanContext; +import org.opensearch.securityanalytics.threatIntel.model.monitor.PerIocTypeScanInput; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.function.BiConsumer; + + +public abstract class IoCScanService implements IoCScanServiceInterface { + private static final Logger log = LogManager.getLogger(IoCScanService.class); + + @Override + public void scanIoCs(IocScanContext iocScanContext, + BiConsumer scanCallback + ) { + try { + List data = iocScanContext.getData(); + if (data.isEmpty()) { + scanCallback.accept(Collections.emptyList(), null); + return; + } + Monitor monitor = iocScanContext.getMonitor(); + + long startTime = System.currentTimeMillis(); + IocLookupDtos iocLookupDtos = extractIocsPerType(data, iocScanContext.getThreatIntelInput().getPerIocTypeScanInputList()); + BiConsumer, Exception> iocScanResultConsumer = (List maliciousIocs, Exception e) -> { + long scanEndTime = System.currentTimeMillis(); + long timeTaken = scanEndTime - startTime; + log.debug("Threat intel monitor {}: scan time taken is {}", monitor.getId(), timeTaken); + if (e == null) { + createIocFindings(maliciousIocs, iocLookupDtos.iocValueToDocIdMap, iocScanContext, + (iocFindings, e1) -> { + if (e1 != null) { + log.error( + () -> new ParameterizedMessage("Threat intel monitor {}: Failed to create ioc findings/ ", + iocScanContext.getMonitor().getId(), data.size()), + e1); + scanCallback.accept(null, e1); + } else { + BiConsumer, Exception> triggerResultConsumer = (alerts, e2) -> { + if (e2 != null) { + log.error( + () -> new ParameterizedMessage("Threat intel monitor {}: Failed to execute threat intel triggers/ ", + iocScanContext.getMonitor().getId(), data.size()), + e2); + scanCallback.accept(null, e2); + return; + } else { + scanCallback.accept(data, null); + } + }; + executeTriggers(maliciousIocs, iocFindings, iocScanContext, data, iocLookupDtos, + triggerResultConsumer); + + } + + } + ); + } else { + log.error( + () -> new ParameterizedMessage("Threat intel monitor {}: Failed to run scan for {} docs", + iocScanContext.getMonitor().getId(), data.size()), + e); + scanCallback.accept(null, e); + + } + }; + matchAgainstThreatIntelAndReturnMaliciousIocs( + iocLookupDtos.getIocsPerIocTypeMap(), monitor, iocScanResultConsumer, iocScanContext.getIocTypeToIndices()); + } catch (Exception e) { + log.error( + () -> new ParameterizedMessage("Threat intel monitor {}: Unexpected failure in running scan for {} docs", + iocScanContext.getMonitor().getId(), iocScanContext.getData().size()), + e); + scanCallback.accept(null, e); + } + } + + + abstract void executeTriggers(List maliciousIocs, + List iocFindings, + IocScanContext iocScanContext, + List data, IocLookupDtos iocLookupDtos, + BiConsumer, Exception> triggerResultConsumer); + abstract void matchAgainstThreatIntelAndReturnMaliciousIocs( + Map> iocsPerType, + Monitor monitor, + BiConsumer, Exception> callback, + Map> iocTypeToIndices); + + /** + * For each doc, we extract different maps for quick look up - + * 1. map of iocs as key to ioc type + * 2. ioc value to doc ids containing the ioc + * 4. doc id to iocs map (reverse mapping of 2) + */ + private IocLookupDtos extractIocsPerType + (List data, List iocTypeToIndexFieldMappings) { + Map> iocsPerIocTypeMap = new HashMap<>(); + Map> iocValueToDocIdMap = new HashMap<>(); + Map> docIdToIocsMap = new HashMap<>(); + for (Data datum : data) { + for (PerIocTypeScanInput iocTypeToIndexFieldMapping : iocTypeToIndexFieldMappings) { + String iocType = iocTypeToIndexFieldMapping.getIocType().toLowerCase(); + String index = getIndexName(datum); + List fields = iocTypeToIndexFieldMapping.getIndexToFieldsMap().get(index); + for (String field : fields) { + List vals = getValuesAsStringList(datum, field); + String id = getId(datum); + String docId = id + ":" + index; + Set iocs = docIdToIocsMap.getOrDefault(docIdToIocsMap.get(docId), new HashSet<>()); + iocs.addAll(vals); + docIdToIocsMap.put(docId, iocs); + for (String ioc : vals) { + Set docIds = iocValueToDocIdMap.getOrDefault(iocValueToDocIdMap.get(ioc), new HashSet<>()); + docIds.add(docId); + iocValueToDocIdMap.put(ioc, docIds); + } + if (false == vals.isEmpty()) { + iocs = iocsPerIocTypeMap.getOrDefault(iocType, new HashSet<>()); + iocs.addAll(vals); + iocsPerIocTypeMap.put(iocType, iocs); + } + } + } + } + return new IocLookupDtos(iocsPerIocTypeMap, iocValueToDocIdMap, docIdToIocsMap); + } + + abstract List getValuesAsStringList(Data datum, String field); + + abstract String getIndexName(Data datum); + + abstract String getId(Data datum); + + private void createIocFindings(List iocs, + Map> iocValueToDocIdMap, + IocScanContext iocScanContext, + BiConsumer, Exception> callback) { + try { + Instant timestamp = Instant.now(); + Monitor monitor = iocScanContext.getMonitor(); + // Map to collect unique IocValue with their respective FeedIds + Map> iocValueToFeedIds = new HashMap<>(); + Map iocValueToType = new HashMap<>(); + for (STIX2IOC ioc : iocs) { + String iocValue = ioc.getValue(); + if (false == iocValueToType.containsKey(iocValue)) + iocValueToType.put(iocValue, ioc.getType().toString()); + iocValueToFeedIds + .computeIfAbsent(iocValue, k -> new HashSet<>()) + .add(new IocWithFeeds(ioc.getId(), ioc.getFeedId(), "")); //todo figure how to store index + } + + List iocFindings = new ArrayList<>(); + + for (Map.Entry> entry : iocValueToFeedIds.entrySet()) { + String iocValue = entry.getKey(); + Set iocWithFeeds = entry.getValue(); + + List relatedDocIds = new ArrayList<>(iocValueToDocIdMap.getOrDefault(iocValue, new HashSet<>())); + List feedIdsList = new ArrayList<>(iocWithFeeds); + try { + IocFinding iocFinding = new IocFinding( + UUID.randomUUID().toString(), // Generating a unique ID + relatedDocIds, + feedIdsList, // update to object + monitor.getId(), + monitor.getName(), + iocValue, + iocValueToType.get(iocValue), + timestamp, + UUID.randomUUID().toString() // TODO execution ID + ); + iocFindings.add(iocFinding); + } catch (Exception e) { + log.error(String.format("skipping creating ioc finding for %s due to unexpected failure.", entry.getKey()), e); + } + } + saveIocFindings(iocFindings, callback, monitor); + } catch (Exception e) { + log.error(() -> new ParameterizedMessage("Failed to create ioc findinges due to unexpected error {}", iocScanContext.getMonitor().getId()), e); + callback.accept(null, e); + } + } + + abstract void saveIocFindings + (List iocs, BiConsumer, Exception> callback, Monitor monitor); + + abstract void saveAlerts(List updatedAlerts, List newAlerts, Monitor monitor, BiConsumer, Exception> callback); + + protected static class IocLookupDtos { + private final Map> iocsPerIocTypeMap; + private final Map> iocValueToDocIdMap; + private final Map> docIdToIocsMap; + + public IocLookupDtos(Map> iocsPerIocTypeMap, Map> iocValueToDocIdMap, Map> docIdToIocsMap) { + this.iocsPerIocTypeMap = iocsPerIocTypeMap; + this.iocValueToDocIdMap = iocValueToDocIdMap; + this.docIdToIocsMap = docIdToIocsMap; + } + + public Map> getIocsPerIocTypeMap() { + return iocsPerIocTypeMap; + } + + public Map> getIocValueToDocIdMap() { + return iocValueToDocIdMap; + } + + public Map> getDocIdToIocsMap() { + return docIdToIocsMap; + } + } + +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/IoCScanServiceInterface.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/IoCScanServiceInterface.java new file mode 100644 index 000000000..1826824d3 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/IoCScanServiceInterface.java @@ -0,0 +1,13 @@ +package org.opensearch.securityanalytics.threatIntel.iocscan.service; + +import org.opensearch.securityanalytics.threatIntel.iocscan.dto.IocScanContext; + +import java.util.function.BiConsumer; + +public interface IoCScanServiceInterface { + + void scanIoCs( + IocScanContext iocScanContext, + BiConsumer scanCallback + ); +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/SaIoCScanService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/SaIoCScanService.java new file mode 100644 index 000000000..81a814915 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/SaIoCScanService.java @@ -0,0 +1,509 @@ +package org.opensearch.securityanalytics.threatIntel.iocscan.service; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.ShardSearchFailure; +import org.opensearch.action.support.GroupedActionListener; +import org.opensearch.client.Client; +import org.opensearch.common.document.DocumentField; +import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.commons.alerting.model.Monitor; +import org.opensearch.commons.alerting.model.Trigger; +import org.opensearch.commons.alerting.model.action.Action; +import org.opensearch.commons.alerting.model.remote.monitors.RemoteMonitorTrigger; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.index.query.TermsQueryBuilder; +import org.opensearch.search.SearchHit; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.securityanalytics.commons.model.STIX2; +import org.opensearch.securityanalytics.correlation.alert.notifications.NotificationService; +import org.opensearch.securityanalytics.model.STIX2IOC; +import org.opensearch.securityanalytics.model.threatintel.IocFinding; +import org.opensearch.securityanalytics.model.threatintel.ThreatIntelAlert; +import org.opensearch.securityanalytics.threatIntel.iocscan.dao.IocFindingService; +import org.opensearch.securityanalytics.threatIntel.iocscan.dao.ThreatIntelAlertService; +import org.opensearch.securityanalytics.threatIntel.iocscan.dto.IocScanContext; +import org.opensearch.securityanalytics.threatIntel.model.monitor.ThreatIntelTrigger; +import org.opensearch.securityanalytics.threatIntel.model.monitor.TransportThreatIntelMonitorFanOutAction.SearchHitsOrException; +import org.opensearch.securityanalytics.threatIntel.util.ThreatIntelMonitorUtils; +import org.opensearch.securityanalytics.util.SecurityAnalyticsException; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.function.BiConsumer; +import java.util.stream.Collectors; + +import static java.util.Collections.emptyList; +import static org.opensearch.securityanalytics.threatIntel.util.ThreatIntelMonitorUtils.getThreatIntelTriggerFromBytesReference; + +public class SaIoCScanService extends IoCScanService { + + private static final Logger log = LogManager.getLogger(SaIoCScanService.class); + public static final int MAX_TERMS = 65536; //TODO make ioc index setting based. use same setting value to create index + private final Client client; + private final NamedXContentRegistry xContentRegistry; + private final IocFindingService iocFindingService; + private final ThreatIntelAlertService threatIntelAlertService; + private final NotificationService notificationService; + + public SaIoCScanService(Client client, NamedXContentRegistry xContentRegistry, IocFindingService iocFindingService, + ThreatIntelAlertService threatIntelAlertService, NotificationService notificationService) { + this.client = client; + this.xContentRegistry = xContentRegistry; + this.iocFindingService = iocFindingService; + this.threatIntelAlertService = threatIntelAlertService; + this.notificationService = notificationService; + } + + @Override + void executeTriggers(List maliciousIocs, List iocFindings, IocScanContext iocScanContext, List searchHits, IoCScanService.IocLookupDtos iocLookupDtos, BiConsumer, Exception> triggerResultConsumer) { + Monitor monitor = iocScanContext.getMonitor(); + if (maliciousIocs.isEmpty() || monitor.getTriggers().isEmpty()) { + triggerResultConsumer.accept(Collections.emptyList(), null); + return; + } + initAlertsIndex( + ActionListener.wrap( + r -> { + GroupedActionListener> allTriggerResultListener = getGroupedListenerForAllTriggersResponse(iocScanContext.getMonitor(), + triggerResultConsumer); + for (Trigger trigger : monitor.getTriggers()) { + executeTrigger(iocFindings, trigger, monitor, allTriggerResultListener); + } + }, + e -> { + log.error(() -> new ParameterizedMessage( + "Threat intel monitor {} Failed to execute triggers . Failed to initialize threat intel alerts index", + monitor.getId()), e); + triggerResultConsumer.accept(Collections.emptyList(), null); + } + ) + ); + } + + private void executeTrigger(List iocFindings, + Trigger trigger, + Monitor monitor, + ActionListener> listener) { + try { + RemoteMonitorTrigger remoteMonitorTrigger = (RemoteMonitorTrigger) trigger; + ThreatIntelTrigger threatIntelTrigger = getThreatIntelTriggerFromBytesReference(remoteMonitorTrigger, xContentRegistry); + ArrayList triggerMatchedFindings = ThreatIntelMonitorUtils.getTriggerMatchedFindings(iocFindings, threatIntelTrigger); + if (triggerMatchedFindings.isEmpty()) { + log.debug("Threat intel monitor {} no matches for trigger {}", monitor.getId(), trigger.getName()); + listener.onResponse(emptyList()); + } else { + fetchExistingAlertsForTrigger(monitor, triggerMatchedFindings, trigger, ActionListener.wrap( + existingAlerts -> { + executeActionsAndSaveAlerts(iocFindings, trigger, monitor, existingAlerts, triggerMatchedFindings, threatIntelTrigger, listener); + }, + e -> { + log.error(() -> new ParameterizedMessage( + "Threat intel monitor {} Failed to execute trigger {}. Failure while fetching existing alerts", + monitor.getId(), trigger.getName()), e); + listener.onFailure(e); + } + )); + } + } catch (Exception e) { + log.error(() -> new ParameterizedMessage( + "Threat intel monitor {} Failed to execute trigger {}", monitor.getId(), trigger.getName()), + e + ); + listener.onFailure(e); + } + } + + private void executeActionsAndSaveAlerts(List iocFindings, + Trigger trigger, + Monitor monitor, + List existingAlerts, + ArrayList triggerMatchedFindings, + ThreatIntelTrigger threatIntelTrigger, ActionListener> listener) { + Map iocToUpdatedAlertsMap = ThreatIntelMonitorUtils.prepareAlertsToUpdate(triggerMatchedFindings, existingAlerts); + List newAlerts = ThreatIntelMonitorUtils.prepareNewAlerts(monitor, trigger, triggerMatchedFindings, iocToUpdatedAlertsMap); + ThreatIntelAlertContext ctx = new ThreatIntelAlertContext(threatIntelTrigger, + trigger, + iocFindings, + monitor, + newAlerts, + existingAlerts); + if (false == trigger.getActions().isEmpty()) { + GroupedActionListener notifsListener = new GroupedActionListener<>(ActionListener.wrap( + r -> { + saveAlerts(new ArrayList<>(iocToUpdatedAlertsMap.values()), + newAlerts, + monitor, + (threatIntelAlerts, e) -> { + if (e != null) { + log.error(String.format("Threat intel monitor %s: Failed to save alerts for trigger {}", monitor.getId(), trigger.getId()), e); + listener.onFailure(e); + } else { + listener.onResponse(threatIntelAlerts); + } + }); + }, e -> { + log.error(String.format("Threat intel monitor %s: Failed to send notification for trigger {}", monitor.getId(), trigger.getId()), e); + listener.onFailure(new SecurityAnalyticsException("Failed to send notification", RestStatus.INTERNAL_SERVER_ERROR, e)); + } + ), trigger.getActions().size()); + for (Action action : trigger.getActions()) { + try { + String transformedSubject = NotificationService.compileTemplate(ctx, action.getSubjectTemplate()); + String transformedMessage = NotificationService.compileTemplate(ctx, action.getMessageTemplate()); + String configId = action.getDestinationId(); + notificationService.sendNotification(configId, trigger.getSeverity(), transformedSubject, transformedMessage, notifsListener); + } catch (Exception e) { + log.error(String.format("Threat intel monitor %s: Failed to send notification to %s for trigger %s", monitor.getId(), action.getDestinationId(), trigger.getId()), e); + notifsListener.onFailure(new SecurityAnalyticsException("Failed to send notification", RestStatus.INTERNAL_SERVER_ERROR, e)); + } + + } + } else { + saveAlerts(new ArrayList<>(iocToUpdatedAlertsMap.values()), + newAlerts, + monitor, + (threatIntelAlerts, e) -> { + if (e != null) { + log.error(String.format("Threat intel monitor %s: Failed to save alerts for trigger %s", monitor.getId(), trigger.getId()), e); + listener.onFailure(e); + } else { + listener.onResponse(threatIntelAlerts); + } + }); + } + } + + private void fetchExistingAlertsForTrigger(Monitor monitor, + ArrayList findings, + Trigger trigger, + ActionListener> listener) { + if (findings.isEmpty()) { + listener.onResponse(emptyList()); + return; + } + SearchSourceBuilder ssb = ThreatIntelMonitorUtils.getSearchSourceBuilderForExistingAlertsQuery(findings, trigger); + threatIntelAlertService.search(ssb, ActionListener.wrap( + searchResponse -> { + List alerts = new ArrayList<>(); + if (searchResponse.getHits() == null || searchResponse.getHits().getHits() == null) { + listener.onResponse(alerts); + return; + } + for (SearchHit hit : searchResponse.getHits().getHits()) { + XContentParser xcp = XContentType.JSON.xContent().createParser( + xContentRegistry, + LoggingDeprecationHandler.INSTANCE, hit.getSourceAsString() + ); + if(xcp.currentToken() == null) + xcp.nextToken(); + ThreatIntelAlert alert = ThreatIntelAlert.parse(xcp, hit.getVersion()); + alerts.add(alert); + } + listener.onResponse(alerts); + }, + e -> { + log.error(() -> new ParameterizedMessage( + "Threat intel monitor {} Failed to execute trigger {}. Unexpected error in fetching existing alerts for dedupe", monitor.getId(), trigger.getName()), + e + ); + listener.onFailure(e); + } + )); + } + + private GroupedActionListener> getGroupedListenerForAllTriggersResponse(Monitor monitor, BiConsumer, Exception> triggerResultConsumer) { + return new GroupedActionListener<>(ActionListener.wrap( + r -> { + List list = new ArrayList<>(); + r.forEach(list::addAll); + triggerResultConsumer.accept(list, null); //todo change emptylist to actual response + }, e -> { + log.error(() -> new ParameterizedMessage( + "Threat intel monitor {} Failed to execute triggers {}", monitor.getId()), + e + ); + triggerResultConsumer.accept(emptyList(), e); + } + ), monitor.getTriggers().size()); + } + + @Override + void matchAgainstThreatIntelAndReturnMaliciousIocs( + Map> iocsPerType, + Monitor monitor, + BiConsumer, Exception> callback, + Map> iocTypeToIndices) { + long startTime = System.currentTimeMillis(); + int numIocs = iocsPerType.values().stream().mapToInt(Set::size).sum(); + GroupedActionListener groupedListenerForAllIocTypes = getGroupedListenerForIocScanFromAllIocTypes(iocsPerType, monitor, callback, startTime, numIocs); + for (String iocType : iocsPerType.keySet()) { + List indices = iocTypeToIndices.get(iocType); + Set iocs = iocsPerType.get(iocType); + if (iocTypeToIndices.containsKey(iocType.toLowerCase())) { + if (indices.isEmpty()) { + log.debug( + "Threat intel monitor {} : No ioc indices of type {} found so no scan performed.", + monitor.getId(), + iocType + ); + groupedListenerForAllIocTypes.onResponse(new SearchHitsOrException(emptyList(), null)); + } else if (iocs.isEmpty()) { + log.debug( + "Threat intel monitor {} : No iocs of type {} found in user data so no scan performed.", + monitor.getId(), + iocType + ); + groupedListenerForAllIocTypes.onResponse(new SearchHitsOrException(emptyList(), null)); + } else { + performScanForMaliciousIocsPerIocType(indices, iocs, monitor, iocType, groupedListenerForAllIocTypes); + } + } else { + groupedListenerForAllIocTypes.onResponse(new SearchHitsOrException(emptyList(), null)); + } + } + } + + private GroupedActionListener getGroupedListenerForIocScanFromAllIocTypes(Map> iocsPerType, Monitor monitor, BiConsumer, Exception> callback, long startTime, int numIocs) { + return new GroupedActionListener<>( + ActionListener.wrap( + lists -> { + long endTime = System.currentTimeMillis(); + long timetaken = endTime - startTime; + log.debug("IOC_SCAN: Threat intel monitor {} completed Ioc match phase in {} millis for {} iocs", + monitor.getId(), timetaken, numIocs); + List hits = new ArrayList<>(); + lists.forEach(hitsOrException -> + hits.addAll(hitsOrException.getHits() == null ? + emptyList() : + hitsOrException.getHits())); + List iocs = new ArrayList<>(); + hits.forEach(hit -> { + try { + XContentParser xcp = XContentType.JSON.xContent().createParser( + xContentRegistry, + LoggingDeprecationHandler.INSTANCE, + hit.getSourceAsString()); + xcp.nextToken(); + + STIX2IOC ioc = STIX2IOC.parse(xcp, hit.getId(), hit.getVersion()); + iocs.add(ioc); + } catch (Exception e) { + log.error(() -> new ParameterizedMessage( + "Failed to parse IOC doc from hit {} index {}", hit.getId(), hit.getIndex()), + e + ); + } + }); + callback.accept(iocs, null); + }, + e -> { + log.error("Threat intel monitor {} :Unexpected error while scanning data for malicious Iocs", e); + callback.accept(emptyList(), e); + } + ), + iocsPerType.size() + ); + } + + private void performScanForMaliciousIocsPerIocType( + List indices, + Set iocs, + Monitor monitor, + String iocType, + GroupedActionListener listener) { + // TODO change ioc indices max terms count to 100k and experiment + // TODO add fuzzy postings on ioc value field to enable bloomfilter on iocs as an index data structure and benchmark performance + GroupedActionListener perIocTypeListener = getGroupedListenerForIocScanPerIocType(iocs, monitor, iocType, listener); + List iocList = new ArrayList<>(iocs); + int totalIocs = iocList.size(); + + for (int start = 0; start < totalIocs; start += MAX_TERMS) { + int end = Math.min(start + MAX_TERMS, totalIocs); + List iocsSublist = iocList.subList(start, end); + SearchRequest searchRequest = getSearchRequestForIocType(indices, iocType, iocsSublist); + client.search(searchRequest, ActionListener.wrap( + searchResponse -> { + if (searchResponse.isTimedOut()) { + log.error("Threat intel monitor {} scan with {} user data indicators TIMED OUT for ioc Type {}", + monitor.getId(), + iocsSublist.size(), + iocType + ); + } + if (searchResponse.getFailedShards() > 0) { + for (ShardSearchFailure shardFailure : searchResponse.getShardFailures()) { + log.error("Threat intel monitor {} scan with {} user data indicators for ioc Type {} has Shard failures {}", + monitor.getId(), + iocsSublist.size(), + iocType, + shardFailure.toString() + ); + } + } + listener.onResponse(new SearchHitsOrException( + searchResponse.getHits() == null || searchResponse.getHits().getHits() == null ? + emptyList() : Arrays.asList(searchResponse.getHits().getHits()), null)); + }, + e -> { + log.error(() -> new ParameterizedMessage("Threat intel monitor {} scan with {} user data indicators failed for ioc Type {}", + monitor.getId(), + iocsSublist.size(), + iocType), e + ); + listener.onResponse(new SearchHitsOrException(emptyList(), e)); + } + )); + } + } + + private static SearchRequest getSearchRequestForIocType(List indices, String iocType, List iocsSublist) { + SearchRequest searchRequest = new SearchRequest(indices.toArray(new String[0])); + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); + // add the iocs sublist + boolQueryBuilder.must(new TermsQueryBuilder(STIX2.VALUE_FIELD, iocsSublist)); + // add ioc type filter + boolQueryBuilder.must(new TermsQueryBuilder(STIX2.TYPE_FIELD, iocType.toLowerCase())); + searchRequest.source().query(boolQueryBuilder); + return searchRequest; + } + + /** + * grouped listener for a given ioc type to listen and collate malicious iocs in search hits from batched search calls. + * batching done for every 65536 or MAX_TERMS setting number of iocs in a list. + */ + private GroupedActionListener getGroupedListenerForIocScanPerIocType(Set iocs, Monitor monitor, String iocType, GroupedActionListener groupedListenerForAllIocTypes) { + return new GroupedActionListener<>( + ActionListener.wrap( + (Collection searchHitsOrExceptions) -> { + if (false == searchHitsOrExceptions.stream().allMatch(shoe -> shoe.getException() != null)) { + List searchHits = new ArrayList<>(); + searchHitsOrExceptions.forEach(searchHitsOrException -> { + if (searchHitsOrException.getException() != null) { + log.error( + () -> new ParameterizedMessage( + "Threat intel monitor {}: Failed to perform ioc scan on one batch for ioc type : ", + monitor.getId(), iocType), searchHitsOrException.getException()); + } else { + searchHits.addAll(searchHitsOrException.getHits() != null ? + searchHitsOrException.getHits() : emptyList()); + } + }); + // we collect all hits we can and log all exceptions and submit to outer listener + groupedListenerForAllIocTypes.onResponse(new SearchHitsOrException(searchHits, null)); + } else { + // we collect all exceptions under one exception and respond to outer listener + groupedListenerForAllIocTypes.onResponse(new SearchHitsOrException(emptyList(), buildException(searchHitsOrExceptions)) + ); + } + }, e -> { + log.error( + () -> new ParameterizedMessage( + "Threat intel monitor {}: Failed to perform ioc scan for ioc type : ", + monitor.getId(), iocType), e); + groupedListenerForAllIocTypes.onResponse(new SearchHitsOrException(emptyList(), e)); + } + ), + //TODO fix groupsize + getGroupSizeForIocs(iocs) // batch into #MAX_TERMS setting + ); + } + + private Exception buildException(Collection searchHitsOrExceptions) { + Exception e = null; + for (SearchHitsOrException searchHitsOrException : searchHitsOrExceptions) { + if (e == null) + e = searchHitsOrException.getException(); + else { + e.addSuppressed(searchHitsOrException.getException()); + } + } + return e; + } + + private static int getGroupSizeForIocs(Set iocs) { + return iocs.size() / MAX_TERMS + (iocs.size() % MAX_TERMS == 0 ? 0 : 1); + } + + @Override + public List getValuesAsStringList(SearchHit hit, String field) { + if (hit.getFields().containsKey(field)) { + DocumentField documentField = hit.getFields().get(field); + return documentField.getValues().stream().filter(Objects::nonNull).map(Object::toString).collect(Collectors.toList()); + } else return emptyList(); + } + + @Override + public String getIndexName(SearchHit hit) { + return hit.getIndex(); + } + + @Override + public String getId(SearchHit hit) { + return hit.getId(); + } + + @Override + void saveIocFindings(List iocFindings, BiConsumer, Exception> callback, Monitor monitor) { + if (iocFindings == null || iocFindings.isEmpty()) { + callback.accept(emptyList(), null); + return; + } + log.debug("Threat intel monitor {}: Indexing {} ioc findings", monitor.getId(), iocFindings.size()); + iocFindingService.bulkIndexEntities(iocFindings, ActionListener.wrap( + v -> { + callback.accept(iocFindings, null); + }, + e -> { + log.error( + () -> new ParameterizedMessage( + "Threat intel monitor {}: Failed to index ioc findings ", + monitor.getId()), e + ); + callback.accept(emptyList(), e); + } + )); + } + + @Override + void saveAlerts(List updatedAlerts, List newAlerts, Monitor monitor, BiConsumer, Exception> callback) { + if ((newAlerts == null || newAlerts.isEmpty()) && (updatedAlerts == null || updatedAlerts.isEmpty())) { + callback.accept(emptyList(), null); + return; + } + log.debug("Threat intel monitor {}: Indexing {} new threat intel alerts and updating {} existing alerts", monitor.getId(), newAlerts.size(), updatedAlerts.size()); + threatIntelAlertService.bulkIndexEntities(newAlerts, updatedAlerts, ActionListener.wrap( + v -> { + ArrayList threatIntelAlerts = new ArrayList<>(newAlerts); + threatIntelAlerts.addAll(updatedAlerts); + callback.accept(threatIntelAlerts, null); + }, + e -> { + log.error( + () -> new ParameterizedMessage( + "Threat intel monitor {}: Failed to index alerts ", + monitor.getId()), e + ); + callback.accept(emptyList(), e); + } + )); + } + + private void initAlertsIndex(ActionListener listener) { + threatIntelAlertService.createIndexIfNotExists(listener); + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/ThreatIntelAlertContext.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/ThreatIntelAlertContext.java new file mode 100644 index 000000000..c2cdbaf65 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/ThreatIntelAlertContext.java @@ -0,0 +1,60 @@ +package org.opensearch.securityanalytics.threatIntel.iocscan.service; + +import org.opensearch.commons.alerting.model.Alert; +import org.opensearch.commons.alerting.model.Monitor; +import org.opensearch.commons.alerting.model.Trigger; +import org.opensearch.securityanalytics.model.threatintel.IocFinding; +import org.opensearch.securityanalytics.model.threatintel.ThreatIntelAlert; +import org.opensearch.securityanalytics.threatIntel.model.monitor.ThreatIntelTrigger; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * context that stores information for sending threat intel monitor notification. + * It is available to use in Threat intel monitor runner in mustache template. + */ + +public class ThreatIntelAlertContext { + public static final String MONITOR_FIELD = "monitor"; + public static final String NEW_ALERTS_FIELD = "new_alerts"; + public static final String EXISTING_ALERTS_FIELD = "existing_alerts"; + + private final List dataSources; + private final List iocTypes; + private final String triggerName; + private final String triggerId; + private final List newAlerts; + private final List existingAlerts; + private final String severity; + private final List findingIds; + private final Monitor monitor; + + public ThreatIntelAlertContext(ThreatIntelTrigger threatIntelTrigger, Trigger trigger, List findingIds, Monitor monitor, List newAlerts, List existingAlerts) { + this.dataSources = threatIntelTrigger.getDataSources(); + this.iocTypes = threatIntelTrigger.getIocTypes(); + this.triggerName = trigger.getName(); + this.triggerId = trigger.getId(); + this.newAlerts = newAlerts; + this.existingAlerts = existingAlerts; + this.severity = triggerId; + this.findingIds = findingIds; + this.monitor = monitor; + } + + //cannot add trigger as Remote Trigger holds bytereference of object and not object itself + public Map asTemplateArg() { + return Map.of( + ThreatIntelTrigger.DATA_SOURCES, dataSources, + ThreatIntelTrigger.IOC_TYPES, iocTypes, + Trigger.NAME_FIELD, triggerName, + Trigger.ID_FIELD, triggerId, + Trigger.SEVERITY_FIELD, severity, + Alert.FINDING_IDS, findingIds.stream().map(IocFinding::asTemplateArg).collect(Collectors.toList()), + MONITOR_FIELD, monitor.asTemplateArg(), + NEW_ALERTS_FIELD, newAlerts.stream().map(ThreatIntelAlert::asTemplateArg).collect(Collectors.toList()), + EXISTING_ALERTS_FIELD, existingAlerts.stream().map(ThreatIntelAlert::asTemplateArg).collect(Collectors.toList()) + ); + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/monitor/SampleRemoteDocLevelMonitorRunner.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/ThreatIntelMonitorRunner.java similarity index 56% rename from src/main/java/org/opensearch/securityanalytics/threatIntel/model/monitor/SampleRemoteDocLevelMonitorRunner.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/ThreatIntelMonitorRunner.java index d7dd5b656..3d89ab905 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/monitor/SampleRemoteDocLevelMonitorRunner.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/ThreatIntelMonitorRunner.java @@ -1,38 +1,38 @@ -package org.opensearch.securityanalytics.threatIntel.model.monitor; +package org.opensearch.securityanalytics.threatIntel.iocscan.service; import org.opensearch.action.ActionType; import org.opensearch.alerting.spi.RemoteMonitorRunner; import org.opensearch.commons.alerting.action.DocLevelMonitorFanOutResponse; -public class SampleRemoteDocLevelMonitorRunner extends RemoteMonitorRunner { +public class ThreatIntelMonitorRunner extends RemoteMonitorRunner { public static final String THREAT_INTEL_MONITOR_ACTION_NAME = "cluster:admin/opensearch/security_analytics/threatIntel/monitor/fanout"; - public static final String REMOTE_DOC_LEVEL_MONITOR_ACTION_NAME = "cluster:admin/security_analytics/threatIntel/monitor/fanout"; + public static final String FAN_OUT_ACTION_NAME = "cluster:admin/security_analytics/threatIntel/monitor/fanout"; public static final String THREAT_INTEL_MONITOR_TYPE = "ti_doc_level_monitor"; public static final String SAMPLE_REMOTE_DOC_LEVEL_MONITOR_RUNNER_INDEX = ".opensearch-alerting-sample-remote-doc-level-monitor"; - public static final ActionType REMOTE_DOC_LEVEL_MONITOR_ACTION_INSTANCE = new ActionType<>(REMOTE_DOC_LEVEL_MONITOR_ACTION_NAME, + public static final ActionType REMOTE_DOC_LEVEL_MONITOR_ACTION_INSTANCE = new ActionType<>(FAN_OUT_ACTION_NAME, DocLevelMonitorFanOutResponse::new); - private static SampleRemoteDocLevelMonitorRunner INSTANCE; + private static ThreatIntelMonitorRunner INSTANCE; - public static SampleRemoteDocLevelMonitorRunner getMonitorRunner() { + public static ThreatIntelMonitorRunner getMonitorRunner() { if (INSTANCE != null) { return INSTANCE; } - synchronized (SampleRemoteDocLevelMonitorRunner.class) { + synchronized (ThreatIntelMonitorRunner.class) { if (INSTANCE != null) { return INSTANCE; } - INSTANCE = new SampleRemoteDocLevelMonitorRunner(); + INSTANCE = new ThreatIntelMonitorRunner(); return INSTANCE; } } @Override public String getFanOutAction() { - return REMOTE_DOC_LEVEL_MONITOR_ACTION_NAME; + return FAN_OUT_ACTION_NAME; } } \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfig.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfig.java index 3605adfb0..a011b25a5 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfig.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfig.java @@ -284,6 +284,9 @@ public static SATIFSourceConfig parse(XContentParser xcp, String id, Long versio case NAME_FIELD: name = xcp.text(); break; + case VERSION_FIELD: + version = xcp.longValue(); + break; case FORMAT_FIELD: format = xcp.text(); break; diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/monitor/TransportRemoteDocLevelMonitorFanOutAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/monitor/TransportRemoteDocLevelMonitorFanOutAction.java deleted file mode 100644 index 72d0c9c2b..000000000 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/monitor/TransportRemoteDocLevelMonitorFanOutAction.java +++ /dev/null @@ -1,97 +0,0 @@ -package org.opensearch.securityanalytics.threatIntel.model.monitor; - -import org.opensearch.action.index.IndexRequest; -import org.opensearch.action.index.IndexResponse; -import org.opensearch.action.support.ActionFilters; -import org.opensearch.action.support.HandledTransportAction; -import org.opensearch.action.support.WriteRequest; -import org.opensearch.client.Client; -import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.inject.Inject; -import org.opensearch.common.settings.Settings; -import org.opensearch.commons.alerting.action.DocLevelMonitorFanOutRequest; -import org.opensearch.commons.alerting.action.DocLevelMonitorFanOutResponse; -import org.opensearch.commons.alerting.model.DocLevelMonitorInput; -import org.opensearch.commons.alerting.model.InputRunResults; -import org.opensearch.commons.alerting.model.Monitor; -import org.opensearch.commons.alerting.model.Trigger; -import org.opensearch.commons.alerting.model.remote.monitors.RemoteDocLevelMonitorInput; -import org.opensearch.commons.alerting.model.remote.monitors.RemoteMonitorTrigger; -import org.opensearch.core.action.ActionListener; -import org.opensearch.core.common.bytes.BytesReference; -import org.opensearch.core.common.io.stream.StreamInput; -import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.tasks.Task; -import org.opensearch.transport.TransportService; - -import java.util.HashMap; -import java.util.Map; - -public class TransportRemoteDocLevelMonitorFanOutAction extends HandledTransportAction { - - private final ClusterService clusterService; - - private final Settings settings; - - private final Client client; - - private final NamedXContentRegistry xContentRegistry; - - @Inject - public TransportRemoteDocLevelMonitorFanOutAction( - TransportService transportService, - Client client, - NamedXContentRegistry xContentRegistry, - ClusterService clusterService, - Settings settings, - ActionFilters actionFilters - ) { - super(SampleRemoteDocLevelMonitorRunner.REMOTE_DOC_LEVEL_MONITOR_ACTION_NAME, transportService, actionFilters, DocLevelMonitorFanOutRequest::new); - this.clusterService = clusterService; - this.client = client; - this.xContentRegistry = xContentRegistry; - this.settings = settings; - } - - @Override - protected void doExecute(Task task, DocLevelMonitorFanOutRequest request, ActionListener actionListener) { - try { - Monitor monitor = request.getMonitor(); - Map lastRunContext = request.getMonitorMetadata().getLastRunContext(); - - RemoteDocLevelMonitorInput input = (RemoteDocLevelMonitorInput) monitor.getInputs().get(0); - BytesReference customInputSerialized = input.getInput(); - StreamInput sin = StreamInput.wrap(customInputSerialized.toBytesRef().bytes); - ThreatIntelInput sampleRemoteDocLevelMonitorInput = new ThreatIntelInput(sin); - DocLevelMonitorInput docLevelMonitorInput = input.getDocLevelMonitorInput(); - String index = docLevelMonitorInput.getIndices().get(0); - - - ((Map) lastRunContext.get(index)).put("0", 0); - IndexRequest indexRequest = new IndexRequest(SampleRemoteDocLevelMonitorRunner.SAMPLE_REMOTE_DOC_LEVEL_MONITOR_RUNNER_INDEX) - .source(Map.of()).setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL); - this.client.index(indexRequest, new ActionListener<>() { - @Override - public void onResponse(IndexResponse indexResponse) { - DocLevelMonitorFanOutResponse response = new DocLevelMonitorFanOutResponse( - clusterService.localNode().getId(), - request.getExecutionId(), - monitor.getId(), - lastRunContext, - new InputRunResults(), - new HashMap<>(), - null - ); - actionListener.onResponse(response); - } - - @Override - public void onFailure(Exception e) { - actionListener.onFailure(e); - } - }); - } catch (Exception ex) { - actionListener.onFailure(ex); - } - } -} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/monitor/TransportThreatIntelMonitorFanOutAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/monitor/TransportThreatIntelMonitorFanOutAction.java new file mode 100644 index 000000000..30cadd556 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/monitor/TransportThreatIntelMonitorFanOutAction.java @@ -0,0 +1,363 @@ +package org.opensearch.securityanalytics.threatIntel.model.monitor; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.GroupedActionListener; +import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.client.Client; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.inject.Inject; +import org.opensearch.common.settings.Settings; +import org.opensearch.commons.alerting.action.DocLevelMonitorFanOutRequest; +import org.opensearch.commons.alerting.action.DocLevelMonitorFanOutResponse; +import org.opensearch.commons.alerting.model.DocumentLevelTriggerRunResult; +import org.opensearch.commons.alerting.model.InputRunResults; +import org.opensearch.commons.alerting.model.Monitor; +import org.opensearch.commons.alerting.model.MonitorRunResult; +import org.opensearch.commons.alerting.model.remote.monitors.RemoteDocLevelMonitorInput; +import org.opensearch.commons.alerting.util.AlertingException; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.index.seqno.SequenceNumbers; +import org.opensearch.search.SearchHit; +import org.opensearch.search.SearchHits; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.search.sort.SortOrder; +import org.opensearch.securityanalytics.threatIntel.iocscan.dto.IocScanContext; +import org.opensearch.securityanalytics.threatIntel.iocscan.service.SaIoCScanService; +import org.opensearch.securityanalytics.threatIntel.iocscan.service.ThreatIntelMonitorRunner; +import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigService; +import org.opensearch.tasks.Task; +import org.opensearch.transport.TransportService; + +import java.io.IOException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.BiConsumer; + +import static org.opensearch.securityanalytics.threatIntel.util.ThreatIntelMonitorUtils.getThreatIntelInputFromBytesReference; + +public class TransportThreatIntelMonitorFanOutAction extends HandledTransportAction { + private static final Logger log = LogManager.getLogger(TransportThreatIntelMonitorFanOutAction.class); + private final ClusterService clusterService; + + private final Settings settings; + private final SATIFSourceConfigService saTifSourceConfigService; + + private final Client client; + + private final NamedXContentRegistry xContentRegistry; + private final SaIoCScanService saIoCScanService; + + @Inject + public TransportThreatIntelMonitorFanOutAction( + TransportService transportService, + Client client, + NamedXContentRegistry xContentRegistry, + ClusterService clusterService, + Settings settings, + ActionFilters actionFilters, + SATIFSourceConfigService saTifSourceConfigService, + SaIoCScanService saIoCScanService + ) { + super(ThreatIntelMonitorRunner.FAN_OUT_ACTION_NAME, transportService, actionFilters, DocLevelMonitorFanOutRequest::new); + this.clusterService = clusterService; + this.client = client; + this.xContentRegistry = xContentRegistry; + this.settings = settings; + this.saTifSourceConfigService = saTifSourceConfigService; + this.saIoCScanService = saIoCScanService; + } + + @Override + protected void doExecute(Task task, DocLevelMonitorFanOutRequest request, ActionListener actionListener) { + try { + Monitor monitor = request.getMonitor(); + MonitorRunResult monitorResult = new MonitorRunResult<>( + monitor.getName(), + Instant.now(), + Instant.now(), + null, + new InputRunResults(Collections.emptyList(), null, null), + Collections.emptyMap() + ); + + // fetch list of threat intel data containing indices per indicator type + + saTifSourceConfigService.getIocTypeToIndices(ActionListener.wrap( + iocTypeToIndicesMap -> { + onGetIocTypeToIndices(iocTypeToIndicesMap, request, actionListener); + }, e -> { + log.error(() -> new ParameterizedMessage("Unexpected Failure in threat intel monitor {} fan out action", request.getMonitor().getId()), e); + actionListener.onResponse( + new DocLevelMonitorFanOutResponse( + clusterService.localNode().getId(), + request.getExecutionId(), + request.getMonitor().getId(), + request.getMonitorMetadata().getLastRunContext(), + new InputRunResults(Collections.emptyList(), null, null), + Collections.emptyMap(),//TODO trigger results, + new AlertingException("Fan action of threat intel monitor failed", RestStatus.INTERNAL_SERVER_ERROR, e) + ) + ); + } + )); + + } catch (Exception ex) { + log.error(() -> new ParameterizedMessage("Unexpected Failure in threat intel monitor {} fan out action", request.getMonitor().getId()), ex); + actionListener.onFailure(ex); + } + } + + private void onGetIocTypeToIndices(Map> iocTypeToIndicesMap, DocLevelMonitorFanOutRequest request, ActionListener actionListener) throws IOException { + RemoteDocLevelMonitorInput remoteDocLevelMonitorInput = (RemoteDocLevelMonitorInput) request.getMonitor().getInputs().get(0); + List indices = remoteDocLevelMonitorInput.getDocLevelMonitorInput().getIndices(); + ThreatIntelInput threatIntelInput = getThreatIntelInputFromBytesReference(remoteDocLevelMonitorInput.getInput(), xContentRegistry); + // TODO update fanout request to add mapping of monitor.input.indices' index to concrete index name. + // right now we can't say which one of aliases/index pattern has resolved to this concrete index name + // + // Map> fieldsToFetchPerIndex = new HashMap<>(); alias -> fields mapping is given but we have concrete index name + List fieldsToFetch = new ArrayList<>(); + threatIntelInput.getPerIocTypeScanInputList().forEach(perIocTypeScanInput -> { + perIocTypeScanInput.getIndexToFieldsMap().values().forEach(fieldsToFetch::addAll); +// Map> indexToFieldsMapPerInput = perIocTypeScanInput.getIndexToFieldsMap(); +// for (String index : indexToFieldsMapPerInput.keySet()) { +// List strings = fieldsToFetchPerIndex.computeIfAbsent( +// perIocTypeScanInput.getIocType(), +// k -> new ArrayList<>()); +// strings.addAll(indexToFieldsMapPerInput.get(index)); +// } + }); + + // function passed to update last run context with new max sequence number +// Map updatedLastRunContext = request.getIndexExecutionContext().getUpdatedLastRunContext(); + Map updatedLastRunContext = request.getMonitorMetadata().getLastRunContext(); + BiConsumer lastRunContextUpdateConsumer = (shardId, value) -> { + String indexName = shardId.getIndexName(); + if (updatedLastRunContext.containsKey(indexName)) { + HashMap context = (HashMap) updatedLastRunContext.putIfAbsent(indexName, new HashMap()); + context.put(String.valueOf(shardId.getId()), value); + } else { + log.error("monitor metadata for threat intel monitor {} expected to contain last run context for index {}", + request.getMonitor().getId(), indexName); + } + }; + ActionListener> searchHitsListener = ActionListener.wrap( + (List hits) -> { + BiConsumer resultConsumer = (r, e) -> { + if (e == null) { + actionListener.onResponse( + new DocLevelMonitorFanOutResponse( + clusterService.localNode().getId(), + request.getExecutionId(), + request.getMonitor().getId(), + updatedLastRunContext, + new InputRunResults(Collections.emptyList(), null, null), + Collections.emptyMap(),//TODO trigger results, + null + ) + ); + } else { + actionListener.onFailure(e); + } + }; + saIoCScanService.scanIoCs(new IocScanContext<>( + request.getMonitor(), + request.getMonitorMetadata(), + false, + hits, + threatIntelInput, + indices, + iocTypeToIndicesMap + ), resultConsumer); + }, + e -> { + log.error("unexpected error while", e); + actionListener.onFailure(e); + } + ); + fetchDataFromShards(request, + fieldsToFetch, + lastRunContextUpdateConsumer, + searchHitsListener); + } + + private void fetchDataFromShards( + DocLevelMonitorFanOutRequest request, + List fieldsToFetch, + BiConsumer updateLastRunContext, + ActionListener> searchHitsListener) { + if (request.getShardIds().isEmpty()) + return; + GroupedActionListener searchHitsFromAllShardsListener = new GroupedActionListener<>( + ActionListener.wrap( + searchHitsOrExceptionCollection -> { + List hits = new ArrayList<>(); + for (SearchHitsOrException searchHitsOrException : searchHitsOrExceptionCollection) { + if (searchHitsOrException.exception == null) { + hits.addAll(searchHitsOrException.hits); + } // else not logging exception as groupedListener onResponse() will log error message + } + searchHitsListener.onResponse(hits); + }, e -> { + log.error("unexpected failure while fetch documents for threat intel monitor " + request.getMonitor().getId(), e); + searchHitsListener.onResponse(Collections.emptyList()); + } + ), request.getShardIds().size() + ); + for (ShardId shardId : request.getShardIds()) { + int shard = shardId.getId(); + + Map lastRunContext = request.getMonitorMetadata().getLastRunContext(); + Long prevSeqNo = lastRunContext.get(shard) != null ? Long.parseLong(lastRunContext.get(shard).toString()) : null; + long fromSeqNo = prevSeqNo != null ? prevSeqNo : SequenceNumbers.NO_OPS_PERFORMED; + long toSeqNo = Long.MAX_VALUE; + fetchLatestDocsFromShard(shardId, fromSeqNo, toSeqNo, new ArrayList<>(), request.getMonitor(), lastRunContext, updateLastRunContext, fieldsToFetch, searchHitsFromAllShardsListener); + } + } + + /** + * recursive function to keep fetching docs in batches of 10000 per search request. all docs with seq_no greater than + * the last seen seq_no are fetched in descending order of sequence number. + */ + + private void fetchLatestDocsFromShard( + ShardId shardId, + long fromSeqNo, long toSeqNo, List searchHitsSoFar, Monitor monitor, + Map lastRunContext, + BiConsumer updateLastRunContext, + List fieldsToFetch, + GroupedActionListener listener) { + + String shard = shardId.getId() + ""; + try { + if (toSeqNo < fromSeqNo || toSeqNo < 0) { + listener.onResponse(new SearchHitsOrException(searchHitsSoFar, null)); + return; + } + Long prevSeqNo = lastRunContext.get(shard) != null ? Long.parseLong(lastRunContext.get(shard).toString()) : null; + if (toSeqNo >= fromSeqNo) { + + searchShard( + shardId.getIndexName(), + shard, + fromSeqNo, + toSeqNo, + Collections.emptyList(), + fieldsToFetch, + ActionListener.wrap( + hits -> { + if (hits.getHits().length == 0) { + if (toSeqNo == Long.MAX_VALUE) { // didn't find any docs + updateLastRunContext.accept(shardId, prevSeqNo != null ? prevSeqNo.toString() : SequenceNumbers.NO_OPS_PERFORMED + ""); + } + listener.onResponse(new SearchHitsOrException(searchHitsSoFar, null)); + return; + } + searchHitsSoFar.addAll(Arrays.asList(hits.getHits())); + if (toSeqNo == Long.MAX_VALUE) { // max sequence number of shard needs to be computed + updateLastRunContext.accept(shardId, String.valueOf(hits.getHits()[0].getSeqNo())); + } + + long leastSeqNoFromHits = hits.getHits()[hits.getHits().length - 1].getSeqNo(); + long updateToSeqNo = leastSeqNoFromHits - 1; + // recursive call to fetch docs with updated seq no. + fetchLatestDocsFromShard(shardId, fromSeqNo, updateToSeqNo, searchHitsSoFar, monitor, lastRunContext, updateLastRunContext, fieldsToFetch, listener); + }, e -> { + log.error(() -> new ParameterizedMessage("Threat intel Monitor {}: Failed to search shard {} in index {}", monitor.getId(), shard, shardId.getIndexName()), e); + listener.onResponse(new SearchHitsOrException(searchHitsSoFar, e)); + } + ) + ); + } + } catch (Exception e) { + log.error(() -> new ParameterizedMessage("threat intel Monitor {}: Failed to run fetch data from shard [{}] of index [{}]", + monitor.getId(), shardId, shardId.getIndexName()), e); + listener.onResponse(new SearchHitsOrException(searchHitsSoFar, e)); + } + } + + public void searchShard( + String index, + String shard, + Long prevSeqNo, + long maxSeqNo, + List docIds, + List fieldsToFetch, + ActionListener listener) { + + if (prevSeqNo != null && prevSeqNo.equals(maxSeqNo) && maxSeqNo != 0L) { + log.debug("Sequence number unchanged."); + listener.onResponse(SearchHits.empty()); + } + + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery() + .filter(QueryBuilders.rangeQuery("_seq_no").gt(prevSeqNo).lte(maxSeqNo)); + + if (docIds != null && !docIds.isEmpty()) { + boolQueryBuilder.filter(QueryBuilders.termsQuery("_id", docIds)); + } + + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder() + .version(true) + .sort("_seq_no", SortOrder.DESC) + .seqNoAndPrimaryTerm(true) + .query(boolQueryBuilder) + .size(10000); + + if (!fieldsToFetch.isEmpty()) { + searchSourceBuilder.fetchSource(false); + for (String field : fieldsToFetch) { + searchSourceBuilder.fetchField(field); + } + } + + SearchRequest request = new SearchRequest() + .indices(index) + .preference("_shards:" + shard) + .source(searchSourceBuilder); + + client.search(request, ActionListener.wrap( + response -> { + if (response.status() != RestStatus.OK) { + log.error("Fetching docs from shard failed"); + throw new IOException("Failed to search shard: [" + shard + "] in index [" + index + "]. Response status is " + response.status()); + } + listener.onResponse(response.getHits()); + }, + listener::onFailure // exception logged in invoker method + )); + + } + + public static class SearchHitsOrException { + private final List hits; + private final Exception exception; + + public SearchHitsOrException(List hits, Exception exception) { + assert hits == null || hits.isEmpty() || exception == null; // just a verification that only one of the variables is non-null + this.hits = hits; + this.exception = exception; + } + + public List getHits() { + return hits; + } + + public Exception getException() { + return exception; + } + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/monitor/RestGetThreatIntelAlertsAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/monitor/RestGetThreatIntelAlertsAction.java new file mode 100644 index 000000000..1cc5266d9 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/monitor/RestGetThreatIntelAlertsAction.java @@ -0,0 +1,89 @@ +package org.opensearch.securityanalytics.threatIntel.resthandler.monitor; + +import java.io.IOException; +import java.time.DateTimeException; +import java.time.Instant; +import java.util.List; + +import org.opensearch.client.node.NodeClient; +import org.opensearch.commons.alerting.model.Table; +import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.action.RestToXContentListener; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.threatIntel.action.monitor.GetThreatIntelAlertsAction; +import org.opensearch.securityanalytics.threatIntel.action.monitor.request.GetThreatIntelAlertsRequest; + + +import static java.util.Collections.singletonList; +import static org.opensearch.rest.RestRequest.Method.GET; + +public class RestGetThreatIntelAlertsAction extends BaseRestHandler { + + @Override + public String getName() { + return "get_threat_intel_alerts_action"; + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + + String severityLevel = request.param("severityLevel", "ALL"); + String alertState = request.param("alertState", "ALL"); + // Table params + String sortString = request.param("sortString", "start_time"); + String sortOrder = request.param("sortOrder", "asc"); + String missing = request.param("missing"); + int size = request.paramAsInt("size", 20); + int startIndex = request.paramAsInt("startIndex", 0); + String searchString = request.param("searchString", ""); + + Instant startTime = null; + String startTimeParam = request.param("startTime"); + if (startTimeParam != null && !startTimeParam.isEmpty()) { + try { + startTime = Instant.ofEpochMilli(Long.parseLong(startTimeParam)); + } catch (NumberFormatException | NullPointerException | DateTimeException e) { + startTime = Instant.now(); + } + } + + Instant endTime = null; + String endTimeParam = request.param("endTime"); + if (endTimeParam != null && !endTimeParam.isEmpty()) { + try { + endTime = Instant.ofEpochMilli(Long.parseLong(endTimeParam)); + } catch (NumberFormatException | NullPointerException | DateTimeException e) { + endTime = Instant.now(); + } + } + + Table table = new Table( + sortOrder, + sortString, + missing, + size, + startIndex, + searchString + ); + + GetThreatIntelAlertsRequest req = new GetThreatIntelAlertsRequest( + table, + severityLevel, + alertState, + startTime, + endTime + ); + + return channel -> client.execute( + GetThreatIntelAlertsAction.INSTANCE, + req, + new RestToXContentListener<>(channel) + ); + } + + @Override + public List routes() { + return singletonList(new Route(GET, SecurityAnalyticsPlugin.THREAT_INTEL_ALERTS_URI)); + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/monitor/RestSearchThreatIntelMonitorAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/monitor/RestSearchThreatIntelMonitorAction.java index 191af8814..047a4f38b 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/monitor/RestSearchThreatIntelMonitorAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/monitor/RestSearchThreatIntelMonitorAction.java @@ -20,6 +20,7 @@ import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; import org.opensearch.securityanalytics.threatIntel.action.monitor.SearchThreatIntelMonitorAction; import org.opensearch.securityanalytics.threatIntel.action.monitor.request.SearchThreatIntelMonitorRequest; +import org.opensearch.securityanalytics.threatIntel.iocscan.service.ThreatIntelMonitorRunner; import java.io.IOException; import java.util.List; @@ -67,8 +68,8 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli } BoolQueryBuilder bqb = new BoolQueryBuilder(); - bqb.should().add(new BoolQueryBuilder().must(QueryBuilders.matchQuery("monitor.owner", PLUGIN_OWNER_FIELD))); - + bqb.must().add(new BoolQueryBuilder().must(QueryBuilders.matchQuery("monitor.owner", PLUGIN_OWNER_FIELD))); + bqb.must().add(new BoolQueryBuilder().must(QueryBuilders.matchQuery("monitor.monitor_type", ThreatIntelMonitorRunner.THREAT_INTEL_MONITOR_TYPE))); boolQueryBuilder.filter(bqb); searchRequest.source().query(boolQueryBuilder); diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/ThreatIntelAlertDto.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/ThreatIntelAlertDto.java new file mode 100644 index 000000000..4bfc8e502 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/ThreatIntelAlertDto.java @@ -0,0 +1,417 @@ +package org.opensearch.securityanalytics.threatIntel.sacommons.monitor; + +import org.opensearch.commons.alerting.model.Alert; +import org.opensearch.commons.authuser.User; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.index.seqno.SequenceNumbers; +import org.opensearch.securityanalytics.model.threatintel.BaseEntity; +import org.opensearch.securityanalytics.model.threatintel.ThreatIntelAlert; +import org.opensearch.securityanalytics.util.XContentUtils; + +import java.io.IOException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.securityanalytics.util.XContentUtils.getInstant; + +public class ThreatIntelAlertDto extends BaseEntity { + + public static final String ALERT_ID_FIELD = "id"; + public static final String SCHEMA_VERSION_FIELD = "schema_version"; + public static final String SEQ_NO_FIELD = "seq_no"; + public static final String PRIMARY_TERM_FIELD = "primary_term"; + public static final String ALERT_VERSION_FIELD = "version"; + public static final String USER_FIELD = "user"; + public static final String TRIGGER_NAME_FIELD = "trigger_id"; + public static final String TRIGGER_ID_FIELD = "trigger_name"; + public static final String STATE_FIELD = "state"; + public static final String START_TIME_FIELD = "start_time"; + public static final String END_TIME_FIELD = "end_time"; + public static final String LAST_UPDATED_TIME_FIELD = "last_updated_time"; + public static final String ACKNOWLEDGED_TIME_FIELD = "acknowledged_time"; + public static final String ERROR_MESSAGE_FIELD = "error_message"; + public static final String SEVERITY_FIELD = "severity"; + public static final String ACTION_EXECUTION_RESULTS_FIELD = "action_execution_results"; + public static final String IOC_VALUE_FIELD = "ioc_value"; + public static final String IOC_TYPE_FIELD = "ioc_type"; + public static final String FINDING_IDS_FIELD = "finding_ids"; + public static final String NO_ID = ""; + public static final long NO_VERSION = 1L; + public static final long NO_SCHEMA_VERSION = 0; + + private final String id; + private final long version; + private final long schemaVersion; + private final long seqNo; + private final long primaryTerm; + private final User user; + private final String triggerName; + private final String triggerId; + private final Alert.State state; + private final Instant startTime; + private final Instant endTime; + private final Instant acknowledgedTime; + private final Instant lastUpdatedTime; + private final String errorMessage; + private final String severity; + private final String iocValue; + private final String iocType; + private List findingIds; + + public ThreatIntelAlertDto( + String id, + long version, + long schemaVersion, + long seqNo, + long primaryTerm, + User user, + String triggerId, + String triggerName, + Alert.State state, + Instant startTime, + Instant endTime, + Instant lastUpdatedTime, + Instant acknowledgedTime, + String errorMessage, + String severity, + String iocValue, + String iocType, + List findingIds + ) { + + this.id = id != null ? id : NO_ID; + this.version = version != 0 ? version : NO_VERSION; + this.schemaVersion = schemaVersion; + this.seqNo = seqNo; + this.primaryTerm = primaryTerm; + this.user = user; + this.triggerId = triggerId; + this.triggerName = triggerName; + this.state = state; + this.startTime = startTime; + this.endTime = endTime; + this.acknowledgedTime = acknowledgedTime; + this.errorMessage = errorMessage; + this.severity = severity; + this.iocValue = iocValue; + this.iocType = iocType; + this.lastUpdatedTime = lastUpdatedTime; + this.findingIds = findingIds; + } + + public ThreatIntelAlertDto(StreamInput sin) throws IOException { + this.id = sin.readString(); + this.version = sin.readLong(); + this.schemaVersion = sin.readLong(); + this.seqNo = sin.readLong(); + this.primaryTerm = sin.readLong(); + this.user = sin.readBoolean() ? new User(sin) : null; + this.triggerId = sin.readString(); + this.triggerName = sin.readString(); + this.state = sin.readEnum(Alert.State.class); + this.startTime = sin.readInstant(); + this.endTime = sin.readOptionalInstant(); + this.acknowledgedTime = sin.readOptionalInstant(); + this.errorMessage = sin.readOptionalString(); + this.severity = sin.readString(); + this.lastUpdatedTime = sin.readOptionalInstant(); + this.iocType = sin.readString(); + this.iocValue = sin.readString(); + this.findingIds = sin.readStringList(); + } + + public ThreatIntelAlertDto(ThreatIntelAlert alert, long seqNo, long primaryTerm) { + this.id = alert.getId(); + this.version = alert.getVersion(); + this.schemaVersion = alert.getSchemaVersion(); + this.user = alert.getUser(); + this.triggerId = alert.getTriggerId(); + this.triggerName = alert.getTriggerName(); + this.state = alert.getState(); + this.startTime = alert.getStartTime(); + this.endTime = alert.getEndTime(); + this.acknowledgedTime = alert.getAcknowledgedTime(); + this.errorMessage = alert.getErrorMessage(); + this.severity = alert.getSeverity(); + this.iocValue = alert.getIocValue(); + this.iocType = alert.getIocType(); + this.lastUpdatedTime = alert.getLastUpdatedTime(); + this.findingIds = alert.getFindingIds(); + this.seqNo = seqNo; + this.primaryTerm = primaryTerm; + } + + public boolean isAcknowledged() { + return state == Alert.State.ACKNOWLEDGED; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(id); + out.writeLong(version); + out.writeLong(schemaVersion); + out.writeLong(seqNo); + out.writeLong(primaryTerm); + out.writeBoolean(user != null); + if (user != null) { + user.writeTo(out); + } + out.writeString(triggerId); + out.writeString(triggerName); + out.writeEnum(state); + out.writeInstant(startTime); + out.writeOptionalInstant(endTime); + out.writeOptionalInstant(acknowledgedTime); + out.writeOptionalString(errorMessage); + out.writeString(severity); + out.writeOptionalInstant(lastUpdatedTime); + out.writeString(iocType); + out.writeString(iocValue); + out.writeStringCollection(findingIds); + } + + public static ThreatIntelAlertDto parse(XContentParser xcp, long version) throws IOException { + String id = NO_ID; + long schemaVersion = NO_SCHEMA_VERSION; + long seqNo = SequenceNumbers.UNASSIGNED_SEQ_NO; + long primaryTerm = SequenceNumbers.UNASSIGNED_PRIMARY_TERM; + User user = null; + String triggerId = null; + String triggerName = null; + Alert.State state = null; + Instant startTime = null; + String severity = null; + Instant endTime = null; + Instant acknowledgedTime = null; + Instant lastUpdatedTime = null; + String errorMessage = null; + String iocValue = null; + String iocType = null; + List findingIds = new ArrayList<>(); + + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = xcp.currentName(); + xcp.nextToken(); + switch (fieldName) { + case USER_FIELD: + user = xcp.currentToken() == XContentParser.Token.VALUE_NULL ? null : User.parse(xcp); + break; + case ALERT_ID_FIELD: + id = xcp.text(); + break; + case IOC_VALUE_FIELD: + iocValue = xcp.textOrNull(); + break; + case IOC_TYPE_FIELD: + iocType = xcp.textOrNull(); + break; + case ALERT_VERSION_FIELD: + version = xcp.longValue(); + break; + case SCHEMA_VERSION_FIELD: + schemaVersion = xcp.intValue(); + break; + case SEQ_NO_FIELD: + seqNo = xcp.longValue(); + break; + case PRIMARY_TERM_FIELD: + primaryTerm = xcp.longValue(); + break; + case TRIGGER_ID_FIELD: + triggerId = xcp.text(); + break; + case TRIGGER_NAME_FIELD: + triggerName = xcp.text(); + break; + case STATE_FIELD: + state = Alert.State.valueOf(xcp.text()); + break; + case ERROR_MESSAGE_FIELD: + errorMessage = xcp.textOrNull(); + break; + case SEVERITY_FIELD: + severity = xcp.text(); + break; + case START_TIME_FIELD: + startTime = getInstant(xcp); + break; + case END_TIME_FIELD: + endTime = getInstant(xcp); + break; + case ACKNOWLEDGED_TIME_FIELD: + acknowledgedTime = getInstant(xcp); + break; + case LAST_UPDATED_TIME_FIELD: + lastUpdatedTime = getInstant(xcp); + break; + case FINDING_IDS_FIELD: + ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { + findingIds.add(xcp.text()); + } + default: + xcp.skipChildren(); + } + } + + return new ThreatIntelAlertDto(id, + version, + schemaVersion, + seqNo, + primaryTerm, + user, + triggerId, + triggerName, + state, + startTime, + endTime, + acknowledgedTime, + lastUpdatedTime, + errorMessage, + severity, + iocValue, + iocType, + findingIds); + } + + public static Alert readFrom(StreamInput sin) throws IOException { + return new Alert(sin); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return createXContentBuilder(builder, true); + } + + @Override + public String getId() { + return id; + } + + public XContentBuilder toXContentWithUser(XContentBuilder builder) throws IOException { + return createXContentBuilder(builder, false); + } + + private XContentBuilder createXContentBuilder(XContentBuilder builder, boolean secure) throws IOException { + builder.startObject() + .field(ALERT_ID_FIELD, id) + .field(ALERT_VERSION_FIELD, version) + .field(SCHEMA_VERSION_FIELD, schemaVersion) + .field(SEQ_NO_FIELD, seqNo) + .field(PRIMARY_TERM_FIELD, primaryTerm) + .field(TRIGGER_NAME_FIELD, triggerName) + .field(TRIGGER_ID_FIELD, triggerName) + .field(STATE_FIELD, state) + .field(ERROR_MESSAGE_FIELD, errorMessage) + .field(IOC_VALUE_FIELD, iocValue) + .field(IOC_TYPE_FIELD, iocType) + .field(SEVERITY_FIELD, severity) + .field(FINDING_IDS_FIELD, findingIds.toArray(new String[0])); + XContentUtils.buildInstantAsField(builder, acknowledgedTime, ACKNOWLEDGED_TIME_FIELD); + XContentUtils.buildInstantAsField(builder, lastUpdatedTime, LAST_UPDATED_TIME_FIELD); + XContentUtils.buildInstantAsField(builder, startTime, START_TIME_FIELD); + XContentUtils.buildInstantAsField(builder, endTime, END_TIME_FIELD); + if (!secure) { + if (user == null) { + builder.nullField(USER_FIELD); + } else { + builder.field(USER_FIELD, user); + } + } + return builder.endObject(); + } + + public Map asTemplateArg() { + Map map = new HashMap<>(); + map.put(ACKNOWLEDGED_TIME_FIELD, acknowledgedTime != null ? acknowledgedTime.toEpochMilli() : null); + map.put(ALERT_ID_FIELD, id); + map.put(ALERT_VERSION_FIELD, version); + map.put(END_TIME_FIELD, endTime != null ? endTime.toEpochMilli() : null); + map.put(ERROR_MESSAGE_FIELD, errorMessage); + map.put(SEVERITY_FIELD, severity); + map.put(START_TIME_FIELD, startTime.toEpochMilli()); + map.put(STATE_FIELD, state.toString()); + map.put(TRIGGER_ID_FIELD, triggerId); + map.put(TRIGGER_NAME_FIELD, triggerName); + map.put(FINDING_IDS_FIELD, findingIds); + map.put(LAST_UPDATED_TIME_FIELD, lastUpdatedTime); + map.put(IOC_TYPE_FIELD, iocType); + map.put(IOC_VALUE_FIELD, iocValue); + return map; + } + + public long getVersion() { + return version; + } + + public long getSchemaVersion() { + return schemaVersion; + } + + public User getUser() { + return user; + } + + public String getTriggerName() { + return triggerName; + } + + public Alert.State getState() { + return state; + } + + public Instant getStartTime() { + return startTime; + } + + public Instant getEndTime() { + return endTime; + } + + public Instant getAcknowledgedTime() { + return acknowledgedTime; + } + + public String getErrorMessage() { + return errorMessage; + } + + public String getSeverity() { + return severity; + } + + public String getTriggerId() { + return triggerId; + } + + public Instant getLastUpdatedTime() { + return lastUpdatedTime; + } + + public String getIocValue() { + return iocValue; + } + + public String getIocType() { + return iocType; + } + + public List getFindingIds() { + return findingIds; + } + + public long getSeqNo() { + return seqNo; + } + + public long getPrimaryTerm() { + return primaryTerm; + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/ThreatIntelMonitorActions.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/ThreatIntelMonitorActions.java index d90884000..dad6054cc 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/ThreatIntelMonitorActions.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/ThreatIntelMonitorActions.java @@ -4,4 +4,5 @@ public class ThreatIntelMonitorActions { public static final String INDEX_THREAT_INTEL_MONITOR_ACTION_NAME = "cluster:admin/security_analytics/threatIntel/monitor/write"; public static final String SEARCH_THREAT_INTEL_MONITOR_ACTION_NAME = "cluster:admin/security_analytics/threatIntel/monitor/search"; public static final String DELETE_THREAT_INTEL_MONITOR_ACTION_NAME = "cluster:admin/security_analytics/threatIntel/monitor/delete"; + public static final String GET_THREAT_INTEL_ALERTS_ACTION_NAME = "cluster:admin/security_analytics/threatIntel/alerts/get"; } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/ThreatIntelMonitorDto.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/ThreatIntelMonitorDto.java index c4a0b9ed4..b856b4dc4 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/ThreatIntelMonitorDto.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/ThreatIntelMonitorDto.java @@ -22,6 +22,7 @@ import java.util.Map; import java.util.Set; import java.util.UUID; +import java.util.stream.Collectors; public class ThreatIntelMonitorDto implements Writeable, ToXContentObject, ThreatIntelMonitorDtoInterface { diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java index a6bb8b468..ff8cbf570 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java @@ -50,6 +50,7 @@ import org.opensearch.securityanalytics.threatIntel.action.monitor.request.SearchThreatIntelMonitorRequest; import org.opensearch.securityanalytics.threatIntel.common.StashedThreadContext; import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; +import org.opensearch.securityanalytics.threatIntel.model.DefaultIocStoreConfig; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; import org.opensearch.securityanalytics.util.SecurityAnalyticsException; import org.opensearch.threadpool.ThreadPool; @@ -59,11 +60,18 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.stream.Collectors; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.INDEX_TIMEOUT; +import static org.opensearch.securityanalytics.threatIntel.common.TIFJobState.AVAILABLE; +import static org.opensearch.securityanalytics.threatIntel.common.TIFJobState.REFRESHING; +import static org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig.SOURCE_CONFIG_FIELD; +import static org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig.STATE_FIELD; import static org.opensearch.securityanalytics.transport.TransportIndexDetectorAction.PLUGIN_OWNER_FIELD; /** @@ -252,7 +260,7 @@ public void searchTIFSourceConfigs( } // convert search hits to threat intel source configs - for (SearchHit hit: searchResponse.getHits()) { + for (SearchHit hit : searchResponse.getHits()) { XContentParser xcp = XContentType.JSON.xContent().createParser( xContentRegistry, LoggingDeprecationHandler.INSTANCE, hit.getSourceAsString() @@ -394,8 +402,6 @@ public void getClusterState( ); } - - public void checkAndEnsureThreatIntelMonitorsDeleted( ActionListener listener ) { @@ -463,4 +469,41 @@ public void checkAndEnsureThreatIntelMonitorsDeleted( } + public void getIocTypeToIndices(ActionListener>> listener) { + SearchRequest searchRequest = new SearchRequest(SecurityAnalyticsPlugin.JOB_INDEX_NAME); + + String stateFieldName = String.format("%s.%s", SOURCE_CONFIG_FIELD, STATE_FIELD); + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery() + .should(QueryBuilders.matchQuery(stateFieldName, AVAILABLE.toString())); + queryBuilder.should(QueryBuilders.matchQuery(stateFieldName, REFRESHING)); + + searchRequest.source().query(queryBuilder); + client.search(searchRequest, ActionListener.wrap( + searchResponse -> { + Map> cumulativeIocTypeToIndices = new HashMap<>(); + for (SearchHit hit : searchResponse.getHits().getHits()) { + XContentParser xcp = XContentType.JSON.xContent().createParser( + xContentRegistry, + LoggingDeprecationHandler.INSTANCE, hit.getSourceAsString() + ); + SATIFSourceConfig config = SATIFSourceConfig.docParse(xcp, hit.getId(), hit.getVersion()); + if (config.getIocStoreConfig() instanceof DefaultIocStoreConfig) { + DefaultIocStoreConfig iocStoreConfig = (DefaultIocStoreConfig) config.getIocStoreConfig(); + Map> iocTypeToIndices = iocStoreConfig.getIocMapStore(); + for (String iocType : iocTypeToIndices.keySet()) { + if (iocTypeToIndices.get(iocType).isEmpty()) + continue; + List strings = cumulativeIocTypeToIndices.computeIfAbsent(iocType, k -> new ArrayList<>()); + strings.addAll(iocTypeToIndices.get(iocType)); + } + } + } + listener.onResponse(cumulativeIocTypeToIndices); + }, + e -> { + log.error("Failed to fetch ioc indices", e); + listener.onFailure(e); + } + )); + } } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportGetIocFindingsAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportGetIocFindingsAction.java index 2c4792650..213a43a44 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportGetIocFindingsAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportGetIocFindingsAction.java @@ -6,27 +6,35 @@ import org.apache.lucene.search.join.ScoreMode; import org.opensearch.OpenSearchStatusException; +import org.opensearch.action.search.SearchResponse; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.HandledTransportAction; import org.opensearch.client.Client; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; import org.opensearch.common.settings.Settings; +import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.common.xcontent.XContentType; import org.opensearch.commons.alerting.model.Table; import org.opensearch.commons.authuser.User; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.Strings; import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; +import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.query.BoolQueryBuilder; import org.opensearch.index.query.Operator; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; +import org.opensearch.search.SearchHit; import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.search.fetch.subphase.FetchSourceContext; import org.opensearch.search.sort.FieldSortBuilder; import org.opensearch.search.sort.SortBuilders; import org.opensearch.search.sort.SortOrder; +import org.opensearch.securityanalytics.model.threatintel.IocFinding; import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; import org.opensearch.securityanalytics.threatIntel.action.GetIocFindingsAction; import org.opensearch.securityanalytics.threatIntel.action.GetIocFindingsRequest; @@ -38,6 +46,7 @@ import org.opensearch.transport.TransportService; import java.time.Instant; +import java.util.ArrayList; import java.util.List; public class TransportGetIocFindingsAction extends HandledTransportAction implements SecureTransportAction { @@ -48,6 +57,7 @@ public class TransportGetIocFindingsAction extends HandledTransportAction() { + @Override + public void onResponse(SearchResponse searchResponse) { + try { + long totalIocFindingsCount = searchResponse.getHits().getTotalHits().value; + List iocFindings = new ArrayList<>(); + + for (SearchHit hit : searchResponse.getHits()) { + XContentParser xcp = XContentType.JSON.xContent() + .createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, hit.getSourceAsString()); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp); + IocFinding iocFinding = IocFinding.parse(xcp); + iocFindings.add(iocFinding); + } + actionListener.onResponse(new GetIocFindingsResponse((int) totalIocFindingsCount, iocFindings)); + } catch (Exception ex) { + this.onFailure(ex); + } + } + + @Override + public void onFailure(Exception e) { + if (e instanceof IndexNotFoundException) { + actionListener.onResponse(new GetIocFindingsResponse(0, List.of())); + return; + } + actionListener.onFailure(e); + } + }); } private void setFilterByEnabled(boolean filterByEnabled) { diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportGetThreatIntelAlertsAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportGetThreatIntelAlertsAction.java new file mode 100644 index 000000000..71fb4a71f --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportGetThreatIntelAlertsAction.java @@ -0,0 +1,185 @@ +package org.opensearch.securityanalytics.threatIntel.transport.monitor; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.OpenSearchStatusException; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.client.Client; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.inject.Inject; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.commons.alerting.model.Table; +import org.opensearch.commons.authuser.User; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.index.IndexNotFoundException; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.search.SearchHit; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.search.sort.FieldSortBuilder; +import org.opensearch.search.sort.SortBuilders; +import org.opensearch.search.sort.SortOrder; +import org.opensearch.securityanalytics.model.threatintel.ThreatIntelAlert; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.securityanalytics.threatIntel.action.monitor.GetThreatIntelAlertsAction; +import org.opensearch.securityanalytics.threatIntel.action.monitor.request.GetThreatIntelAlertsRequest; +import org.opensearch.securityanalytics.threatIntel.action.monitor.request.SearchThreatIntelMonitorRequest; +import org.opensearch.securityanalytics.threatIntel.action.monitor.response.GetThreatIntelAlertsResponse; +import org.opensearch.securityanalytics.threatIntel.iocscan.dao.ThreatIntelAlertService; +import org.opensearch.securityanalytics.threatIntel.iocscan.service.ThreatIntelMonitorRunner; +import org.opensearch.securityanalytics.threatIntel.sacommons.monitor.ThreatIntelAlertDto; +import org.opensearch.securityanalytics.transport.SecureTransportAction; +import org.opensearch.tasks.Task; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import static org.opensearch.securityanalytics.transport.TransportIndexDetectorAction.PLUGIN_OWNER_FIELD; + +public class TransportGetThreatIntelAlertsAction extends HandledTransportAction implements SecureTransportAction { + + private final Client client; + private final TransportSearchThreatIntelMonitorAction transportSearchThreatIntelMonitorAction; + + private final NamedXContentRegistry xContentRegistry; + + private final ClusterService clusterService; + + private final Settings settings; + + private final ThreadPool threadPool; + + private final ThreatIntelAlertService alertsService; + + private volatile Boolean filterByEnabled; + + private static final Logger log = LogManager.getLogger(TransportGetThreatIntelAlertsAction.class); + + + @Inject + public TransportGetThreatIntelAlertsAction(TransportService transportService, + ActionFilters actionFilters, + ClusterService clusterService, + ThreadPool threadPool, + Settings settings, + NamedXContentRegistry xContentRegistry, + Client client, + TransportSearchThreatIntelMonitorAction transportSearchThreatIntelMonitorAction1, ThreatIntelAlertService alertsService) { + super(GetThreatIntelAlertsAction.NAME, transportService, actionFilters, GetThreatIntelAlertsRequest::new); + this.client = client; + this.transportSearchThreatIntelMonitorAction = transportSearchThreatIntelMonitorAction1; + this.xContentRegistry = xContentRegistry; + this.clusterService = clusterService; + this.threadPool = threadPool; + this.settings = settings; + this.alertsService = alertsService; + this.filterByEnabled = SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES.get(this.settings); + this.clusterService.getClusterSettings().addSettingsUpdateConsumer(SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES, this::setFilterByEnabled); + } + + private void setFilterByEnabled(boolean filterByEnabled) { + this.filterByEnabled = filterByEnabled; + } + + @Override + protected void doExecute(Task task, GetThreatIntelAlertsRequest request, ActionListener listener) { + User user = readUserFromThreadContext(this.threadPool); + + String validateBackendRoleMessage = validateUserBackendRoles(user, this.filterByEnabled); + if (!"".equals(validateBackendRoleMessage)) { + listener.onFailure(new OpenSearchStatusException("Do not have permissions to resource", RestStatus.FORBIDDEN)); + return; + } + //fetch monitors and search + SearchRequest threatIntelMonitorsSearchRequest = new SearchRequest(); + threatIntelMonitorsSearchRequest.indices(".opendistro-alerting-config"); + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); + boolQueryBuilder.should().add(new BoolQueryBuilder().must(QueryBuilders.matchQuery("monitor.owner", PLUGIN_OWNER_FIELD))); + boolQueryBuilder.should().add(new BoolQueryBuilder().must(QueryBuilders.matchQuery("monitor.monitor_type", ThreatIntelMonitorRunner.THREAT_INTEL_MONITOR_TYPE))); + threatIntelMonitorsSearchRequest.source(new SearchSourceBuilder().query(boolQueryBuilder)); + transportSearchThreatIntelMonitorAction.execute(new SearchThreatIntelMonitorRequest(threatIntelMonitorsSearchRequest), ActionListener.wrap( + searchResponse -> { + List monitorIds = searchResponse.getHits() == null || searchResponse.getHits().getHits() == null ? new ArrayList<>() : + Arrays.stream(searchResponse.getHits().getHits()).map(SearchHit::getId).collect(Collectors.toList()); + if (monitorIds.isEmpty()) { + listener.onResponse(new GetThreatIntelAlertsResponse(Collections.emptyList(), 0)); + return; + } + getAlerts(monitorIds, request, listener); + }, + + e -> { + if (e instanceof IndexNotFoundException) { + log.debug("Monitor index not created. Returning 0 threat intel alerts"); + listener.onResponse(new GetThreatIntelAlertsResponse(Collections.emptyList(), 0)); + return; + } + log.error("Failed to get threat intel monitor alerts", e); + listener.onFailure(e); + } + )); + } + + private void getAlerts(List monitorIds, + GetThreatIntelAlertsRequest request, + ActionListener listener) { + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + BoolQueryBuilder monitorIdMatchQuery = QueryBuilders.boolQuery(); + for (String monitorId : monitorIds) { + monitorIdMatchQuery.should(QueryBuilders.boolQuery() + .must(QueryBuilders.matchQuery("monitor_id", monitorId))); + + } + queryBuilder.filter(monitorIdMatchQuery); + Table tableProp = request.getTable(); + FieldSortBuilder sortBuilder = SortBuilders + .fieldSort(tableProp.getSortString()) + .order(SortOrder.fromString(tableProp.getSortOrder())); + if (tableProp.getMissing() != null && !tableProp.getMissing().isEmpty()) { + sortBuilder.missing(tableProp.getMissing()); + } + + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder() + .version(true) + .seqNoAndPrimaryTerm(true) + .query(queryBuilder) + .sort(sortBuilder) + .size(tableProp.getSize()) + .from(tableProp.getStartIndex()); + alertsService.search(searchSourceBuilder, ActionListener.wrap( + searchResponse -> { + List alerts = new ArrayList<>(); + if (searchResponse.getHits() == null || searchResponse.getHits().getHits() == null || searchResponse.getHits().getHits().length == 0) { + listener.onResponse(new GetThreatIntelAlertsResponse(Collections.emptyList(), 0)); + return; + } + for (SearchHit hit : searchResponse.getHits().getHits()) { + XContentParser xcp = XContentType.JSON.xContent().createParser( + xContentRegistry, + LoggingDeprecationHandler.INSTANCE, hit.getSourceAsString() + ); + if (xcp.currentToken() == null) + xcp.nextToken(); + ThreatIntelAlert alert = ThreatIntelAlert.parse(xcp, hit.getVersion()); + alerts.add(new ThreatIntelAlertDto(alert, hit.getSeqNo(), hit.getPrimaryTerm())); + } + listener.onResponse(new GetThreatIntelAlertsResponse(alerts, (int) searchResponse.getHits().getTotalHits().value)); + }, e -> { + log.error("Failed to search for threat intel alerts", e); + listener.onFailure(e); + } + )); + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportIndexThreatIntelMonitorAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportIndexThreatIntelMonitorAction.java index 7a0cb390f..9b325a828 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportIndexThreatIntelMonitorAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportIndexThreatIntelMonitorAction.java @@ -49,7 +49,7 @@ import java.util.List; import java.util.stream.Collectors; -import static org.opensearch.securityanalytics.threatIntel.model.monitor.SampleRemoteDocLevelMonitorRunner.THREAT_INTEL_MONITOR_TYPE; +import static org.opensearch.securityanalytics.threatIntel.iocscan.service.ThreatIntelMonitorRunner.THREAT_INTEL_MONITOR_TYPE; import static org.opensearch.securityanalytics.transport.TransportIndexDetectorAction.PLUGIN_OWNER_FIELD; public class TransportIndexThreatIntelMonitorAction extends HandledTransportAction implements SecureTransportAction { @@ -109,7 +109,7 @@ protected void doExecute(Task task, IndexThreatIntelMonitorRequest request, Acti } )); } catch (Exception e) { - log.error(() -> new ParameterizedMessage("Unexpected failure while indexing threat intel monitor {} named {}", request.getId(), request.getThreatIntelMonitor().getName())); + log.error(() -> new ParameterizedMessage("Unexpected failure while indexing threat intel monitor {} named {}", request.getId(), request.getMonitor().getName())); listener.onFailure(new SecurityAnalyticsException("Unexpected failure while indexing threat intel monitor", RestStatus.INTERNAL_SERVER_ERROR, e)); } } @@ -136,11 +136,11 @@ private IndexMonitorRequest buildIndexMonitorRequest(IndexThreatIntelMonitorRequ private Monitor buildThreatIntelMonitor(IndexThreatIntelMonitorRequest request) throws IOException { //TODO replace with threat intel monitor DocLevelMonitorInput docLevelMonitorInput = new DocLevelMonitorInput( - String.format("threat intel input for monitor named %s", request.getThreatIntelMonitor().getName()), - request.getThreatIntelMonitor().getIndices(), + String.format("threat intel input for monitor named %s", request.getMonitor().getName()), + request.getMonitor().getIndices(), Collections.emptyList() // no percolate queries ); - List perIocTypeScanInputs = request.getThreatIntelMonitor().getPerIocTypeScanInputList().stream().map( + List perIocTypeScanInputs = request.getMonitor().getPerIocTypeScanInputList().stream().map( it -> new PerIocTypeScanInput(it.getIocType(), it.getIndexToFieldsMap()) ).collect(Collectors.toList()); ThreatIntelInput threatIntelInput = new ThreatIntelInput(perIocTypeScanInputs); @@ -148,7 +148,7 @@ private Monitor buildThreatIntelMonitor(IndexThreatIntelMonitorRequest request) threatIntelInput.getThreatIntelInputAsBytesReference(), docLevelMonitorInput); List triggers = new ArrayList<>(); - for (ThreatIntelTriggerDto it : request.getThreatIntelMonitor().getTriggers()) { + for (ThreatIntelTriggerDto it : request.getMonitor().getTriggers()) { try { RemoteMonitorTrigger trigger = ThreatIntelMonitorUtils.buildRemoteMonitorTrigger(it); triggers.add(trigger); @@ -160,13 +160,13 @@ private Monitor buildThreatIntelMonitor(IndexThreatIntelMonitorRequest request) return new Monitor( request.getMethod() == RestRequest.Method.POST ? Monitor.NO_ID : request.getId(), Monitor.NO_VERSION, - StringUtils.isBlank(request.getThreatIntelMonitor().getName()) ? "threat_intel_monitor" : request.getThreatIntelMonitor().getName(), - request.getThreatIntelMonitor().isEnabled(), - request.getThreatIntelMonitor().getSchedule(), + StringUtils.isBlank(request.getMonitor().getName()) ? "threat_intel_monitor" : request.getMonitor().getName(), + request.getMonitor().isEnabled(), + request.getMonitor().getSchedule(), Instant.now(), - request.getThreatIntelMonitor().isEnabled() ? Instant.now() : null, + request.getMonitor().isEnabled() ? Instant.now() : null, THREAT_INTEL_MONITOR_TYPE, - request.getThreatIntelMonitor().getUser(), + request.getMonitor().getUser(), 1, List.of(remoteDocLevelMonitorInput), triggers, diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/util/ThreatIntelMonitorUtils.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/util/ThreatIntelMonitorUtils.java index 4e149e1fd..0204b8488 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/util/ThreatIntelMonitorUtils.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/util/ThreatIntelMonitorUtils.java @@ -1,5 +1,8 @@ package org.opensearch.securityanalytics.threatIntel.util; +import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.commons.alerting.model.Alert; import org.opensearch.commons.alerting.model.Monitor; import org.opensearch.commons.alerting.model.Trigger; import org.opensearch.commons.alerting.model.remote.monitors.RemoteDocLevelMonitorInput; @@ -7,6 +10,15 @@ import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.index.query.TermQueryBuilder; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.securityanalytics.model.threatintel.IocFinding; +import org.opensearch.securityanalytics.model.threatintel.ThreatIntelAlert; import org.opensearch.securityanalytics.threatIntel.iocscan.dto.PerIocTypeScanInputDto; import org.opensearch.securityanalytics.threatIntel.model.monitor.ThreatIntelInput; import org.opensearch.securityanalytics.threatIntel.model.monitor.ThreatIntelTrigger; @@ -14,13 +26,19 @@ import org.opensearch.securityanalytics.threatIntel.sacommons.monitor.ThreatIntelTriggerDto; import java.io.IOException; +import java.time.Instant; import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.UUID; import java.util.stream.Collectors; import static org.opensearch.securityanalytics.util.XContentUtils.getBytesReference; -public class ThreatIntelMonitorUtils { +public class ThreatIntelMonitorUtils { public static RemoteMonitorTrigger buildRemoteMonitorTrigger(ThreatIntelTriggerDto trigger) throws IOException { return new RemoteMonitorTrigger(trigger.getId(), trigger.getName(), trigger.getSeverity(), trigger.getActions(), getBytesReference(new ThreatIntelTrigger(trigger.getDataSources(), trigger.getIocTypes()))); @@ -41,6 +59,14 @@ public static List buildThreatIntelTriggerDtos(List dataSources = new ArrayList<>(); + List iocTypes = new ArrayList<>(); + triggerDtos.add(new ThreatIntelTriggerDto(dataSources, + iocTypes, + remoteMonitorTrigger.getActions(), + remoteMonitorTrigger.getName(), + remoteMonitorTrigger.getId(), + remoteMonitorTrigger.getSeverity())); } return triggerDtos; } @@ -50,7 +76,7 @@ public static ThreatIntelTrigger getThreatIntelTriggerFromBytesReference(RemoteM return new ThreatIntelTrigger(triggerSin); } - public static ThreatIntelInput getThreatIntelInputFromBytesReference(BytesReference bytes) throws IOException { + public static ThreatIntelInput getThreatIntelInputFromBytesReference(BytesReference bytes, NamedXContentRegistry namedXContentRegistry) throws IOException { StreamInput sin = StreamInput.wrap(bytes.toBytesRef().bytes); ThreatIntelInput threatIntelInput = new ThreatIntelInput(sin); return threatIntelInput; @@ -59,7 +85,7 @@ public static ThreatIntelInput getThreatIntelInputFromBytesReference(BytesRefere public static ThreatIntelMonitorDto buildThreatIntelMonitorDto(String id, Monitor monitor, NamedXContentRegistry namedXContentRegistry) throws IOException { RemoteDocLevelMonitorInput remoteDocLevelMonitorInput = (RemoteDocLevelMonitorInput) monitor.getInputs().get(0); List indices = remoteDocLevelMonitorInput.getDocLevelMonitorInput().getIndices(); - ThreatIntelInput threatIntelInput = getThreatIntelInputFromBytesReference(remoteDocLevelMonitorInput.getInput()); + ThreatIntelInput threatIntelInput = getThreatIntelInputFromBytesReference(remoteDocLevelMonitorInput.getInput(), namedXContentRegistry); return new ThreatIntelMonitorDto( id, monitor.getName(), @@ -70,4 +96,114 @@ public static ThreatIntelMonitorDto buildThreatIntelMonitorDto(String id, Monito buildThreatIntelTriggerDtos(monitor.getTriggers(), namedXContentRegistry) ); } + + /** + * Fetch ACTIVE or ACKNOWLEDGED state alerts for the triggre. Criteria is they should match the ioc value+type from findings + */ + public static SearchSourceBuilder getSearchSourceBuilderForExistingAlertsQuery(ArrayList findings, Trigger trigger) { + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + queryBuilder.must(QueryBuilders.matchQuery(ThreatIntelAlert.TRIGGER_NAME_FIELD, trigger.getName())); + BoolQueryBuilder iocQueryBuilder = QueryBuilders.boolQuery(); + for (IocFinding finding : findings) { + BoolQueryBuilder innerQb = QueryBuilders.boolQuery(); + innerQb.must(QueryBuilders.matchQuery(ThreatIntelAlert.IOC_TYPE_FIELD, finding.getIocType())); + innerQb.must(QueryBuilders.matchQuery(ThreatIntelAlert.IOC_VALUE_FIELD, finding.getIocValue())); + iocQueryBuilder.should(innerQb); + } + queryBuilder.must(iocQueryBuilder); + BoolQueryBuilder stateQueryBuilder = QueryBuilders.boolQuery(); + stateQueryBuilder.should(QueryBuilders.matchQuery(ThreatIntelAlert.STATE_FIELD, Alert.State.ACTIVE.toString())); + stateQueryBuilder.should(QueryBuilders.matchQuery(ThreatIntelAlert.STATE_FIELD, Alert.State.ACKNOWLEDGED.toString())); + queryBuilder.must(stateQueryBuilder); + + SearchSourceBuilder ssb = new SearchSourceBuilder(); + ssb.query(queryBuilder); + ssb.size(9999); + return ssb; + } + + + public static Map prepareAlertsToUpdate(ArrayList triggerMatchedFindings, + List existingAlerts) { + Map updatedAlerts = new HashMap<>(); + for (ThreatIntelAlert existingAlert : existingAlerts) { + String iocType = existingAlert.getIocType(); + String iocValue = existingAlert.getIocValue(); + if (iocType == null || iocValue == null) + continue; + for (IocFinding finding : triggerMatchedFindings) { + if (iocType.equals(finding.getIocType()) && iocValue.equals(finding.getIocValue())) { + List findingIds = new ArrayList<>(existingAlert.getFindingIds()); + findingIds.add(finding.getId()); + updatedAlerts.put(existingAlert.getIocValue() + existingAlert.getIocType(), new ThreatIntelAlert(existingAlert, findingIds)); + } + } + } + return updatedAlerts; + + } + + public static List prepareNewAlerts(Monitor monitor, + Trigger trigger, + ArrayList findings, + Map updatedAlerts) { + List alerts = new ArrayList<>(); + for (IocFinding finding : findings) { + if (updatedAlerts.containsKey(finding.getIocValue() + finding.getIocType())) + continue; + Instant now = Instant.now(); + alerts.add(new ThreatIntelAlert( + UUID.randomUUID().toString(), + ThreatIntelAlert.NO_VERSION, + ThreatIntelAlert.NO_SCHEMA_VERSION, + monitor.getUser(), + trigger.getId(), + trigger.getName(), + monitor.getId(), + monitor.getName(), + Alert.State.ACTIVE, + now, + null, + now, + null, + null, + trigger.getSeverity(), + finding.getIocValue(), + finding.getIocType(), + Collections.emptyList(), + List.of(finding.getId()) + )); + } + return alerts; + } + + public static ArrayList getTriggerMatchedFindings(List iocFindings, ThreatIntelTrigger threatIntelTrigger) { + ArrayList triggerMatchedFindings = new ArrayList(); + for (IocFinding iocFinding : iocFindings) { + boolean iocTypeConditionMatch = false; + if (threatIntelTrigger.getIocTypes() == null || threatIntelTrigger.getIocTypes().isEmpty()) { + iocTypeConditionMatch = true; + } else if (threatIntelTrigger.getIocTypes().contains(iocFinding.getIocType().toLowerCase())) { + iocTypeConditionMatch = true; + } + boolean dataSourcesConditionMatch = false; + if (threatIntelTrigger.getDataSources() == null || threatIntelTrigger.getDataSources().isEmpty()) { + dataSourcesConditionMatch = true; + } else { + List dataSources = iocFinding.getRelatedDocIds().stream().map(it -> { + String[] parts = it.split(":"); + if (parts.length == 2) { + return parts[1]; + } else return null; + }).filter(Objects::nonNull).collect(Collectors.toList()); + if (threatIntelTrigger.getDataSources().stream().anyMatch(dataSources::contains)) { + dataSourcesConditionMatch = true; + } + } + if (dataSourcesConditionMatch && iocTypeConditionMatch) { + triggerMatchedFindings.add(iocFinding); + } + } + return triggerMatchedFindings; + } } diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportAckCorrelationAlertsAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportAckCorrelationAlertsAction.java new file mode 100644 index 000000000..917d0349c --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportAckCorrelationAlertsAction.java @@ -0,0 +1,81 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.transport; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.OpenSearchStatusException; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.client.Client; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.inject.Inject; +import org.opensearch.common.settings.Settings; +import org.opensearch.commons.authuser.User; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.securityanalytics.action.AckCorrelationAlertsAction; +import org.opensearch.securityanalytics.action.AckCorrelationAlertsRequest; +import org.opensearch.securityanalytics.action.AckCorrelationAlertsResponse; +import org.opensearch.securityanalytics.correlation.alert.CorrelationAlertService; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.tasks.Task; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; + +public class TransportAckCorrelationAlertsAction extends HandledTransportAction implements SecureTransportAction { + + private final NamedXContentRegistry xContentRegistry; + + private final ClusterService clusterService; + + private final Settings settings; + + private final ThreadPool threadPool; + + private final CorrelationAlertService correlationAlertService; + + private volatile Boolean filterByEnabled; + + private static final Logger log = LogManager.getLogger(TransportGetCorrelationAlertsAction.class); + + + @Inject + public TransportAckCorrelationAlertsAction(TransportService transportService, CorrelationAlertService correlationAlertService, ActionFilters actionFilters, ClusterService clusterService, AckCorrelationAlertsAction correlationAckAlertsAction, ThreadPool threadPool, Settings settings, NamedXContentRegistry xContentRegistry, Client client) { + super(correlationAckAlertsAction.NAME, transportService, actionFilters, AckCorrelationAlertsRequest::new); + this.xContentRegistry = xContentRegistry; + this.correlationAlertService = correlationAlertService; + this.clusterService = clusterService; + this.threadPool = threadPool; + this.settings = settings; + this.filterByEnabled = SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES.get(this.settings); + this.clusterService.getClusterSettings().addSettingsUpdateConsumer(SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES, this::setFilterByEnabled); + } + + @Override + protected void doExecute(Task task, AckCorrelationAlertsRequest request, ActionListener actionListener) { + + User user = readUserFromThreadContext(this.threadPool); + + String validateBackendRoleMessage = validateUserBackendRoles(user, this.filterByEnabled); + if (!"".equals(validateBackendRoleMessage)) { + actionListener.onFailure(new OpenSearchStatusException("Do not have permissions to resource", RestStatus.FORBIDDEN)); + return; + } + + if (!request.getCorrelationAlertIds().isEmpty()) { + correlationAlertService.acknowledgeAlerts( + request.getCorrelationAlertIds(), + actionListener + ); + } + } + + private void setFilterByEnabled(boolean filterByEnabled) { + this.filterByEnabled = filterByEnabled; + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportCorrelateFindingAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportCorrelateFindingAction.java index 910794556..e84d8b3e9 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportCorrelateFindingAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportCorrelateFindingAction.java @@ -35,6 +35,7 @@ import org.opensearch.commons.alerting.action.PublishFindingsRequest; import org.opensearch.commons.alerting.action.SubscribeFindingsResponse; import org.opensearch.commons.alerting.action.AlertingActions; +import org.opensearch.commons.authuser.User; import org.opensearch.core.common.io.stream.InputStreamStreamInput; import org.opensearch.core.common.io.stream.OutputStreamStreamOutput; import org.opensearch.core.xcontent.NamedXContentRegistry; @@ -49,6 +50,8 @@ import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.securityanalytics.correlation.JoinEngine; import org.opensearch.securityanalytics.correlation.VectorEmbeddingsEngine; +import org.opensearch.securityanalytics.correlation.alert.CorrelationAlertService; +import org.opensearch.securityanalytics.correlation.alert.notifications.NotificationService; import org.opensearch.securityanalytics.logtype.LogTypeService; import org.opensearch.securityanalytics.model.CustomLogType; import org.opensearch.securityanalytics.model.Detector; @@ -99,6 +102,10 @@ public class TransportCorrelateFindingAction extends HandledTransportAction actionListener) { try { PublishFindingsRequest transformedRequest = transformRequest(request); - AsyncCorrelateFindingAction correlateFindingAction = new AsyncCorrelateFindingAction(task, transformedRequest, actionListener); + AsyncCorrelateFindingAction correlateFindingAction = new AsyncCorrelateFindingAction(task, transformedRequest, readUserFromThreadContext(this.threadPool), actionListener); if (!this.correlationIndices.correlationIndexExists()) { try { @@ -146,7 +155,6 @@ protected void doExecute(Task task, ActionRequest request, ActionListener { @@ -168,6 +176,19 @@ protected void doExecute(Task task, ActionRequest request, ActionListener { + if (createIndexResponse.isAcknowledged()) { + IndexUtils.correlationAlertIndexUpdated(); + } else { + correlateFindingAction.onFailures(new OpenSearchStatusException("Failed to create correlation metadata Index", RestStatus.INTERNAL_SERVER_ERROR)); + } + }, correlateFindingAction::onFailures)); + } catch (Exception ex) { + correlateFindingAction.onFailures(ex); + } + } } else { correlateFindingAction.onFailures(new OpenSearchStatusException("Failed to create correlation Index", RestStatus.INTERNAL_SERVER_ERROR)); } @@ -193,14 +214,12 @@ public class AsyncCorrelateFindingAction { private final AtomicBoolean counter = new AtomicBoolean(); private final Task task; - AsyncCorrelateFindingAction(Task task, PublishFindingsRequest request, ActionListener listener) { + AsyncCorrelateFindingAction(Task task, PublishFindingsRequest request, User user, ActionListener listener) { this.task = task; this.request = request; this.listener = listener; - this.response =new AtomicReference<>(); - - this.joinEngine = new JoinEngine(client, request, xContentRegistry, corrTimeWindow, this, logTypeService, enableAutoCorrelation); + this.joinEngine = new JoinEngine(client, request, xContentRegistry, corrTimeWindow, indexTimeout, this, logTypeService, enableAutoCorrelation, correlationAlertService, notificationService, user); this.vectorEmbeddingsEngine = new VectorEmbeddingsEngine(client, indexTimeout, corrTimeWindow, this); } diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportDeleteCorrelationRuleAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportDeleteCorrelationRuleAction.java index d3c21cf1c..c8f9273e9 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportDeleteCorrelationRuleAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportDeleteCorrelationRuleAction.java @@ -8,6 +8,7 @@ package org.opensearch.securityanalytics.transport; +import java.util.Collections; import java.util.Locale; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -19,6 +20,7 @@ import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.client.Client; import org.opensearch.common.inject.Inject; +import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.index.query.QueryBuilders; import org.opensearch.index.reindex.BulkByScrollResponse; import org.opensearch.index.reindex.DeleteByQueryAction; @@ -26,6 +28,7 @@ import org.opensearch.core.rest.RestStatus; import org.opensearch.securityanalytics.action.DeleteCorrelationRuleAction; import org.opensearch.securityanalytics.action.DeleteCorrelationRuleRequest; +import org.opensearch.securityanalytics.correlation.alert.CorrelationAlertService; import org.opensearch.securityanalytics.model.CorrelationRule; import org.opensearch.securityanalytics.util.SecurityAnalyticsException; import org.opensearch.tasks.Task; @@ -37,14 +40,19 @@ public class TransportDeleteCorrelationRuleAction extends HandledTransportAction private final Client client; + private CorrelationAlertService correlationAlertService; + + @Inject public TransportDeleteCorrelationRuleAction( TransportService transportService, Client client, - ActionFilters actionFilters + ActionFilters actionFilters, + CorrelationAlertService correlationAlertService ) { super(DeleteCorrelationRuleAction.NAME, transportService, actionFilters, DeleteCorrelationRuleRequest::new); this.client = client; + this.correlationAlertService = correlationAlertService; } @Override @@ -72,6 +80,9 @@ public void onResponse(BulkByScrollResponse response) { ); return; } + // update the alerts assosciated with correlation Rules, with error STATE and errorMessage + log.debug("Updating Correlation Alerts with error Message for ruleId: " + correlationRuleId); + correlationAlertService.updateCorrelationAlertsWithError(correlationRuleId); listener.onResponse(new AcknowledgedResponse(true)); } diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportGetCorrelationAlertsAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportGetCorrelationAlertsAction.java new file mode 100644 index 000000000..cdca86a23 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportGetCorrelationAlertsAction.java @@ -0,0 +1,85 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.transport; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.OpenSearchStatusException; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.client.Client; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.inject.Inject; +import org.opensearch.common.settings.Settings; +import org.opensearch.commons.authuser.User; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.securityanalytics.action.*; +import org.opensearch.securityanalytics.correlation.alert.CorrelationAlertService; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.tasks.Task; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; + +public class TransportGetCorrelationAlertsAction extends HandledTransportAction implements SecureTransportAction { + + private final NamedXContentRegistry xContentRegistry; + + private final ClusterService clusterService; + + private final Settings settings; + + private final ThreadPool threadPool; + + private final CorrelationAlertService correlationAlertService; + + private volatile Boolean filterByEnabled; + + private static final Logger log = LogManager.getLogger(TransportGetCorrelationAlertsAction.class); + + + @Inject + public TransportGetCorrelationAlertsAction(TransportService transportService, CorrelationAlertService correlationAlertService, ActionFilters actionFilters, ClusterService clusterService, GetCorrelationAlertsAction getCorrelationAlertsAction, ThreadPool threadPool, Settings settings, NamedXContentRegistry xContentRegistry, Client client) { + super(getCorrelationAlertsAction.NAME, transportService, actionFilters, GetCorrelationAlertsRequest::new); + this.xContentRegistry = xContentRegistry; + this.correlationAlertService = correlationAlertService; + this.clusterService = clusterService; + this.threadPool = threadPool; + this.settings = settings; + this.filterByEnabled = SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES.get(this.settings); + this.clusterService.getClusterSettings().addSettingsUpdateConsumer(SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES, this::setFilterByEnabled); + } + + @Override + protected void doExecute(Task task, GetCorrelationAlertsRequest request, ActionListener actionListener) { + + User user = readUserFromThreadContext(this.threadPool); + + String validateBackendRoleMessage = validateUserBackendRoles(user, this.filterByEnabled); + if (!"".equals(validateBackendRoleMessage)) { + actionListener.onFailure(new OpenSearchStatusException("Do not have permissions to resource", RestStatus.FORBIDDEN)); + return; + } + + if (request.getCorrelationRuleId() != null) { + correlationAlertService.getCorrelationAlerts( + request.getCorrelationRuleId(), + request.getTable(), + actionListener + ); + } else { + correlationAlertService.getCorrelationAlerts( + null, + request.getTable(), + actionListener + ); + } + } + + private void setFilterByEnabled(boolean filterByEnabled) { + this.filterByEnabled = filterByEnabled; + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportListIOCsAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportListIOCsAction.java index 8da415714..c64a0195d 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportListIOCsAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportListIOCsAction.java @@ -122,11 +122,10 @@ void start() { } - SortBuilder sortBuilder = SortBuilders .fieldSort(STIX2_IOC_NESTED_PATH + request.getSortString()) .order(SortOrder.fromString(request.getSortOrder().toString())); - + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder() .version(true) .seqNoAndPrimaryTerm(true) diff --git a/src/main/java/org/opensearch/securityanalytics/util/CorrelationIndices.java b/src/main/java/org/opensearch/securityanalytics/util/CorrelationIndices.java index 624d76d58..375342d09 100644 --- a/src/main/java/org/opensearch/securityanalytics/util/CorrelationIndices.java +++ b/src/main/java/org/opensearch/securityanalytics/util/CorrelationIndices.java @@ -36,6 +36,8 @@ public class CorrelationIndices { public static final String CORRELATION_HISTORY_INDEX_PATTERN_REGEXP = ".opensearch-sap-correlation-history*"; public static final String CORRELATION_HISTORY_WRITE_INDEX = ".opensearch-sap-correlation-history-write"; + + public static final String CORRELATION_ALERT_INDEX = ".opensearch-sap-correlation-alerts"; public static final long FIXED_HISTORICAL_INTERVAL = 24L * 60L * 60L * 20L * 1000L; private final Client client; @@ -84,6 +86,11 @@ public boolean correlationMetadataIndexExists() { return clusterState.metadata().hasIndex(CORRELATION_METADATA_INDEX); } + public boolean correlationAlertIndexExists() { + ClusterState clusterState = clusterService.state(); + return clusterState.metadata().hasIndex(CORRELATION_ALERT_INDEX); + } + public void setupCorrelationIndex(TimeValue indexTimeout, Long setupTimestamp, ActionListener listener) throws IOException { try { long currentTimestamp = System.currentTimeMillis(); @@ -122,4 +129,17 @@ public void setupCorrelationIndex(TimeValue indexTimeout, Long setupTimestamp, A throw ex; } } + + public static String correlationAlertIndexMappings() throws IOException { + return new String(Objects.requireNonNull(CorrelationIndices.class.getClassLoader().getResourceAsStream("mappings/correlation_alert_mapping.json")).readAllBytes(), Charset.defaultCharset()); + } + public void initCorrelationAlertIndex(ActionListener actionListener) throws IOException { + Settings correlationAlertSettings = Settings.builder() + .put("index.hidden", true) + .build(); + CreateIndexRequest indexRequest = new CreateIndexRequest(CORRELATION_ALERT_INDEX) + .mapping(correlationAlertIndexMappings()) + .settings(correlationAlertSettings); + client.admin().indices().create(indexRequest, actionListener); + } } \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/util/IndexUtils.java b/src/main/java/org/opensearch/securityanalytics/util/IndexUtils.java index ce358591e..a24286fda 100644 --- a/src/main/java/org/opensearch/securityanalytics/util/IndexUtils.java +++ b/src/main/java/org/opensearch/securityanalytics/util/IndexUtils.java @@ -45,6 +45,8 @@ public class IndexUtils { public static String lastUpdatedCorrelationHistoryIndex = null; public static Boolean correlationRuleIndexUpdated = false; + public static Boolean correlationAlertIndexUpdated = false; + public static Boolean customLogTypeIndexUpdated = false; public static void detectorIndexUpdated() { @@ -65,6 +67,10 @@ public static void correlationMetadataIndexUpdated() { correlationMetadataIndexUpdated = true; } + public static void correlationAlertIndexUpdated() { + correlationAlertIndexUpdated = true; + } + public static void correlationRuleIndexUpdated() { correlationRuleIndexUpdated = true; } diff --git a/src/main/java/org/opensearch/securityanalytics/util/XContentUtils.java b/src/main/java/org/opensearch/securityanalytics/util/XContentUtils.java index d4cd4b06b..6c56af6fc 100644 --- a/src/main/java/org/opensearch/securityanalytics/util/XContentUtils.java +++ b/src/main/java/org/opensearch/securityanalytics/util/XContentUtils.java @@ -6,16 +6,18 @@ package org.opensearch.securityanalytics.util; import java.io.IOException; +import java.time.Instant; +import java.util.Locale; import java.util.Map; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.common.bytes.BytesReference; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentHelper; -import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; public class XContentUtils { @@ -37,4 +39,25 @@ public static BytesReference getBytesReference(Writeable writeable) throws IOExc return bytes; } + public static Instant getInstant(XContentParser xcp) throws IOException { + Instant lastUpdateTime; + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + lastUpdateTime = null; + } else if (xcp.currentToken().isValue()) { + lastUpdateTime = Instant.ofEpochMilli(xcp.longValue()); + } else { + XContentParserUtils.throwUnknownToken(xcp.currentToken(), xcp.getTokenLocation()); + lastUpdateTime = null; + } + return lastUpdateTime; + } + + public static void buildInstantAsField(XContentBuilder builder, Instant instant, String fieldName) throws IOException { + if (instant == null) { + builder.nullField(fieldName); + } else { + builder.timeField(fieldName, String.format(Locale.getDefault(), "%s_in_millis", fieldName), instant.toEpochMilli()); + } + } + } \ No newline at end of file diff --git a/src/main/resources/mappings/correlation_alert_mapping.json b/src/main/resources/mappings/correlation_alert_mapping.json new file mode 100644 index 000000000..5245edba8 --- /dev/null +++ b/src/main/resources/mappings/correlation_alert_mapping.json @@ -0,0 +1,102 @@ +{ + "_meta": { + "schema_version": 1 + }, + "properties": { + "acknowledged_time": { + "type": "date" + }, + "action_execution_results": { + "type": "nested", + "properties": { + "action_id": { + "type": "keyword" + }, + "last_execution_time": { + "type": "date" + }, + "throttled_count": { + "type": "integer" + } + } + }, + "error_message": { + "type": "text" + }, + "correlated_finding_ids": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "correlation_rule_id": { + "type": "keyword" + }, + "correlation_rule_name": { + "type": "text" + }, + "user": { + "properties": { + "backend_roles": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "custom_attribute_names": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "roles": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + } + } + }, + "schema_version": { + "type": "integer" + }, + "severity": { + "type": "keyword" + }, + "state": { + "type": "keyword" + }, + "id": { + "type": "keyword" + }, + "trigger_name": { + "type": "text" + }, + "version": { + "type": "long" + }, + "start_time": { + "type": "date" + }, + "end_time": { + "type": "date" + } + } +} diff --git a/src/main/resources/mappings/threat_intel_alert_mapping.json b/src/main/resources/mappings/threat_intel_alert_mapping.json new file mode 100644 index 000000000..f59dcf9dc --- /dev/null +++ b/src/main/resources/mappings/threat_intel_alert_mapping.json @@ -0,0 +1,110 @@ +{ + "dynamic": "strict", + "_meta": { + "schema_version": 0 + }, + "properties": { + "id": { + "type": "keyword" + }, + "version": { + "type": "long" + }, + "schema_version": { + "type": "long" + }, + "user": { + "properties": { + "name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "backend_roles": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "roles": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "custom_attribute_names": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + } + } + }, + "trigger_id": { + "type": "keyword" + }, + "trigger_name": { + "type": "keyword" + }, + "monitor_id": { + "type": "keyword" + }, + "monitor_name": { + "type": "keyword" + }, + "state": { + "type": "keyword" + }, + "start_time": { + "type": "date" + }, + "end_time": { + "type": "date" + }, + "acknowledged_time": { + "type": "date" + }, + "last_updated_time": { + "type": "date" + }, + "error_message": { + "type": "text" + }, + "severity": { + "type": "keyword" + }, + "action_execution_results": { + "type": "nested", + "properties": { + "action_id": { + "type": "keyword" + }, + "last_execution_time": { + "type": "date" + }, + "throttled_count": { + "type": "integer" + } + } + }, + "ioc_value": { + "type": "keyword" + }, + "ioc_type": { + "type": "keyword" + }, + "finding_ids": { + "type": "text" + } + } +} \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java index 52f1d4b5d..12568d95b 100644 --- a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java +++ b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java @@ -61,7 +61,7 @@ import org.opensearch.securityanalytics.model.Detector; import org.opensearch.securityanalytics.model.Rule; import org.opensearch.securityanalytics.model.ThreatIntelFeedData; -import org.opensearch.securityanalytics.model.IocFinding; +import org.opensearch.securityanalytics.model.threatintel.IocFinding; import org.opensearch.securityanalytics.threatIntel.iocscan.dao.IocFindingService; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; import org.opensearch.securityanalytics.threatIntel.sacommons.monitor.ThreatIntelMonitorDto; @@ -669,14 +669,14 @@ protected HttpEntity toHttpEntity(SATIFSourceConfigDto saTifSourceConfigDto) thr return new StringEntity(toJsonString(saTifSourceConfigDto), ContentType.APPLICATION_JSON); } + protected HttpEntity toHttpEntity(IocFinding iocFinding) throws IOException { + return new StringEntity(toJsonString(iocFinding), ContentType.APPLICATION_JSON); + } + protected HttpEntity toHttpEntity(ThreatIntelMonitorDto threatIntelMonitorDto) throws IOException { return new StringEntity(toJsonString(threatIntelMonitorDto), ContentType.APPLICATION_JSON); } - protected HttpEntity toHttpEntity(IocFinding iocFinding) throws IOException { - return new StringEntity(toJsonString(iocFinding), ContentType.APPLICATION_JSON); - } - protected HttpEntity toHttpEntity(TestS3ConnectionRequest testS3ConnectionRequest) throws IOException { return new StringEntity(toJsonString(testS3ConnectionRequest), ContentType.APPLICATION_JSON); } @@ -738,7 +738,7 @@ private String toJsonString(IocFinding iocFinding) throws IOException { XContentBuilder builder = XContentFactory.jsonBuilder(); return IndexUtilsKt.string(shuffleXContent(iocFinding.toXContent(builder, ToXContent.EMPTY_PARAMS))); } - + private String toJsonString(TestS3ConnectionRequest testS3ConnectionRequest) throws IOException { XContentBuilder builder = XContentFactory.jsonBuilder(); return IndexUtilsKt.string(shuffleXContent(testS3ConnectionRequest.toXContent(builder, ToXContent.EMPTY_PARAMS))); @@ -1632,8 +1632,8 @@ protected void createNetflowLogIndex(String indexName) throws IOException { Response response = client().performRequest(indexRequest); assertEquals(HttpStatus.SC_CREATED, response.getStatusLine().getStatusCode()); // Refresh everything - response = client().performRequest(new Request("POST", "_refresh")); - assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + //response = client().performRequest(new Request("POST", "_refresh")); + //assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); } diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index 3dccd142c..426262c63 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -6,19 +6,19 @@ import com.carrotsearch.randomizedtesting.generators.RandomNumbers; import org.apache.lucene.tests.util.LuceneTestCase; -import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.xcontent.LoggingDeprecationHandler; -import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.core.xcontent.ToXContent; import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.commons.alerting.model.IntervalSchedule; import org.opensearch.commons.alerting.model.Schedule; import org.opensearch.commons.alerting.model.action.Action; import org.opensearch.commons.alerting.model.action.Throttle; import org.opensearch.commons.authuser.User; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; import org.opensearch.script.Script; import org.opensearch.script.ScriptType; import org.opensearch.securityanalytics.model.CorrelationQuery; @@ -28,10 +28,11 @@ import org.opensearch.securityanalytics.model.DetectorInput; import org.opensearch.securityanalytics.model.DetectorRule; import org.opensearch.securityanalytics.model.DetectorTrigger; -import org.opensearch.securityanalytics.model.IocFinding; import org.opensearch.securityanalytics.model.ThreatIntelFeedData; -import org.opensearch.securityanalytics.threatIntel.common.SourceConfigType; +import org.opensearch.securityanalytics.model.threatintel.IocFinding; +import org.opensearch.securityanalytics.model.threatintel.ThreatIntelAlert; import org.opensearch.securityanalytics.threatIntel.common.RefreshType; +import org.opensearch.securityanalytics.threatIntel.common.SourceConfigType; import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; import org.opensearch.securityanalytics.threatIntel.model.DefaultIocStoreConfig; import org.opensearch.securityanalytics.threatIntel.model.IocStoreConfig; @@ -88,25 +89,28 @@ public static Detector randomDetectorWithInputsAndThreatIntelAndTriggers(List inputs, List triggers) { return randomDetector(null, null, null, inputs, triggers, null, null, null, null, false); } + public static Detector randomDetectorWithInputs(List inputs, String detectorType) { return randomDetector(null, detectorType, null, inputs, List.of(), null, null, null, null, false); } - public static Detector randomDetectorWithTriggers(List triggers) { return randomDetector(null, null, null, List.of(), triggers, null, null, null, null, false); } + public static Detector randomDetectorWithTriggers(List rules, List triggers) { DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), Collections.emptyList(), rules.stream().map(DetectorRule::new).collect(Collectors.toList())); return randomDetector(null, null, null, List.of(input), triggers, null, null, null, null, false); } + public static Detector randomDetectorWithTriggers(List rules, List triggers, List inputIndices) { DetectorInput input = new DetectorInput("windows detector for security analytics", inputIndices, Collections.emptyList(), rules.stream().map(DetectorRule::new).collect(Collectors.toList())); return randomDetector(null, null, null, List.of(input), triggers, null, true, null, null, false); } + public static Detector randomDetectorWithTriggersAndScheduleAndEnabled(List rules, List triggers, Schedule schedule, boolean enabled) { DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), Collections.emptyList(), rules.stream().map(DetectorRule::new).collect(Collectors.toList())); @@ -207,37 +211,37 @@ public static Detector randomDetectorWithNoUser() { Instant lastUpdateTime = Instant.now().truncatedTo(ChronoUnit.MILLIS); return new Detector( - null, - null, - name, - enabled, - schedule, - lastUpdateTime, - enabledTime, - detectorType, - null, - inputs, - Collections.emptyList(), - Collections.singletonList(""), - "", - "", - "", - "", - "", - "", - Collections.emptyMap(), - Collections.emptyList(), - false + null, + null, + name, + enabled, + schedule, + lastUpdateTime, + enabledTime, + detectorType, + null, + inputs, + Collections.emptyList(), + Collections.singletonList(""), + "", + "", + "", + "", + "", + "", + Collections.emptyMap(), + Collections.emptyList(), + false ); } public static CorrelationRule randomCorrelationRule(String name) { - name = name.isEmpty()? ">": name; + name = name.isEmpty() ? ">" : name; return new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, name, List.of( new CorrelationQuery("vpc_flow1", "dstaddr:192.168.1.*", "network", null), new CorrelationQuery("ad_logs1", "azure.platformlogs.result_type:50126", "ad_ldap", null) - ), 300000L); + ), 300000L, null); } public static String randomRule() { @@ -330,8 +334,8 @@ public static String randomRuleWithNotCondition() { " - Legitimate usage of remote file encryption\n" + "level: high"; } - - public static String randomRuleWithCriticalSeverity() { + + public static String randomRuleWithCriticalSeverity() { return "title: Remote Encrypting File System Abuse\n" + "id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" + "description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" + @@ -468,7 +472,7 @@ public static String randomRuleForMappingView(String field) { " definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" + "detection:\n" + " selection:\n" + - " "+ field + ": 'ACL'\n" + + " " + field + ": 'ACL'\n" + " condition: selection\n" + "falsepositives:\n" + " - Legitimate usage of remote file encryption\n" + @@ -685,7 +689,7 @@ public static String productIndexMaxAggRule() { " condition: sel | max(fieldA) by fieldB > 110"; } - public static String randomProductDocument(){ + public static String randomProductDocument() { return "{\n" + " \"name\": \"laptop\",\n" + " \"fieldA\": 123,\n" + @@ -694,7 +698,7 @@ public static String randomProductDocument(){ "}\n"; } - public static String randomProductDocumentWithTime(long time){ + public static String randomProductDocumentWithTime(long time) { return "{\n" + " \"fieldA\": 123,\n" + " \"mappedB\": 111,\n" + @@ -815,6 +819,12 @@ public static String toJsonString(IocFinding iocFinding) throws IOException { return BytesReference.bytes(builder).utf8ToString(); } + public static String toJsonString(ThreatIntelAlert alert) throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder = alert.toXContent(builder, ToXContent.EMPTY_PARAMS); + return BytesReference.bytes(builder).utf8ToString(); + } + public static String toJsonString(ThreatIntelFeedData threatIntelFeedData) throws IOException { XContentBuilder builder = XContentFactory.jsonBuilder(); builder = threatIntelFeedData.toXContent(builder, ToXContent.EMPTY_PARAMS); @@ -959,7 +969,7 @@ public static String netFlowMappings() { " }"; } - public static String productIndexMapping(){ + public static String productIndexMapping() { return "\"properties\":{\n" + " \"name\":{\n" + " \"type\":\"keyword\"\n" + @@ -980,7 +990,7 @@ public static String productIndexMapping(){ "}"; } - public static String productIndexAvgAggRule(){ + public static String productIndexAvgAggRule() { return " title: Test\n" + " id: 39f918f3-981b-4e6f-a975-8af7e507ef2b\n" + " status: test\n" + @@ -1000,7 +1010,7 @@ public static String productIndexAvgAggRule(){ " condition: sel | avg(fieldA) by fieldC > 110"; } - public static String productIndexCountAggRule(){ + public static String productIndexCountAggRule() { return " title: Test\n" + " id: 39f918f3-981b-4e6f-a975-8af7e507ef2b\n" + " status: test\n" + @@ -1018,7 +1028,7 @@ public static String productIndexCountAggRule(){ " condition: sel | count(*) by name > 2"; } - public static String randomAggregationRule(String aggFunction, String signAndValue) { + public static String randomAggregationRule(String aggFunction, String signAndValue) { String rule = "title: Remote Encrypting File System Abuse\n" + "id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" + "description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" + @@ -1049,7 +1059,7 @@ public static String randomAggregationRule(String aggFunction, String signAndVa return String.format(Locale.ROOT, rule, aggFunction, signAndValue); } - public static String randomAggregationRule(String aggFunction, String signAndValue, String opCode) { + public static String randomAggregationRule(String aggFunction, String signAndValue, String opCode) { String rule = "title: Remote Encrypting File System Abuse\n" + "id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" + "description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" + @@ -1081,7 +1091,7 @@ public static String randomAggregationRule(String aggFunction, String signAndVa } public static String randomCloudtrailAggrRule() { - return "id: c64c5175-5189-431b-a55e-6d9882158250\n" + + return "id: c64c5175-5189-431b-a55e-6d9882158250\n" + "logsource:\n" + " product: cloudtrail\n" + "title: Accounts created and deleted within 24h\n" + @@ -1852,8 +1862,8 @@ public static String windowsIndexMappingOnlyNumericAndText() { } - public static String randomDoc(int severity, int version, String opCode) { - String doc = "{\n" + + public static String randomDoc(int severity, int version, String opCode) { + String doc = "{\n" + "\"EventTime\":\"2020-02-04T14:59:39.343541+00:00\",\n" + "\"HostName\":\"EC2AMAZ-EPO7HKA\",\n" + "\"Keywords\":\"9223372036854775808\",\n" + @@ -1892,7 +1902,7 @@ public static String randomDoc(int severity, int version, String opCode) { } public static String randomDocForNotCondition(int severity, int version, String opCode) { - String doc = "{\n" + + String doc = "{\n" + "\"EventTime\":\"2020-02-04T14:59:39.343541+00:00\",\n" + "\"HostName\":\"EC2AMAZ-EPO7HKA\",\n" + "\"Keywords\":\"9223372036854775808\",\n" + @@ -1930,7 +1940,7 @@ public static String randomDocForNotCondition(int severity, int version, String } public static String randomDocOnlyNumericAndDate(int severity, int version, String opCode) { - String doc = "{\n" + + String doc = "{\n" + "\"EventTime\":\"2020-02-04T14:59:39.343541+00:00\",\n" + "\"ExecutionProcessID\":2001,\n" + "\"ExecutionThreadID\":2616,\n" + @@ -1941,7 +1951,7 @@ public static String randomDocOnlyNumericAndDate(int severity, int version, Stri } public static String randomDocOnlyNumericAndText(int severity, int version, String opCode) { - String doc = "{\n" + + String doc = "{\n" + "\"TaskName\":\"SYSTEM\",\n" + "\"ExecutionProcessID\":2001,\n" + "\"ExecutionThreadID\":2616,\n" + @@ -1952,8 +1962,8 @@ public static String randomDocOnlyNumericAndText(int severity, int version, Stri } //Add IPs in HostName field. - public static String randomDocWithIpIoc(int severity, int version, String ioc) { - String doc = "{\n" + + public static String randomDocWithIpIoc(int severity, int version, String ioc) { + String doc = "{\n" + "\"EventTime\":\"2020-02-04T14:59:39.343541+00:00\",\n" + "\"HostName\":\"%s\",\n" + "\"Keywords\":\"9223372036854775808\",\n" + @@ -2880,7 +2890,7 @@ public static SATIFSourceConfig randomSATIFSourceConfig( feedFormat, sourceConfigType, description, - createdByUser, + new User("wrgrer", List.of("b1"), List.of("r1"), List.of("ca")), createdAt, source, enabledTime, diff --git a/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java b/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java index f9f3e25d5..46279da01 100644 --- a/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java +++ b/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java @@ -358,7 +358,7 @@ public void testAckAlerts_WithInvalidDetectorAlertsCombination() throws IOExcept indexDoc(index, "1", randomDoc()); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); Response executeResponse = null; @@ -566,7 +566,7 @@ public void testGetAlerts_byDetectorType_success() throws IOException, Interrupt indexDoc(index, "1", randomDoc()); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); Map executeResults = entityAsMap(executeResponse); @@ -682,7 +682,7 @@ public void testGetAlerts_byDetectorType_multipleDetectors_success() throws IOEx noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); Assert.assertEquals(1, noOfSigmaRuleMatches); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); request = "{\n" + " \"query\" : {\n" + @@ -700,7 +700,7 @@ public void testGetAlerts_byDetectorType_multipleDetectors_success() throws IOEx hits = executeSearch(DetectorMonitorConfig.getAlertsIndex("network"), request); } - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); // Call GetAlerts API for WINDOWS detector Map params = new HashMap<>(); @@ -1081,7 +1081,7 @@ public void testAlertHistoryRollover_maxDocs() throws IOException, InterruptedEx indexDoc(index, "1", randomDoc()); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); Map executeResults = entityAsMap(executeResponse); diff --git a/src/test/java/org/opensearch/securityanalytics/alerts/SecureAlertsRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/alerts/SecureAlertsRestApiIT.java index 20e526697..f96fb5bec 100644 --- a/src/test/java/org/opensearch/securityanalytics/alerts/SecureAlertsRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/alerts/SecureAlertsRestApiIT.java @@ -258,7 +258,7 @@ public void testGetAlerts_byDetectorType_success() throws IOException, Interrupt indexDoc(index, "1", randomDoc()); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); Map executeResults = entityAsMap(executeResponse); diff --git a/src/test/java/org/opensearch/securityanalytics/correlation/CorrelationEngineRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/correlation/CorrelationEngineRestApiIT.java index a2979a231..a7eda56aa 100644 --- a/src/test/java/org/opensearch/securityanalytics/correlation/CorrelationEngineRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/correlation/CorrelationEngineRestApiIT.java @@ -968,7 +968,7 @@ private String createNetworkToWindowsFieldBasedRule(LogIndices indices) throws I CorrelationQuery query1 = new CorrelationQuery(indices.vpcFlowsIndex, null, "network", "srcaddr"); CorrelationQuery query4 = new CorrelationQuery(indices.windowsIndex, null, "test_windows", "SourceIp"); - CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "network to windows", List.of(query1, query4), 300000L); + CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "network to windows", List.of(query1, query4), 300000L, null); Request request = new Request("POST", "/_plugins/_security_analytics/correlation/rules"); request.setJsonEntity(toJsonString(rule)); Response response = client().performRequest(request); @@ -981,7 +981,7 @@ private String createNetworkToWindowsFilterQueryBasedRule(LogIndices indices) th CorrelationQuery query1 = new CorrelationQuery(indices.vpcFlowsIndex, "srcaddr:1.2.3.4", "network", null); CorrelationQuery query4 = new CorrelationQuery(indices.windowsIndex, "SourceIp:1.2.3.4", "test_windows", null); - CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "network to windows", List.of(query1, query4), 300000L); + CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "network to windows", List.of(query1, query4), 300000L, null); Request request = new Request("POST", "/_plugins/_security_analytics/correlation/rules"); request.setJsonEntity(toJsonString(rule)); Response response = client().performRequest(request); @@ -994,7 +994,7 @@ private String createNetworkToCustomLogTypeFieldBasedRule(LogIndices indices, St CorrelationQuery query1 = new CorrelationQuery(indices.vpcFlowsIndex, null, "network", "srcaddr"); CorrelationQuery query4 = new CorrelationQuery(customLogTypeIndex, null, customLogTypeName, "SourceIp"); - CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "network to custom log type", List.of(query1, query4), 300000L); + CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "network to custom log type", List.of(query1, query4), 300000L, null); Request request = new Request("POST", "/_plugins/_security_analytics/correlation/rules"); request.setJsonEntity(toJsonString(rule)); Response response = client().performRequest(request); @@ -1008,7 +1008,7 @@ private String createNetworkToAdLdapToWindowsRule(LogIndices indices) throws IOE CorrelationQuery query2 = new CorrelationQuery(indices.adLdapLogsIndex, "ResultType:50126", "ad_ldap", null); CorrelationQuery query4 = new CorrelationQuery(indices.windowsIndex, "Domain:NTAUTHORI*", "test_windows", null); - CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "network to ad_ldap to windows", List.of(query1, query2, query4), 300000L); + CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "network to ad_ldap to windows", List.of(query1, query2, query4), 300000L, null); Request request = new Request("POST", "/_plugins/_security_analytics/correlation/rules"); request.setJsonEntity(toJsonString(rule)); Response response = client().performRequest(request); @@ -1022,7 +1022,7 @@ private String createWindowsToAppLogsToS3LogsRule(LogIndices indices) throws IOE CorrelationQuery query2 = new CorrelationQuery(indices.appLogsIndex, "endpoint:\\/customer_records.txt", "others_application", null); CorrelationQuery query4 = new CorrelationQuery(indices.s3AccessLogsIndex, "aws.cloudtrail.eventName:ReplicateObject", "s3", null); - CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "windows to app_logs to s3 logs", List.of(query1, query2, query4), 300000L); + CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "windows to app_logs to s3 logs", List.of(query1, query2, query4), 300000L, null); Request request = new Request("POST", "/_plugins/_security_analytics/correlation/rules"); request.setJsonEntity(toJsonString(rule)); Response response = client().performRequest(request); @@ -1035,7 +1035,7 @@ private String createCloudtrailFieldBasedRule(String index, String field, Long t CorrelationQuery query1 = new CorrelationQuery(index, "EventName:CreateUser", "cloudtrail", field); CorrelationQuery query2 = new CorrelationQuery(index, "EventName:DeleteUser", "cloudtrail", field); - CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "cloudtrail field based", List.of(query1, query2), timeWindow); + CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "cloudtrail field based", List.of(query1, query2), timeWindow, null); Request request = new Request("POST", "/_plugins/_security_analytics/correlation/rules"); request.setJsonEntity(toJsonString(rule)); Response response = client().performRequest(request); diff --git a/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java b/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java index c00eb9653..12264c4b3 100644 --- a/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java +++ b/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java @@ -257,7 +257,7 @@ public void testGetFindings_byDetectorType_success() throws IOException { noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); Assert.assertEquals(1, noOfSigmaRuleMatches); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); // Call GetFindings API for first detector Map params = new HashMap<>(); @@ -367,7 +367,7 @@ public void testGetAllFindings_success() throws IOException { noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); // Assert.assertEquals(1, noOfSigmaRuleMatches); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); // Call GetFindings API for all the detectors Map params = new HashMap<>(); @@ -586,7 +586,7 @@ public void testGetFindings_bySeverity_success() throws IOException { noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); Assert.assertEquals(1, noOfSigmaRuleMatches); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); // Call GetFindings API for first detector by severity Map params = new HashMap<>(); @@ -707,7 +707,7 @@ public void testGetFindings_bySearchString_success() throws IOException { noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); Assert.assertEquals(1, noOfSigmaRuleMatches); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); // Call GetFindings API for first detector by searchString 'high' Map params = new HashMap<>(); @@ -823,7 +823,7 @@ public void testGetFindings_byStartTimeAndEndTime_success() throws IOException { int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); Assert.assertEquals(1, noOfSigmaRuleMatches); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); // Call GetFindings API for first detector by startTime and endTime Map params = new HashMap<>(); params.put("startTime", String.valueOf(startTime1.toEpochMilli())); @@ -834,7 +834,7 @@ public void testGetFindings_byStartTimeAndEndTime_success() throws IOException { Map getFindingsBody = entityAsMap(getFindingsResponse); Assert.assertEquals(1, getFindingsBody.get("total_findings")); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); Instant startTime2 = Instant.now(); // execute monitor 2 executeResponse = executeAlertingMonitor(monitorId2, Collections.emptyMap()); @@ -1335,7 +1335,7 @@ public void testGetFindings_rolloverByMaxDoc_short_retention_success() throws IO // Call GetFindings API Map params = new HashMap<>(); params.put("detector_id", detectorId); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); Map getFindingsBody = entityAsMap(getFindingsResponse); Assert.assertEquals(1, getFindingsBody.get("total_findings")); @@ -1364,7 +1364,7 @@ public void testGetFindings_rolloverByMaxDoc_short_retention_success() throws IO noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); Assert.assertEquals(5, noOfSigmaRuleMatches); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); getFindingsBody = entityAsMap(getFindingsResponse); Assert.assertEquals(1, getFindingsBody.get("total_findings")); diff --git a/src/test/java/org/opensearch/securityanalytics/findings/SecureFindingRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/findings/SecureFindingRestApiIT.java index 41b3d2742..6e1f62a53 100644 --- a/src/test/java/org/opensearch/securityanalytics/findings/SecureFindingRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/findings/SecureFindingRestApiIT.java @@ -263,7 +263,7 @@ public void testGetFindings_byDetectorType_success() throws IOException { noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); Assert.assertEquals(5, noOfSigmaRuleMatches); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); // try to do get finding as a user with read access diff --git a/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java index 3b064f308..1b59944eb 100644 --- a/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java @@ -723,7 +723,7 @@ public void testCreateMappings_withIndexPattern_differentMappings_indexTemplateC createIndex(indexName1, Settings.EMPTY, null); createIndex(indexName2, Settings.EMPTY, null); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); // Insert sample docs String sampleDoc1 = "{" + @@ -738,7 +738,7 @@ public void testCreateMappings_withIndexPattern_differentMappings_indexTemplateC indexDoc(indexName1, "1", sampleDoc1); indexDoc(indexName2, "1", sampleDoc2); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); // Execute CreateMappingsAction to add alias mapping for index createMappingsAPI(indexPattern, "netflow"); @@ -801,7 +801,7 @@ public void testCreateMappings_withIndexPattern_indexTemplate_createAndUpdate_su createIndex(indexName1, Settings.EMPTY, null); createIndex(indexName2, Settings.EMPTY, null); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); // Insert sample doc String sampleDoc1 = "{" + @@ -812,7 +812,7 @@ public void testCreateMappings_withIndexPattern_indexTemplate_createAndUpdate_su indexDoc(indexName1, "1", sampleDoc1); indexDoc(indexName2, "1", sampleDoc1); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); // Execute CreateMappingsAction to add alias mapping for index createMappingsAPI(indexPattern, "netflow"); @@ -888,7 +888,7 @@ public void testCreateMappings_withIndexPattern_oneNoMappings_failure() throws I createIndex(indexName1, Settings.EMPTY, null); createIndex(indexName2, Settings.EMPTY, null); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); // Insert sample docs String sampleDoc1 = "{" + @@ -898,7 +898,7 @@ public void testCreateMappings_withIndexPattern_oneNoMappings_failure() throws I "}"; indexDoc(indexName1, "1", sampleDoc1); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); // Execute CreateMappingsAction to add alias mapping for index try { @@ -1112,7 +1112,7 @@ public void testCreateMappings_withIndexPattern_success() throws IOException { createIndex(indexName1, Settings.EMPTY, null); createIndex(indexName2, Settings.EMPTY, null); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); // Insert sample doc String sampleDoc = "{" + @@ -1125,7 +1125,7 @@ public void testCreateMappings_withIndexPattern_success() throws IOException { indexDoc(indexName1, "1", sampleDoc); indexDoc(indexName2, "1", sampleDoc); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); // Execute CreateMappingsAction to add alias mapping for index Request request = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); @@ -1151,7 +1151,7 @@ public void testCreateMappings_withIndexPattern_conflictingTemplates_success() t createIndex(indexName1, Settings.EMPTY, null); createIndex(indexName2, Settings.EMPTY, null); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); // Insert sample doc String sampleDoc = "{" + @@ -1162,7 +1162,7 @@ public void testCreateMappings_withIndexPattern_conflictingTemplates_success() t indexDoc(indexName1, "1", sampleDoc); indexDoc(indexName2, "1", sampleDoc); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); // Execute CreateMappingsAction with first index pattern createMappingsAPI(indexPattern1, "netflow"); @@ -1206,7 +1206,7 @@ public void testCreateMappings_withIndexPattern_conflictingTemplates_failure_1() createIndex(indexName1, Settings.EMPTY, null); createIndex(indexName2, Settings.EMPTY, null); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); // Insert sample doc String sampleDoc = "{" + @@ -1217,7 +1217,7 @@ public void testCreateMappings_withIndexPattern_conflictingTemplates_failure_1() indexDoc(indexName1, "1", sampleDoc); indexDoc(indexName2, "1", sampleDoc); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); // Execute CreateMappingsAction with first index pattern createMappingsAPI(indexPattern1, "netflow"); @@ -1245,7 +1245,7 @@ public void testCreateMappings_withIndexPattern_conflictingTemplates_failure_2() createIndex(indexName1, Settings.EMPTY, null); createIndex(indexName2, Settings.EMPTY, null); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); // Insert sample doc String sampleDoc = "{" + @@ -1256,7 +1256,7 @@ public void testCreateMappings_withIndexPattern_conflictingTemplates_failure_2() indexDoc(indexName1, "1", sampleDoc); indexDoc(indexName2, "1", sampleDoc); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); // User-create template with conflicting pattern but higher priority @@ -1280,7 +1280,7 @@ public void testCreateMappings_withIndexPattern_oneNoMatches_success() throws IO createIndex(indexName1, Settings.EMPTY, null); createIndex(indexName2, Settings.EMPTY, null); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); // Insert sample docs String sampleDoc1 = "{" + @@ -1295,7 +1295,7 @@ public void testCreateMappings_withIndexPattern_oneNoMatches_success() throws IO indexDoc(indexName1, "1", sampleDoc1); indexDoc(indexName2, "1", sampleDoc2); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); // Execute CreateMappingsAction to add alias mapping for index Request request = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); @@ -1382,8 +1382,8 @@ private void createSampleIndex(String indexName, Settings settings, String alias Response response = client().performRequest(indexRequest); assertEquals(HttpStatus.SC_CREATED, response.getStatusLine().getStatusCode()); // Refresh everything - response = client().performRequest(new Request("POST", "_refresh")); - assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + //response = client().performRequest(new Request("POST", "_refresh")); + //assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); } private void createSampleWindex(String indexName) throws IOException { @@ -1445,8 +1445,8 @@ private void createSampleWindex(String indexName, Settings settings, String alia Response response = client().performRequest(indexRequest); assertEquals(HttpStatus.SC_CREATED, response.getStatusLine().getStatusCode()); // Refresh everything - response = client().performRequest(new Request("POST", "_refresh")); - assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + //response = client().performRequest(new Request("POST", "_refresh")); + //assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); } private void createSampleDatastream(String datastreamName) throws IOException { @@ -1534,8 +1534,8 @@ private void createSampleDatastream(String datastreamName) throws IOException { response = client().performRequest(indexRequest); assertEquals(HttpStatus.SC_CREATED, response.getStatusLine().getStatusCode()); // Refresh everything - response = client().performRequest(new Request("POST", "_refresh")); - assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + //response = client().performRequest(new Request("POST", "_refresh")); + //assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); } private void deleteDatastream(String datastreamName) throws IOException { @@ -1614,8 +1614,8 @@ public void testCreateDNSMapping() throws IOException{ }); // Refresh everything - response = client().performRequest(new Request("POST", "_refresh")); - assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + //response = client().performRequest(new Request("POST", "_refresh")); + //assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); } diff --git a/src/test/java/org/opensearch/securityanalytics/model/IocFindingTests.java b/src/test/java/org/opensearch/securityanalytics/model/IocFindingTests.java index 8acf10744..afbc4c6f0 100644 --- a/src/test/java/org/opensearch/securityanalytics/model/IocFindingTests.java +++ b/src/test/java/org/opensearch/securityanalytics/model/IocFindingTests.java @@ -5,6 +5,8 @@ import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.securityanalytics.model.threatintel.IocFinding; +import org.opensearch.securityanalytics.model.threatintel.IocWithFeeds; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; diff --git a/src/test/java/org/opensearch/securityanalytics/model/threatintel/ThreatIntelAlertTests.java b/src/test/java/org/opensearch/securityanalytics/model/threatintel/ThreatIntelAlertTests.java new file mode 100644 index 000000000..0e945d217 --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/model/threatintel/ThreatIntelAlertTests.java @@ -0,0 +1,110 @@ +package org.opensearch.securityanalytics.model.threatintel; + +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.commons.alerting.model.Alert; +import org.opensearch.commons.authuser.User; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.time.Instant; +import java.util.Collections; +import java.util.List; + +public class ThreatIntelAlertTests extends OpenSearchTestCase { + + public void testAlertAsStream() throws IOException { + ThreatIntelAlert alert = getRandomAlert(); + BytesStreamOutput out = new BytesStreamOutput(); + alert.writeTo(out); + StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); + ThreatIntelAlert newThreatIntelAlert = new ThreatIntelAlert(sin); + asserts(alert, newThreatIntelAlert); + } + + private static void asserts(ThreatIntelAlert alert, ThreatIntelAlert newThreatIntelAlert) { + assertEquals(alert.getId(), newThreatIntelAlert.getId()); + assertEquals(alert.getErrorMessage(), newThreatIntelAlert.getErrorMessage()); + assertEquals(alert.getSeverity(), newThreatIntelAlert.getSeverity()); + assertEquals(alert.getSchemaVersion(), newThreatIntelAlert.getSchemaVersion()); + assertEquals(alert.getTriggerName(), newThreatIntelAlert.getTriggerName()); + assertEquals(alert.getTriggerId(), newThreatIntelAlert.getTriggerId()); + assertEquals(alert.getMonitorId(), newThreatIntelAlert.getMonitorId()); + assertEquals(alert.getMonitorName(), newThreatIntelAlert.getMonitorName()); + assertEquals(alert.getVersion(), newThreatIntelAlert.getVersion()); + assertEquals(alert.getActionExecutionResults(), newThreatIntelAlert.getActionExecutionResults()); + assertEquals(alert.getStartTime(), newThreatIntelAlert.getStartTime()); + assertEquals(alert.getAcknowledgedTime(), newThreatIntelAlert.getAcknowledgedTime()); + assertEquals(alert.getState(), newThreatIntelAlert.getState()); + assertEquals(alert.getIocValue(), newThreatIntelAlert.getIocValue()); + assertEquals(alert.getIocType(), newThreatIntelAlert.getIocType()); + assertEquals(alert.getLastUpdatedTime(), newThreatIntelAlert.getLastUpdatedTime()); + assertTrue(alert.getFindingIds().containsAll(newThreatIntelAlert.getFindingIds())); + } + + public void testThreatIntelAlertParse() throws IOException { + long now = System.currentTimeMillis(); + String threatIntelAlertString = "{\n" + + " \"id\": \"example-id\",\n" + + " \"version\": 1,\n" + + " \"schema_version\": 1,\n" + + " \"user\": null,\n" + + " \"trigger_name\": \"example-trigger-name\",\n" + + " \"trigger_id\": \"example-trigger-id\",\n" + + " \"monitor_id\": \"example-monitor-id\",\n" + + " \"monitor_name\": \"example-monitor-name\",\n" + + " \"state\": \"ACTIVE\",\n" + + " \"start_time\": \"" + now + "\",\n" + + " \"end_time\": \"" + now + "\",\n" + + " \"acknowledged_time\": \"" + now + "\",\n" + + " \"last_updated_time\": \"" + now + "\",\n" + + " \"ioc_value\": \"" + now + "\",\n" + + " \"ioc_type\": \"" + now + "\",\n" + + " \"error_message\": \"example-error-message\",\n" + + " \"severity\": \"high\",\n" + + " \"action_execution_results\": [],\n" + + " \"finding_id\": [ \"f1\", \"f2\"]\n" + + "}\n"; + + ThreatIntelAlert alert = ThreatIntelAlert.parse(getParser(threatIntelAlertString), 1l); + BytesStreamOutput out = new BytesStreamOutput(); + alert.writeTo(out); + StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); + ThreatIntelAlert newThreatIntelAlert = new ThreatIntelAlert(sin); + asserts(alert, newThreatIntelAlert); + } + + public XContentParser getParser(String xc) throws IOException { + XContentParser parser = XContentType.JSON.xContent().createParser(xContentRegistry(), LoggingDeprecationHandler.INSTANCE, xc); + parser.nextToken(); + return parser; + + } + + private static ThreatIntelAlert getRandomAlert() { + return new ThreatIntelAlert( + randomAlphaOfLength(10), + randomLong(), + randomLong(), + new User(randomAlphaOfLength(10), List.of(randomAlphaOfLength(10)), List.of(randomAlphaOfLength(10)), List.of(randomAlphaOfLength(10))), + randomAlphaOfLength(10), + randomAlphaOfLength(10), + randomAlphaOfLength(10), + randomAlphaOfLength(10), + Alert.State.ACKNOWLEDGED, + Instant.now(), + Instant.now(), + Instant.now(), + Instant.now(), + randomAlphaOfLength(10), + randomAlphaOfLength(10), + randomAlphaOfLength(10), + randomAlphaOfLength(10), + Collections.emptyList(), + List.of(randomAlphaOfLength(10), randomAlphaOfLength(10)) + ); + } +} \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/ListIOCsRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/ListIOCsRestApiIT.java index 68c8ca0a1..331f9d11d 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/ListIOCsRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/ListIOCsRestApiIT.java @@ -143,9 +143,9 @@ public void test_retrievesIOCs() throws IOException { Instant.parse((String) hit.get(STIX2IOC.MODIFIED_FIELD)), (String) hit.get(STIX2IOC.DESCRIPTION_FIELD), (List) hit.get(STIX2IOC.LABELS_FIELD), + (String) hit.get(STIX2IOC.SPEC_VERSION_FIELD), (String) hit.get(STIX2IOC.FEED_ID_FIELD), (String) hit.get(STIX2IOC.FEED_NAME_FIELD), - (String) hit.get(STIX2IOC.SPEC_VERSION_FIELD), Long.parseLong(String.valueOf(hit.get(STIX2IOC.VERSION_FIELD))) // TODO implement DetailedSTIX2IOCDto.NUM_FINDINGS_FIELD check when GetFindings API is added ); diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelMonitorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelMonitorRestApiIT.java index 78036373f..f6c490bb5 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelMonitorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelMonitorRestApiIT.java @@ -6,11 +6,25 @@ import org.apache.logging.log4j.Logger; import org.junit.Assert; import org.opensearch.client.Response; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.commons.alerting.model.IntervalSchedule; import org.opensearch.commons.alerting.model.Monitor; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.search.SearchHit; import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; import org.opensearch.securityanalytics.SecurityAnalyticsRestTestCase; +import org.opensearch.securityanalytics.commons.model.IOCType; +import org.opensearch.securityanalytics.model.STIX2IOC; +import org.opensearch.securityanalytics.threatIntel.common.RefreshType; +import org.opensearch.securityanalytics.threatIntel.common.SourceConfigType; +import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; +import org.opensearch.securityanalytics.threatIntel.iocscan.dao.ThreatIntelAlertService; import org.opensearch.securityanalytics.threatIntel.iocscan.dto.PerIocTypeScanInputDto; +import org.opensearch.securityanalytics.threatIntel.model.DefaultIocStoreConfig; +import org.opensearch.securityanalytics.threatIntel.model.S3Source; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; import org.opensearch.securityanalytics.threatIntel.sacommons.monitor.ThreatIntelMonitorDto; +import org.opensearch.securityanalytics.threatIntel.sacommons.monitor.ThreatIntelTriggerDto; import java.io.IOException; import java.time.Instant; @@ -20,6 +34,7 @@ import java.util.List; import java.util.Map; +import static java.util.Collections.emptyList; import static org.opensearch.securityanalytics.TestHelpers.randomIndex; import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping; import static org.opensearch.securityanalytics.threatIntel.resthandler.monitor.RestSearchThreatIntelMonitorAction.SEARCH_THREAT_INTEL_MONITOR_PATH; @@ -27,7 +42,72 @@ public class ThreatIntelMonitorRestApiIT extends SecurityAnalyticsRestTestCase { private static final Logger log = LogManager.getLogger(ThreatIntelMonitorRestApiIT.class); + public void indexSourceConfigsAndIocs(int num, List iocVals) throws IOException { + for (int i = 0; i < num; i++) { + String configId = "id" + i; + String iocIndexName = ".opensearch-sap-ioc-" + configId; + indexTifSourceConfig(num, configId, iocIndexName, i); + for (int i1 = 0; i1 < iocVals.size(); i1++) { + indexIocs(iocVals, iocIndexName, i1, configId); + } + } + } + + private void indexIocs(List iocVals, String iocIndexName, int i1, String configId) throws IOException { + String iocId = iocIndexName + i1; + STIX2IOC stix2IOC = new STIX2IOC( + iocId, + "random", + IOCType.ip, + iocVals.get(i1), + "", + Instant.now(), + Instant.now(), + "", + emptyList(), + "spec", + configId, + "", + STIX2IOC.NO_VERSION + ); + indexDoc(iocIndexName, iocId, stix2IOC.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS).toString()); + List searchHits = executeSearch(iocIndexName, getMatchAllSearchRequestString(iocVals.size())); + assertEquals(searchHits.size(), i1 + 1); + } + + private void indexTifSourceConfig(int num, String configId, String iocIndexName, int i) throws IOException { + SATIFSourceConfig config = new SATIFSourceConfig( + configId, + SATIFSourceConfig.NO_VERSION, + "name1", + "STIX2", + SourceConfigType.S3_CUSTOM, + "description", + null, + Instant.now(), + new S3Source("bucketname", "key", "region", "roleArn"), + null, + Instant.now(), + new org.opensearch.jobscheduler.spi.schedule.IntervalSchedule(Instant.now(), 1, ChronoUnit.MINUTES), + TIFJobState.AVAILABLE, + RefreshType.FULL, + null, + null, + false, + new DefaultIocStoreConfig(Map.of("ip", List.of(iocIndexName))), + List.of("ip") + ); + String indexName = SecurityAnalyticsPlugin.JOB_INDEX_NAME; + Response response = indexDoc(indexName, configId, config.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS).toString()); + } + public void testCreateThreatIntelMonitor() throws IOException { + Response iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search", + Map.of(), null); + Map responseAsMap = responseAsMap(iocFindingsResponse); + Assert.assertEquals(0, ((List>) responseAsMap.get("ioc_findings")).size()); + List vals = List.of("ip1", "ip2"); + indexSourceConfigsAndIocs(1, vals); String index = createTestIndex(randomIndex(), windowsIndexMapping()); String monitorName = "test_monitor_name"; @@ -41,17 +121,21 @@ public void testCreateThreatIntelMonitor() throws IOException { Response alertingMonitorResponse = getAlertingMonitor(client(), monitorId); Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode()); + int i = 1; + for (String val : vals) { + String doc = String.format("{\"ip\":\"%s\", \"ip1\":\"%s\"}", val, val); + try { + indexDoc(index, "" + i++, doc); + } catch (IOException e) { + fail(); + } + } Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); Map executeResults = entityAsMap(executeResponse); assertEquals(1, 1); - String matchAllRequest = "{\n" + - " \"query\" : {\n" + - " \"match_all\":{\n" + - " }\n" + - " }\n" + - "}"; + String matchAllRequest = getMatchAllRequest(); Response searchMonitorResponse = makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, ContentType.APPLICATION_JSON, false)); Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode()); HashMap hits = (HashMap) asMap(searchMonitorResponse).get("hits"); @@ -60,6 +144,39 @@ public void testCreateThreatIntelMonitor() throws IOException { assertEquals(totalHitsVal.intValue(), 1); makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, ContentType.APPLICATION_JSON, false)); + + iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search", + Map.of(), null); + responseAsMap = responseAsMap(iocFindingsResponse); + Assert.assertEquals(2, ((List>) responseAsMap.get("ioc_findings")).size()); + + //alerts + List searchHits = executeSearch(ThreatIntelAlertService.THREAT_INTEL_ALERT_ALIAS_NAME, matchAllRequest); + Assert.assertEquals(4, searchHits.size()); + + for (String val : vals) { + String doc = String.format("{\"ip\":\"%s\", \"ip1\":\"%s\"}", val, val); + try { + indexDoc(index, "" + i++, doc); + } catch (IOException e) { + fail(); + } + } + executeAlertingMonitor(monitorId, Collections.emptyMap()); + iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search", + Map.of(), null); + responseAsMap = responseAsMap(iocFindingsResponse); + Assert.assertEquals(4, ((List>) responseAsMap.get("ioc_findings")).size()); + //alerts via system index search + searchHits = executeSearch(ThreatIntelAlertService.THREAT_INTEL_ALERT_ALIAS_NAME, matchAllRequest); + Assert.assertEquals(4, searchHits.size()); + + // alerts via API + Map params = new HashMap<>(); + Response getAlertsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_ALERTS_URI, params, null); + Map getAlertsBody = asMap(getAlertsResponse); + Assert.assertEquals(4, getAlertsBody.get("total_alerts")); + //delete Response delete = makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI + "/" + monitorId, Collections.emptyMap(), null); Assert.assertEquals(200, delete.getStatusLine().getStatusCode()); @@ -69,16 +186,33 @@ public void testCreateThreatIntelMonitor() throws IOException { totalHits = (HashMap) hits.get("total"); totalHitsVal = (Integer) totalHits.get("value"); assertEquals(totalHitsVal.intValue(), 0); + + + } + + public static String getMatchAllRequest() { + return "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; } private ThreatIntelMonitorDto randomIocScanMonitorDto(String index) { + ThreatIntelTriggerDto t1 = new ThreatIntelTriggerDto(List.of(index, "randomIndex"), List.of("ip", "domain"), emptyList(), "match", null, "severity"); + ThreatIntelTriggerDto t2 = new ThreatIntelTriggerDto(List.of("randomIndex"), List.of("domain"), emptyList(), "nomatch", null, "severity"); + ThreatIntelTriggerDto t3 = new ThreatIntelTriggerDto(emptyList(), List.of("domain"), emptyList(), "domainmatchsonomatch", null, "severity"); + ThreatIntelTriggerDto t4 = new ThreatIntelTriggerDto(List.of(index), emptyList(), emptyList(), "indexmatch", null, "severity"); + return new ThreatIntelMonitorDto( Monitor.NO_ID, randomAlphaOfLength(10), - List.of(new PerIocTypeScanInputDto("IP", Map.of(index, List.of("abc")))), - new org.opensearch.commons.alerting.model.IntervalSchedule(1, ChronoUnit.MINUTES, Instant.now()), - true, - null , Collections.emptyList()); + List.of(new PerIocTypeScanInputDto("IP", Map.of(index, List.of("ip")))), + new IntervalSchedule(1, ChronoUnit.MINUTES, Instant.now()), + false, + null, + List.of(t1, t2, t3, t4)); } } diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocFindingServiceRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocFindingServiceRestApiIT.java index 5c66d50bb..0c2225323 100644 --- a/src/test/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocFindingServiceRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocFindingServiceRestApiIT.java @@ -8,8 +8,8 @@ import org.opensearch.client.Response; import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; import org.opensearch.securityanalytics.SecurityAnalyticsRestTestCase; -import org.opensearch.securityanalytics.model.IocFinding; -import org.opensearch.securityanalytics.model.IocWithFeeds; +import org.opensearch.securityanalytics.model.threatintel.IocFinding; +import org.opensearch.securityanalytics.model.threatintel.IocWithFeeds; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/model/monitor/ThreatIntelInputTests.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/model/monitor/ThreatIntelInputTests.java index 925df5e15..b965cf0d9 100644 --- a/src/test/java/org/opensearch/securityanalytics/threatIntel/model/monitor/ThreatIntelInputTests.java +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/model/monitor/ThreatIntelInputTests.java @@ -25,7 +25,7 @@ import static java.util.Collections.emptyList; import static java.util.Collections.emptyMap; -import static org.opensearch.securityanalytics.threatIntel.model.monitor.SampleRemoteDocLevelMonitorRunner.THREAT_INTEL_MONITOR_TYPE; +import static org.opensearch.securityanalytics.threatIntel.iocscan.service.ThreatIntelMonitorRunner.THREAT_INTEL_MONITOR_TYPE; public class ThreatIntelInputTests extends OpenSearchTestCase {