diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d71e1a1..bc7299c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ - TBD ## New features -- TBD +- Add indexer alerter - [#1451](https://github.com/jertel/elastalert2/pull/1451) - @olehpalanskyi ## Other changes - [Docs] Fixed typo in Alerta docs with incorrect number of seconds in a day. - @jertel diff --git a/docs/source/alerts.rst b/docs/source/alerts.rst index cdbc5efb..c8a15d13 100644 --- a/docs/source/alerts.rst +++ b/docs/source/alerts.rst @@ -28,6 +28,7 @@ or - googlechat - gelf - hivealerter + - indexer - iris - jira - lark @@ -49,7 +50,7 @@ or - tencent_sms - twilio - victorops - - workwechat + - workwechat - zabbix Options for each alerter can either defined at the top level of the YAML file, or nested within the alert name, allowing for different settings @@ -1085,8 +1086,91 @@ Example usage with json string formatting:: "X-custom-{{key}}": "{{type}}" } +Indexer +~~~~~~~ + +Description: Creates a record in an arbitrary index within an Elasticsearch or OpenSearch index. + +Indexer alerter can be used to create a new alert in existing Opensearch/Elasticsearch. The alerter supports +custom fields, and observables from the alert matches and rule data. + +Required: + +``indexer_alert_config``: Configuration options for the alert, see example below for structure. + +``customFields`` Fields must be manually added, all of them will exist in the newly created index. You can set own field or use existing field fron match (see example below for structure). + +``indexer_alerts_name``: The index to use for creating the new alert records. + +One of below is required: + +``indexer_connection``: Options the connection details to your server instance (see example below for the required syntax Example 1). + +``indexer_config``: Options for loading the connection details to your server instance from a file (see example below for the required syntax Example 2). + + +Example 1 usage:: + + alert: indexer + + indexer_connection: + es_host: localhost + es_port: es_port + ssl_show_warn: False + use_ssl: True + verify_certs: False + es_username: user + es_password: password + indexer_alerts_name: elastalert2 # You can create own config or use global config just added ``indexer_alerts_name`` in global config + + indexer_alert_config: + #Existing fields from match alert + message: message + host.name: host.name + event.action: event.action + event.type: event.type + winlog.computer_name: winlog.computer_name + winlog.event_id: winlog.event_id + winlog.task: winlog.task + #Enrich existing event with additional fields + customFields: + - name: original_time + value: "@timestamp" + - name: severity + value: high + - name: risk_score + value: 73 + - name: description + value: General description. + +Example 2 usage:: + + alert: indexer + + indexer_config: /opt/elastalert/config/config.yaml # Uses the ElastAlert 2 global config, with an added ``indexer_alerts_name`` parameter + + indexer_alert_config: + #Existing fields from match alert + message: message + host.name: host.name + event.action: event.action + event.type: event.type + winlog.computer_name: winlog.computer_name + winlog.event_id: winlog.event_id + winlog.task: winlog.task + #Enrich existing event with additional fields + customFields: + - name: original_time + value: "@timestamp" + - name: severity + value: high + - name: risk_score + value: 73 + - name: description + value: General description. + IRIS -~~~~~~~~~ +~~~~ The Iris alerter can be used to create a new alert or case in `Iris IRP System `_. The alerter supports adding tags, IOCs, and context from the alert matches and rule data. The alerter requires the following option: diff --git a/docs/source/elastalert.rst b/docs/source/elastalert.rst index d5aae3d2..548de232 100755 --- a/docs/source/elastalert.rst +++ b/docs/source/elastalert.rst @@ -44,6 +44,7 @@ Currently, we have support built in for these alert types: - Graylog GELF - HTTP POST - HTTP POST 2 +- Indexer - Iris - Jira - Lark diff --git a/elastalert/alerters/indexer.py b/elastalert/alerters/indexer.py new file mode 100644 index 00000000..599a4446 --- /dev/null +++ b/elastalert/alerters/indexer.py @@ -0,0 +1,113 @@ +import os +import yaml +from datetime import datetime +from elasticsearch.exceptions import TransportError +from elastalert.alerts import Alerter +from elastalert.util import lookup_es_key, EAException, elastalert_logger, elasticsearch_client + +class IndexerAlerter(Alerter): + """ + Use matched data to create alerts on Opensearch/Elasticsearch + """ + required_options = frozenset(['indexer_alert_config']) + + def lookup_field(self, match: dict, field_name: str, default): + field_value = lookup_es_key(match, field_name) + if field_value is None: + field_value = self.rule.get(field_name, default) + + return field_value + + def get_query(self,body_request_raw): + original = body_request_raw[0] + for orig in original.values(): + for query_string in orig.values(): + query = query_string + return query['query'] + + def lookup_list_fields(self, original_fields_raw: list, match: dict): + original_fields = {} + for field in original_fields_raw: + if field.get('value'): + if (isinstance(field['value'], str)): + if field['value'] == 'filter': + body_request_raw = self.rule.get(field['value']) + value = self.get_query(body_request_raw) + else: + value = self.lookup_field(match, field['value'], field['value']) + else: + value = field['value'] + original_fields[field['name']] = value + else: + for k,v in field.items(): + original_fields[k] = self.lookup_list_fields(v, match) + + return original_fields + + def event_orig_fields(self, original_fields_raw, match: dict): + if (isinstance(original_fields_raw, str)): + value = self.lookup_field(match, original_fields_raw, original_fields_raw) + elif (isinstance(original_fields_raw, list)): + value = self.lookup_list_fields(original_fields_raw, match) + else: + value = original_fields_raw + return value + + def make_nested_fields(self, data): + nested_data = {} + for key, value in data.items(): + keys = key.split(".") + current_nested_data = nested_data + for nested_key in keys[:-1]: + current_nested_data = current_nested_data.setdefault(nested_key, {}) + current_nested_data[keys[-1]] = value + return nested_data + + def flatten_dict(self, data, prefix='', sep='.'): + nd = {} + for k, v in data.items(): + if isinstance(v, dict): + nd.update(self.flatten_dict(v, f'{prefix}{k}{sep}')) + else: + nd[f'{prefix}{k}'] = v + return nd + + def remove_matching_pairs(self, input_dict): + return {key: value for key, value in input_dict.items() if key != value} + + def alert(self, matches): + alert_config = { + '@timestamp': datetime.now().strftime('%Y-%m-%dT%H:%M:%S.%fZ') + } + alert_config.update(self.rule.get('indexer_alert_config', {})) + + if len(matches) > 0: + alert_config = self.flatten_dict(alert_config) + for event_orig in alert_config: + alert_config[event_orig] = self.event_orig_fields(alert_config[event_orig],matches[0]) + alert_config = self.remove_matching_pairs(self.flatten_dict(alert_config)) + alert_config = self.make_nested_fields(alert_config) + + + # POST the alert to SIEM + try: + data = self.rule.get('indexer_connection', '') + if not data: + if os.path.isfile(self.rule.get('indexer_config', '')): + filename = self.rule.get('indexer_config', '') + else: + filename = '' + + if filename: + with open(filename) as config_file: + data = yaml.load(config_file, Loader=yaml.FullLoader) + elasticsearch_client(data).index(index = data.get('indexer_alerts_name'), + body = alert_config, + refresh = True) + + except TransportError as e: + raise EAException(f"Error posting to SIEM: {e}") + elastalert_logger.info("Alert sent to SIEM") + + def get_info(self): + return {'type': 'indexer'} diff --git a/elastalert/loaders.py b/elastalert/loaders.py index 1bd18aab..599de406 100644 --- a/elastalert/loaders.py +++ b/elastalert/loaders.py @@ -52,6 +52,7 @@ from elastalert.alerters.teams import MsTeamsAlerter from elastalert.alerters.zabbix import ZabbixAlerter from elastalert.alerters.tencentsms import TencentSMSAlerter +from elastalert.alerters.indexer import IndexerAlerter from elastalert.util import dt_to_ts from elastalert.util import dt_to_ts_with_format from elastalert.util import dt_to_unix @@ -137,6 +138,7 @@ class RulesLoader(object): 'rocketchat': elastalert.alerters.rocketchat.RocketChatAlerter, 'gelf': elastalert.alerters.gelf.GelfAlerter, 'iris': elastalert.alerters.iris.IrisAlerter, + 'indexer': IndexerAlerter, } # A partial ordering of alert types. Relative order will be preserved in the resulting alerts list diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index 365363df..f974ec4b 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -538,6 +538,35 @@ properties: http_post2_ignore_ssl_errors: {type: boolean} http_post2_timeout: {type: integer} + ### INDEXER + indexer_alerts_name: {type: string} + indexer_config: {type: string} + indexer_connection: + type: object + properties: + es_host: {type: string} + es_hosts: {type: array, items: {type: string}} + es_port: {type: integer} + ssl_show_warn: {type: boolean} + use_ssl: {type: boolean} + verify_certs: {type: boolean} + ca_cert: {type: string} + es_username: {type: string} + es_password: {type: string} + index_alerts_name: {type: string} + indexer_alert_config: + type: object + minProperties: 1 + patternProperties: + "^.+$": + oneOf: + - type: [boolean, string, integer] + - type: object + additionalProperties: false + required: [ field ] + properties: + field: { type: [boolean, string, integer], minLength: 1 } + ### IRIS iris_host: {type: string} iris_api_token: {type: string} diff --git a/tests/alerters/indexer_test.py b/tests/alerters/indexer_test.py new file mode 100644 index 00000000..3c6f747a --- /dev/null +++ b/tests/alerters/indexer_test.py @@ -0,0 +1,362 @@ +import logging +from unittest import mock +import pytest +from elasticsearch.exceptions import TransportError +from elastalert.util import EAException +from elastalert.loaders import FileRulesLoader +from elastalert.alerters.indexer import IndexerAlerter + + +def rule_config(): + return { + 'alert': [], + 'name': 'test-alert', + 'index': 'my-index', + 'filter': [{'key': {'query': {'query': 'test_query'}}}], + 'description': 'test', + 'indexer_connection': { + 'es_host': 'localhost', + 'es_port': 9200, + 'indexer_alerts_name': 'test_index' + }, + 'indexer_alert_config': { + 'get_index': 'index', + 'get_type': 'type', + 'get_field1': 'field1', + 'get_field2': 'field2', + '@timestamp': '@timestamp' + }, + 'type': 'any' + } + + +def test_indexer_alerter(caplog): + + caplog.set_level(logging.INFO) + rule = rule_config() + + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = IndexerAlerter(rule) + + match = { + 'index': 'test-index', + 'type': 'test-type', + 'field1': 'value1', + 'field2': 'value2', + '@timestamp': '2021-05-09T14:43:30' + } + + with mock.patch('elasticsearch.Elasticsearch.index') as mock_create: + alert.alert([match]) + + expected_data = { + 'get_index': 'test-index', + 'get_type': 'test-type', + 'get_field1': 'value1', + 'get_field2': 'value2', + '@timestamp': '2021-05-09T14:43:30' + } + + mock_create.assert_called_once_with( + index='test_index', body=mock.ANY, refresh=True + ) + actual_data = mock_create.call_args_list[0][1]['body'] + assert expected_data == actual_data + assert ('elastalert', logging.INFO, 'Alert sent to SIEM') == caplog.record_tuples[0] + + +def test_alert_with_file_config(): + + rule = rule_config() + rule.pop('indexer_connection') + rule['indexer_config'] = 'config.yaml' + + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = IndexerAlerter(rule) + + match = { + 'index': 'test-index', + 'type': 'test-type', + 'field1': 'value1', + 'field2': 'value2', + '@timestamp': '2021-05-09T14:43:30' + } + + with mock.patch('elasticsearch.Elasticsearch.index') as mock_create, \ + mock.patch('os.path.isfile', return_value=True), \ + mock.patch('builtins.open', new_callable=mock.mock_open, + read_data='indexer_connection:\n es_host: localhost\n es_port: 9200\n indexer_alerts_name: test_index'), \ + mock.patch('yaml.load', return_value={'es_host': 'localhost', 'es_port': 9200, 'indexer_alerts_name': 'test_index'}): + alert.alert([match]) + + expected_data = { + 'get_index': 'test-index', + 'get_type': 'test-type', + 'get_field1': 'value1', + 'get_field2': 'value2', + '@timestamp': '2021-05-09T14:43:30' + } + + mock_create.assert_called_once_with( + index='test_index', body=mock.ANY, refresh=True + ) + + actual_data = mock_create.call_args_list[0][1]['body'] + assert expected_data == actual_data + + +def test_alert_with_transport_error(): + + with pytest.raises(EAException) as ea: + rule = rule_config() + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = IndexerAlerter(rule) + match = { + '@timestamp': '2021-01-01T00:00:00', + 'somefield': 'foobarbaz' + } + + mock_run = mock.MagicMock(side_effect=TransportError(500, "Error creating index")) + # Mocking the Elasticsearch create method to raise TransportError + with mock.patch('elasticsearch.Elasticsearch.index', mock_run), pytest.raises(TransportError): + alert.alert([match]) + + assert "Error posting to SIEM" in str(ea) + + +def test_get_query(): + + body_request_raw = [{'key': {'query': {'query': 'test_query'}}}] + rule = rule_config() + + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = IndexerAlerter(rule) + expected_data = { + 'get_query': 'test_query' + } + + actual_data = {'get_query': alert.get_query(body_request_raw)} + assert expected_data == actual_data + + +def test_lookup_field(): + + rule = rule_config() + + match = {'field1': 'some important'} + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = IndexerAlerter(rule) + expected_data1 = { + 'get_field1': 'some important', + } + expected_data2 = { + 'get_field1': 'field2' + } + + actual_data = {'get_field1': alert.lookup_field(match, expected_data1['get_field1'], expected_data1['get_field1'])} + assert expected_data1 == actual_data + actual_data = {'get_field1': alert.lookup_field(match, expected_data2['get_field1'], expected_data2['get_field1'])} + assert expected_data2 == actual_data + + +def test_lookup_list_fields(): + + rule = rule_config() + match = {'field1': 'value1'} + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = IndexerAlerter(rule) + + # Test simple case with direct value lookup + original_fields_raw = [{'name': 'field1', 'value': 'field1'}] + expected_data = {'field1': 'value1'} + actual_data = alert.lookup_list_fields(original_fields_raw, match) + assert expected_data == actual_data + + # Test simple case with direct value lookup not str + original_fields_raw = [{'name': 'field1', 'value': 123}] + expected_data = {'field1': 123} + actual_data = alert.lookup_list_fields(original_fields_raw, match) + assert expected_data == actual_data + + # Test with 'filter' keyword + original_fields_raw = [{'name': 'query', 'value': 'filter'}] + expected_data = {'query': 'test_query'} + actual_data = alert.lookup_list_fields(original_fields_raw, match) + assert actual_data == expected_data + + original_fields_raw = [{'name': 'query', 'value': 'filter'}] + expected_data = {'query': 'test_query'} + actual_data = alert.lookup_list_fields(original_fields_raw, match) + assert actual_data == expected_data + + original_fields_raw = [{'test_event_data': [{'name': 'test', 'value': 'test_event'}]}] + expected_data = {'test_event_data': {'test': 'test_event'}} + actual_data = alert.lookup_list_fields(original_fields_raw, match) + assert actual_data == expected_data + + +def test_event_orig_fields(): + + rule = rule_config() + match = {'field1': 'value1'} + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = IndexerAlerter(rule) + + # test if (isinstance(original_fields_raw, str)) + expected_data = {'field1': 'value1'} + actual_data = {'field1': alert.event_orig_fields('field1', match)} + assert expected_data == actual_data + + # test elif (isinstance(original_fields_raw, list)) + list_data = [{'name': 'field1', 'value': 'value1'}] + expected_data = {'field1': 'value1'} + actual_data = alert.event_orig_fields(list_data, match) + assert expected_data == actual_data + + # test else not str or list + expected_data = {'test_data': 10} + actual_data = alert.event_orig_fields(expected_data, match) + assert expected_data == actual_data + + +def test_make_nested_fields(): + + rule = rule_config() + data = { + 'a.b.c': 1, 'a.b.d': 2, 'e': 3 + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = IndexerAlerter(rule) + expected_data = {'a': {'b': {'c': 1, 'd': 2}}, 'e': 3} + actual_data = alert.make_nested_fields(data) + assert expected_data == actual_data + + +def test_flatten_dict(): + + rule = rule_config() + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = IndexerAlerter(rule) + data = {'a': {'b': {'c': 1, 'd': 2}}, 'e': 3} + expected_data = {'a.b.c': 1, 'a.b.d': 2, 'e': 3} + actual_data = alert.flatten_dict(data) + assert expected_data == actual_data + + +def test_remove_matching_pairs(): + + rule = rule_config() + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = IndexerAlerter(rule) + data = {'a': 'a', 'b': 'c'} + expected_data = {'b': 'c'} + actual_data = alert.remove_matching_pairs(data) + assert actual_data == expected_data + + +def test_indexer_getinfo(): + + rule = rule_config() + + alert = IndexerAlerter(rule) + expected_data = { + 'type': 'indexer' + } + actual_data = alert.get_info() + assert expected_data == actual_data + + +def test_alert_with_matches(): + + rule = rule_config() + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = IndexerAlerter(rule) + + match = {'field1': 'value1'} + alert_config = { + '@timestamp': '2021-01-01T00:00:00', + 'key1': 'value1', + 'key2': 'value2' + } + + with mock.patch('elasticsearch.Elasticsearch.index') as mock_create, \ + mock.patch.object(alert, 'flatten_dict', return_value=alert_config) as mock_flatten, \ + mock.patch.object(alert, 'event_orig_fields', side_effect=lambda x, y: f"processed_{y}") as mock_event_orig_fields, \ + mock.patch.object(alert, 'remove_matching_pairs', return_value=alert_config) as mock_remove_matching_pairs, \ + mock.patch.object(alert, 'make_nested_fields', return_value=alert_config) as mock_make_nested_fields: + + alert.alert([match]) + + mock_flatten.assert_called() + mock_event_orig_fields.assert_called() + mock_remove_matching_pairs.assert_called() + mock_make_nested_fields.assert_called() + + # Ensure that flatten_dict was called twice + expected_data = 2 + actual_data = mock_flatten.call_count + assert actual_data == expected_data + + # Check if event_orig_fields was called for each key in alert_config + expected_data = len(alert_config) + actual_data = mock_event_orig_fields.call_count + assert actual_data == expected_data + + # Verify if the transformed alert_config is passed to remove_matching_pairs and make_nested_fields + mock_remove_matching_pairs.assert_called_with(alert_config) + mock_make_nested_fields.assert_called_with(alert_config) + + mock_create.assert_called_once_with( + index='test_index', body=alert_config, refresh=True + ) + + +def test_alert_with_empty_matches(): + + rule = rule_config() + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = IndexerAlerter(rule) + match = [] + alert_config = { + '@timestamp': '2021-01-01T00:00:00', + 'key1': 'value1', + 'key2': 'value2' + } + with mock.patch('elasticsearch.Elasticsearch.index') as mock_create, \ + mock.patch.object(alert, 'flatten_dict', return_value=alert_config) as mock_flatten, \ + mock.patch.object(alert, 'remove_matching_pairs', return_value=alert_config) as mock_remove_matching_pairs, \ + mock.patch.object(alert, 'make_nested_fields', return_value=alert_config) as mock_make_nested_fields, \ + mock.patch.object(alert, 'event_orig_fields') as mock_event_orig_fields: + alert.alert(match) + + mock_flatten.assert_called() + mock_remove_matching_pairs.assert_called() + mock_make_nested_fields.assert_called() + mock_event_orig_fields.assert_not_called() + # Ensure that flatten_dict was called twice + expected_data = 1 + actual_data = mock_flatten.call_count + assert actual_data == expected_data + + expected_data = 0 + actual_data = mock_event_orig_fields.call_count + assert expected_data == actual_data + # Verify if the transformed alert_config is passed to remove_matching_pairs and make_nested_fields + mock_remove_matching_pairs.assert_called_with(alert_config) + mock_make_nested_fields.assert_called_with(alert_config) + + mock_create.assert_called_once_with( + index='test_index', body=alert_config, refresh=True + )