diff --git a/CHANGELOG.md b/CHANGELOG.md index 516a80ef..f0328c19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,18 +15,24 @@ ## Breaking changes - [VictorOps] Changed `state_message` and `entity_display_name` values to be taken from an alert rule. - [#329](https://github.com/jertel/elastalert2/pull/329) - @ChristophShyper - Potentially a breaking change if the alert subject changes due to the new default behavior. +- Change metric/percentage rule types to store query_key as dict, instead of string, for consistency with other rule types. [#340](https://github.com/jertel/elastalert2/issues/340) - @AntoineBlaud ## New features - None ## Other changes +- [Tests] Improve test code coverage - [#331](https://github.com/jertel/elastalert2/pull/331) - @nsano-rururu +- [Docs] Upgrade Sphinx from 4.0.2 to 4.1.1 - [#344](https://github.com/jertel/elastalert2/pull/344) - @nsano-rururu +- [Tests] Upgrade Tox from 3.23.1 to 3.24.0 - [#345](https://github.com/jertel/elastalert2/pull/345) - @nsano-rururu +- Upgrade Jinja from 2.11.3 to 3.0.1 - [#350](https://github.com/jertel/elastalert2/pull/350) - @mrfroggg - [Tests] Add test code. Changed ubuntu version of Dockerfile-test from latest to 21.10. - [#354](https://github.com/jertel/elastalert2/pull/354) - @nsano-rururu - [Tests] Improved test coverage for opsgenie.py 96% to 100% - [#364](https://github.com/jertel/elastalert2/pull/364) - @nsano-rururu - Remove Python 2.x compatibility code - [#354](https://github.com/jertel/elastalert2/pull/354) - @nsano-rururu - [Docs] Added Chatwork proxy settings to documentation - [#360](https://github.com/jertel/elastalert2/pull/360) - @nsano-rururu +- Add settings to schema.yaml(Chatwork proxy, Dingtalk proxy) - [#361](https://github.com/jertel/elastalert2/pull/361) - @nsano-rururu - [Docs] Tidy Twilio alerter documentation - [#363](https://github.com/jertel/elastalert2/pull/363) - @ferozsalam +- [Tests] Improve opsgenie test code coverage - [#364](https://github.com/jertel/elastalert2/pull/364) - @nsano-rururu - [Docs] Update mentions of JIRA to Jira - [#365](https://github.com/jertel/elastalert2/pull/365) - @ferozsalam -- Add settings to schema.yaml(Chatwork proxy, Dingtalk proxy) - [#361](https://github.com/jertel/elastalert2/pull/361) - @nsano-rururu # 2.1.2 ## Breaking changes diff --git a/elastalert/ruletypes.py b/elastalert/ruletypes.py index fce33f50..08d02b98 100644 --- a/elastalert/ruletypes.py +++ b/elastalert/ruletypes.py @@ -7,7 +7,7 @@ from elastalert.util import (add_raw_postfix, dt_to_ts, EAException, elastalert_logger, elasticsearch_client, format_index, hashable, lookup_es_key, new_get_event_ts, pretty_ts, total_seconds, - ts_now, ts_to_dt) + ts_now, ts_to_dt, expand_string_into_dict) class RuleType(object): @@ -1096,7 +1096,7 @@ def check_matches(self, timestamp, query_key, aggregation_data): match = {self.rules['timestamp_field']: timestamp, self.metric_key: metric_val} if query_key is not None: - match[self.rules['query_key']] = query_key + match = expand_string_into_dict(match, self.rules['query_key'], query_key) self.add_match(match) def check_matches_recursive(self, timestamp, query_key, aggregation_data, compound_keys, match_data): @@ -1286,7 +1286,7 @@ def check_matches(self, timestamp, query_key, aggregation_data): if self.percentage_violation(match_percentage): match = {self.rules['timestamp_field']: timestamp, 'percentage': match_percentage, 'denominator': total_count} if query_key is not None: - match[self.rules['query_key']] = query_key + match = expand_string_into_dict(match, self.rules['query_key'], query_key) self.add_match(match) def percentage_violation(self, match_percentage): diff --git a/elastalert/util.py b/elastalert/util.py index ea50fd52..82d84888 100644 --- a/elastalert/util.py +++ b/elastalert/util.py @@ -481,3 +481,40 @@ def should_scrolling_continue(rule_conf): stop_the_scroll = 0 < max_scrolling <= rule_conf.get('scrolling_cycle') return not stop_the_scroll + + +def _expand_string_into_dict(string, value, sep='.'): + """ + Converts a encapsulated string-dict to a sequence of dict. Use separator (default '.') to split the string. + Example: + string1.string2.stringN : value -> {string1: {string2: {string3: value}} + + :param string: The encapsulated "string-dict" + :param value: Value associated to the last field of the "string-dict" + :param sep: Separator character. Default: '.' + :rtype: dict + """ + if sep not in string: + return {string : value} + key, val = string.split(sep, 1) + return {key: _expand_string_into_dict(val, value)} + + +def expand_string_into_dict(dictionary, string , value, sep='.'): + """ + Useful function to "compile" a string-dict string used in metric and percentage rules into a dictionary sequence. + + :param dictionary: The dictionary dict + :param string: String Key + :param value: String Value + :param sep: Separator character. Default: '.' + :rtype: dict + """ + + if sep not in string: + dictionary[string] = value + return dictionary + else: + field1, new_string = string.split(sep, 1) + dictionary[field1] = _expand_string_into_dict(new_string, value) + return dictionary diff --git a/tests/rules_test.py b/tests/rules_test.py index 8e1401d3..0683b4fa 100644 --- a/tests/rules_test.py +++ b/tests/rules_test.py @@ -1163,10 +1163,15 @@ def test_metric_aggregation(): rule.check_matches(datetime.datetime.now(), None, {'metric_cpu_pct_avg': {'value': 0.95}}) assert len(rule.matches) == 2 - rules['query_key'] = 'qk' + rules['query_key'] = 'subdict' rule = MetricAggregationRule(rules) rule.check_matches(datetime.datetime.now(), 'qk_val', {'metric_cpu_pct_avg': {'value': 0.95}}) - assert rule.matches[0]['qk'] == 'qk_val' + assert rule.matches[0]['subdict'] == 'qk_val' + + rules['query_key'] = 'subdict1.subdict2.subdict3' + rule = MetricAggregationRule(rules) + rule.check_matches(datetime.datetime.now(), 'qk_val', {'metric_cpu_pct_avg': {'value': 0.95}}) + assert rule.matches[0]['subdict1']['subdict2']['subdict3'] == 'qk_val' def test_metric_aggregation_complex_query_key(): @@ -1279,6 +1284,12 @@ def test_percentage_match(): rule = PercentageMatchRule(rules) rule.check_matches(datetime.datetime.now(), 'qk_val', create_percentage_match_agg(76.666666667, 24)) assert rule.matches[0]['qk'] == 'qk_val' + + rules['query_key'] = 'subdict1.subdict2' + rule = PercentageMatchRule(rules) + rule.check_matches(datetime.datetime.now(), 'qk_val', create_percentage_match_agg(76.666666667, 24)) + assert rule.matches[0]['subdict1']['subdict2'] == 'qk_val' + assert '76.1589403974' in rule.get_match_str(rule.matches[0]) rules['percentage_format_string'] = '%.2f' assert '76.16' in rule.get_match_str(rule.matches[0]) diff --git a/tests/util_test.py b/tests/util_test.py index 5727a3d9..89090de2 100644 --- a/tests/util_test.py +++ b/tests/util_test.py @@ -33,6 +33,7 @@ from elastalert.util import total_seconds from elastalert.util import ts_to_dt_with_format from elastalert.util import ts_utc_to_tz +from elastalert.util import expand_string_into_dict from elastalert.util import unixms_to_dt @@ -460,6 +461,15 @@ def test_elasticsearch_client(es_host, es_port, es_bearer, es_api_key): assert None is not acutual +def test_expand_string_into_dict(): + dictionnary = {'@timestamp': '2021-07-06 01:00:00', 'metric_netfilter.ipv4_dst_cardinality': 401} + string = 'metadata.source.ip' + value = '0.0.0.0' + + expand_string_into_dict(dictionnary, string, value) + assert dictionnary['metadata']['source']['ip'] == value + + def test_inc_ts(): dt = datetime(2021, 7, 6, hour=0, minute=0, second=0) actual = inc_ts(dt)