#340 fix match["query_key"] malformation

CHANGELOG.md

@@ -15,18 +15,24 @@
 ## Breaking changes
 - [VictorOps] Changed `state_message` and `entity_display_name` values to be taken from an alert rule. - [#329](https://github.com/jertel/elastalert2/pull/329) - @ChristophShyper
  - Potentially a breaking change if the alert subject changes due to the new default behavior.
+- Change metric/percentage rule types to store query_key as dict, instead of string, for consistency with other rule types. [#340](https://github.com/jertel/elastalert2/issues/340) - @AntoineBlaud
 
 ## New features
 - None
 
 ## Other changes
+- [Tests] Improve test code coverage - [#331](https://github.com/jertel/elastalert2/pull/331) - @nsano-rururu
+- [Docs] Upgrade Sphinx from 4.0.2 to 4.1.1 - [#344](https://github.com/jertel/elastalert2/pull/344) - @nsano-rururu
+- [Tests] Upgrade Tox from 3.23.1 to 3.24.0 - [#345](https://github.com/jertel/elastalert2/pull/345) - @nsano-rururu
+- Upgrade Jinja from 2.11.3 to 3.0.1 - [#350](https://github.com/jertel/elastalert2/pull/350) - @mrfroggg
 - [Tests] Add test code. Changed ubuntu version of Dockerfile-test from latest to 21.10. - [#354](https://github.com/jertel/elastalert2/pull/354) - @nsano-rururu
 - [Tests] Improved test coverage for opsgenie.py 96% to 100% - [#364](https://github.com/jertel/elastalert2/pull/364) - @nsano-rururu
 - Remove Python 2.x compatibility code - [#354](https://github.com/jertel/elastalert2/pull/354) - @nsano-rururu
 - [Docs] Added Chatwork proxy settings to documentation - [#360](https://github.com/jertel/elastalert2/pull/360) - @nsano-rururu
+- Add settings to schema.yaml(Chatwork proxy, Dingtalk proxy) - [#361](https://github.com/jertel/elastalert2/pull/361) - @nsano-rururu
 - [Docs] Tidy Twilio alerter documentation - [#363](https://github.com/jertel/elastalert2/pull/363) - @ferozsalam
+- [Tests] Improve opsgenie test code coverage - [#364](https://github.com/jertel/elastalert2/pull/364) - @nsano-rururu
 - [Docs] Update mentions of JIRA to Jira - [#365](https://github.com/jertel/elastalert2/pull/365) - @ferozsalam
-- Add settings to schema.yaml(Chatwork proxy, Dingtalk proxy) - [#361](https://github.com/jertel/elastalert2/pull/361) - @nsano-rururu
 
 # 2.1.2
 ## Breaking changes

elastalert/ruletypes.py

@@ -7,7 +7,7 @@
 
 from elastalert.util import (add_raw_postfix, dt_to_ts, EAException, elastalert_logger, elasticsearch_client,
  format_index, hashable, lookup_es_key, new_get_event_ts, pretty_ts, total_seconds,
- ts_now, ts_to_dt)
+ ts_now, ts_to_dt, expand_string_into_dict)
 
 
 class RuleType(object):
@@ -1096,7 +1096,7 @@ def check_matches(self, timestamp, query_key, aggregation_data):
  match = {self.rules['timestamp_field']: timestamp,
  self.metric_key: metric_val}
  if query_key is not None:
- match[self.rules['query_key']] = query_key
+ match = expand_string_into_dict(match, self.rules['query_key'], query_key)
  self.add_match(match)
 
  def check_matches_recursive(self, timestamp, query_key, aggregation_data, compound_keys, match_data):
@@ -1286,7 +1286,7 @@ def check_matches(self, timestamp, query_key, aggregation_data):
  if self.percentage_violation(match_percentage):
  match = {self.rules['timestamp_field']: timestamp, 'percentage': match_percentage, 'denominator': total_count}
  if query_key is not None:
- match[self.rules['query_key']] = query_key
+ match = expand_string_into_dict(match, self.rules['query_key'], query_key)
  self.add_match(match)
 
  def percentage_violation(self, match_percentage):

elastalert/util.py

@@ -481,3 +481,40 @@ def should_scrolling_continue(rule_conf):
  stop_the_scroll = 0 < max_scrolling <= rule_conf.get('scrolling_cycle')
 
  return not stop_the_scroll
+
+
+def _expand_string_into_dict(string, value, sep='.'):
+ """
+ Converts a encapsulated string-dict to a sequence of dict. Use separator (default '.') to split the string.
+ Example: 
+ string1.string2.stringN : value -> {string1: {string2: {string3: value}}
+ 
+ :param string: The encapsulated "string-dict"
+ :param value: Value associated to the last field of the "string-dict"
+ :param sep: Separator character. Default: '.'
+ :rtype: dict
+ """
+ if sep not in string:
+ return {string : value}
+ key, val = string.split(sep, 1)
+ return {key: _expand_string_into_dict(val, value)}
+
+
+def expand_string_into_dict(dictionary, string , value, sep='.'):
+ """
+ Useful function to "compile" a string-dict string used in metric and percentage rules into a dictionary sequence.
+ 
+ :param dictionary: The dictionary dict
+ :param string: String Key 
+ :param value: String Value
+ :param sep: Separator character. Default: '.'
+ :rtype: dict
+ """
+
+ if sep not in string:
+ dictionary[string] = value
+ return dictionary
+ else:
+ field1, new_string = string.split(sep, 1)
+ dictionary[field1] = _expand_string_into_dict(new_string, value)
+ return dictionary

tests/rules_test.py

@@ -1163,10 +1163,15 @@ def test_metric_aggregation():
  rule.check_matches(datetime.datetime.now(), None, {'metric_cpu_pct_avg': {'value': 0.95}})
  assert len(rule.matches) == 2
 
- rules['query_key'] = 'qk'
+ rules['query_key'] = 'subdict'
  rule = MetricAggregationRule(rules)
  rule.check_matches(datetime.datetime.now(), 'qk_val', {'metric_cpu_pct_avg': {'value': 0.95}})
- assert rule.matches[0]['qk'] == 'qk_val'
+ assert rule.matches[0]['subdict'] == 'qk_val'
+
+ rules['query_key'] = 'subdict1.subdict2.subdict3'
+ rule = MetricAggregationRule(rules)
+ rule.check_matches(datetime.datetime.now(), 'qk_val', {'metric_cpu_pct_avg': {'value': 0.95}})
+ assert rule.matches[0]['subdict1']['subdict2']['subdict3'] == 'qk_val'
 
 
 def test_metric_aggregation_complex_query_key():
@@ -1279,6 +1284,12 @@ def test_percentage_match():
  rule = PercentageMatchRule(rules)
  rule.check_matches(datetime.datetime.now(), 'qk_val', create_percentage_match_agg(76.666666667, 24))
  assert rule.matches[0]['qk'] == 'qk_val'
+
+ rules['query_key'] = 'subdict1.subdict2'
+ rule = PercentageMatchRule(rules)
+ rule.check_matches(datetime.datetime.now(), 'qk_val', create_percentage_match_agg(76.666666667, 24))
+ assert rule.matches[0]['subdict1']['subdict2'] == 'qk_val'
+
  assert '76.1589403974' in rule.get_match_str(rule.matches[0])
  rules['percentage_format_string'] = '%.2f'
  assert '76.16' in rule.get_match_str(rule.matches[0])

tests/util_test.py

@@ -33,6 +33,7 @@
 from elastalert.util import total_seconds
 from elastalert.util import ts_to_dt_with_format
 from elastalert.util import ts_utc_to_tz
+from elastalert.util import expand_string_into_dict
 from elastalert.util import unixms_to_dt
 
 
@@ -460,6 +461,15 @@ def test_elasticsearch_client(es_host, es_port, es_bearer, es_api_key):
  assert None is not acutual
 
 
+def test_expand_string_into_dict():
+ dictionnary = {'@timestamp': '2021-07-06 01:00:00', 'metric_netfilter.ipv4_dst_cardinality': 401}
+ string = 'metadata.source.ip'
+ value = '0.0.0.0'
+
+ expand_string_into_dict(dictionnary, string, value)
+ assert dictionnary['metadata']['source']['ip'] == value
+
+
 def test_inc_ts():
  dt = datetime(2021, 7, 6, hour=0, minute=0, second=0)
  actual = inc_ts(dt)