[Response Ops][Alerting] Adding group by options to ES query rule type

docs/user/alerting/rule-types/es-query.asciidoc

@@ -12,33 +12,41 @@ threshold condition is met.
 Fill in the <<defining-rules-general-details, rule details>>, then select 
 *{es} query*.
 
+[float]
+=== Select the query type
+
+{es} query rule can be defined using KQL/Lucene or Query DSL.
+
+[role="screenshot"]
+image::user/alerting/images/rule-types-es-query-query-type.png[Select KQL or DSL query type]
+
 
 [float]
 === Define the conditions
 
 Define properties to detect the condition.
 
 [role="screenshot"]
-image::user/alerting/images/rule-types-es-query-conditions.png[Six clauses define the condition to detect]
+image::user/alerting/images/rule-types-es-query-conditions.png[Eight clauses define the condition to detect]
 
 Index:: Specifies an *index or data view* and a *time field* that is used for 
 the *time window*.
-Size:: Specifies the number of documents to pass to the configured actions when 
-the threshold condition is met.
-{es} query:: Specifies the ES DSL query. The number of documents that 
-match this query is evaluated against the threshold condition. Only the `query`, `fields`, `_source` and `runtime_mappings`
-fields are used, other DSL fields are not considered. 
+{es} query:: Specifies the ES DSL query. Only the `query`, `fields`, `_source` and `runtime_mappings` fields are used, other DSL fields are not considered.
+When:: Specifies how the value to be compared to the threshold is calculated. The value is calculated by aggregating a numeric field a the *time window*. The aggregation options are: `count`, `average`, `sum`, `min`, and `max`. When using `count` the document count is used, and an aggregation field is not necessary. 
+Over/Grouped Over:: Specifies whether the aggregation is applied over all documents, or should be split into groups using a grouping field. If grouping is used, an <<alerting-concepts-alerts, alert>> will be created for each group when it meets the condition. To limit the number of alerts on high cardinality fields, you must specify the number of groups to check against the threshold. Only the *top* groups are checked.
 Threshold:: Defines a threshold value and a comparison operator (`is above`, 
-`is above or equals`, `is below`, `is below or equals`, or `is between`). The 
-number of documents that match the specified query is compared to this 
-threshold.
+`is above or equals`, `is below`, `is below or equals`, or `is between`). The value
+calculated by the aggregation is compared to this threshold.
 Time window:: Defines how far back to search for documents, using the 
 *time field* set in the *index* clause. Generally this value should be set to a 
 value higher than the *check every* value in the 
 <<defining-rules-general-details, general rule details>>, to avoid gaps in 
 detection.
+Size:: Specifies the number of documents to pass to the configured actions when 
+the threshold condition is met.
 Exclude matches from previous run:: Turn on to avoid alert duplication by
-excluding documents that have already been detected by the previous rule run.
+excluding documents that have already been detected by the previous rule run. This
+option is not available when a grouping field is specified.
 
 [float]
 === Add action variables

src/core/server/integration_tests/saved_objects/migrations/check_registered_types.test.ts

@@ -57,7 +57,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
  Object {
  "action": "7858e6d5a9f231bf23f6f2e57328eb0095b26735",
  "action_task_params": "bbd38cbfd74bf6713586fe078e3fa92db2234299",
- "alert": "eefada4a02ce05962387c0679d7b292771a931c4",
+ "alert": "90ae0b15c8666e4266a5340f9bc0d017fa86dee7",
  "api_key_pending_invalidation": "9b4bc1235337da9a87ef05a1d1f4858b2a3b77c6",
  "apm-indices": "ceb0870f3a74e2ffc3a1cd3a3c73af76baca0999",
  "apm-server-schema": "2bfd2998d3873872e1366458ce553def85418f91",

x-pack/plugins/alerting/server/saved_objects/migrations/8.7/index.ts

@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { SavedObjectUnsanitizedDoc } from '@kbn/core-saved-objects-server';
+import { EncryptedSavedObjectsPluginSetup } from '@kbn/encrypted-saved-objects-plugin/server';
+import { createEsoMigration, isEsQueryRuleType, pipeMigrations } from '../utils';
+import { RawRule } from '../../../types';
+
+function addGroupByToEsQueryRule(
+ doc: SavedObjectUnsanitizedDoc<RawRule>
+): SavedObjectUnsanitizedDoc<RawRule> {
+ // Adding another check in for isEsQueryRuleType in case we add more migrations
+ if (isEsQueryRuleType(doc)) {
+ return {
+ ...doc,
+ attributes: {
+ ...doc.attributes,
+ params: {
+ ...doc.attributes.params,
+ aggType: 'count',
+ groupBy: 'all',
+ },
+ },
+ };
+ }
+
+ return doc;
+}
+
+export const getMigrations870 = (encryptedSavedObjects: EncryptedSavedObjectsPluginSetup) =>
+ createEsoMigration(
+ encryptedSavedObjects,
+ (doc): doc is SavedObjectUnsanitizedDoc<RawRule> => isEsQueryRuleType(doc),
+ pipeMigrations(addGroupByToEsQueryRule)
+ );

x-pack/plugins/alerting/server/saved_objects/migrations/index.test.ts

@@ -2508,6 +2508,41 @@ describe('successful migrations', () => {
  });
  });
 
+ describe('8.7.0', () => {
+ test('migrates es_query rule params and adds group by fields', () => {
+ const migration870 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)['8.7.0'];
+ const rule = getMockData(
+ {
+ params: { esQuery: '{ "query": "test-query" }', searchType: 'esQuery' },
+ alertTypeId: '.es-query',
+ },
+ true
+ );
+ const migratedAlert870 = migration870(rule, migrationContext);
+
+ expect(migratedAlert870.attributes.params).toEqual({
+ esQuery: '{ "query": "test-query" }',
+ searchType: 'esQuery',
+ aggType: 'count',
+ groupBy: 'all',
+ });
+ });
+
+ test('does not migrate rule params if rule is not es query', () => {
+ const migration870 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)['8.7.0'];
+ const rule = getMockData(
+ {
+ params: { foo: true },
+ alertTypeId: '.not-es-query',
+ },
+ true
+ );
+ const migratedAlert870 = migration870(rule, migrationContext);
+
+ expect(migratedAlert870.attributes.params).toEqual({ foo: true });
+ });
+ });
+
  describe('Metrics Inventory Threshold rule', () => {
  test('Migrates incorrect action group spelling', () => {
  const migration800 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)['8.0.0'];

x-pack/plugins/alerting/server/saved_objects/migrations/index.ts

@@ -29,6 +29,7 @@ import { getMigrations830 } from './8.3';
 import { getMigrations841 } from './8.4';
 import { getMigrations850 } from './8.5';
 import { getMigrations860 } from './8.6';
+import { getMigrations870 } from './8.7';
 import { AlertLogMeta, AlertMigration } from './types';
 import { MINIMUM_SS_MIGRATION_VERSION } from './constants';
 import { createEsoMigration, isEsQueryRuleType, pipeMigrations } from './utils';
@@ -77,6 +78,7 @@ export function getMigrations(
  '8.4.1': executeMigrationWithErrorHandling(getMigrations841(encryptedSavedObjects), '8.4.1'),
  '8.5.0': executeMigrationWithErrorHandling(getMigrations850(encryptedSavedObjects), '8.5.0'),
  '8.6.0': executeMigrationWithErrorHandling(getMigrations860(encryptedSavedObjects), '8.6.0'),
+ '8.7.0': executeMigrationWithErrorHandling(getMigrations870(encryptedSavedObjects), '8.7.0'),
  },
  getSearchSourceMigrations(encryptedSavedObjects, searchSourceMigrations)
  );

x-pack/plugins/stack_alerts/common/build_sorted_events_query.test.ts

@@ -57,9 +57,6 @@ describe('buildSortedEventsQuery', () => {
  ],
  },
  },
- {
- match_all: {},
- },
  ],
  },
  },
@@ -108,9 +105,6 @@ describe('buildSortedEventsQuery', () => {
  ],
  },
  },
- {
- match_all: {},
- },
  ],
  },
  },
@@ -160,9 +154,6 @@ describe('buildSortedEventsQuery', () => {
  ],
  },
  },
- {
- match_all: {},
- },
  ],
  },
  },
@@ -213,9 +204,6 @@ describe('buildSortedEventsQuery', () => {
  ],
  },
  },
- {
- match_all: {},
- },
  ],
  },
  },
@@ -271,9 +259,6 @@ describe('buildSortedEventsQuery', () => {
  ],
  },
  },
- {
- match_all: {},
- },
  ],
  },
  },
@@ -329,9 +314,6 @@ describe('buildSortedEventsQuery', () => {
  ],
  },
  },
- {
- match_all: {},
- },
  ],
  },
  },
@@ -380,9 +362,6 @@ describe('buildSortedEventsQuery', () => {
  ],
  },
  },
- {
- match_all: {},
- },
  ],
  },
  },

x-pack/plugins/stack_alerts/common/build_sorted_events_query.ts

@@ -72,12 +72,7 @@ export const buildSortedEventsQuery = ({
  docvalue_fields: docFields,
  query: {
  bool: {
- filter: [
- ...filterWithTime,
- {
- match_all: {},
- },
- ],
+ filter: [...filterWithTime],
  },
  },
  ...(aggs ? { aggs } : {}),

x-pack/plugins/stack_alerts/server/rule_types/lib/comparator.test.ts → x-pack/plugins/stack_alerts/common/comparator.test.ts

@@ -6,7 +6,7 @@
  */
 
 import { getComparatorScript } from './comparator';
-import { Comparator } from '../../../common/comparator_types';
+import { Comparator } from './comparator_types';
 
 describe('getComparatorScript', () => {
  it('correctly returns script when comparator is LT', () => {

x-pack/plugins/stack_alerts/common/comparator.ts

@@ -0,0 +1,81 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { Comparator } from './comparator_types';
+
+export type ComparatorFn = (value: number, threshold: number[]) => boolean;
+
+const humanReadableComparators = new Map<Comparator, string>([
+ [Comparator.LT, 'less than'],
+ [Comparator.LT_OR_EQ, 'less than or equal to'],
+ [Comparator.GT_OR_EQ, 'greater than or equal to'],
+ [Comparator.GT, 'greater than'],
+ [Comparator.BETWEEN, 'between'],
+ [Comparator.NOT_BETWEEN, 'not between'],
+]);
+
+export const ComparatorFns = new Map<Comparator, ComparatorFn>([
+ [Comparator.LT, (value: number, threshold: number[]) => value < threshold[0]],
+ [Comparator.LT_OR_EQ, (value: number, threshold: number[]) => value <= threshold[0]],
+ [Comparator.GT_OR_EQ, (value: number, threshold: number[]) => value >= threshold[0]],
+ [Comparator.GT, (value: number, threshold: number[]) => value > threshold[0]],
+ [
+ Comparator.BETWEEN,
+ (value: number, threshold: number[]) => value >= threshold[0] && value <= threshold[1],
+ ],
+ [
+ Comparator.NOT_BETWEEN,
+ (value: number, threshold: number[]) => value < threshold[0] || value > threshold[1],
+ ],
+]);
+
+export const getComparatorScript = (
+ comparator: Comparator,
+ threshold: number[],
+ fieldName: string
+) => {
+ if (threshold.length === 0) {
+ throw new Error('Threshold value required');
+ }
+
+ function getThresholdString(thresh: number) {
+ return Number.isInteger(thresh) ? `${thresh}L` : `${thresh}`;
+ }
+
+ switch (comparator) {
+ case Comparator.LT:
+ return `${fieldName} < ${getThresholdString(threshold[0])}`;
+ case Comparator.LT_OR_EQ:
+ return `${fieldName} <= ${getThresholdString(threshold[0])}`;
+ case Comparator.GT:
+ return `${fieldName} > ${getThresholdString(threshold[0])}`;
+ case Comparator.GT_OR_EQ:
+ return `${fieldName} >= ${getThresholdString(threshold[0])}`;
+ case Comparator.BETWEEN:
+ if (threshold.length < 2) {
+ throw new Error('Threshold values required');
+ }
+ return `${fieldName} >= ${getThresholdString(
+ threshold[0]
+ )} && ${fieldName} <= ${getThresholdString(threshold[1])}`;
+ case Comparator.NOT_BETWEEN:
+ if (threshold.length < 2) {
+ throw new Error('Threshold values required');
+ }
+ return `${fieldName} < ${getThresholdString(
+ threshold[0]
+ )} || ${fieldName} > ${getThresholdString(threshold[1])}`;
+ }
+};
+
+export const ComparatorFnNames = new Set(ComparatorFns.keys());
+
+export function getHumanReadableComparator(comparator: Comparator) {
+ return humanReadableComparators.has(comparator)
+ ? humanReadableComparators.get(comparator)
+ : comparator;
+}

x-pack/plugins/stack_alerts/common/index.ts

@@ -5,4 +5,10 @@
  * 2.0.
  */
 
+export {
+ ComparatorFns,
+ getComparatorScript,
+ ComparatorFnNames,
+ getHumanReadableComparator,
+} from './comparator';
 export { STACK_ALERTS_FEATURE_ID } from './constants';

x-pack/plugins/stack_alerts/public/rule_types/es_query/constants.ts

@@ -19,6 +19,9 @@ export const DEFAULT_VALUES = {
  TIME_WINDOW_SIZE: 5,
  TIME_WINDOW_UNIT: 'm',
  THRESHOLD: [1000],
+ AGGREGATION_TYPE: 'count',
+ TERM_SIZE: 5,
+ GROUP_BY: 'all',
  EXCLUDE_PREVIOUS_HITS: true,
 };
 
@@ -33,6 +36,11 @@ export const EXPRESSION_ERRORS = {
  timeWindowSize: new Array<string>(),
  searchConfiguration: new Array<string>(),
  searchType: new Array<string>(),
+ aggField: new Array<string>(),
+ aggType: new Array<string>(),
+ groupBy: new Array<string>(),
+ termSize: new Array<string>(),
+ termField: new Array<string>(),
 };
 
 export const EXPRESSION_ERROR_KEYS = Object.keys(EXPRESSION_ERRORS) as ErrorKey[];