[ML] Anomaly detection rule lookback interval improvements (#97370) (#…

…97595)

* [ML] add advanced settings

* [ML] default advanced settings

* [ML] advanced settings validators

* [ML] range control for top n buckets

* [ML] execute rule with a new query for most recent anomalies

* [ML] find most anomalous bucket from the top N

* Revert "[ML] range control for top n buckets"

This reverts commit e039f25

* [ML] validate check interval against the lookback interval

* [ML] update descriptions

* [ML] fix test subjects

* [ML] update warning message

* [ML] add functional tests

* [ML] adjust unit tests, mark getLookbackInterval

* [ML] update lookback interval description and warning message

* [ML] update fetchResult tsDoc

* [ML] cleanup

* [ML] fix imports to reduce bundle size

* [ML] round up lookback interval

* [ML] update functional test assertion

* [ML] async import for validator

Co-authored-by: Dima Arnautov <dmitrii.arnautov@elastic.co>

x-pack/plugins/ml/common/constants/alerts.ts

@@ -47,3 +47,5 @@ export const ML_ALERT_TYPES_CONFIG: Record<
 };
 
 export const ALERT_PREVIEW_SAMPLE_SIZE = 5;
+
+export const TOP_N_BUCKETS_COUNT = 1;

x-pack/plugins/ml/common/types/alerts.ts

@@ -93,4 +93,11 @@ export type MlAnomalyDetectionAlertParams = {
   severity: number;
   resultType: AnomalyResultType;
   includeInterim: boolean;
+  lookbackInterval: string | null | undefined;
+  topNBuckets: number | null | undefined;
 } & AlertTypeParams;
+
+export type MlAnomalyDetectionAlertAdvancedSettings = Pick<
+  MlAnomalyDetectionAlertParams,
+  'lookbackInterval' | 'topNBuckets'
+>;

x-pack/plugins/ml/common/util/alerts.test.ts

@@ -0,0 +1,78 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { getLookbackInterval, resolveLookbackInterval } from './alerts';
+import type { CombinedJobWithStats, Datafeed, Job } from '../types/anomaly_detection_jobs';
+
+describe('resolveLookbackInterval', () => {
+  test('resolves interval for bucket spans bigger than 1m', () => {
+    const testJobs = [
+      {
+        analysis_config: {
+          bucket_span: '15m',
+        },
+      },
+    ] as Job[];
+
+    const testDatafeeds = [
+      {
+        query_delay: '65630ms',
+      },
+    ] as Datafeed[];
+
+    expect(resolveLookbackInterval(testJobs, testDatafeeds)).toBe('32m');
+  });
+
+  test('resolves interval for bucket spans smaller than 1m', () => {
+    const testJobs = [
+      {
+        analysis_config: {
+          bucket_span: '50s',
+        },
+      },
+    ] as Job[];
+
+    const testDatafeeds = [
+      {
+        query_delay: '20s',
+      },
+    ] as Datafeed[];
+
+    expect(resolveLookbackInterval(testJobs, testDatafeeds)).toBe('3m');
+  });
+
+  test('resolves interval for bucket spans smaller than 1m without query dealay', () => {
+    const testJobs = [
+      {
+        analysis_config: {
+          bucket_span: '59s',
+        },
+      },
+    ] as Job[];
+
+    const testDatafeeds = [{}] as Datafeed[];
+
+    expect(resolveLookbackInterval(testJobs, testDatafeeds)).toBe('3m');
+  });
+});
+
+describe('getLookbackInterval', () => {
+  test('resolves interval for bucket spans bigger than 1m', () => {
+    const testJobs = [
+      {
+        analysis_config: {
+          bucket_span: '15m',
+        },
+        datafeed_config: {
+          query_delay: '65630ms',
+        },
+      },
+    ] as CombinedJobWithStats[];
+
+    expect(getLookbackInterval(testJobs)).toBe('32m');
+  });
+});

x-pack/plugins/ml/common/util/alerts.ts

@@ -0,0 +1,53 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { CombinedJobWithStats, Datafeed, Job } from '../types/anomaly_detection_jobs';
+import { resolveMaxTimeInterval } from './job_utils';
+import { isDefined } from '../types/guards';
+import { parseInterval } from './parse_interval';
+
+const narrowBucketLength = 60;
+
+/**
+ * Resolves the lookback interval for the rule
+ * using the formula max(2m, 2 * bucket_span) + query_delay + 1s.
+ * and rounds up to a whole number of minutes.
+ */
+export function resolveLookbackInterval(jobs: Job[], datafeeds: Datafeed[]): string {
+  const bucketSpanInSeconds = Math.ceil(
+    resolveMaxTimeInterval(jobs.map((v) => v.analysis_config.bucket_span)) ?? 0
+  );
+  const queryDelayInSeconds = Math.ceil(
+    resolveMaxTimeInterval(datafeeds.map((v) => v.query_delay).filter(isDefined)) ?? 0
+  );
+
+  const result =
+    Math.max(2 * narrowBucketLength, 2 * bucketSpanInSeconds) + queryDelayInSeconds + 1;
+
+  return `${Math.ceil(result / 60)}m`;
+}
+
+/**
+ * @deprecated We should avoid using {@link CombinedJobWithStats}. Replace usages with {@link resolveLookbackInterval} when
+ * Kibana API returns mapped job and the datafeed configs.
+ */
+export function getLookbackInterval(jobs: CombinedJobWithStats[]): string {
+  return resolveLookbackInterval(
+    jobs,
+    jobs.map((v) => v.datafeed_config)
+  );
+}
+
+export function getTopNBuckets(job: Job): number {
+  const bucketSpan = parseInterval(job.analysis_config.bucket_span);
+
+  if (bucketSpan === null) {
+    throw new Error('Unable to resolve a bucket span length');
+  }
+
+  return Math.ceil(narrowBucketLength / bucketSpan.asSeconds());
+}

x-pack/plugins/ml/common/util/job_utils.test.ts

@@ -20,7 +20,7 @@ import {
   getSafeAggregationName,
   getLatestDataOrBucketTimestamp,
   getEarliestDatafeedStartTime,
-  resolveBucketSpanInSeconds,
+  resolveMaxTimeInterval,
 } from './job_utils';
 import { CombinedJob, Job } from '../types/anomaly_detection_jobs';
 import moment from 'moment';
@@ -606,7 +606,10 @@ describe('ML - job utils', () => {
 
   describe('resolveBucketSpanInSeconds', () => {
     test('should resolve maximum bucket interval', () => {
-      expect(resolveBucketSpanInSeconds(['15m', '1h', '6h', '90s'])).toBe(21600);
+      expect(resolveMaxTimeInterval(['15m', '1h', '6h', '90s'])).toBe(21600);
+    });
+    test('returns undefined for an empty array', () => {
+      expect(resolveMaxTimeInterval([])).toBe(undefined);
     });
   });
 });

x-pack/plugins/ml/common/util/job_utils.ts

@@ -831,14 +831,16 @@ export function splitIndexPatternNames(indexPatternName: string): string[] {
 }
 
 /**
- * Resolves the longest bucket span from the list.
- * @param bucketSpans Collection of bucket spans
+ * Resolves the longest time interval from the list.
+ * @param timeIntervals Collection of the strings representing time intervals, e.g. ['15m', '1h', '2d']
  */
-export function resolveBucketSpanInSeconds(bucketSpans: string[]): number {
-  return Math.max(
-    ...bucketSpans
+export function resolveMaxTimeInterval(timeIntervals: string[]): number | undefined {
+  const result = Math.max(
+    ...timeIntervals
       .map((b) => parseInterval(b))
       .filter(isDefined)
       .map((v) => v.asSeconds())
   );
+
+  return Number.isFinite(result) ? result : undefined;
 }

x-pack/plugins/ml/common/util/validators.ts

@@ -7,6 +7,7 @@
 
 import { ALLOWED_DATA_UNITS } from '../constants/validation';
 import { parseInterval } from './parse_interval';
+import { isPopulatedObject } from './object_utils';
 
 /**
  * Provides a validator function for maximum allowed input length.
@@ -85,6 +86,10 @@ export function memoryInputValidator(allowedUnits = ALLOWED_DATA_UNITS) {
 
 export function timeIntervalInputValidator() {
   return (value: string) => {
+    if (value === '') {
+      return null;
+    }
+
     const r = parseInterval(value);
     if (r === null) {
       return {
@@ -95,3 +100,32 @@ export function timeIntervalInputValidator() {
     return null;
   };
 }
+
+export interface NumberValidationResult {
+  min: boolean;
+  max: boolean;
+}
+
+export function numberValidator(conditions?: { min?: number; max?: number }) {
+  if (
+    conditions?.min !== undefined &&
+    conditions.max !== undefined &&
+    conditions.min > conditions.max
+  ) {
+    throw new Error('Invalid validator conditions');
+  }
+
+  return (value: number): NumberValidationResult | null => {
+    const result = {} as NumberValidationResult;
+    if (conditions?.min !== undefined && value < conditions.min) {
+      result.min = true;
+    }
+    if (conditions?.max !== undefined && value > conditions.max) {
+      result.max = true;
+    }
+    if (isPopulatedObject(result)) {
+      return result;
+    }
+    return null;
+  };
+}

x-pack/plugins/ml/public/alerting/advanced_settings.tsx

@@ -0,0 +1,117 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import React, { FC } from 'react';
+import { FormattedMessage } from '@kbn/i18n/react';
+import {
+  EuiAccordion,
+  EuiDescribedFormGroup,
+  EuiFieldNumber,
+  EuiFormRow,
+  EuiHorizontalRule,
+  EuiSpacer,
+  EuiText,
+} from '@elastic/eui';
+import { MlAnomalyDetectionAlertAdvancedSettings } from '../../common/types/alerts';
+import { TimeIntervalControl } from './time_interval_control';
+import { TOP_N_BUCKETS_COUNT } from '../../common/constants/alerts';
+
+interface AdvancedSettingsProps {
+  value: MlAnomalyDetectionAlertAdvancedSettings;
+  onChange: (update: Partial<MlAnomalyDetectionAlertAdvancedSettings>) => void;
+}
+
+export const AdvancedSettings: FC<AdvancedSettingsProps> = React.memo(({ value, onChange }) => {
+  return (
+    <EuiAccordion
+      id="mlAnomalyAlertAdvancedSettings"
+      buttonContent={
+        <FormattedMessage
+          id="xpack.ml.anomalyDetectionAlert.advancedSettingsLabel"
+          defaultMessage="Advanced settings"
+        />
+      }
+      data-test-subj={'mlAnomalyAlertAdvancedSettingsTrigger'}
+    >
+      <EuiSpacer size="m" />
+      <EuiDescribedFormGroup
+        gutterSize={'s'}
+        titleSize={'xxs'}
+        title={
+          <h4>
+            <FormattedMessage
+              id="xpack.ml.anomalyDetectionAlert.lookbackIntervalLabel"
+              defaultMessage="Lookback interval"
+            />
+          </h4>
+        }
+        description={
+          <EuiText size={'xs'}>
+            <FormattedMessage
+              id="xpack.ml.anomalyDetectionAlert.lookbackIntervalDescription"
+              defaultMessage="Time interval to query the anomalies data during each rule condition check. By default, is derived from the bucket span of the job and the query delay of the datafeed."
+            />
+          </EuiText>
+        }
+      >
+        <TimeIntervalControl
+          value={value.lookbackInterval}
+          label={
+            <FormattedMessage
+              id="xpack.ml.anomalyDetectionAlert.lookbackIntervalLabel"
+              defaultMessage="Lookback interval"
+            />
+          }
+          onChange={(update) => {
+            onChange({ lookbackInterval: update });
+          }}
+          data-test-subj={'mlAnomalyAlertLookbackInterval'}
+        />
+      </EuiDescribedFormGroup>
+
+      <EuiDescribedFormGroup
+        gutterSize={'s'}
+        titleSize={'xxs'}
+        title={
+          <h4>
+            <FormattedMessage
+              id="xpack.ml.anomalyDetectionAlert.topNBucketsLabel"
+              defaultMessage="Number of latest buckets"
+            />
+          </h4>
+        }
+        description={
+          <EuiText size={'xs'}>
+            <FormattedMessage
+              id="xpack.ml.anomalyDetectionAlert.topNBucketsDescription"
+              defaultMessage="The number of latest buckets to check to obtain the highest anomaly."
+            />
+          </EuiText>
+        }
+      >
+        <EuiFormRow
+          label={
+            <FormattedMessage
+              id="xpack.ml.anomalyDetectionAlert.topNBucketsLabel"
+              defaultMessage="Number of latest buckets"
+            />
+          }
+        >
+          <EuiFieldNumber
+            value={value.topNBuckets ?? TOP_N_BUCKETS_COUNT}
+            min={1}
+            onChange={(e) => {
+              onChange({ topNBuckets: Number(e.target.value) });
+            }}
+            data-test-subj={'mlAnomalyAlertTopNBuckets'}
+          />
+        </EuiFormRow>
+      </EuiDescribedFormGroup>
+      <EuiHorizontalRule margin={'m'} />
+    </EuiAccordion>
+  );
+});