diff --git a/src/core/server/integration_tests/saved_objects/migrations/check_registered_types.test.ts b/src/core/server/integration_tests/saved_objects/migrations/check_registered_types.test.ts index 1f0a043782311..03718a71cdb74 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/check_registered_types.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/check_registered_types.test.ts @@ -53,7 +53,7 @@ describe('checking migration metadata changes on all registered SO types', () => Object { "action": "7858e6d5a9f231bf23f6f2e57328eb0095b26735", "action_task_params": "bbd38cbfd74bf6713586fe078e3fa92db2234299", - "alert": "48461f3375d9ba22882ea23a318b62a5b0921a9b", + "alert": "eefada4a02ce05962387c0679d7b292771a931c4", "api_key_pending_invalidation": "9b4bc1235337da9a87ef05a1d1f4858b2a3b77c6", "apm-indices": "ceb0870f3a74e2ffc3a1cd3a3c73af76baca0999", "apm-server-schema": "2bfd2998d3873872e1366458ce553def85418f91", diff --git a/x-pack/plugins/alerting/common/rule.ts b/x-pack/plugins/alerting/common/rule.ts index 56fcaa8832792..bd479b96f9b1d 100644 --- a/x-pack/plugins/alerting/common/rule.ts +++ b/x-pack/plugins/alerting/common/rule.ts @@ -32,6 +32,9 @@ export const RuleExecutionStatusValues = [ ] as const; export type RuleExecutionStatuses = typeof RuleExecutionStatusValues[number]; +export const RuleLastRunOutcomeValues = ['succeeded', 'warning', 'failed'] as const; +export type RuleLastRunOutcomes = typeof RuleLastRunOutcomeValues[number]; + export enum RuleExecutionStatusErrorReasons { Read = 'read', Decrypt = 'decrypt', @@ -76,12 +79,25 @@ export interface RuleAction { export interface RuleAggregations { alertExecutionStatus: { [status: string]: number }; + ruleLastRunOutcome: { [status: string]: number }; ruleEnabledStatus: { enabled: number; disabled: number }; ruleMutedStatus: { muted: number; unmuted: number }; ruleSnoozedStatus: { snoozed: number }; ruleTags: string[]; } +export interface RuleLastRun { + outcome: RuleLastRunOutcomes; + warning?: RuleExecutionStatusErrorReasons | RuleExecutionStatusWarningReasons | null; + outcomeMsg?: string | null; + alertsCount: { + active?: number | null; + new?: number | null; + recovered?: number | null; + ignored?: number | null; + }; +} + export interface MappedParamsProperties { risk_score?: number; severity?: string; @@ -116,6 +132,8 @@ export interface Rule { snoozeSchedule?: RuleSnooze; // Remove ? when this parameter is made available in the public API activeSnoozes?: string[]; isSnoozedUntil?: Date | null; + lastRun?: RuleLastRun | null; + nextRun?: Date | null; } export type SanitizedRule = Omit, 'apiKey'>; @@ -175,16 +193,34 @@ export interface RuleMonitoringHistory extends SavedObjectAttributes { success: boolean; timestamp: number; duration?: number; + outcome?: RuleLastRunOutcomes; +} + +export interface RuleMonitoringCalculatedMetrics extends SavedObjectAttributes { + p50?: number; + p95?: number; + p99?: number; + success_ratio: number; +} + +export interface RuleMonitoringLastRunMetrics extends SavedObjectAttributes { + duration?: number; + total_search_duration_ms?: number | null; + total_indexing_duration_ms?: number | null; + total_alerts_detected?: number | null; + total_alerts_created?: number | null; + gap_duration_s?: number | null; +} + +export interface RuleMonitoringLastRun extends SavedObjectAttributes { + timestamp: string; + metrics: RuleMonitoringLastRunMetrics; } -export interface RuleMonitoring extends SavedObjectAttributes { - execution: { +export interface RuleMonitoring { + run: { history: RuleMonitoringHistory[]; - calculated_metrics: { - p50?: number; - p95?: number; - p99?: number; - success_ratio: number; - }; + calculated_metrics: RuleMonitoringCalculatedMetrics; + last_run: RuleMonitoringLastRun; }; } diff --git a/x-pack/plugins/alerting/public/lib/common_transformations.test.ts b/x-pack/plugins/alerting/public/lib/common_transformations.test.ts index 51d24538b449e..6b7026f1ea593 100644 --- a/x-pack/plugins/alerting/public/lib/common_transformations.test.ts +++ b/x-pack/plugins/alerting/public/lib/common_transformations.test.ts @@ -6,7 +6,7 @@ */ import { ApiRule, transformRule } from './common_transformations'; -import { RuleExecutionStatusErrorReasons } from '../../common'; +import { RuleExecutionStatusErrorReasons, RuleLastRunOutcomeValues } from '../../common'; beforeEach(() => jest.resetAllMocks()); @@ -54,6 +54,43 @@ describe('common_transformations', () => { message: 'this is just a test', }, }, + monitoring: { + run: { + history: [ + { + timestamp: dateExecuted.getTime(), + duration: 42, + success: false, + outcome: RuleLastRunOutcomeValues[2], + }, + ], + calculated_metrics: { + success_ratio: 0, + p50: 0, + p95: 42, + p99: 42, + }, + last_run: { + timestamp: dateExecuted.toISOString(), + metrics: { + duration: 42, + total_search_duration_ms: 100, + }, + }, + }, + }, + last_run: { + outcome: RuleLastRunOutcomeValues[2], + outcome_msg: 'this is just a test', + warning: RuleExecutionStatusErrorReasons.Unknown, + alerts_count: { + new: 1, + active: 2, + recovered: 3, + ignored: 4, + }, + }, + next_run: dateUpdated.toISOString(), }; expect(transformRule(apiRule)).toMatchInlineSnapshot(` Object { @@ -89,12 +126,49 @@ describe('common_transformations', () => { "status": "error", }, "id": "some-id", + "lastRun": Object { + "alertsCount": Object { + "active": 2, + "ignored": 4, + "new": 1, + "recovered": 3, + }, + "outcome": "failed", + "outcomeMsg": "this is just a test", + "warning": "unknown", + }, + "monitoring": Object { + "run": Object { + "calculated_metrics": Object { + "p50": 0, + "p95": 42, + "p99": 42, + "success_ratio": 0, + }, + "history": Array [ + Object { + "duration": 42, + "outcome": "failed", + "success": false, + "timestamp": 1639571696789, + }, + ], + "last_run": Object { + "metrics": Object { + "duration": 42, + "total_search_duration_ms": 100, + }, + "timestamp": "2021-12-15T12:34:56.789Z", + }, + }, + }, "muteAll": false, "mutedInstanceIds": Array [ "bob", "jim", ], "name": "some-name", + "nextRun": 2021-12-15T12:34:55.789Z, "notifyWhen": "onActiveAlert", "params": Object { "bar": "foo", @@ -152,6 +226,43 @@ describe('common_transformations', () => { last_execution_date: dateExecuted.toISOString(), status: 'error', }, + monitoring: { + run: { + history: [ + { + timestamp: dateExecuted.getTime(), + duration: 42, + success: false, + outcome: 'failed', + }, + ], + calculated_metrics: { + success_ratio: 0, + p50: 0, + p95: 42, + p99: 42, + }, + last_run: { + timestamp: dateExecuted.toISOString(), + metrics: { + duration: 42, + total_search_duration_ms: 100, + }, + }, + }, + }, + last_run: { + outcome: 'failed', + outcome_msg: 'this is just a test', + warning: RuleExecutionStatusErrorReasons.Unknown, + alerts_count: { + new: 1, + active: 2, + recovered: 3, + ignored: 4, + }, + }, + next_run: dateUpdated.toISOString(), }; expect(transformRule(apiRule)).toMatchInlineSnapshot(` Object { @@ -176,12 +287,49 @@ describe('common_transformations', () => { "status": "error", }, "id": "some-id", + "lastRun": Object { + "alertsCount": Object { + "active": 2, + "ignored": 4, + "new": 1, + "recovered": 3, + }, + "outcome": "failed", + "outcomeMsg": "this is just a test", + "warning": "unknown", + }, + "monitoring": Object { + "run": Object { + "calculated_metrics": Object { + "p50": 0, + "p95": 42, + "p99": 42, + "success_ratio": 0, + }, + "history": Array [ + Object { + "duration": 42, + "outcome": "failed", + "success": false, + "timestamp": 1639571696789, + }, + ], + "last_run": Object { + "metrics": Object { + "duration": 42, + "total_search_duration_ms": 100, + }, + "timestamp": "2021-12-15T12:34:56.789Z", + }, + }, + }, "muteAll": false, "mutedInstanceIds": Array [ "bob", "jim", ], "name": "some-name", + "nextRun": 2021-12-15T12:34:55.789Z, "notifyWhen": "onActiveAlert", "params": Object {}, "schedule": Object { diff --git a/x-pack/plugins/alerting/public/lib/common_transformations.ts b/x-pack/plugins/alerting/public/lib/common_transformations.ts index 1b306aae0ae2f..c48c1f882eaed 100644 --- a/x-pack/plugins/alerting/public/lib/common_transformations.ts +++ b/x-pack/plugins/alerting/public/lib/common_transformations.ts @@ -5,7 +5,14 @@ * 2.0. */ import { AsApiContract } from '@kbn/actions-plugin/common'; -import { RuleExecutionStatus, Rule, RuleAction, RuleType } from '../../common'; +import { + RuleExecutionStatus, + RuleMonitoring, + Rule, + RuleLastRun, + RuleAction, + RuleType, +} from '../../common'; function transformAction(input: AsApiContract): RuleAction { const { connector_type_id: actionTypeId, ...rest } = input; @@ -27,6 +34,31 @@ function transformExecutionStatus(input: ApiRuleExecutionStatus): RuleExecutionS }; } +function transformMonitoring(input: RuleMonitoring): RuleMonitoring { + const { run } = input; + const { last_run: lastRun, ...rest } = run; + const { timestamp, ...restLastRun } = lastRun; + + return { + run: { + last_run: { + timestamp: input.run.last_run.timestamp, + ...restLastRun, + }, + ...rest, + }, + }; +} + +function transformLastRun(input: AsApiContract): RuleLastRun { + const { outcome_msg: outcomeMsg, alerts_count: alertsCount, ...rest } = input; + return { + outcomeMsg, + alertsCount, + ...rest, + }; +} + // AsApiContract does not deal with object properties that also // need snake -> camel conversion, Dates, are renamed, etc, so we do by hand export type ApiRule = Omit< @@ -37,6 +69,8 @@ export type ApiRule = Omit< | 'updated_at' | 'alert_type_id' | 'muted_instance_ids' + | 'last_run' + | 'next_run' > & { execution_status: ApiRuleExecutionStatus; actions: Array>; @@ -44,6 +78,8 @@ export type ApiRule = Omit< updated_at: string; rule_type_id: string; muted_alert_ids: string[]; + last_run?: AsApiContract; + next_run?: string; }; export function transformRule(input: ApiRule): Rule { @@ -61,6 +97,9 @@ export function transformRule(input: ApiRule): Rule { scheduled_task_id: scheduledTaskId, execution_status: executionStatusAPI, actions: actionsAPI, + next_run: nextRun, + last_run: lastRun, + monitoring: monitoring, ...rest } = input; @@ -78,6 +117,9 @@ export function transformRule(input: ApiRule): Rule { executionStatus: transformExecutionStatus(executionStatusAPI), actions: actionsAPI ? actionsAPI.map((action) => transformAction(action)) : [], scheduledTaskId, + ...(nextRun ? { nextRun: new Date(nextRun) } : {}), + ...(monitoring ? { monitoring: transformMonitoring(monitoring) } : {}), + ...(lastRun ? { lastRun: transformLastRun(lastRun) } : {}), ...rest, }; } diff --git a/x-pack/plugins/alerting/server/lib/index.ts b/x-pack/plugins/alerting/server/lib/index.ts index 825a9863badbb..de4c89e06f33d 100644 --- a/x-pack/plugins/alerting/server/lib/index.ts +++ b/x-pack/plugins/alerting/server/lib/index.ts @@ -25,6 +25,13 @@ export { ruleExecutionStatusToRaw, ruleExecutionStatusFromRaw, } from './rule_execution_status'; +export { lastRunFromState, lastRunFromError, lastRunToRaw } from './last_run_status'; +export { + updateMonitoring, + getDefaultMonitoring, + convertMonitoringFromRawAndVerify, +} from './monitoring'; +export { getNextRun } from './next_run'; export { processAlerts } from './process_alerts'; export { createWrappedScopedClusterClientFactory } from './wrap_scoped_cluster_client'; export { isRuleSnoozed, getRuleSnoozeEndTime } from './is_rule_snoozed'; diff --git a/x-pack/plugins/alerting/server/lib/last_run_status.test.ts b/x-pack/plugins/alerting/server/lib/last_run_status.test.ts new file mode 100644 index 0000000000000..da44325ba3cbd --- /dev/null +++ b/x-pack/plugins/alerting/server/lib/last_run_status.test.ts @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { lastRunFromState } from './last_run_status'; +import { ActionsCompletion } from '../../common'; +import { RuleRunMetrics } from './rule_run_metrics_store'; +const getMetrics = (): RuleRunMetrics => { + return { + triggeredActionsStatus: ActionsCompletion.COMPLETE, + esSearchDurationMs: 3, + numSearches: 1, + numberOfActiveAlerts: 10, + numberOfGeneratedActions: 15, + numberOfNewAlerts: 12, + numberOfRecoveredAlerts: 11, + numberOfTriggeredActions: 5, + totalSearchDurationMs: 2, + hasReachedAlertLimit: false, + }; +}; + +describe('lastRunFromState', () => { + it('successfuly outcome', () => { + const result = lastRunFromState({ metrics: getMetrics() }); + + expect(result.lastRun.outcome).toEqual('succeeded'); + expect(result.lastRun.outcomeMsg).toEqual(null); + expect(result.lastRun.warning).toEqual(null); + + expect(result.lastRun.alertsCount).toEqual({ + active: 10, + new: 12, + recovered: 11, + ignored: 0, + }); + }); + + it('limited reached outcome', () => { + const result = lastRunFromState({ + metrics: { + ...getMetrics(), + hasReachedAlertLimit: true, + }, + }); + + expect(result.lastRun.outcome).toEqual('warning'); + expect(result.lastRun.outcomeMsg).toEqual( + 'Rule reported more than the maximum number of alerts in a single run. Alerts may be missed and recovery notifications may be delayed' + ); + expect(result.lastRun.warning).toEqual('maxAlerts'); + + expect(result.lastRun.alertsCount).toEqual({ + active: 10, + new: 12, + recovered: 11, + ignored: 0, + }); + }); + + it('partial triggered actions status outcome', () => { + const result = lastRunFromState({ + metrics: { + ...getMetrics(), + triggeredActionsStatus: ActionsCompletion.PARTIAL, + }, + }); + + expect(result.lastRun.outcome).toEqual('warning'); + expect(result.lastRun.outcomeMsg).toEqual( + 'The maximum number of actions for this rule type was reached; excess actions were not triggered.' + ); + expect(result.lastRun.warning).toEqual('maxExecutableActions'); + + expect(result.lastRun.alertsCount).toEqual({ + active: 10, + new: 12, + recovered: 11, + ignored: 0, + }); + }); +}); diff --git a/x-pack/plugins/alerting/server/lib/last_run_status.ts b/x-pack/plugins/alerting/server/lib/last_run_status.ts new file mode 100644 index 0000000000000..d7027fc7c2770 --- /dev/null +++ b/x-pack/plugins/alerting/server/lib/last_run_status.ts @@ -0,0 +1,87 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { RuleTaskStateAndMetrics } from '../task_runner/types'; +import { getReasonFromError } from './error_with_reason'; +import { getEsErrorMessage } from './errors'; +import { ActionsCompletion } from '../../common'; +import { + RuleLastRunOutcomeValues, + RuleLastRunOutcomes, + RuleExecutionStatusWarningReasons, + RawRuleLastRun, + RuleLastRun, +} from '../types'; +import { translations } from '../constants/translations'; +import { RuleRunMetrics } from './rule_run_metrics_store'; + +export interface ILastRun { + lastRun: RuleLastRun; + metrics: RuleRunMetrics | null; +} + +export const lastRunFromState = (stateWithMetrics: RuleTaskStateAndMetrics): ILastRun => { + const { metrics } = stateWithMetrics; + let outcome: RuleLastRunOutcomes = RuleLastRunOutcomeValues[0]; + // Check for warning states + let warning = null; + let outcomeMsg = null; + + // We only have a single warning field so prioritizing the alert circuit breaker over the actions circuit breaker + if (metrics.hasReachedAlertLimit) { + outcome = RuleLastRunOutcomeValues[1]; + warning = RuleExecutionStatusWarningReasons.MAX_ALERTS; + outcomeMsg = translations.taskRunner.warning.maxAlerts; + } else if (metrics.triggeredActionsStatus === ActionsCompletion.PARTIAL) { + outcome = RuleLastRunOutcomeValues[1]; + warning = RuleExecutionStatusWarningReasons.MAX_EXECUTABLE_ACTIONS; + outcomeMsg = translations.taskRunner.warning.maxExecutableActions; + } + + return { + lastRun: { + outcome, + outcomeMsg: outcomeMsg || null, + warning: warning || null, + alertsCount: { + active: metrics.numberOfActiveAlerts, + new: metrics.numberOfNewAlerts, + recovered: metrics.numberOfRecoveredAlerts, + ignored: 0, + }, + }, + metrics, + }; +}; + +export const lastRunFromError = (error: Error): ILastRun => { + return { + lastRun: { + outcome: RuleLastRunOutcomeValues[2], + warning: getReasonFromError(error), + outcomeMsg: getEsErrorMessage(error), + alertsCount: {}, + }, + metrics: null, + }; +}; + +export const lastRunToRaw = (lastRun: ILastRun['lastRun']): RawRuleLastRun => { + const { warning, alertsCount, outcomeMsg } = lastRun; + + return { + ...lastRun, + alertsCount: { + active: alertsCount.active || 0, + new: alertsCount.new || 0, + recovered: alertsCount.recovered || 0, + ignored: alertsCount.ignored || 0, + }, + warning: warning ?? null, + outcomeMsg: outcomeMsg ?? null, + }; +}; diff --git a/x-pack/plugins/alerting/server/lib/monitoring.test.ts b/x-pack/plugins/alerting/server/lib/monitoring.test.ts index eeafdfcff1cbe..492e205a99508 100644 --- a/x-pack/plugins/alerting/server/lib/monitoring.test.ts +++ b/x-pack/plugins/alerting/server/lib/monitoring.test.ts @@ -5,7 +5,11 @@ * 2.0. */ -import { getExecutionDurationPercentiles } from './monitoring'; +import { + getExecutionDurationPercentiles, + updateMonitoring, + convertMonitoringFromRawAndVerify, +} from './monitoring'; import { RuleMonitoring } from '../types'; const mockHistory = [ @@ -42,17 +46,23 @@ const mockHistory = [ ]; const mockRuleMonitoring = { - execution: { + run: { history: mockHistory, calculated_metrics: { success_ratio: 0, }, + last_run: { + timestamp: '2022-06-18T01:00:00.000Z', + metrics: { + duration: 123, + }, + }, }, } as RuleMonitoring; describe('getExecutionDurationPercentiles', () => { it('Calculates the percentile given partly undefined durations', () => { - const percentiles = getExecutionDurationPercentiles(mockRuleMonitoring); + const percentiles = getExecutionDurationPercentiles(mockRuleMonitoring.run.history); expect(percentiles.p50).toEqual(250); expect(percentiles.p95).toEqual(500); expect(percentiles.p99).toEqual(500); @@ -66,13 +76,53 @@ describe('getExecutionDurationPercentiles', () => { const newMockRuleMonitoring = { ...mockRuleMonitoring, - execution: { - ...mockRuleMonitoring.execution, + run: { + ...mockRuleMonitoring.run, history: nullDurationHistory, }, } as RuleMonitoring; - const percentiles = getExecutionDurationPercentiles(newMockRuleMonitoring); + const percentiles = getExecutionDurationPercentiles(newMockRuleMonitoring.run.history); expect(Object.keys(percentiles).length).toEqual(0); }); }); + +describe('updateMonitoring', () => { + it('can update monitoring', () => { + const result = updateMonitoring({ + monitoring: mockRuleMonitoring, + timestamp: '2022-07-18T01:00:00.000Z', + duration: 1000, + }); + + expect(result.run.history).toEqual(mockRuleMonitoring.run.history); + expect(result.run.calculated_metrics).toEqual(mockRuleMonitoring.run.calculated_metrics); + expect(result.run.last_run.timestamp).toEqual('2022-07-18T01:00:00.000Z'); + expect(result.run.last_run.metrics.duration).toEqual(1000); + }); +}); + +describe('convertMonitoringFromRawAndVerify', () => { + it('can convert monitoring to raw and verify the duration', () => { + const monitoring = { + run: { + ...mockRuleMonitoring.run, + last_run: { + ...mockRuleMonitoring.run.last_run, + timestamp: 'invalid', + }, + }, + }; + + const mockLoggerDebug = jest.fn(); + const mockLogger = { + debug: mockLoggerDebug, + }; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const result = convertMonitoringFromRawAndVerify(mockLogger as any, '123', monitoring); + expect(mockLoggerDebug).toHaveBeenCalledWith( + 'invalid monitoring last_run.timestamp "invalid" in raw rule 123' + ); + expect(Date.parse(result!.run.last_run.timestamp)).toBeTruthy(); + }); +}); diff --git a/x-pack/plugins/alerting/server/lib/monitoring.ts b/x-pack/plugins/alerting/server/lib/monitoring.ts index d817b10225ee0..93da6e2283152 100644 --- a/x-pack/plugins/alerting/server/lib/monitoring.ts +++ b/x-pack/plugins/alerting/server/lib/monitoring.ts @@ -4,19 +4,37 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - +import { Logger } from '@kbn/core/server'; import stats from 'stats-lite'; -import { RuleMonitoring } from '../types'; +import { RuleMonitoring, RawRuleMonitoring, RuleMonitoringHistory } from '../types'; + +export const INITIAL_METRICS = { + total_search_duration_ms: null, + total_indexing_duration_ms: null, + total_alerts_detected: null, + total_alerts_created: null, + gap_duration_s: null, +}; -export const getExecutionSuccessRatio = (ruleMonitoring: RuleMonitoring) => { - const { history } = ruleMonitoring.execution; - return history.filter(({ success }) => success).length / history.length; +export const getDefaultMonitoring = (timestamp: string): RawRuleMonitoring => { + return { + run: { + history: [], + calculated_metrics: { + success_ratio: 0, + }, + last_run: { + timestamp, + metrics: INITIAL_METRICS, + }, + }, + }; }; -export const getExecutionDurationPercentiles = (ruleMonitoring: RuleMonitoring) => { - const durationSamples = ruleMonitoring.execution.history.reduce((duration, history) => { - if (typeof history.duration === 'number') { - return [...duration, history.duration]; +export const getExecutionDurationPercentiles = (history: RuleMonitoringHistory[]) => { + const durationSamples = history.reduce((duration, historyItem) => { + if (typeof historyItem.duration === 'number') { + return [...duration, historyItem.duration]; } return duration; }, []); @@ -31,3 +49,56 @@ export const getExecutionDurationPercentiles = (ruleMonitoring: RuleMonitoring) return {}; }; + +// Immutably updates the monitoring object with timestamp and duration. +// Used when converting from and between raw monitoring object +export const updateMonitoring = ({ + monitoring, + timestamp, + duration, +}: { + monitoring: RuleMonitoring; + timestamp: string; + duration?: number; +}) => { + const { run } = monitoring; + const { last_run: lastRun, ...rest } = run; + const { metrics = INITIAL_METRICS } = lastRun; + + return { + run: { + last_run: { + timestamp, + metrics: { + ...metrics, + duration, + }, + }, + ...rest, + }, + }; +}; + +export const convertMonitoringFromRawAndVerify = ( + logger: Logger, + ruleId: string, + monitoring: RawRuleMonitoring +): RuleMonitoring | undefined => { + if (!monitoring) { + return undefined; + } + + const lastRunDate = monitoring.run.last_run.timestamp; + + let parsedDateMillis = lastRunDate ? Date.parse(lastRunDate) : Date.now(); + if (isNaN(parsedDateMillis)) { + logger.debug(`invalid monitoring last_run.timestamp "${lastRunDate}" in raw rule ${ruleId}`); + parsedDateMillis = Date.now(); + } + + return updateMonitoring({ + monitoring, + timestamp: new Date(parsedDateMillis).toISOString(), + duration: monitoring.run.last_run.metrics.duration, + }); +}; diff --git a/x-pack/plugins/alerting/server/lib/next_run.ts b/x-pack/plugins/alerting/server/lib/next_run.ts new file mode 100644 index 0000000000000..8ce87e9db6883 --- /dev/null +++ b/x-pack/plugins/alerting/server/lib/next_run.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import moment from 'moment'; +import { parseDuration } from '../../common'; + +export const getNextRun = ({ + startDate, + interval, +}: { + startDate?: Date | null; + interval: string; +}) => { + return moment(startDate || new Date()) + .add(parseDuration(interval), 'ms') + .toISOString(); +}; diff --git a/x-pack/plugins/alerting/server/mocks.ts b/x-pack/plugins/alerting/server/mocks.ts index 0a99489822d38..18f0a66bb1657 100644 --- a/x-pack/plugins/alerting/server/mocks.ts +++ b/x-pack/plugins/alerting/server/mocks.ts @@ -14,7 +14,7 @@ import { searchSourceCommonMock } from '@kbn/data-plugin/common/search/search_so import { rulesClientMock } from './rules_client.mock'; import { PluginSetupContract, PluginStartContract } from './plugin'; import { Alert, AlertFactoryDoneUtils } from './alert'; -import { AlertInstanceContext, AlertInstanceState } from './types'; +import { AlertInstanceContext, AlertInstanceState, PublicRuleMonitoringService } from './types'; export { rulesClientMock }; @@ -96,6 +96,18 @@ const createAbortableSearchServiceMock = () => { }; }; +const createRuleMonitoringServiceMock = () => { + const mock = { + setLastRunMetricsTotalSearchDurationMs: jest.fn(), + setLastRunMetricsTotalIndexingDurationMs: jest.fn(), + setLastRunMetricsTotalAlertsDetected: jest.fn(), + setLastRunMetricsTotalAlertsCreated: jest.fn(), + setLastRunMetricsGapDurationS: jest.fn(), + } as unknown as jest.Mocked; + + return mock; +}; + const createRuleExecutorServicesMock = < InstanceState extends AlertInstanceState = AlertInstanceState, InstanceContext extends AlertInstanceContext = AlertInstanceContext @@ -118,6 +130,7 @@ const createRuleExecutorServicesMock = < shouldStopExecution: () => true, search: createAbortableSearchServiceMock(), searchSourceClient: searchSourceCommonMock, + ruleMonitoringService: createRuleMonitoringServiceMock(), }; }; export type RuleExecutorServicesMock = ReturnType; @@ -128,3 +141,5 @@ export const alertsMock = { createStart: createStartMock, createRuleExecutorServices: createRuleExecutorServicesMock, }; + +export const ruleMonitoringServiceMock = { create: createRuleMonitoringServiceMock }; diff --git a/x-pack/plugins/alerting/server/monitoring/rule_monitoring_service.test.ts b/x-pack/plugins/alerting/server/monitoring/rule_monitoring_service.test.ts new file mode 100644 index 0000000000000..54fc6552308b5 --- /dev/null +++ b/x-pack/plugins/alerting/server/monitoring/rule_monitoring_service.test.ts @@ -0,0 +1,175 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { RuleMonitoringService } from './rule_monitoring_service'; +import { getDefaultMonitoring } from '../lib/monitoring'; + +const mockNow = '2020-01-01T02:00:00.000Z'; + +const ONE_MINUTE = 60 * 1000; +const ONE_HOUR = 60 * ONE_MINUTE; + +describe('RuleMonitoringService', () => { + beforeAll(() => { + jest.useFakeTimers('modern'); + jest.setSystemTime(new Date(mockNow).getTime()); + }); + + afterAll(() => { + jest.useRealTimers(); + }); + + it('should initialize with default monitoring', () => { + const ruleMonitoringService = new RuleMonitoringService(); + expect(ruleMonitoringService.getMonitoring()).toEqual( + getDefaultMonitoring(new Date(mockNow).toISOString()) + ); + }); + + it('should add history', () => { + const ruleMonitoringService = new RuleMonitoringService(); + + jest.advanceTimersByTime(ONE_HOUR); + const firstRunDate = new Date(); + + ruleMonitoringService.addHistory({ + duration: ONE_MINUTE, + hasError: false, + runDate: firstRunDate, + }); + + jest.advanceTimersByTime(ONE_HOUR); + const secondRunDate = new Date(); + + ruleMonitoringService.addHistory({ + duration: 2 * ONE_MINUTE, + hasError: true, + runDate: secondRunDate, + }); + + const { run } = ruleMonitoringService.getMonitoring(); + const { history, last_run: lastRun, calculated_metrics: calculatedMetrics } = run; + const { timestamp, metrics } = lastRun; + + expect(history.length).toEqual(2); + expect(history[0]).toEqual({ + success: true, + timestamp: firstRunDate.getTime(), + duration: ONE_MINUTE, + }); + expect(history[1]).toEqual({ + success: false, + timestamp: secondRunDate.getTime(), + duration: 2 * ONE_MINUTE, + }); + + expect(timestamp).toEqual(secondRunDate.toISOString()); + expect(metrics.duration).toEqual(2 * ONE_MINUTE); + + expect(calculatedMetrics).toEqual({ success_ratio: 0.5, p50: 90000, p95: 120000, p99: 120000 }); + }); + + describe('setters', () => { + it('should set monitoring', () => { + const ruleMonitoringService = new RuleMonitoringService(); + const customMonitoring = { + run: { + history: [ + { + success: true, + duration: 100000, + timestamp: 0, + }, + ], + calculated_metrics: { + success_ratio: 1, + p50: 100, + p95: 1000, + p99: 10000, + }, + last_run: { + timestamp: mockNow, + metrics: { + duration: 100000, + }, + }, + }, + }; + ruleMonitoringService.setMonitoring(customMonitoring); + expect(ruleMonitoringService.getMonitoring()).toEqual(customMonitoring); + }); + + it('should set totalSearchDurationMs', () => { + const ruleMonitoringService = new RuleMonitoringService(); + const { setLastRunMetricsTotalSearchDurationMs } = + ruleMonitoringService.getLastRunMetricsSetters(); + setLastRunMetricsTotalSearchDurationMs(123); + + const { + run: { + last_run: { metrics }, + }, + } = ruleMonitoringService.getMonitoring(); + expect(metrics.total_search_duration_ms).toEqual(123); + }); + + it('should set totalIndexDurationMs', () => { + const ruleMonitoringService = new RuleMonitoringService(); + const { setLastRunMetricsTotalIndexingDurationMs } = + ruleMonitoringService.getLastRunMetricsSetters(); + setLastRunMetricsTotalIndexingDurationMs(234); + + const { + run: { + last_run: { metrics }, + }, + } = ruleMonitoringService.getMonitoring(); + expect(metrics.total_indexing_duration_ms).toEqual(234); + }); + + it('should set totalAlertsDetected', () => { + const ruleMonitoringService = new RuleMonitoringService(); + const { setLastRunMetricsTotalAlertsDetected } = + ruleMonitoringService.getLastRunMetricsSetters(); + setLastRunMetricsTotalAlertsDetected(345); + + const { + run: { + last_run: { metrics }, + }, + } = ruleMonitoringService.getMonitoring(); + expect(metrics.total_alerts_detected).toEqual(345); + }); + + it('should set totalAlertsCreated', () => { + const ruleMonitoringService = new RuleMonitoringService(); + const { setLastRunMetricsTotalAlertsCreated } = + ruleMonitoringService.getLastRunMetricsSetters(); + setLastRunMetricsTotalAlertsCreated(456); + + const { + run: { + last_run: { metrics }, + }, + } = ruleMonitoringService.getMonitoring(); + expect(metrics.total_alerts_created).toEqual(456); + }); + + it('should set gapDurationS', () => { + const ruleMonitoringService = new RuleMonitoringService(); + const { setLastRunMetricsGapDurationS } = ruleMonitoringService.getLastRunMetricsSetters(); + setLastRunMetricsGapDurationS(567); + + const { + run: { + last_run: { metrics }, + }, + } = ruleMonitoringService.getMonitoring(); + expect(metrics.gap_duration_s).toEqual(567); + }); + }); +}); diff --git a/x-pack/plugins/alerting/server/monitoring/rule_monitoring_service.ts b/x-pack/plugins/alerting/server/monitoring/rule_monitoring_service.ts new file mode 100644 index 0000000000000..0043f47c51633 --- /dev/null +++ b/x-pack/plugins/alerting/server/monitoring/rule_monitoring_service.ts @@ -0,0 +1,98 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getDefaultMonitoring, getExecutionDurationPercentiles } from '../lib/monitoring'; +import { RuleMonitoring, RuleMonitoringHistory, PublicRuleMonitoringService } from '../types'; + +export class RuleMonitoringService { + private monitoring: RuleMonitoring = getDefaultMonitoring(new Date().toISOString()); + + public setLastRunMetricsDuration(duration: number) { + this.monitoring.run.last_run.metrics.duration = duration; + } + + public setMonitoring(monitoringFromSO: RuleMonitoring | undefined) { + if (monitoringFromSO) { + this.monitoring = monitoringFromSO; + } + } + + public getMonitoring(): RuleMonitoring { + return this.monitoring; + } + + public addHistory({ + duration, + hasError = true, + runDate, + }: { + duration: number | undefined; + hasError: boolean; + runDate: Date; + }) { + const date = runDate ?? new Date(); + const monitoringHistory: RuleMonitoringHistory = { + success: true, + timestamp: date.getTime(), + }; + if (null != duration) { + monitoringHistory.duration = duration; + this.setLastRunMetricsDuration(duration); + } + if (hasError) { + monitoringHistory.success = false; + } + this.monitoring.run.last_run.timestamp = date.toISOString(); + this.monitoring.run.history.push(monitoringHistory); + this.monitoring.run.calculated_metrics = { + success_ratio: this.buildExecutionSuccessRatio(), + ...this.buildExecutionDurationPercentiles(), + }; + } + + public getLastRunMetricsSetters(): PublicRuleMonitoringService { + return { + setLastRunMetricsTotalSearchDurationMs: + this.setLastRunMetricsTotalSearchDurationMs.bind(this), + setLastRunMetricsTotalIndexingDurationMs: + this.setLastRunMetricsTotalIndexingDurationMs.bind(this), + setLastRunMetricsTotalAlertsDetected: this.setLastRunMetricsTotalAlertsDetected.bind(this), + setLastRunMetricsTotalAlertsCreated: this.setLastRunMetricsTotalAlertsCreated.bind(this), + setLastRunMetricsGapDurationS: this.setLastRunMetricsGapDurationS.bind(this), + }; + } + + private setLastRunMetricsTotalSearchDurationMs(totalSearchDurationMs: number) { + this.monitoring.run.last_run.metrics.total_search_duration_ms = totalSearchDurationMs; + } + + private setLastRunMetricsTotalIndexingDurationMs(totalIndexingDurationMs: number) { + this.monitoring.run.last_run.metrics.total_indexing_duration_ms = totalIndexingDurationMs; + } + + private setLastRunMetricsTotalAlertsDetected(totalAlertDetected: number) { + this.monitoring.run.last_run.metrics.total_alerts_detected = totalAlertDetected; + } + + private setLastRunMetricsTotalAlertsCreated(totalAlertCreated: number) { + this.monitoring.run.last_run.metrics.total_alerts_created = totalAlertCreated; + } + + private setLastRunMetricsGapDurationS(gapDurationS: number) { + this.monitoring.run.last_run.metrics.gap_duration_s = gapDurationS; + } + + private buildExecutionSuccessRatio() { + const { history } = this.monitoring.run; + return history.filter(({ success }) => success).length / history.length; + } + + private buildExecutionDurationPercentiles = () => { + const { history } = this.monitoring.run; + return getExecutionDurationPercentiles(history); + }; +} diff --git a/x-pack/plugins/alerting/server/routes/aggregate_rules.test.ts b/x-pack/plugins/alerting/server/routes/aggregate_rules.test.ts index 8c24b457df565..26210e9bed285 100644 --- a/x-pack/plugins/alerting/server/routes/aggregate_rules.test.ts +++ b/x-pack/plugins/alerting/server/routes/aggregate_rules.test.ts @@ -49,6 +49,11 @@ describe('aggregateRulesRoute', () => { pending: 1, unknown: 0, }, + ruleLastRunOutcome: { + succeeded: 1, + failed: 2, + warning: 3, + }, ruleEnabledStatus: { disabled: 1, enabled: 40, @@ -88,6 +93,11 @@ describe('aggregateRulesRoute', () => { "pending": 1, "unknown": 0, }, + "rule_last_run_outcome": Object { + "failed": 2, + "succeeded": 1, + "warning": 3, + }, "rule_muted_status": Object { "muted": 2, "unmuted": 39, @@ -128,6 +138,11 @@ describe('aggregateRulesRoute', () => { pending: 1, unknown: 0, }, + rule_last_run_outcome: { + succeeded: 1, + failed: 2, + warning: 3, + }, rule_muted_status: { muted: 2, unmuted: 39, @@ -156,6 +171,11 @@ describe('aggregateRulesRoute', () => { pending: 1, unknown: 0, }, + ruleLastRunOutcome: { + succeeded: 2, + failed: 4, + warning: 6, + }, }); const [context, req, res] = mockHandlerArguments( @@ -209,6 +229,11 @@ describe('aggregateRulesRoute', () => { pending: 1, unknown: 0, }, + ruleLastRunOutcome: { + succeeded: 2, + failed: 4, + warning: 6, + }, }; rulesClient.aggregate.mockResolvedValueOnce(aggregateResult); const [, handler] = router.get.mock.calls[0]; diff --git a/x-pack/plugins/alerting/server/routes/aggregate_rules.ts b/x-pack/plugins/alerting/server/routes/aggregate_rules.ts index c48c74fc28754..acab5ca75d2e0 100644 --- a/x-pack/plugins/alerting/server/routes/aggregate_rules.ts +++ b/x-pack/plugins/alerting/server/routes/aggregate_rules.ts @@ -47,6 +47,7 @@ const rewriteQueryReq: RewriteRequestCase = ({ }); const rewriteBodyRes: RewriteResponseCase = ({ alertExecutionStatus, + ruleLastRunOutcome, ruleEnabledStatus, ruleMutedStatus, ruleSnoozedStatus, @@ -55,6 +56,7 @@ const rewriteBodyRes: RewriteResponseCase = ({ }) => ({ ...rest, rule_execution_status: alertExecutionStatus, + rule_last_run_outcome: ruleLastRunOutcome, rule_enabled_status: ruleEnabledStatus, rule_muted_status: ruleMutedStatus, rule_snoozed_status: ruleSnoozedStatus, diff --git a/x-pack/plugins/alerting/server/routes/create_rule.ts b/x-pack/plugins/alerting/server/routes/create_rule.ts index 442162ae21cbb..1b114dc54d26b 100644 --- a/x-pack/plugins/alerting/server/routes/create_rule.ts +++ b/x-pack/plugins/alerting/server/routes/create_rule.ts @@ -14,6 +14,7 @@ import { handleDisabledApiKeysError, verifyAccessAndContext, countUsageOfPredefinedIds, + rewriteRuleLastRun, } from './lib'; import { SanitizedRule, @@ -68,6 +69,8 @@ const rewriteBodyRes: RewriteResponseCase> = ({ muteAll, mutedInstanceIds, snoozeSchedule, + lastRun, + nextRun, executionStatus: { lastExecutionDate, lastDuration, ...executionStatus }, ...rest }) => ({ @@ -94,6 +97,8 @@ const rewriteBodyRes: RewriteResponseCase> = ({ params, connector_type_id: actionTypeId, })), + ...(lastRun ? { last_run: rewriteRuleLastRun(lastRun) } : {}), + ...(nextRun ? { next_run: nextRun } : {}), }); export const createRuleRoute = ({ router, licenseState, usageCounter }: RouteOptions) => { diff --git a/x-pack/plugins/alerting/server/routes/get_rule.ts b/x-pack/plugins/alerting/server/routes/get_rule.ts index f7834ec52b9ed..0b529b35fa466 100644 --- a/x-pack/plugins/alerting/server/routes/get_rule.ts +++ b/x-pack/plugins/alerting/server/routes/get_rule.ts @@ -9,7 +9,7 @@ import { omit } from 'lodash'; import { schema } from '@kbn/config-schema'; import { IRouter } from '@kbn/core/server'; import { ILicenseState } from '../lib'; -import { verifyAccessAndContext, RewriteResponseCase } from './lib'; +import { verifyAccessAndContext, RewriteResponseCase, rewriteRuleLastRun } from './lib'; import { RuleTypeParams, AlertingRequestHandlerContext, @@ -37,6 +37,8 @@ const rewriteBodyRes: RewriteResponseCase> = ({ scheduledTaskId, snoozeSchedule, isSnoozedUntil, + lastRun, + nextRun, ...rest }) => ({ ...rest, @@ -63,6 +65,8 @@ const rewriteBodyRes: RewriteResponseCase> = ({ params, connector_type_id: actionTypeId, })), + ...(lastRun ? { last_run: rewriteRuleLastRun(lastRun) } : {}), + ...(nextRun ? { next_run: nextRun } : {}), }); interface BuildGetRulesRouteParams { diff --git a/x-pack/plugins/alerting/server/routes/legacy/aggregate.test.ts b/x-pack/plugins/alerting/server/routes/legacy/aggregate.test.ts index a2fad24b24c0f..1fc3ee17ab51e 100644 --- a/x-pack/plugins/alerting/server/routes/legacy/aggregate.test.ts +++ b/x-pack/plugins/alerting/server/routes/legacy/aggregate.test.ts @@ -54,6 +54,11 @@ describe('aggregateAlertRoute', () => { pending: 1, unknown: 0, }, + ruleLastRunOutcome: { + succeeded: 1, + failed: 2, + warning: 3, + }, }; rulesClient.aggregate.mockResolvedValueOnce(aggregateResult); @@ -77,6 +82,11 @@ describe('aggregateAlertRoute', () => { "pending": 1, "unknown": 0, }, + "ruleLastRunOutcome": Object { + "failed": 2, + "succeeded": 1, + "warning": 3, + }, }, } `); @@ -113,6 +123,11 @@ describe('aggregateAlertRoute', () => { pending: 1, unknown: 0, }, + ruleLastRunOutcome: { + succeeded: 1, + failed: 2, + warning: 3, + }, }); const [context, req, res] = mockHandlerArguments( diff --git a/x-pack/plugins/alerting/server/routes/lib/index.ts b/x-pack/plugins/alerting/server/routes/lib/index.ts index 90d903ada6eed..cda768e7b363b 100644 --- a/x-pack/plugins/alerting/server/routes/lib/index.ts +++ b/x-pack/plugins/alerting/server/routes/lib/index.ts @@ -18,5 +18,5 @@ export type { } from './rewrite_request_case'; export { verifyAccessAndContext } from './verify_access_and_context'; export { countUsageOfPredefinedIds } from './count_usage_of_predefined_ids'; -export { rewriteRule } from './rewrite_rule'; +export { rewriteRule, rewriteRuleLastRun } from './rewrite_rule'; export { rewriteNamespaces } from './rewrite_namespaces'; diff --git a/x-pack/plugins/alerting/server/routes/lib/rewrite_request_case.ts b/x-pack/plugins/alerting/server/routes/lib/rewrite_request_case.ts index 2874f9567231b..4eb352d2b2b8c 100644 --- a/x-pack/plugins/alerting/server/routes/lib/rewrite_request_case.ts +++ b/x-pack/plugins/alerting/server/routes/lib/rewrite_request_case.ts @@ -24,7 +24,7 @@ type RenameAlertToRule = K extends `alertTypeId` export type AsApiContract< T, - ComplexPropertyKeys = `actions` | `executionStatus`, + ComplexPropertyKeys = `actions` | `executionStatus` | 'lastRun', OpaquePropertyKeys = `params` > = T extends Array ? Array> diff --git a/x-pack/plugins/alerting/server/routes/lib/rewrite_rule.ts b/x-pack/plugins/alerting/server/routes/lib/rewrite_rule.ts index 4b3ab65e3e8e4..270db64812b28 100644 --- a/x-pack/plugins/alerting/server/routes/lib/rewrite_rule.ts +++ b/x-pack/plugins/alerting/server/routes/lib/rewrite_rule.ts @@ -6,7 +6,16 @@ */ import { omit } from 'lodash'; -import { RuleTypeParams, SanitizedRule } from '../../types'; +import { RuleTypeParams, SanitizedRule, RuleLastRun } from '../../types'; + +export const rewriteRuleLastRun = (lastRun: RuleLastRun) => { + const { outcomeMsg, alertsCount, ...rest } = lastRun; + return { + alerts_count: alertsCount, + outcome_msg: outcomeMsg, + ...rest, + }; +}; export const rewriteRule = ({ alertTypeId, @@ -24,6 +33,8 @@ export const rewriteRule = ({ snoozeSchedule, isSnoozedUntil, activeSnoozes, + lastRun, + nextRun, ...rest }: SanitizedRule & { activeSnoozes?: string[] }) => ({ ...rest, @@ -51,4 +62,6 @@ export const rewriteRule = ({ params, connector_type_id: actionTypeId, })), + ...(lastRun ? { last_run: rewriteRuleLastRun(lastRun) } : {}), + ...(nextRun ? { next_run: nextRun } : {}), }); diff --git a/x-pack/plugins/alerting/server/routes/resolve_rule.ts b/x-pack/plugins/alerting/server/routes/resolve_rule.ts index b3576c0c5ed44..48d2253a03fc9 100644 --- a/x-pack/plugins/alerting/server/routes/resolve_rule.ts +++ b/x-pack/plugins/alerting/server/routes/resolve_rule.ts @@ -9,7 +9,7 @@ import { omit } from 'lodash'; import { schema } from '@kbn/config-schema'; import { IRouter } from '@kbn/core/server'; import { ILicenseState } from '../lib'; -import { verifyAccessAndContext, RewriteResponseCase } from './lib'; +import { verifyAccessAndContext, RewriteResponseCase, rewriteRuleLastRun } from './lib'; import { RuleTypeParams, AlertingRequestHandlerContext, @@ -34,6 +34,8 @@ const rewriteBodyRes: RewriteResponseCase> executionStatus, actions, scheduledTaskId, + lastRun, + nextRun, ...rest }) => ({ ...rest, @@ -58,6 +60,8 @@ const rewriteBodyRes: RewriteResponseCase> params, connector_type_id: actionTypeId, })), + ...(lastRun ? { last_run: rewriteRuleLastRun(lastRun) } : {}), + ...(nextRun ? { next_run: nextRun } : {}), }); export const resolveRuleRoute = ( diff --git a/x-pack/plugins/alerting/server/routes/update_rule.ts b/x-pack/plugins/alerting/server/routes/update_rule.ts index 1faddd66c8f0e..2b7f6b3c98b39 100644 --- a/x-pack/plugins/alerting/server/routes/update_rule.ts +++ b/x-pack/plugins/alerting/server/routes/update_rule.ts @@ -15,6 +15,7 @@ import { RewriteResponseCase, RewriteRequestCase, handleDisabledApiKeysError, + rewriteRuleLastRun, } from './lib'; import { RuleTypeParams, @@ -72,6 +73,8 @@ const rewriteBodyRes: RewriteResponseCase> = ({ executionStatus, snoozeSchedule, isSnoozedUntil, + lastRun, + nextRun, ...rest }) => ({ ...rest, @@ -106,6 +109,8 @@ const rewriteBodyRes: RewriteResponseCase> = ({ })), } : {}), + ...(lastRun ? { last_run: rewriteRuleLastRun(lastRun) } : {}), + ...(nextRun ? { next_run: nextRun } : {}), }); export const updateRuleRoute = ( diff --git a/x-pack/plugins/alerting/server/rules_client/rules_client.ts b/x-pack/plugins/alerting/server/rules_client/rules_client.ts index 0d1a03ff6f3ed..67a347f5d45c1 100644 --- a/x-pack/plugins/alerting/server/rules_client/rules_client.ts +++ b/x-pack/plugins/alerting/server/rules_client/rules_client.ts @@ -65,6 +65,7 @@ import { RuleTaskState, AlertSummary, RuleExecutionStatusValues, + RuleLastRunOutcomeValues, RuleNotifyWhenType, RuleTypeParams, ResolvedSanitizedRule, @@ -83,6 +84,10 @@ import { convertRuleIdsToKueryNode, getRuleSnoozeEndTime, convertEsSortToEventLogSort, + getDefaultMonitoring, + updateMonitoring, + convertMonitoringFromRawAndVerify, + getNextRun, } from '../lib'; import { taskInstanceToAlertTaskInstance } from '../task_runner/alert_task_instance'; import { RegistryRuleType, UntypedNormalizedRuleType } from '../rule_type_registry'; @@ -113,7 +118,6 @@ import { getRuleExecutionStatusPending } from '../lib/rule_execution_status'; import { Alert } from '../alert'; import { EVENT_LOG_ACTIONS } from '../plugin'; import { createAlertEventLogRecordObject } from '../lib/create_alert_event_log_record_object'; -import { getDefaultRuleMonitoring } from '../task_runner/task_runner'; import { getMappedParams, getModifiedField, @@ -153,6 +157,12 @@ export interface RuleAggregation { doc_count: number; }>; }; + outcome: { + buckets: Array<{ + key: string; + doc_count: number; + }>; + }; muted: { buckets: Array<{ key: number; @@ -335,6 +345,7 @@ interface IndexType { export interface AggregateResult { alertExecutionStatus: { [status: string]: number }; + ruleLastRunOutcome: { [status: string]: number }; ruleEnabledStatus?: { enabled: number; disabled: number }; ruleMutedStatus?: { muted: number; unmuted: number }; ruleSnoozedStatus?: { snoozed: number }; @@ -364,6 +375,8 @@ export interface CreateOptions { | 'executionStatus' | 'snoozeSchedule' | 'isSnoozedUntil' + | 'lastRun' + | 'nextRun' > & { actions: NormalizedAlertAction[] }; options?: { id?: string; @@ -584,6 +597,7 @@ export class RulesClient { } = await this.extractReferences(ruleType, data.actions, validatedAlertTypeParams); const createTime = Date.now(); + const lastRunTimestamp = new Date(); const legacyId = Semver.lt(this.kibanaVersion, '8.0.0') ? id : null; const notifyWhen = getRuleNotifyWhenType(data.notifyWhen, data.throttle); @@ -601,8 +615,8 @@ export class RulesClient { muteAll: false, mutedInstanceIds: [], notifyWhen, - executionStatus: getRuleExecutionStatusPending(new Date().toISOString()), - monitoring: getDefaultRuleMonitoring(), + executionStatus: getRuleExecutionStatusPending(lastRunTimestamp.toISOString()), + monitoring: getDefaultMonitoring(lastRunTimestamp.toISOString()), }; const mappedParams = getMappedParams(updatedParams); @@ -1433,6 +1447,9 @@ export class RulesClient { status: { terms: { field: 'alert.attributes.executionStatus.status' }, }, + outcome: { + terms: { field: 'alert.attributes.lastRun.outcome' }, + }, enabled: { terms: { field: 'alert.attributes.enabled' }, }, @@ -1463,6 +1480,7 @@ export class RulesClient { // Return a placeholder with all zeroes const placeholder: AggregateResult = { alertExecutionStatus: {}, + ruleLastRunOutcome: {}, ruleEnabledStatus: { enabled: 0, disabled: 0, @@ -1487,11 +1505,21 @@ export class RulesClient { }) ); + const ruleLastRunOutcome = resp.aggregations.outcome.buckets.map( + ({ key, doc_count: docCount }) => ({ + [key]: docCount, + }) + ); + const ret: AggregateResult = { alertExecutionStatus: alertExecutionStatus.reduce( (acc, curr: { [status: string]: number }) => Object.assign(acc, curr), {} ), + ruleLastRunOutcome: ruleLastRunOutcome.reduce( + (acc, curr: { [status: string]: number }) => Object.assign(acc, curr), + {} + ), }; // Fill missing keys with zeroes @@ -1500,6 +1528,11 @@ export class RulesClient { ret.alertExecutionStatus[key] = 0; } } + for (const key of RuleLastRunOutcomeValues) { + if (!ret.ruleLastRunOutcome.hasOwnProperty(key)) { + ret.ruleLastRunOutcome[key] = 0; + } + } const enabledBuckets = resp.aggregations.enabled.buckets; ret.ruleEnabledStatus = { @@ -2606,17 +2639,28 @@ export class RulesClient { if (attributes.enabled === false) { const username = await this.getUserName(); + const now = new Date(); + + const schedule = attributes.schedule as IntervalSchedule; const updateAttributes = this.updateMeta({ ...attributes, ...(!existingApiKey && (await this.createNewAPIKeySet({ attributes, username }))), + ...(attributes.monitoring && { + monitoring: updateMonitoring({ + monitoring: attributes.monitoring, + timestamp: now.toISOString(), + duration: 0, + }), + }), + nextRun: getNextRun({ interval: schedule.interval }), enabled: true, updatedBy: username, - updatedAt: new Date().toISOString(), + updatedAt: now.toISOString(), executionStatus: { status: 'pending', lastDuration: 0, - lastExecutionDate: new Date().toISOString(), + lastExecutionDate: now.toISOString(), error: null, warning: null, }, @@ -2793,6 +2837,7 @@ export class RulesClient { scheduledTaskId: attributes.scheduledTaskId === id ? attributes.scheduledTaskId : null, updatedBy: await this.getUserName(), updatedAt: new Date().toISOString(), + nextRun: null, }), { version } ); @@ -3442,6 +3487,8 @@ export class RulesClient { scheduledTaskId, params, executionStatus, + monitoring, + nextRun, schedule, actions, snoozeSchedule, @@ -3468,6 +3515,7 @@ export class RulesClient { snoozeSchedule, }) : null; + const includeMonitoring = monitoring && !excludeFromPublicApi; const rule = { id, notifyWhen, @@ -3493,6 +3541,10 @@ export class RulesClient { ...(executionStatus ? { executionStatus: ruleExecutionStatusFromRaw(this.logger, id, executionStatus) } : {}), + ...(includeMonitoring + ? { monitoring: convertMonitoringFromRawAndVerify(this.logger, id, monitoring) } + : {}), + ...(nextRun ? { nextRun: new Date(nextRun) } : {}), }; return includeLegacyId diff --git a/x-pack/plugins/alerting/server/rules_client/tests/aggregate.test.ts b/x-pack/plugins/alerting/server/rules_client/tests/aggregate.test.ts index 2826ba0284b90..494af8f668bfb 100644 --- a/x-pack/plugins/alerting/server/rules_client/tests/aggregate.test.ts +++ b/x-pack/plugins/alerting/server/rules_client/tests/aggregate.test.ts @@ -90,6 +90,13 @@ describe('aggregate()', () => { { key: 'warning', doc_count: 1 }, ], }, + outcome: { + buckets: [ + { key: 'succeeded', doc_count: 2 }, + { key: 'failed', doc_count: 4 }, + { key: 'warning', doc_count: 6 }, + ], + }, enabled: { buckets: [ { key: 0, key_as_string: '0', doc_count: 2 }, @@ -165,6 +172,11 @@ describe('aggregate()', () => { "disabled": 2, "enabled": 28, }, + "ruleLastRunOutcome": Object { + "failed": 4, + "succeeded": 2, + "warning": 6, + }, "ruleMutedStatus": Object { "muted": 3, "unmuted": 27, @@ -191,6 +203,9 @@ describe('aggregate()', () => { status: { terms: { field: 'alert.attributes.executionStatus.status' }, }, + outcome: { + terms: { field: 'alert.attributes.lastRun.outcome' }, + }, enabled: { terms: { field: 'alert.attributes.enabled' }, }, @@ -246,6 +261,9 @@ describe('aggregate()', () => { status: { terms: { field: 'alert.attributes.executionStatus.status' }, }, + outcome: { + terms: { field: 'alert.attributes.lastRun.outcome' }, + }, enabled: { terms: { field: 'alert.attributes.enabled' }, }, diff --git a/x-pack/plugins/alerting/server/rules_client/tests/create.test.ts b/x-pack/plugins/alerting/server/rules_client/tests/create.test.ts index f5192bf6cbe65..aad483f09fe9e 100644 --- a/x-pack/plugins/alerting/server/rules_client/tests/create.test.ts +++ b/x-pack/plugins/alerting/server/rules_client/tests/create.test.ts @@ -19,7 +19,7 @@ import { TaskStatus } from '@kbn/task-manager-plugin/server'; import { auditLoggerMock } from '@kbn/security-plugin/server/audit/mocks'; import { getBeforeSetup, setGlobalDate } from './lib'; import { RecoveredActionGroup } from '../../../common'; -import { getDefaultRuleMonitoring } from '../../task_runner/task_runner'; +import { getDefaultMonitoring } from '../../lib/monitoring'; import { bulkMarkApiKeysForInvalidation } from '../../invalidate_pending_api_keys/bulk_mark_api_keys_for_invalidation'; jest.mock('../../invalidate_pending_api_keys/bulk_mark_api_keys_for_invalidation', () => ({ @@ -421,11 +421,21 @@ describe('create()', () => { "versionApiKeyLastmodified": "v8.0.0", }, "monitoring": Object { - "execution": Object { + "run": Object { "calculated_metrics": Object { "success_ratio": 0, }, "history": Array [], + "last_run": Object { + "metrics": Object { + "gap_duration_s": null, + "total_alerts_created": null, + "total_alerts_detected": null, + "total_indexing_duration_ms": null, + "total_search_duration_ms": null, + }, + "timestamp": "2019-02-12T21:01:22.479Z", + }, }, }, "muteAll": false, @@ -628,11 +638,21 @@ describe('create()', () => { "versionApiKeyLastmodified": "v7.10.0", }, "monitoring": Object { - "execution": Object { + "run": Object { "calculated_metrics": Object { "success_ratio": 0, }, "history": Array [], + "last_run": Object { + "metrics": Object { + "gap_duration_s": null, + "total_alerts_created": null, + "total_alerts_detected": null, + "total_indexing_duration_ms": null, + "total_search_duration_ms": null, + }, + "timestamp": "2019-02-12T21:01:22.479Z", + }, }, }, "muteAll": false, @@ -1056,7 +1076,7 @@ describe('create()', () => { status: 'pending', warning: null, }, - monitoring: getDefaultRuleMonitoring(), + monitoring: getDefaultMonitoring('2019-02-12T21:01:22.479Z'), meta: { versionApiKeyLastmodified: kibanaVersion }, muteAll: false, snoozeSchedule: [], @@ -1255,7 +1275,7 @@ describe('create()', () => { status: 'pending', warning: null, }, - monitoring: getDefaultRuleMonitoring(), + monitoring: getDefaultMonitoring('2019-02-12T21:01:22.479Z'), meta: { versionApiKeyLastmodified: kibanaVersion }, muteAll: false, snoozeSchedule: [], @@ -1423,7 +1443,7 @@ describe('create()', () => { status: 'pending', warning: null, }, - monitoring: getDefaultRuleMonitoring(), + monitoring: getDefaultMonitoring('2019-02-12T21:01:22.479Z'), meta: { versionApiKeyLastmodified: kibanaVersion }, muteAll: false, snoozeSchedule: [], @@ -1601,7 +1621,7 @@ describe('create()', () => { error: null, warning: null, }, - monitoring: getDefaultRuleMonitoring(), + monitoring: getDefaultMonitoring('2019-02-12T21:01:22.479Z'), }, { id: 'mock-saved-object-id', @@ -1733,7 +1753,7 @@ describe('create()', () => { error: null, warning: null, }, - monitoring: getDefaultRuleMonitoring(), + monitoring: getDefaultMonitoring('2019-02-12T21:01:22.479Z'), }, { id: 'mock-saved-object-id', @@ -1865,7 +1885,7 @@ describe('create()', () => { error: null, warning: null, }, - monitoring: getDefaultRuleMonitoring(), + monitoring: getDefaultMonitoring('2019-02-12T21:01:22.479Z'), }, { id: 'mock-saved-object-id', @@ -2013,11 +2033,21 @@ describe('create()', () => { warning: null, }, monitoring: { - execution: { + run: { history: [], calculated_metrics: { success_ratio: 0, }, + last_run: { + timestamp: '2019-02-12T21:01:22.479Z', + metrics: { + gap_duration_s: null, + total_alerts_created: null, + total_alerts_detected: null, + total_indexing_duration_ms: null, + total_search_duration_ms: null, + }, + }, }, }, mapped_params: { @@ -2375,7 +2405,7 @@ describe('create()', () => { error: null, warning: null, }, - monitoring: getDefaultRuleMonitoring(), + monitoring: getDefaultMonitoring('2019-02-12T21:01:22.479Z'), }, { id: 'mock-saved-object-id', @@ -2477,7 +2507,7 @@ describe('create()', () => { error: null, warning: null, }, - monitoring: getDefaultRuleMonitoring(), + monitoring: getDefaultMonitoring('2019-02-12T21:01:22.479Z'), }, { id: 'mock-saved-object-id', diff --git a/x-pack/plugins/alerting/server/rules_client/tests/disable.test.ts b/x-pack/plugins/alerting/server/rules_client/tests/disable.test.ts index 558d33ecca87c..5af5ec3e60bd1 100644 --- a/x-pack/plugins/alerting/server/rules_client/tests/disable.test.ts +++ b/x-pack/plugins/alerting/server/rules_client/tests/disable.test.ts @@ -224,6 +224,7 @@ describe('disable()', () => { }, }, ], + nextRun: null, }, { version: '123', @@ -294,6 +295,7 @@ describe('disable()', () => { }, }, ], + nextRun: null, }, { version: '123', @@ -375,6 +377,7 @@ describe('disable()', () => { }, }, ], + nextRun: null, }, { version: '123', @@ -418,6 +421,7 @@ describe('disable()', () => { }, }, ], + nextRun: null, }, { version: '123', @@ -509,6 +513,7 @@ describe('disable()', () => { }, }, ], + nextRun: null, }, { version: '123', @@ -556,6 +561,7 @@ describe('disable()', () => { }, }, ], + nextRun: null, }, { version: '123', diff --git a/x-pack/plugins/alerting/server/rules_client/tests/enable.test.ts b/x-pack/plugins/alerting/server/rules_client/tests/enable.test.ts index 1c96241b15800..9cb6c356edaed 100644 --- a/x-pack/plugins/alerting/server/rules_client/tests/enable.test.ts +++ b/x-pack/plugins/alerting/server/rules_client/tests/enable.test.ts @@ -234,6 +234,7 @@ describe('enable()', () => { error: null, warning: null, }, + nextRun: '2019-02-12T21:01:32.479Z', }, { version: '123', @@ -290,6 +291,7 @@ describe('enable()', () => { error: null, warning: null, }, + nextRun: '2019-02-12T21:01:32.479Z', }, { version: '123', @@ -356,6 +358,7 @@ describe('enable()', () => { error: null, warning: null, }, + nextRun: '2019-02-12T21:01:32.479Z', }, { version: '123', diff --git a/x-pack/plugins/alerting/server/saved_objects/index.ts b/x-pack/plugins/alerting/server/saved_objects/index.ts index 31f6f96d67c8b..76fc730409294 100644 --- a/x-pack/plugins/alerting/server/saved_objects/index.ts +++ b/x-pack/plugins/alerting/server/saved_objects/index.ts @@ -36,6 +36,8 @@ export const AlertAttributesExcludedFromAAD = [ 'snoozeEndTime', // field removed in 8.2, but must be retained in case an rule created/updated in 8.2 is being migrated 'snoozeSchedule', 'isSnoozedUntil', + 'lastRun', + 'nextRun', ]; // useful for Pick which is a @@ -52,7 +54,9 @@ export type AlertAttributesExcludedFromAADType = | 'monitoring' | 'snoozeEndTime' | 'snoozeSchedule' - | 'isSnoozedUntil'; + | 'isSnoozedUntil' + | 'lastRun' + | 'nextRun'; export function setupSavedObjects( savedObjects: SavedObjectsServiceSetup, diff --git a/x-pack/plugins/alerting/server/saved_objects/mappings.ts b/x-pack/plugins/alerting/server/saved_objects/mappings.ts index f33dbdd788534..727e70e26055e 100644 --- a/x-pack/plugins/alerting/server/saved_objects/mappings.ts +++ b/x-pack/plugins/alerting/server/saved_objects/mappings.ts @@ -114,7 +114,7 @@ export const alertMappings: SavedObjectsTypeMappingDefinition = { }, monitoring: { properties: { - execution: { + run: { properties: { history: { properties: { @@ -127,6 +127,9 @@ export const alertMappings: SavedObjectsTypeMappingDefinition = { timestamp: { type: 'date', }, + outcome: { + type: 'keyword', + }, }, }, calculated_metrics: { @@ -145,41 +148,34 @@ export const alertMappings: SavedObjectsTypeMappingDefinition = { }, }, }, - }, - }, - }, - }, - executionStatus: { - properties: { - numberOfTriggeredActions: { - type: 'long', - }, - status: { - type: 'keyword', - }, - lastExecutionDate: { - type: 'date', - }, - lastDuration: { - type: 'long', - }, - error: { - properties: { - reason: { - type: 'keyword', - }, - message: { - type: 'keyword', - }, - }, - }, - warning: { - properties: { - reason: { - type: 'keyword', - }, - message: { - type: 'keyword', + last_run: { + properties: { + timestamp: { + type: 'date', + }, + metrics: { + properties: { + duration: { + type: 'long', + }, + total_search_duration_ms: { + type: 'long', + }, + total_indexing_duration_ms: { + type: 'long', + }, + total_alerts_detected: { + type: 'float', + }, + total_alerts_created: { + type: 'float', + }, + gap_duration_s: { + type: 'float', + }, + }, + }, + }, }, }, }, @@ -255,5 +251,77 @@ export const alertMappings: SavedObjectsTypeMappingDefinition = { }, }, }, + nextRun: { + type: 'date', + }, + // Deprecated, if you need to add new property please do it in `last_run` + executionStatus: { + properties: { + numberOfTriggeredActions: { + type: 'long', + }, + status: { + type: 'keyword', + }, + lastExecutionDate: { + type: 'date', + }, + lastDuration: { + type: 'long', + }, + error: { + properties: { + reason: { + type: 'keyword', + }, + message: { + type: 'keyword', + }, + }, + }, + warning: { + properties: { + reason: { + type: 'keyword', + }, + message: { + type: 'keyword', + }, + }, + }, + }, + }, + lastRun: { + properties: { + outcome: { + type: 'keyword', + }, + outcomeOrder: { + type: 'float', + }, + warning: { + type: 'text', + }, + outcomeMsg: { + type: 'text', + }, + alertsCount: { + properties: { + active: { + type: 'float', + }, + new: { + type: 'float', + }, + recovered: { + type: 'float', + }, + ignored: { + type: 'float', + }, + }, + }, + }, + }, }, }; diff --git a/x-pack/plugins/alerting/server/saved_objects/migrations/8.6/index.ts b/x-pack/plugins/alerting/server/saved_objects/migrations/8.6/index.ts new file mode 100644 index 0000000000000..d4535008bfbe6 --- /dev/null +++ b/x-pack/plugins/alerting/server/saved_objects/migrations/8.6/index.ts @@ -0,0 +1,101 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SavedObjectUnsanitizedDoc } from '@kbn/core-saved-objects-server'; +import { EncryptedSavedObjectsPluginSetup } from '@kbn/encrypted-saved-objects-plugin/server'; +import { createEsoMigration, pipeMigrations } from '../utils'; +import { RawRule, RuleLastRunOutcomeValues } from '../../../types'; +import { getDefaultMonitoring } from '../../../lib/monitoring'; + +const succeededStatus = ['ok', 'active', 'succeeded']; +const warningStatus = ['warning']; +const failedStatus = ['error', 'failed']; + +const getLastRun = (attributes: RawRule) => { + const { executionStatus } = attributes; + const { status, warning, error } = executionStatus || {}; + + let outcome; + if (succeededStatus.includes(status)) { + outcome = RuleLastRunOutcomeValues[0]; + } else if (warningStatus.includes(status) || warning) { + outcome = RuleLastRunOutcomeValues[1]; + } else if (failedStatus.includes(status) || error) { + outcome = RuleLastRunOutcomeValues[2]; + } + + // Don't set last run if status is unknown or pending, let the + // task runner do it instead + if (!outcome) { + return null; + } + + return { + outcome, + outcomeMsg: warning?.message || error?.message || null, + warning: warning?.reason || error?.reason || null, + alertsCount: {}, + }; +}; + +const getMonitoring = (attributes: RawRule) => { + const { executionStatus, monitoring } = attributes; + if (!monitoring) { + if (!executionStatus) { + return null; + } + + // monitoring now has data from executionStatus, therefore, we should migrate + // these fields even if monitoring doesn't exist. + const defaultMonitoring = getDefaultMonitoring(executionStatus.lastExecutionDate); + if (executionStatus.lastDuration) { + defaultMonitoring.run.last_run.metrics.duration = executionStatus.lastDuration; + } + return defaultMonitoring; + } + + const { lastExecutionDate, lastDuration } = executionStatus; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const monitoringExecution = (monitoring as any).execution; + + return { + run: { + ...monitoringExecution, + last_run: { + timestamp: lastExecutionDate, + metrics: { + ...(lastDuration ? { duration: lastDuration } : {}), + }, + }, + }, + }; +}; + +function migrateLastRun( + doc: SavedObjectUnsanitizedDoc +): SavedObjectUnsanitizedDoc { + const { attributes } = doc; + const lastRun = getLastRun(attributes); + const monitoring = getMonitoring(attributes); + + return { + ...doc, + attributes: { + ...attributes, + ...(lastRun ? { lastRun } : {}), + ...(monitoring ? { monitoring } : {}), + }, + }; +} + +export const getMigrations860 = (encryptedSavedObjects: EncryptedSavedObjectsPluginSetup) => + createEsoMigration( + encryptedSavedObjects, + (doc: SavedObjectUnsanitizedDoc): doc is SavedObjectUnsanitizedDoc => true, + pipeMigrations(migrateLastRun) + ); diff --git a/x-pack/plugins/alerting/server/saved_objects/migrations/index.test.ts b/x-pack/plugins/alerting/server/saved_objects/migrations/index.test.ts index 09f466a8e9a37..f7f31d1d06786 100644 --- a/x-pack/plugins/alerting/server/saved_objects/migrations/index.test.ts +++ b/x-pack/plugins/alerting/server/saved_objects/migrations/index.test.ts @@ -2150,339 +2150,416 @@ describe('successful migrations', () => { expect(migratedAlert.attributes.params.outputIndex).toEqual(''); } ); + }); - describe('8.0.1', () => { - describe.each(Object.keys(ruleTypeMappings) as RuleType[])( - 'auto_disabled %p rule tags', - (ruleType) => { - const alert717Enabled = getMockData( - { - params: { outputIndex: 'output-index', type: ruleType }, - alertTypeId: 'siem.signals', - enabled: true, - scheduledTaskId: 'abcd', - }, - true - ); - const alert717Disabled = getMockData( - { - params: { outputIndex: 'output-index', type: ruleType }, - alertTypeId: 'siem.signals', - enabled: false, - }, - true - ); - const alert800 = getMockData( - { - params: { outputIndex: '', type: ruleType }, - alertTypeId: ruleTypeMappings[ruleType], - enabled: false, - scheduledTaskId: 'abcd', - }, - true - ); + describe('8.0.1', () => { + describe.each(Object.keys(ruleTypeMappings) as RuleType[])( + 'auto_disabled %p rule tags', + (ruleType) => { + const alert717Enabled = getMockData( + { + params: { outputIndex: 'output-index', type: ruleType }, + alertTypeId: 'siem.signals', + enabled: true, + scheduledTaskId: 'abcd', + }, + true + ); + const alert717Disabled = getMockData( + { + params: { outputIndex: 'output-index', type: ruleType }, + alertTypeId: 'siem.signals', + enabled: false, + }, + true + ); + const alert800 = getMockData( + { + params: { outputIndex: '', type: ruleType }, + alertTypeId: ruleTypeMappings[ruleType], + enabled: false, + scheduledTaskId: 'abcd', + }, + true + ); - test('Does not update rule tags if rule has already been enabled', () => { - const migrations = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured); - const migration800 = migrations['8.0.0']; - const migration801 = migrations['8.0.1']; + test('Does not update rule tags if rule has already been enabled', () => { + const migrations = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured); + const migration800 = migrations['8.0.0']; + const migration801 = migrations['8.0.1']; - // migrate to 8.0.0 - const migratedAlert800 = migration800(alert717Enabled, migrationContext); - expect(migratedAlert800.attributes.enabled).toEqual(false); + // migrate to 8.0.0 + const migratedAlert800 = migration800(alert717Enabled, migrationContext); + expect(migratedAlert800.attributes.enabled).toEqual(false); - // reenable rule - migratedAlert800.attributes.enabled = true; + // reenable rule + migratedAlert800.attributes.enabled = true; - // migrate to 8.0.1 - const migratedAlert801 = migration801(migratedAlert800, migrationContext); + // migrate to 8.0.1 + const migratedAlert801 = migration801(migratedAlert800, migrationContext); - expect(migratedAlert801.attributes.alertTypeId).toEqual(ruleTypeMappings[ruleType]); - expect(migratedAlert801.attributes.enabled).toEqual(true); - expect(migratedAlert801.attributes.params.outputIndex).toEqual(''); + expect(migratedAlert801.attributes.alertTypeId).toEqual(ruleTypeMappings[ruleType]); + expect(migratedAlert801.attributes.enabled).toEqual(true); + expect(migratedAlert801.attributes.params.outputIndex).toEqual(''); - // tags not updated - expect(migratedAlert801.attributes.tags).toEqual(['foo']); - }); + // tags not updated + expect(migratedAlert801.attributes.tags).toEqual(['foo']); + }); - test('Does not update rule tags if rule was already disabled before upgrading to 8.0', () => { - const migrations = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured); - const migration800 = migrations['8.0.0']; - const migration801 = migrations['8.0.1']; + test('Does not update rule tags if rule was already disabled before upgrading to 8.0', () => { + const migrations = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured); + const migration800 = migrations['8.0.0']; + const migration801 = migrations['8.0.1']; - // migrate to 8.0.0 - const migratedAlert800 = migration800(alert717Disabled, migrationContext); - expect(migratedAlert800.attributes.enabled).toEqual(false); + // migrate to 8.0.0 + const migratedAlert800 = migration800(alert717Disabled, migrationContext); + expect(migratedAlert800.attributes.enabled).toEqual(false); - // migrate to 8.0.1 - const migratedAlert801 = migration801(migratedAlert800, migrationContext); + // migrate to 8.0.1 + const migratedAlert801 = migration801(migratedAlert800, migrationContext); - expect(migratedAlert801.attributes.alertTypeId).toEqual(ruleTypeMappings[ruleType]); - expect(migratedAlert801.attributes.enabled).toEqual(false); - expect(migratedAlert801.attributes.params.outputIndex).toEqual(''); + expect(migratedAlert801.attributes.alertTypeId).toEqual(ruleTypeMappings[ruleType]); + expect(migratedAlert801.attributes.enabled).toEqual(false); + expect(migratedAlert801.attributes.params.outputIndex).toEqual(''); - // tags not updated - expect(migratedAlert801.attributes.tags).toEqual(['foo']); - }); + // tags not updated + expect(migratedAlert801.attributes.tags).toEqual(['foo']); + }); - test('Updates rule tags if rule was auto-disabled in 8.0 upgrade and not reenabled', () => { - const migrations = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured); - const migration800 = migrations['8.0.0']; - const migration801 = migrations['8.0.1']; + test('Updates rule tags if rule was auto-disabled in 8.0 upgrade and not reenabled', () => { + const migrations = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured); + const migration800 = migrations['8.0.0']; + const migration801 = migrations['8.0.1']; - // migrate to 8.0.0 - const migratedAlert800 = migration800(alert717Enabled, migrationContext); - expect(migratedAlert800.attributes.enabled).toEqual(false); + // migrate to 8.0.0 + const migratedAlert800 = migration800(alert717Enabled, migrationContext); + expect(migratedAlert800.attributes.enabled).toEqual(false); - // migrate to 8.0.1 - const migratedAlert801 = migration801(migratedAlert800, migrationContext); + // migrate to 8.0.1 + const migratedAlert801 = migration801(migratedAlert800, migrationContext); - expect(migratedAlert801.attributes.alertTypeId).toEqual(ruleTypeMappings[ruleType]); - expect(migratedAlert801.attributes.enabled).toEqual(false); - expect(migratedAlert801.attributes.params.outputIndex).toEqual(''); + expect(migratedAlert801.attributes.alertTypeId).toEqual(ruleTypeMappings[ruleType]); + expect(migratedAlert801.attributes.enabled).toEqual(false); + expect(migratedAlert801.attributes.params.outputIndex).toEqual(''); - // tags updated - expect(migratedAlert801.attributes.tags).toEqual(['foo', 'auto_disabled_8.0']); - }); + // tags updated + expect(migratedAlert801.attributes.tags).toEqual(['foo', 'auto_disabled_8.0']); + }); - test('Updates rule tags correctly if tags are undefined', () => { - const migrations = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured); - const migration801 = migrations['8.0.1']; + test('Updates rule tags correctly if tags are undefined', () => { + const migrations = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured); + const migration801 = migrations['8.0.1']; - const alert = { - ...alert800, - attributes: { - ...alert800.attributes, - tags: undefined, - }, - }; + const alert = { + ...alert800, + attributes: { + ...alert800.attributes, + tags: undefined, + }, + }; - // migrate to 8.0.1 - const migratedAlert801 = migration801(alert, migrationContext); + // migrate to 8.0.1 + const migratedAlert801 = migration801(alert, migrationContext); - expect(migratedAlert801.attributes.alertTypeId).toEqual(ruleTypeMappings[ruleType]); - expect(migratedAlert801.attributes.enabled).toEqual(false); - expect(migratedAlert801.attributes.params.outputIndex).toEqual(''); + expect(migratedAlert801.attributes.alertTypeId).toEqual(ruleTypeMappings[ruleType]); + expect(migratedAlert801.attributes.enabled).toEqual(false); + expect(migratedAlert801.attributes.params.outputIndex).toEqual(''); - // tags updated - expect(migratedAlert801.attributes.tags).toEqual(['auto_disabled_8.0']); - }); + // tags updated + expect(migratedAlert801.attributes.tags).toEqual(['auto_disabled_8.0']); + }); - test('Updates rule tags correctly if tags are null', () => { - const migrations = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured); - const migration801 = migrations['8.0.1']; + test('Updates rule tags correctly if tags are null', () => { + const migrations = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured); + const migration801 = migrations['8.0.1']; - const alert = { - ...alert800, - attributes: { - ...alert800.attributes, - tags: null, - }, - }; + const alert = { + ...alert800, + attributes: { + ...alert800.attributes, + tags: null, + }, + }; - // migrate to 8.0.1 - const migratedAlert801 = migration801(alert, migrationContext); + // migrate to 8.0.1 + const migratedAlert801 = migration801(alert, migrationContext); - expect(migratedAlert801.attributes.alertTypeId).toEqual(ruleTypeMappings[ruleType]); - expect(migratedAlert801.attributes.enabled).toEqual(false); - expect(migratedAlert801.attributes.params.outputIndex).toEqual(''); + expect(migratedAlert801.attributes.alertTypeId).toEqual(ruleTypeMappings[ruleType]); + expect(migratedAlert801.attributes.enabled).toEqual(false); + expect(migratedAlert801.attributes.params.outputIndex).toEqual(''); - // tags updated - expect(migratedAlert801.attributes.tags).toEqual(['auto_disabled_8.0']); - }); - } - ); - }); + // tags updated + expect(migratedAlert801.attributes.tags).toEqual(['auto_disabled_8.0']); + }); + } + ); + }); - describe('8.2.0', () => { - test('migrates params to mapped_params', () => { - const migration820 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)[ - '8.2.0' - ]; - const alert = getMockData( - { - params: { - risk_score: 60, - severity: 'high', - foo: 'bar', - }, - alertTypeId: 'siem.signals', + describe('8.2.0', () => { + test('migrates params to mapped_params', () => { + const migration820 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)['8.2.0']; + const alert = getMockData( + { + params: { + risk_score: 60, + severity: 'high', + foo: 'bar', }, - true - ); + alertTypeId: 'siem.signals', + }, + true + ); - const migratedAlert820 = migration820(alert, migrationContext); + const migratedAlert820 = migration820(alert, migrationContext); - expect(migratedAlert820.attributes.mapped_params).toEqual({ - risk_score: 60, - severity: '60-high', - }); + expect(migratedAlert820.attributes.mapped_params).toEqual({ + risk_score: 60, + severity: '60-high', }); }); + }); - describe('8.3.0', () => { - test('migrates snoozed rules to the new data model', () => { - const fakeTimer = sinon.useFakeTimers(); - const migration830 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)[ - '8.3.0' - ]; - const mutedAlert = getMockData( - { - snoozeEndTime: '1970-01-02T00:00:00.000Z', - }, - true - ); - const migratedMutedAlert830 = migration830(mutedAlert, migrationContext); + describe('8.3.0', () => { + test('migrates snoozed rules to the new data model', () => { + const fakeTimer = sinon.useFakeTimers(); + const migration830 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)['8.3.0']; + const mutedAlert = getMockData( + { + snoozeEndTime: '1970-01-02T00:00:00.000Z', + }, + true + ); + const migratedMutedAlert830 = migration830(mutedAlert, migrationContext); - expect(migratedMutedAlert830.attributes.snoozeSchedule.length).toEqual(1); - expect(migratedMutedAlert830.attributes.snoozeSchedule[0].rRule.dtstart).toEqual( - '1970-01-01T00:00:00.000Z' - ); - expect(migratedMutedAlert830.attributes.snoozeSchedule[0].duration).toEqual(86400000); - fakeTimer.restore(); - }); + expect(migratedMutedAlert830.attributes.snoozeSchedule.length).toEqual(1); + expect(migratedMutedAlert830.attributes.snoozeSchedule[0].rRule.dtstart).toEqual( + '1970-01-01T00:00:00.000Z' + ); + expect(migratedMutedAlert830.attributes.snoozeSchedule[0].duration).toEqual(86400000); + fakeTimer.restore(); + }); - test('migrates es_query alert params', () => { - const migration830 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)[ - '8.3.0' - ]; - const alert = getMockData( - { - params: { esQuery: '{ "query": "test-query" }' }, - alertTypeId: '.es-query', - }, - true - ); - const migratedAlert820 = migration830(alert, migrationContext); + test('migrates es_query alert params', () => { + const migration830 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)['8.3.0']; + const alert = getMockData( + { + params: { esQuery: '{ "query": "test-query" }' }, + alertTypeId: '.es-query', + }, + true + ); + const migratedAlert820 = migration830(alert, migrationContext); - expect(migratedAlert820.attributes.params).toEqual({ - esQuery: '{ "query": "test-query" }', - searchType: 'esQuery', - }); + expect(migratedAlert820.attributes.params).toEqual({ + esQuery: '{ "query": "test-query" }', + searchType: 'esQuery', }); + }); - test('removes internal tags', () => { - const migration830 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)[ - '8.3.0' - ]; - const alert = getMockData( - { - tags: [ - '__internal_immutable:false', - '__internal_rule_id:064e3fed-6328-416b-bb85-c08265088f41', - 'test-tag', - ], - alertTypeId: 'siem.queryRule', - }, - true - ); + test('removes internal tags', () => { + const migration830 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)['8.3.0']; + const alert = getMockData( + { + tags: [ + '__internal_immutable:false', + '__internal_rule_id:064e3fed-6328-416b-bb85-c08265088f41', + 'test-tag', + ], + alertTypeId: 'siem.queryRule', + }, + true + ); - const migratedAlert830 = migration830(alert, migrationContext); + const migratedAlert830 = migration830(alert, migrationContext); - expect(migratedAlert830.attributes.tags).toEqual(['test-tag']); - }); + expect(migratedAlert830.attributes.tags).toEqual(['test-tag']); + }); - test('do not remove internal tags if rule is not Security solution rule', () => { - const migration830 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)[ - '8.3.0' - ]; - const alert = getMockData( - { - tags: ['__internal_immutable:false', 'tag-1'], - }, - true - ); + test('do not remove internal tags if rule is not Security solution rule', () => { + const migration830 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)['8.3.0']; + const alert = getMockData( + { + tags: ['__internal_immutable:false', 'tag-1'], + }, + true + ); - const migratedAlert830 = migration830(alert, migrationContext); + const migratedAlert830 = migration830(alert, migrationContext); - expect(migratedAlert830.attributes.tags).toEqual(['__internal_immutable:false', 'tag-1']); - }); + expect(migratedAlert830.attributes.tags).toEqual(['__internal_immutable:false', 'tag-1']); }); + }); - describe('8.4.1', () => { - test('removes isSnoozedUntil', () => { - const migration841 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)[ - '8.4.1' - ]; - const mutedAlert = getMockData( - { - isSnoozedUntil: '1970-01-02T00:00:00.000Z', - }, - true - ); - expect(mutedAlert.attributes.isSnoozedUntil).toBeTruthy(); - const migratedAlert841 = migration841(mutedAlert, migrationContext); + describe('8.4.1', () => { + test('removes isSnoozedUntil', () => { + const migration841 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)['8.4.1']; + const mutedAlert = getMockData( + { + isSnoozedUntil: '1970-01-02T00:00:00.000Z', + }, + true + ); + expect(mutedAlert.attributes.isSnoozedUntil).toBeTruthy(); + const migratedAlert841 = migration841(mutedAlert, migrationContext); - expect(migratedAlert841.attributes.isSnoozedUntil).toBeFalsy(); + expect(migratedAlert841.attributes.isSnoozedUntil).toBeFalsy(); + }); + + test('works as expected if isSnoozedUntil is not populated', () => { + const migration841 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)['8.4.1']; + const mutedAlert = getMockData({}, true); + expect(mutedAlert.attributes.isSnoozedUntil).toBeFalsy(); + expect(() => migration841(mutedAlert, migrationContext)).not.toThrowError(); + }); + }); + + describe('8.6.0', () => { + test('migrates executionStatus success', () => { + const migration860 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)['8.6.0']; + + let ruleWithExecutionStatus = getMockData({ + executionStatus: { + status: 'ok', + lastExecutionDate: '2022-01-02T00:00:00.000Z', + lastDuration: 60000, + }, }); - test('works as expected if isSnoozedUntil is not populated', () => { - const migration841 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)[ - '8.4.1' - ]; - const mutedAlert = getMockData({}, true); - expect(mutedAlert.attributes.isSnoozedUntil).toBeFalsy(); - expect(() => migration841(mutedAlert, migrationContext)).not.toThrowError(); + let migratedRule = migration860(ruleWithExecutionStatus, migrationContext); + expect(migratedRule.attributes.lastRun.outcome).toEqual('succeeded'); + expect(migratedRule.attributes.lastRun.warning).toEqual(null); + ruleWithExecutionStatus = getMockData({ + executionStatus: { + status: 'active', + lastExecutionDate: '2022-01-02T00:00:00.000Z', + lastDuration: 60000, + }, }); + + migratedRule = migration860(ruleWithExecutionStatus, migrationContext); + expect(migratedRule.attributes.lastRun.outcome).toEqual('succeeded'); + expect(migratedRule.attributes.lastRun.warning).toEqual(null); }); - describe('Metrics Inventory Threshold rule', () => { - test('Migrates incorrect action group spelling', () => { - const migration800 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)[ - '8.0.0' - ]; + test('migrates executionStatus warning and error', () => { + const migration860 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)['8.6.0']; - const actions = [ - { - group: 'metrics.invenotry_threshold.fired', - params: { - level: 'info', - message: - '""{{alertName}} - {{context.group}} is in a state of {{context.alertState}} Reason: {{context.reason}}""', - }, - actionRef: 'action_0', - actionTypeId: '.server-log', + let ruleWithExecutionStatus = getMockData({ + executionStatus: { + status: 'warning', + lastExecutionDate: '2022-01-02T00:00:00.000Z', + lastDuration: 60000, + warning: { + reason: 'warning reason', + message: 'warning message', }, - ]; + }, + }); - const alert = getMockData({ alertTypeId: 'metrics.alert.inventory.threshold', actions }); + let migratedRule = migration860(ruleWithExecutionStatus, migrationContext); + expect(migratedRule.attributes.lastRun.outcome).toEqual('warning'); + expect(migratedRule.attributes.lastRun.outcomeMsg).toEqual('warning message'); + expect(migratedRule.attributes.lastRun.warning).toEqual('warning reason'); - expect(migration800(alert, migrationContext)).toMatchObject({ - ...alert, - attributes: { - ...alert.attributes, - actions: [{ ...actions[0], group: 'metrics.inventory_threshold.fired' }], + ruleWithExecutionStatus = getMockData({ + executionStatus: { + status: 'error', + lastExecutionDate: '2022-01-02T00:00:00.000Z', + lastDuration: 60000, + error: { + reason: 'failed reason', + message: 'failed message', }, - }); + }, }); - test('Works with the correct action group spelling', () => { - const migration800 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)[ - '8.0.0' - ]; + migratedRule = migration860(ruleWithExecutionStatus, migrationContext); + expect(migratedRule.attributes.lastRun.outcome).toEqual('failed'); + expect(migratedRule.attributes.lastRun.outcomeMsg).toEqual('failed message'); + expect(migratedRule.attributes.lastRun.warning).toEqual('failed reason'); + }); - const actions = [ - { - group: 'metrics.inventory_threshold.fired', - params: { - level: 'info', - message: - '""{{alertName}} - {{context.group}} is in a state of {{context.alertState}} Reason: {{context.reason}}""', - }, - actionRef: 'action_0', - actionTypeId: '.server-log', + test('migrates empty monitoring', () => { + const migration860 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)['8.6.0']; + + const ruleWithoutMonitoring = getMockData(); + const migratedRule = migration860(ruleWithoutMonitoring, migrationContext); + + expect(migratedRule.attributes.monitoring).toBeUndefined(); + }); + + test('migrates empty monitoring when executionStatus exists', () => { + const migration860 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)['8.6.0']; + + const ruleWithMonitoring = getMockData({ + executionStatus: { + status: 'ok', + lastExecutionDate: '2022-01-02T00:00:00.000Z', + lastDuration: 60000, + }, + }); + const migratedRule = migration860(ruleWithMonitoring, migrationContext); + + expect(migratedRule.attributes.monitoring.run.history).toEqual([]); + expect(migratedRule.attributes.monitoring.run.last_run.timestamp).toEqual( + '2022-01-02T00:00:00.000Z' + ); + expect(migratedRule.attributes.monitoring.run.last_run.metrics.duration).toEqual(60000); + }); + }); + + describe('Metrics Inventory Threshold rule', () => { + test('Migrates incorrect action group spelling', () => { + const migration800 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)['8.0.0']; + + const actions = [ + { + group: 'metrics.invenotry_threshold.fired', + params: { + level: 'info', + message: + '""{{alertName}} - {{context.group}} is in a state of {{context.alertState}} Reason: {{context.reason}}""', }, - ]; + actionRef: 'action_0', + actionTypeId: '.server-log', + }, + ]; + + const alert = getMockData({ alertTypeId: 'metrics.alert.inventory.threshold', actions }); + + expect(migration800(alert, migrationContext)).toMatchObject({ + ...alert, + attributes: { + ...alert.attributes, + actions: [{ ...actions[0], group: 'metrics.inventory_threshold.fired' }], + }, + }); + }); - const alert = getMockData({ alertTypeId: 'metrics.alert.inventory.threshold', actions }); + test('Works with the correct action group spelling', () => { + const migration800 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)['8.0.0']; - expect(migration800(alert, migrationContext)).toMatchObject({ - ...alert, - attributes: { - ...alert.attributes, - actions: [{ ...actions[0], group: 'metrics.inventory_threshold.fired' }], + const actions = [ + { + group: 'metrics.inventory_threshold.fired', + params: { + level: 'info', + message: + '""{{alertName}} - {{context.group}} is in a state of {{context.alertState}} Reason: {{context.reason}}""', }, - }); + actionRef: 'action_0', + actionTypeId: '.server-log', + }, + ]; + + const alert = getMockData({ alertTypeId: 'metrics.alert.inventory.threshold', actions }); + + expect(migration800(alert, migrationContext)).toMatchObject({ + ...alert, + attributes: { + ...alert.attributes, + actions: [{ ...actions[0], group: 'metrics.inventory_threshold.fired' }], + }, }); }); }); diff --git a/x-pack/plugins/alerting/server/saved_objects/migrations/index.ts b/x-pack/plugins/alerting/server/saved_objects/migrations/index.ts index a3a06614ec4c5..b44e6e4c09c6e 100644 --- a/x-pack/plugins/alerting/server/saved_objects/migrations/index.ts +++ b/x-pack/plugins/alerting/server/saved_objects/migrations/index.ts @@ -28,6 +28,7 @@ import { getMigrations820 } from './8.2'; import { getMigrations830 } from './8.3'; import { getMigrations841 } from './8.4'; import { getMigrations850 } from './8.5'; +import { getMigrations860 } from './8.6'; import { AlertLogMeta, AlertMigration } from './types'; import { MINIMUM_SS_MIGRATION_VERSION } from './constants'; import { createEsoMigration, isEsQueryRuleType, pipeMigrations } from './utils'; @@ -75,6 +76,7 @@ export function getMigrations( '8.3.0': executeMigrationWithErrorHandling(getMigrations830(encryptedSavedObjects), '8.3.0'), '8.4.1': executeMigrationWithErrorHandling(getMigrations841(encryptedSavedObjects), '8.4.1'), '8.5.0': executeMigrationWithErrorHandling(getMigrations850(encryptedSavedObjects), '8.5.0'), + '8.6.0': executeMigrationWithErrorHandling(getMigrations860(encryptedSavedObjects), '8.6.0'), }, getSearchSourceMigrations(encryptedSavedObjects, searchSourceMigrations) ); diff --git a/x-pack/plugins/alerting/server/task_runner/fixtures.ts b/x-pack/plugins/alerting/server/task_runner/fixtures.ts index ca9ddd0c48db6..b0e98263591a9 100644 --- a/x-pack/plugins/alerting/server/task_runner/fixtures.ts +++ b/x-pack/plugins/alerting/server/task_runner/fixtures.ts @@ -6,8 +6,8 @@ */ import { TaskStatus } from '@kbn/task-manager-plugin/server'; -import { Rule, RuleTypeParams, RecoveredActionGroup } from '../../common'; -import { getDefaultRuleMonitoring } from './task_runner'; +import { Rule, RuleTypeParams, RecoveredActionGroup, RuleMonitoring } from '../../common'; +import { getDefaultMonitoring } from '../lib/monitoring'; import { UntypedNormalizedRuleType } from '../rule_type_registry'; import { EVENT_LOG_ACTIONS } from '../plugin'; @@ -54,29 +54,51 @@ export const RULE_ACTIONS = [ }, ]; +const defaultHistory = [ + { + success: true, + timestamp: 0, + }, +]; + export const generateSavedObjectParams = ({ error = null, warning = null, status = 'ok', + outcome = 'succeeded', + nextRun = '1970-01-01T00:00:10.000Z', + successRatio = 1, + history = defaultHistory, + alertsCount, }: { error?: null | { reason: string; message: string }; warning?: null | { reason: string; message: string }; status?: string; + outcome?: string; + nextRun?: string | null; + successRatio?: number; + history?: RuleMonitoring['run']['history']; + alertsCount?: Record; }) => [ 'alert', '1', { monitoring: { - execution: { + run: { calculated_metrics: { - success_ratio: 1, + success_ratio: successRatio, }, - history: [ - { - success: true, - timestamp: 0, + history, + last_run: { + timestamp: '1970-01-01T00:00:00.000Z', + metrics: { + gap_duration_s: null, + total_alerts_created: null, + total_alerts_detected: null, + total_indexing_duration_ms: null, + total_search_duration_ms: null, }, - ], + }, }, }, executionStatus: { @@ -86,6 +108,19 @@ export const generateSavedObjectParams = ({ status, warning, }, + lastRun: { + outcome, + outcomeMsg: error?.message || warning?.message || null, + warning: error?.reason || warning?.reason || null, + alertsCount: { + active: 0, + ignored: 0, + new: 0, + recovered: 0, + ...(alertsCount || {}), + }, + }, + nextRun, }, { refresh: false, namespace: undefined }, ]; @@ -155,7 +190,7 @@ export const mockedRuleTypeSavedObject: Rule = { status: 'unknown', lastExecutionDate: new Date('2020-08-20T19:23:38Z'), }, - monitoring: getDefaultRuleMonitoring(), + monitoring: getDefaultMonitoring('2020-08-20T19:23:38Z'), }; export const mockTaskInstance = () => ({ @@ -218,12 +253,22 @@ export const generateRunnerResult = ({ }: GeneratorParams = {}) => { return { monitoring: { - execution: { + run: { calculated_metrics: { success_ratio: successRatio, }, // @ts-ignore history: history.map((success) => ({ success, timestamp: 0 })), + last_run: { + metrics: { + gap_duration_s: null, + total_alerts_created: null, + total_alerts_detected: null, + total_indexing_duration_ms: null, + total_search_duration_ms: null, + }, + timestamp: '1970-01-01T00:00:00.000Z', + }, }, }, schedule: { diff --git a/x-pack/plugins/alerting/server/task_runner/rule_loader.test.ts b/x-pack/plugins/alerting/server/task_runner/rule_loader.test.ts index 96815743daaef..1cbe2bb2ae3b9 100644 --- a/x-pack/plugins/alerting/server/task_runner/rule_loader.test.ts +++ b/x-pack/plugins/alerting/server/task_runner/rule_loader.test.ts @@ -79,7 +79,7 @@ describe('rule_loader', () => { expect(result.rule.alertTypeId).toBe(ruleTypeId); expect(result.rule.name).toBe(ruleName); expect(result.rule.params).toBe(ruleParams); - expect(result.rule.monitoring?.execution.history.length).toBe(MONITORING_HISTORY_LIMIT - 1); + expect(result.rule.monitoring?.run.history.length).toBe(MONITORING_HISTORY_LIMIT - 1); }); test('without API key, any execution history, or validator', async () => { @@ -102,7 +102,7 @@ describe('rule_loader', () => { expect(result.rule.alertTypeId).toBe(ruleTypeId); expect(result.rule.name).toBe(ruleName); expect(result.rule.params).toBe(ruleParams); - expect(result.rule.monitoring?.execution.history.length).toBe(0); + expect(result.rule.monitoring?.run.history.length).toBe(0); }); }); @@ -348,7 +348,7 @@ function getTaskRunnerContext(ruleParameters: unknown, historyElements: number) alertTypeId: ruleTypeId, params: ruleParameters, monitoring: { - execution: { + run: { history: new Array(historyElements), }, }, diff --git a/x-pack/plugins/alerting/server/task_runner/rule_loader.ts b/x-pack/plugins/alerting/server/task_runner/rule_loader.ts index 75fcff9f4510f..8d4b00a54e094 100644 --- a/x-pack/plugins/alerting/server/task_runner/rule_loader.ts +++ b/x-pack/plugins/alerting/server/task_runner/rule_loader.ts @@ -79,9 +79,9 @@ export async function loadRule(params: LoadRulePa } if (rule.monitoring) { - if (rule.monitoring.execution.history.length >= MONITORING_HISTORY_LIMIT) { + if (rule.monitoring.run.history.length >= MONITORING_HISTORY_LIMIT) { // Remove the first (oldest) record - rule.monitoring.execution.history.shift(); + rule.monitoring.run.history.shift(); } } diff --git a/x-pack/plugins/alerting/server/task_runner/task_runner.test.ts b/x-pack/plugins/alerting/server/task_runner/task_runner.test.ts index 54573f1b7e2b2..69b648f099e44 100644 --- a/x-pack/plugins/alerting/server/task_runner/task_runner.test.ts +++ b/x-pack/plugins/alerting/server/task_runner/task_runner.test.ts @@ -197,8 +197,8 @@ describe('Task Runner', () => { taskRunnerFactoryInitializerParams.executionContext.withContext.mockImplementation((ctx, fn) => fn() ); - mockedRuleTypeSavedObject.monitoring!.execution.history = []; - mockedRuleTypeSavedObject.monitoring!.execution.calculated_metrics.success_ratio = 0; + mockedRuleTypeSavedObject.monitoring!.run.history = []; + mockedRuleTypeSavedObject.monitoring!.run.calculated_metrics.success_ratio = 0; alertingEventLogger.getStartAndDuration.mockImplementation(() => ({ start: new Date() })); (AlertingEventLogger as jest.Mock).mockImplementation(() => alertingEventLogger); @@ -252,14 +252,18 @@ describe('Task Runner', () => { expect(call.services.scopedClusterClient).toBeTruthy(); expect(call.services).toBeTruthy(); - expect(logger.debug).toHaveBeenCalledTimes(4); + expect(logger.debug).toHaveBeenCalledTimes(5); expect(logger.debug).nthCalledWith(1, 'executing rule test:1 at 1970-01-01T00:00:00.000Z'); expect(logger.debug).nthCalledWith( 2, - 'ruleRunStatus for test:1: {"lastExecutionDate":"1970-01-01T00:00:00.000Z","status":"ok"}' + 'deprecated ruleRunStatus for test:1: {"lastExecutionDate":"1970-01-01T00:00:00.000Z","status":"ok"}' ); expect(logger.debug).nthCalledWith( 3, + 'ruleRunStatus for test:1: {"outcome":"succeeded","outcomeMsg":null,"warning":null,"alertsCount":{"active":0,"new":0,"recovered":0,"ignored":0}}' + ); + expect(logger.debug).nthCalledWith( + 4, 'ruleRunMetrics for test:1: {"numSearches":3,"totalSearchDurationMs":23423,"esSearchDurationMs":33,"numberOfTriggeredActions":0,"numberOfGeneratedActions":0,"numberOfActiveAlerts":0,"numberOfRecoveredAlerts":0,"numberOfNewAlerts":0,"hasReachedAlertLimit":false,"triggeredActionsStatus":"complete"}' ); @@ -327,7 +331,7 @@ describe('Task Runner', () => { expect(enqueueFunction).toHaveBeenCalledTimes(1); expect(enqueueFunction).toHaveBeenCalledWith(generateEnqueueFunctionInput(inputIsArray)); - expect(logger.debug).toHaveBeenCalledTimes(5); + expect(logger.debug).toHaveBeenCalledTimes(6); expect(logger.debug).nthCalledWith(1, 'executing rule test:1 at 1970-01-01T00:00:00.000Z'); expect(logger.debug).nthCalledWith( 2, @@ -335,10 +339,14 @@ describe('Task Runner', () => { ); expect(logger.debug).nthCalledWith( 3, - 'ruleRunStatus for test:1: {"lastExecutionDate":"1970-01-01T00:00:00.000Z","status":"active"}' + 'deprecated ruleRunStatus for test:1: {"lastExecutionDate":"1970-01-01T00:00:00.000Z","status":"active"}' ); expect(logger.debug).nthCalledWith( 4, + 'ruleRunStatus for test:1: {"outcome":"succeeded","outcomeMsg":null,"warning":null,"alertsCount":{"active":1,"new":1,"recovered":0,"ignored":0}}' + ); + expect(logger.debug).nthCalledWith( + 5, 'ruleRunMetrics for test:1: {"numSearches":3,"totalSearchDurationMs":23423,"esSearchDurationMs":33,"numberOfTriggeredActions":1,"numberOfGeneratedActions":1,"numberOfActiveAlerts":1,"numberOfRecoveredAlerts":0,"numberOfNewAlerts":1,"hasReachedAlertLimit":false,"triggeredActionsStatus":"complete"}' ); @@ -405,7 +413,7 @@ describe('Task Runner', () => { await taskRunner.run(); expect(actionsClient.ephemeralEnqueuedExecution).toHaveBeenCalledTimes(0); - expect(logger.debug).toHaveBeenCalledTimes(6); + expect(logger.debug).toHaveBeenCalledTimes(7); expect(logger.debug).nthCalledWith(1, 'executing rule test:1 at 1970-01-01T00:00:00.000Z'); expect(logger.debug).nthCalledWith( 2, @@ -417,10 +425,14 @@ describe('Task Runner', () => { ); expect(logger.debug).nthCalledWith( 4, - 'ruleRunStatus for test:1: {"lastExecutionDate":"1970-01-01T00:00:00.000Z","status":"active"}' + 'deprecated ruleRunStatus for test:1: {"lastExecutionDate":"1970-01-01T00:00:00.000Z","status":"active"}' ); expect(logger.debug).nthCalledWith( 5, + 'ruleRunStatus for test:1: {"outcome":"succeeded","outcomeMsg":null,"warning":null,"alertsCount":{"active":1,"new":1,"recovered":0,"ignored":0}}' + ); + expect(logger.debug).nthCalledWith( + 6, 'ruleRunMetrics for test:1: {"numSearches":3,"totalSearchDurationMs":23423,"esSearchDurationMs":33,"numberOfTriggeredActions":0,"numberOfGeneratedActions":0,"numberOfActiveAlerts":1,"numberOfRecoveredAlerts":0,"numberOfNewAlerts":1,"hasReachedAlertLimit":false,"triggeredActionsStatus":"complete"}' ); @@ -579,7 +591,7 @@ describe('Task Runner', () => { await taskRunner.run(); expect(enqueueFunction).toHaveBeenCalledTimes(1); - expect(logger.debug).toHaveBeenCalledTimes(6); + expect(logger.debug).toHaveBeenCalledTimes(7); expect(logger.debug).nthCalledWith(1, 'executing rule test:1 at 1970-01-01T00:00:00.000Z'); expect(logger.debug).nthCalledWith( 2, @@ -591,10 +603,14 @@ describe('Task Runner', () => { ); expect(logger.debug).nthCalledWith( 4, - 'ruleRunStatus for test:1: {"lastExecutionDate":"1970-01-01T00:00:00.000Z","status":"active"}' + 'deprecated ruleRunStatus for test:1: {"lastExecutionDate":"1970-01-01T00:00:00.000Z","status":"active"}' ); expect(logger.debug).nthCalledWith( 5, + 'ruleRunStatus for test:1: {"outcome":"succeeded","outcomeMsg":null,"warning":null,"alertsCount":{"active":2,"new":2,"recovered":0,"ignored":0}}' + ); + expect(logger.debug).nthCalledWith( + 6, 'ruleRunMetrics for test:1: {"numSearches":3,"totalSearchDurationMs":23423,"esSearchDurationMs":33,"numberOfTriggeredActions":1,"numberOfGeneratedActions":1,"numberOfActiveAlerts":2,"numberOfRecoveredAlerts":0,"numberOfNewAlerts":2,"hasReachedAlertLimit":false,"triggeredActionsStatus":"complete"}' ); expect(mockUsageCounter.incrementCounter).not.toHaveBeenCalled(); @@ -702,7 +718,7 @@ describe('Task Runner', () => { encryptedSavedObjectsClient.getDecryptedAsInternalUser.mockResolvedValue(SAVED_OBJECT); await taskRunner.run(); expect(enqueueFunction).toHaveBeenCalledTimes(1); - expect(logger.debug).toHaveBeenCalledTimes(6); + expect(logger.debug).toHaveBeenCalledTimes(7); expect(logger.debug).nthCalledWith( 3, `skipping scheduling of actions for '2' in rule test:1: '${RULE_NAME}': rule is muted` @@ -1010,7 +1026,7 @@ describe('Task Runner', () => { generateAlertInstance({ id: 1, duration: MOCK_DURATION, start: DATE_1969 }) ); - expect(logger.debug).toHaveBeenCalledTimes(6); + expect(logger.debug).toHaveBeenCalledTimes(7); expect(logger.debug).nthCalledWith(1, 'executing rule test:1 at 1970-01-01T00:00:00.000Z'); expect(logger.debug).nthCalledWith( 2, @@ -1022,10 +1038,14 @@ describe('Task Runner', () => { ); expect(logger.debug).nthCalledWith( 4, - 'ruleRunStatus for test:1: {"lastExecutionDate":"1970-01-01T00:00:00.000Z","status":"active"}' + 'deprecated ruleRunStatus for test:1: {"lastExecutionDate":"1970-01-01T00:00:00.000Z","status":"active"}' ); expect(logger.debug).nthCalledWith( 5, + 'ruleRunStatus for test:1: {"outcome":"succeeded","outcomeMsg":null,"warning":null,"alertsCount":{"active":1,"new":0,"recovered":1,"ignored":0}}' + ); + expect(logger.debug).nthCalledWith( + 6, 'ruleRunMetrics for test:1: {"numSearches":3,"totalSearchDurationMs":23423,"esSearchDurationMs":33,"numberOfTriggeredActions":2,"numberOfGeneratedActions":2,"numberOfActiveAlerts":1,"numberOfRecoveredAlerts":1,"numberOfNewAlerts":0,"hasReachedAlertLimit":false,"triggeredActionsStatus":"complete"}' ); @@ -1135,10 +1155,14 @@ describe('Task Runner', () => { ); expect(logger.debug).nthCalledWith( 4, - `ruleRunStatus for test:1: {"lastExecutionDate":"1970-01-01T00:00:00.000Z","status":"active"}` + `deprecated ruleRunStatus for test:1: {"lastExecutionDate":"1970-01-01T00:00:00.000Z","status":"active"}` ); expect(logger.debug).nthCalledWith( 5, + 'ruleRunStatus for test:1: {"outcome":"succeeded","outcomeMsg":null,"warning":null,"alertsCount":{"active":1,"new":0,"recovered":1,"ignored":0}}' + ); + expect(logger.debug).nthCalledWith( + 6, `ruleRunMetrics for test:1: {"numSearches":3,"totalSearchDurationMs":23423,"esSearchDurationMs":33,"numberOfTriggeredActions":2,"numberOfGeneratedActions":2,"numberOfActiveAlerts":1,"numberOfRecoveredAlerts":1,"numberOfNewAlerts":0,"hasReachedAlertLimit":false,"triggeredActionsStatus":"complete"}` ); @@ -1544,10 +1568,15 @@ describe('Task Runner', () => { return taskRunner.run().catch((ex) => { expect(ex.toString()).toEqual(`Error: Saved object [alert/1] not found`); - const executeRuleDebugLogger = logger.debug.mock.calls[3][0]; + const updateRuleDebugLogger = logger.debug.mock.calls[3][0]; + expect(updateRuleDebugLogger as string).toMatchInlineSnapshot( + `"Updating rule task for test rule with id 1 - {\\"lastExecutionDate\\":\\"1970-01-01T00:00:00.000Z\\",\\"status\\":\\"error\\",\\"error\\":{\\"reason\\":\\"read\\",\\"message\\":\\"Saved object [alert/1] not found\\"}} - {\\"outcome\\":\\"failed\\",\\"warning\\":\\"read\\",\\"outcomeMsg\\":\\"Saved object [alert/1] not found\\",\\"alertsCount\\":{}}"` + ); + const executeRuleDebugLogger = logger.debug.mock.calls[4][0]; expect(executeRuleDebugLogger as string).toMatchInlineSnapshot( `"Executing Rule foo:test:1 has resulted in Error: Saved object [alert/1] not found"` ); + expect(logger.error).not.toHaveBeenCalled(); expect(logger.warn).toHaveBeenCalledTimes(1); expect(logger.warn).nthCalledWith( @@ -1626,10 +1655,15 @@ describe('Task Runner', () => { return taskRunner.run().catch((ex) => { expect(ex.toString()).toEqual(`Error: Saved object [alert/1] not found`); - const ruleExecuteDebugLog = logger.debug.mock.calls[3][0]; + const updateRuleDebugLogger = logger.debug.mock.calls[3][0]; + expect(updateRuleDebugLogger as string).toMatchInlineSnapshot( + `"Updating rule task for test rule with id 1 - {\\"lastExecutionDate\\":\\"1970-01-01T00:00:00.000Z\\",\\"status\\":\\"error\\",\\"error\\":{\\"reason\\":\\"read\\",\\"message\\":\\"Saved object [alert/1] not found\\"}} - {\\"outcome\\":\\"failed\\",\\"warning\\":\\"read\\",\\"outcomeMsg\\":\\"Saved object [alert/1] not found\\",\\"alertsCount\\":{}}"` + ); + const ruleExecuteDebugLog = logger.debug.mock.calls[4][0]; expect(ruleExecuteDebugLog as string).toMatchInlineSnapshot( `"Executing Rule test space:test:1 has resulted in Error: Saved object [alert/1] not found"` ); + expect(logger.error).not.toHaveBeenCalled(); expect(logger.warn).toHaveBeenCalledTimes(1); expect(logger.warn).nthCalledWith( @@ -2077,14 +2111,18 @@ describe('Task Runner', () => { expect(call.services.scopedClusterClient).toBeTruthy(); expect(call.services).toBeTruthy(); - expect(logger.debug).toHaveBeenCalledTimes(4); + expect(logger.debug).toHaveBeenCalledTimes(5); expect(logger.debug).nthCalledWith(1, 'executing rule test:1 at 1970-01-01T00:00:00.000Z'); expect(logger.debug).nthCalledWith( 2, - 'ruleRunStatus for test:1: {"lastExecutionDate":"1970-01-01T00:00:00.000Z","status":"ok"}' + 'deprecated ruleRunStatus for test:1: {"lastExecutionDate":"1970-01-01T00:00:00.000Z","status":"ok"}' ); expect(logger.debug).nthCalledWith( 3, + 'ruleRunStatus for test:1: {"outcome":"succeeded","outcomeMsg":null,"warning":null,"alertsCount":{"active":0,"new":0,"recovered":0,"ignored":0}}' + ); + expect(logger.debug).nthCalledWith( + 4, 'ruleRunMetrics for test:1: {"numSearches":3,"totalSearchDurationMs":23423,"esSearchDurationMs":33,"numberOfTriggeredActions":0,"numberOfGeneratedActions":0,"numberOfActiveAlerts":0,"numberOfRecoveredAlerts":0,"numberOfNewAlerts":0,"hasReachedAlertLimit":false,"triggeredActionsStatus":"complete"}' ); @@ -2176,6 +2214,78 @@ describe('Task Runner', () => { ); }); + test('successfully stores next run', async () => { + const taskRunner = new TaskRunner( + ruleType, + mockedTaskInstance, + taskRunnerFactoryInitializerParams, + inMemoryMetrics + ); + expect(AlertingEventLogger).toHaveBeenCalled(); + rulesClient.get.mockResolvedValue({ + ...mockedRuleTypeSavedObject, + schedule: { interval: '50s' }, + }); + encryptedSavedObjectsClient.getDecryptedAsInternalUser.mockResolvedValueOnce(SAVED_OBJECT); + + await taskRunner.run(); + expect( + taskRunnerFactoryInitializerParams.internalSavedObjectsRepository.update + ).toHaveBeenCalledWith( + ...generateSavedObjectParams({ + nextRun: '1970-01-01T00:00:50.000Z', + }) + ); + }); + + test('updates the rule saved object correctly when failed', async () => { + const taskRunner = new TaskRunner( + ruleType, + mockedTaskInstance, + taskRunnerFactoryInitializerParams, + inMemoryMetrics + ); + expect(AlertingEventLogger).toHaveBeenCalled(); + + rulesClient.get.mockResolvedValue(mockedRuleTypeSavedObject); + encryptedSavedObjectsClient.getDecryptedAsInternalUser.mockResolvedValue(SAVED_OBJECT); + + ruleType.executor.mockImplementation( + async ({ + services: executorServices, + }: RuleExecutorOptions< + RuleTypeParams, + RuleTypeState, + AlertInstanceState, + AlertInstanceContext, + string + >) => { + throw new Error(GENERIC_ERROR_MESSAGE); + } + ); + await taskRunner.run(); + ruleType.executor.mockClear(); + expect( + taskRunnerFactoryInitializerParams.internalSavedObjectsRepository.update + ).toHaveBeenCalledWith( + ...generateSavedObjectParams({ + error: { + message: GENERIC_ERROR_MESSAGE, + reason: 'execute', + }, + outcome: 'failed', + status: 'error', + successRatio: 0, + history: [ + { + success: false, + timestamp: 0, + }, + ], + }) + ); + }); + test('caps monitoring history at 200', async () => { const taskRunner = new TaskRunner( ruleType, @@ -2192,7 +2302,7 @@ describe('Task Runner', () => { await taskRunner.run(); } const runnerResult = await taskRunner.run(); - expect(runnerResult.monitoring?.execution.history.length).toBe(200); + expect(runnerResult.monitoring?.run.history.length).toBe(200); }); test('Actions circuit breaker kicked in, should set status as warning and log a message in event log', async () => { @@ -2276,7 +2386,17 @@ describe('Task Runner', () => { expect( taskRunnerFactoryInitializerParams.internalSavedObjectsRepository.update - ).toHaveBeenCalledWith(...generateSavedObjectParams({ status: 'warning', warning })); + ).toHaveBeenCalledWith( + ...generateSavedObjectParams({ + status: 'warning', + outcome: 'warning', + warning, + alertsCount: { + active: 1, + new: 1, + }, + }) + ); expect(runnerResult).toEqual( generateRunnerResult({ @@ -2299,7 +2419,7 @@ describe('Task Runner', () => { }) ); - expect(logger.debug).toHaveBeenCalledTimes(6); + expect(logger.debug).toHaveBeenCalledTimes(7); expect(logger.debug).nthCalledWith( 3, @@ -2428,7 +2548,17 @@ describe('Task Runner', () => { expect( taskRunnerFactoryInitializerParams.internalSavedObjectsRepository.update - ).toHaveBeenCalledWith(...generateSavedObjectParams({ status: 'warning', warning })); + ).toHaveBeenCalledWith( + ...generateSavedObjectParams({ + status: 'warning', + outcome: 'warning', + warning, + alertsCount: { + active: 2, + new: 2, + }, + }) + ); expect(runnerResult).toEqual( generateRunnerResult({ @@ -2463,7 +2593,7 @@ describe('Task Runner', () => { }) ); - expect(logger.debug).toHaveBeenCalledTimes(6); + expect(logger.debug).toHaveBeenCalledTimes(7); expect(logger.debug).nthCalledWith( 3, diff --git a/x-pack/plugins/alerting/server/task_runner/task_runner.ts b/x-pack/plugins/alerting/server/task_runner/task_runner.ts index 354b18d3d38d4..a06bd01cb9ce0 100644 --- a/x-pack/plugins/alerting/server/task_runner/task_runner.ts +++ b/x-pack/plugins/alerting/server/task_runner/task_runner.ts @@ -23,6 +23,8 @@ import { ruleExecutionStatusToRaw, isRuleSnoozed, processAlerts, + lastRunFromError, + getNextRun, } from '../lib'; import { RuleExecutionStatus, @@ -30,13 +32,12 @@ import { IntervalSchedule, RawAlertInstance, RawRuleExecutionStatus, - RuleMonitoring, - RuleMonitoringHistory, + RawRuleMonitoring, RuleTaskState, RuleTypeRegistry, + RawRuleLastRun, } from '../types'; -import { asErr, asOk, map, resolveErr, Result } from '../lib/result_type'; -import { getExecutionDurationPercentiles, getExecutionSuccessRatio } from '../lib/monitoring'; +import { asErr, asOk, isOk, map, resolveErr, Result } from '../lib/result_type'; import { taskInstanceToAlertTaskInstance } from './alert_task_instance'; import { isAlertSavedObjectNotFoundError, isEsUnavailableError } from '../lib/is_alerting_error'; import { partiallyUpdateAlert } from '../saved_objects'; @@ -66,19 +67,12 @@ import { loadRule } from './rule_loader'; import { logAlerts } from './log_alerts'; import { getPublicAlertFactory } from '../alert/create_alert_factory'; import { TaskRunnerTimer, TaskRunnerTimerSpan } from './task_runner_timer'; +import { RuleMonitoringService } from '../monitoring/rule_monitoring_service'; +import { ILastRun, lastRunFromState, lastRunToRaw } from '../lib/last_run_status'; const FALLBACK_RETRY_INTERVAL = '5m'; const CONNECTIVITY_RETRY_INTERVAL = '5m'; -export const getDefaultRuleMonitoring = (): RuleMonitoring => ({ - execution: { - history: [], - calculated_metrics: { - success_ratio: 0, - }, - }, -}); - interface StackTraceLog { message: ElasticsearchError; stackTrace?: string; @@ -117,6 +111,7 @@ export class TaskRunner< private searchAbortController: AbortController; private cancelled: boolean; private stackTraceLog: StackTraceLog | null; + private ruleMonitoring: RuleMonitoringService; constructor( ruleType: NormalizedRuleType< @@ -149,15 +144,20 @@ export class TaskRunner< this.timer = new TaskRunnerTimer({ logger: this.logger }); this.alertingEventLogger = new AlertingEventLogger(this.context.eventLogger); this.stackTraceLog = null; + this.ruleMonitoring = new RuleMonitoringService(); } private async updateRuleSavedObject( ruleId: string, namespace: string | undefined, - attributes: { executionStatus?: RawRuleExecutionStatus; monitoring?: RuleMonitoring } + attributes: { + executionStatus?: RawRuleExecutionStatus; + monitoring?: RawRuleMonitoring; + nextRun?: string | null; + lastRun?: RawRuleLastRun | null; + } ) { const client = this.context.internalSavedObjectsRepository; - try { await partiallyUpdateAlert(client, ruleId, attributes, { ignore404: true, @@ -324,6 +324,7 @@ export class TaskRunner< alertFactory: getPublicAlertFactory(alertFactory), shouldWriteAlerts: () => this.shouldLogAndScheduleActionsForAlerts(), shouldStopExecution: () => this.cancelled, + ruleMonitoringService: this.ruleMonitoring.getLastRunMetricsSetters(), }, params, state: ruleTypeState as RuleState, @@ -521,13 +522,13 @@ export class TaskRunner< } private async processRunResults({ + nextRun, runDate, stateWithMetrics, - monitoring, }: { + nextRun: string | null; runDate: Date; stateWithMetrics: Result; - monitoring: RuleMonitoring; }) { const { params: { alertId: ruleId, spaceId }, @@ -535,7 +536,8 @@ export class TaskRunner< const namespace = this.context.spaceIdToNamespace(spaceId); - const { status: executionStatus, metrics: executionMetrics } = map< + // Getting executionStatus for backwards compatibility + const { status: executionStatus } = map< RuleTaskStateAndMetrics, ElasticsearchError, IExecutionStatusAndMetrics @@ -545,16 +547,36 @@ export class TaskRunner< (err: ElasticsearchError) => executionStatusFromError(err, runDate) ); + // New consolidated statuses for lastRun + const { lastRun, metrics: executionMetrics } = map< + RuleTaskStateAndMetrics, + ElasticsearchError, + ILastRun + >( + stateWithMetrics, + (ruleRunStateWithMetrics) => lastRunFromState(ruleRunStateWithMetrics), + (err: ElasticsearchError) => lastRunFromError(err) + ); + if (apm.currentTransaction) { if (executionStatus.status === 'ok' || executionStatus.status === 'active') { apm.currentTransaction.setOutcome('success'); } else if (executionStatus.status === 'error' || executionStatus.status === 'unknown') { apm.currentTransaction.setOutcome('failure'); + } else if (lastRun.outcome === 'succeeded') { + apm.currentTransaction.setOutcome('success'); + } else if (lastRun.outcome === 'failed') { + apm.currentTransaction.setOutcome('failure'); } } this.logger.debug( - `ruleRunStatus for ${this.ruleType.id}:${ruleId}: ${JSON.stringify(executionStatus)}` + `deprecated ruleRunStatus for ${this.ruleType.id}:${ruleId}: ${JSON.stringify( + executionStatus + )}` + ); + this.logger.debug( + `ruleRunStatus for ${this.ruleType.id}:${ruleId}: ${JSON.stringify(lastRun)}` ); if (executionMetrics) { this.logger.debug( @@ -562,11 +584,6 @@ export class TaskRunner< ); } - const monitoringHistory: RuleMonitoringHistory = { - success: true, - timestamp: +new Date(), - }; - // set start and duration based on event log const { start, duration } = this.alertingEventLogger.getStartAndDuration(); if (null != start) { @@ -574,34 +591,32 @@ export class TaskRunner< } if (null != duration) { executionStatus.lastDuration = nanosToMillis(duration); - monitoringHistory.duration = executionStatus.lastDuration; } // if executionStatus indicates an error, fill in fields in - // event from it - if (executionStatus.error) { - monitoringHistory.success = false; - } - - monitoring.execution.history.push(monitoringHistory); - monitoring.execution.calculated_metrics = { - success_ratio: getExecutionSuccessRatio(monitoring), - ...getExecutionDurationPercentiles(monitoring), - }; + this.ruleMonitoring.addHistory({ + duration: executionStatus.lastDuration, + hasError: executionStatus.error != null, + runDate, + }); if (!this.cancelled) { this.inMemoryMetrics.increment(IN_MEMORY_METRICS.RULE_EXECUTIONS); - if (executionStatus.error) { + if (lastRun.outcome === 'failed') { + this.inMemoryMetrics.increment(IN_MEMORY_METRICS.RULE_FAILURES); + } else if (executionStatus.error) { this.inMemoryMetrics.increment(IN_MEMORY_METRICS.RULE_FAILURES); } this.logger.debug( `Updating rule task for ${this.ruleType.id} rule with id ${ruleId} - ${JSON.stringify( executionStatus - )}` + )} - ${JSON.stringify(lastRun)}` ); await this.updateRuleSavedObject(ruleId, namespace, { executionStatus: ruleExecutionStatusToRaw(executionStatus), - monitoring, + nextRun, + lastRun: lastRunToRaw(lastRun), + monitoring: this.ruleMonitoring.getMonitoring() as RawRuleMonitoring, }); } @@ -625,15 +640,13 @@ export class TaskRunner< } let stateWithMetrics: Result; - let monitoring: RuleMonitoring = getDefaultRuleMonitoring(); let schedule: Result; try { const preparedResult = await this.timer.runWithTimer( TaskRunnerTimerSpan.PrepareRule, async () => this.prepareToRun() ); - - monitoring = preparedResult.rule.monitoring ?? getDefaultRuleMonitoring(); + this.ruleMonitoring.setMonitoring(preparedResult.rule.monitoring); stateWithMetrics = asOk(await this.runRule(preparedResult)); @@ -645,13 +658,20 @@ export class TaskRunner< schedule = asErr(err); } + let nextRun: string | null = null; + if (isOk(schedule)) { + nextRun = getNextRun({ startDate: startedAt, interval: schedule.value.interval }); + } else if (taskSchedule) { + nextRun = getNextRun({ startDate: startedAt, interval: taskSchedule.interval }); + } + const { executionStatus, executionMetrics } = await this.timer.runWithTimer( TaskRunnerTimerSpan.ProcessRuleRun, async () => this.processRunResults({ + nextRun, runDate, stateWithMetrics, - monitoring, }) ); @@ -721,7 +741,7 @@ export class TaskRunner< return { interval: retryInterval }; }), - monitoring, + monitoring: this.ruleMonitoring.getMonitoring(), }; } @@ -735,6 +755,8 @@ export class TaskRunner< // Write event log entry const { params: { alertId: ruleId, spaceId, consumer }, + schedule: taskSchedule, + startedAt, } = this.taskInstance; const namespace = this.context.spaceIdToNamespace(spaceId); @@ -755,13 +777,20 @@ export class TaskRunner< this.inMemoryMetrics.increment(IN_MEMORY_METRICS.RULE_TIMEOUTS); + let nextRun: string | null = null; + if (taskSchedule) { + nextRun = getNextRun({ startDate: startedAt, interval: taskSchedule.interval }); + } + + const outcomeMsg = `${this.ruleType.id}:${ruleId}: execution cancelled due to timeout - exceeded rule type timeout of ${this.ruleType.ruleTaskTimeout}`; + const date = new Date(); // Update the rule saved object with execution status const executionStatus: RuleExecutionStatus = { - lastExecutionDate: new Date(), + lastExecutionDate: date, status: 'error', error: { reason: RuleExecutionStatusErrorReasons.Timeout, - message: `${this.ruleType.id}:${ruleId}: execution cancelled due to timeout - exceeded rule type timeout of ${this.ruleType.ruleTaskTimeout}`, + message: outcomeMsg, }, }; this.logger.debug( @@ -769,6 +798,14 @@ export class TaskRunner< ); await this.updateRuleSavedObject(ruleId, namespace, { executionStatus: ruleExecutionStatusToRaw(executionStatus), + lastRun: { + outcome: 'failed', + warning: RuleExecutionStatusErrorReasons.Timeout, + outcomeMsg, + alertsCount: {}, + }, + monitoring: this.ruleMonitoring.getMonitoring() as RawRuleMonitoring, + nextRun: nextRun && new Date(nextRun).getTime() > date.getTime() ? nextRun : null, }); } } diff --git a/x-pack/plugins/alerting/server/task_runner/task_runner_cancel.test.ts b/x-pack/plugins/alerting/server/task_runner/task_runner_cancel.test.ts index 92353cb043984..33efc649add26 100644 --- a/x-pack/plugins/alerting/server/task_runner/task_runner_cancel.test.ts +++ b/x-pack/plugins/alerting/server/task_runner/task_runner_cancel.test.ts @@ -214,6 +214,32 @@ describe('Task Runner Cancel', () => { status: 'error', warning: null, }, + lastRun: { + alertsCount: {}, + outcome: 'failed', + outcomeMsg: + 'test:1: execution cancelled due to timeout - exceeded rule type timeout of 5m', + warning: 'timeout', + }, + monitoring: { + run: { + calculated_metrics: { + success_ratio: 0, + }, + history: [], + last_run: { + metrics: { + gap_duration_s: null, + total_alerts_created: null, + total_alerts_detected: null, + total_indexing_duration_ms: null, + total_search_duration_ms: null, + }, + timestamp: '1970-01-01T00:00:00.000Z', + }, + }, + }, + nextRun: '1970-01-01T00:00:10.000Z', }, { refresh: false, namespace: undefined } ); @@ -391,7 +417,7 @@ describe('Task Runner Cancel', () => { }); function testLogger() { - expect(logger.debug).toHaveBeenCalledTimes(7); + expect(logger.debug).toHaveBeenCalledTimes(8); expect(logger.debug).nthCalledWith(1, 'executing rule test:1 at 1970-01-01T00:00:00.000Z'); expect(logger.debug).nthCalledWith( 2, @@ -411,10 +437,10 @@ describe('Task Runner Cancel', () => { ); expect(logger.debug).nthCalledWith( 6, - 'ruleRunStatus for test:1: {"lastExecutionDate":"1970-01-01T00:00:00.000Z","status":"active"}' + 'deprecated ruleRunStatus for test:1: {"lastExecutionDate":"1970-01-01T00:00:00.000Z","status":"active"}' ); expect(logger.debug).nthCalledWith( - 7, + 8, 'ruleRunMetrics for test:1: {"numSearches":3,"totalSearchDurationMs":23423,"esSearchDurationMs":33,"numberOfTriggeredActions":1,"numberOfGeneratedActions":1,"numberOfActiveAlerts":1,"numberOfRecoveredAlerts":0,"numberOfNewAlerts":1,"hasReachedAlertLimit":false,"triggeredActionsStatus":"complete"}' ); } diff --git a/x-pack/plugins/alerting/server/types.ts b/x-pack/plugins/alerting/server/types.ts index 88f5d8d1562de..3eccd1d362127 100644 --- a/x-pack/plugins/alerting/server/types.ts +++ b/x-pack/plugins/alerting/server/types.ts @@ -43,6 +43,7 @@ import { RuleMonitoring, MappedParams, RuleSnooze, + RuleLastRun, } from '../common'; import { PublicAlertFactory } from './alert/create_alert_factory'; export type WithoutQueryAndParams = Pick>; @@ -81,6 +82,7 @@ export interface RuleExecutorServices< alertFactory: PublicAlertFactory; shouldWriteAlerts: () => boolean; shouldStopExecution: () => boolean; + ruleMonitoringService?: PublicRuleMonitoringService; } export interface RuleExecutorOptions< @@ -271,9 +273,11 @@ export interface RawRule extends SavedObjectAttributes { mutedInstanceIds: string[]; meta?: RuleMeta; executionStatus: RawRuleExecutionStatus; - monitoring?: RuleMonitoring; + monitoring?: RawRuleMonitoring; snoozeSchedule?: RuleSnooze; // Remove ? when this parameter is made available in the public API isSnoozedUntil?: string | null; + lastRun?: RawRuleLastRun | null; + nextRun?: string | null; } export interface AlertingPlugin { @@ -302,3 +306,14 @@ export interface InvalidatePendingApiKey { export type RuleTypeRegistry = PublicMethodsOf; export type RulesClientApi = PublicMethodsOf; + +export interface PublicRuleMonitoringService { + setLastRunMetricsTotalSearchDurationMs: (totalSearchDurationMs: number) => void; + setLastRunMetricsTotalIndexingDurationMs: (totalIndexingDurationMs: number) => void; + setLastRunMetricsTotalAlertsDetected: (totalAlertDetected: number) => void; + setLastRunMetricsTotalAlertsCreated: (totalAlertCreated: number) => void; + setLastRunMetricsGapDurationS: (gapDurationS: number) => void; +} + +export interface RawRuleLastRun extends SavedObjectAttributes, RuleLastRun {} +export interface RawRuleMonitoring extends SavedObjectAttributes, RuleMonitoring {} diff --git a/x-pack/plugins/triggers_actions_ui/public/application/mock/rule_details/alert_summary/index.ts b/x-pack/plugins/triggers_actions_ui/public/application/mock/rule_details/alert_summary/index.ts index 2d5af068ae55d..0c00abae8d855 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/mock/rule_details/alert_summary/index.ts +++ b/x-pack/plugins/triggers_actions_ui/public/application/mock/rule_details/alert_summary/index.ts @@ -33,7 +33,7 @@ export const mockRule = (): Rule => { lastExecutionDate: new Date('2020-08-20T19:23:38Z'), }, monitoring: { - execution: { + run: { history: [ { success: true, @@ -55,7 +55,13 @@ export const mockRule = (): Rule => { success_ratio: 0.66, p50: 200000, p95: 300000, - p99: 300000, + p99: 390000, + }, + last_run: { + timestamp: '2020-08-20T19:23:38Z', + metrics: { + duration: 500, + }, }, }, }, diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/rule_details/components/rule_definition.test.tsx b/x-pack/plugins/triggers_actions_ui/public/application/sections/rule_details/components/rule_definition.test.tsx index 5166642eaabba..6f215e139adf0 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/sections/rule_details/components/rule_definition.test.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/rule_details/components/rule_definition.test.tsx @@ -221,7 +221,7 @@ function mockRule(overwrite = {}): Rule { lastExecutionDate: new Date('2020-08-20T19:23:38Z'), }, monitoring: { - execution: { + run: { history: [ { success: true, @@ -245,6 +245,12 @@ function mockRule(overwrite = {}): Rule { p95: 300000, p99: 300000, }, + last_run: { + timestamp: '2020-08-20T19:23:38Z', + metrics: { + duration: 500, + }, + }, }, }, ...overwrite, diff --git a/x-pack/test/alerting_api_integration/common/lib/wait_for_execution_count.ts b/x-pack/test/alerting_api_integration/common/lib/wait_for_execution_count.ts index 76d8a9afb7098..6ad0dab431d81 100644 --- a/x-pack/test/alerting_api_integration/common/lib/wait_for_execution_count.ts +++ b/x-pack/test/alerting_api_integration/common/lib/wait_for_execution_count.ts @@ -28,13 +28,13 @@ export function createWaitForExecutionCount( const prefix = spaceId ? getUrlPrefix(spaceId) : ''; const getResponse = await st.get(`${prefix}/internal/alerting/rule/${id}`); expect(getResponse.status).to.eql(200); - if (getResponse.body.monitoring.execution.history.length >= count) { + if (getResponse.body.monitoring.run.history.length >= count) { attempts = 0; return true; } // eslint-disable-next-line no-console console.log( - `found ${getResponse.body.monitoring.execution.history.length} and looking for ${count}, waiting 3s then retrying` + `found ${getResponse.body.monitoring.run.history.length} and looking for ${count}, waiting 3s then retrying` ); await delay(delayMs); return waitForExecutionCount(count, id); diff --git a/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/create.ts b/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/create.ts index f775b3607fade..cebfe68e279b0 100644 --- a/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/create.ts +++ b/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/create.ts @@ -124,10 +124,15 @@ export default function createAlertTests({ getService }: FtrProviderContext) { mute_all: false, muted_alert_ids: [], execution_status: response.body.execution_status, + ...(response.body.next_run ? { next_run: response.body.next_run } : {}), + ...(response.body.last_run ? { last_run: response.body.last_run } : {}), }); expect(typeof response.body.scheduled_task_id).to.be('string'); expect(Date.parse(response.body.created_at)).to.be.greaterThan(0); expect(Date.parse(response.body.updated_at)).to.be.greaterThan(0); + if (response.body.next_run) { + expect(Date.parse(response.body.next_run)).to.be.greaterThan(0); + } const taskRecord = await getScheduledTask(response.body.scheduled_task_id); expect(taskRecord.type).to.eql('task'); diff --git a/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/find.ts b/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/find.ts index 484d2c9cdac82..b8460e84202c9 100644 --- a/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/find.ts +++ b/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/find.ts @@ -84,6 +84,8 @@ const findTestUtils = ( mute_all: false, muted_alert_ids: [], execution_status: match.execution_status, + ...(match.next_run ? { next_run: match.next_run } : {}), + ...(match.last_run ? { last_run: match.last_run } : {}), ...(describeType === 'internal' ? { monitoring: match.monitoring, @@ -291,6 +293,8 @@ const findTestUtils = ( created_at: match.created_at, updated_at: match.updated_at, execution_status: match.execution_status, + ...(match.next_run ? { next_run: match.next_run } : {}), + ...(match.last_run ? { last_run: match.last_run } : {}), ...(describeType === 'internal' ? { monitoring: match.monitoring, diff --git a/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/get.ts b/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/get.ts index 4b2afe01d7a86..b0900f74993cb 100644 --- a/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/get.ts +++ b/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/get.ts @@ -81,6 +81,8 @@ const getTestUtils = ( mute_all: false, muted_alert_ids: [], execution_status: response.body.execution_status, + ...(response.body.next_run ? { next_run: response.body.next_run } : {}), + ...(response.body.last_run ? { last_run: response.body.last_run } : {}), ...(describeType === 'internal' ? { monitoring: response.body.monitoring, @@ -91,6 +93,9 @@ const getTestUtils = ( }); expect(Date.parse(response.body.created_at)).to.be.greaterThan(0); expect(Date.parse(response.body.updated_at)).to.be.greaterThan(0); + if (response.body.next_run) { + expect(Date.parse(response.body.next_run)).to.be.greaterThan(0); + } break; default: throw new Error(`Scenario untested: ${JSON.stringify(scenario)}`); diff --git a/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/alerting/update.ts b/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/alerting/update.ts index c49fa62c606b6..430d69274041a 100644 --- a/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/alerting/update.ts +++ b/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/alerting/update.ts @@ -132,12 +132,17 @@ export default function createUpdateTests({ getService }: FtrProviderContext) { created_at: response.body.created_at, updated_at: response.body.updated_at, execution_status: response.body.execution_status, + ...(response.body.next_run ? { next_run: response.body.next_run } : {}), + ...(response.body.last_run ? { last_run: response.body.last_run } : {}), }); expect(Date.parse(response.body.created_at)).to.be.greaterThan(0); expect(Date.parse(response.body.updated_at)).to.be.greaterThan(0); expect(Date.parse(response.body.updated_at)).to.be.greaterThan( Date.parse(response.body.created_at) ); + if (response.body.next_run) { + expect(Date.parse(response.body.next_run)).to.be.greaterThan(0); + } // Ensure AAD isn't broken await checkAAD({ supertest, @@ -216,12 +221,17 @@ export default function createUpdateTests({ getService }: FtrProviderContext) { created_at: response.body.created_at, updated_at: response.body.updated_at, execution_status: response.body.execution_status, + ...(response.body.next_run ? { next_run: response.body.next_run } : {}), + ...(response.body.last_run ? { last_run: response.body.last_run } : {}), }); expect(Date.parse(response.body.created_at)).to.be.greaterThan(0); expect(Date.parse(response.body.updated_at)).to.be.greaterThan(0); expect(Date.parse(response.body.updated_at)).to.be.greaterThan( Date.parse(response.body.created_at) ); + if (response.body.next_run) { + expect(Date.parse(response.body.next_run)).to.be.greaterThan(0); + } // Ensure AAD isn't broken await checkAAD({ supertest, @@ -311,12 +321,17 @@ export default function createUpdateTests({ getService }: FtrProviderContext) { created_at: response.body.created_at, updated_at: response.body.updated_at, execution_status: response.body.execution_status, + ...(response.body.next_run ? { next_run: response.body.next_run } : {}), + ...(response.body.last_run ? { last_run: response.body.last_run } : {}), }); expect(Date.parse(response.body.created_at)).to.be.greaterThan(0); expect(Date.parse(response.body.updated_at)).to.be.greaterThan(0); expect(Date.parse(response.body.updated_at)).to.be.greaterThan( Date.parse(response.body.created_at) ); + if (response.body.next_run) { + expect(Date.parse(response.body.next_run)).to.be.greaterThan(0); + } // Ensure AAD isn't broken await checkAAD({ supertest, @@ -406,12 +421,17 @@ export default function createUpdateTests({ getService }: FtrProviderContext) { created_at: response.body.created_at, updated_at: response.body.updated_at, execution_status: response.body.execution_status, + ...(response.body.next_run ? { next_run: response.body.next_run } : {}), + ...(response.body.last_run ? { last_run: response.body.last_run } : {}), }); expect(Date.parse(response.body.created_at)).to.be.greaterThan(0); expect(Date.parse(response.body.updated_at)).to.be.greaterThan(0); expect(Date.parse(response.body.updated_at)).to.be.greaterThan( Date.parse(response.body.created_at) ); + if (response.body.next_run) { + expect(Date.parse(response.body.next_run)).to.be.greaterThan(0); + } // Ensure AAD isn't broken await checkAAD({ supertest, @@ -499,12 +519,17 @@ export default function createUpdateTests({ getService }: FtrProviderContext) { created_at: response.body.created_at, updated_at: response.body.updated_at, execution_status: response.body.execution_status, + ...(response.body.next_run ? { next_run: response.body.next_run } : {}), + ...(response.body.last_run ? { last_run: response.body.last_run } : {}), }); expect(Date.parse(response.body.created_at)).to.be.greaterThan(0); expect(Date.parse(response.body.updated_at)).to.be.greaterThan(0); expect(Date.parse(response.body.updated_at)).to.be.greaterThan( Date.parse(response.body.created_at) ); + if (response.body.next_run) { + expect(Date.parse(response.body.next_run)).to.be.greaterThan(0); + } // Ensure AAD isn't broken await checkAAD({ supertest, diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/aggregate.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/aggregate.ts index 4424175e36953..ff24b25d89fa2 100644 --- a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/aggregate.ts +++ b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/aggregate.ts @@ -37,6 +37,11 @@ export default function createAggregateTests({ getService }: FtrProviderContext) unknown: 0, warning: 0, }, + rule_last_run_outcome: { + succeeded: 0, + warning: 0, + failed: 0, + }, rule_muted_status: { muted: 0, unmuted: 0, @@ -116,6 +121,11 @@ export default function createAggregateTests({ getService }: FtrProviderContext) unknown: 0, warning: 0, }, + rule_last_run_outcome: { + succeeded: 5, + warning: 0, + failed: 2, + }, rule_muted_status: { muted: 0, unmuted: 7, @@ -200,6 +210,11 @@ export default function createAggregateTests({ getService }: FtrProviderContext) disabled: 0, enabled: 7, }, + ruleLastRunOutcome: { + succeeded: 5, + warning: 0, + failed: 2, + }, ruleMutedStatus: { muted: 0, unmuted: 7, diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/create.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/create.ts index 7860bf15dc8e5..5cc7316f1c6e3 100644 --- a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/create.ts +++ b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/create.ts @@ -94,11 +94,15 @@ export default function createAlertTests({ getService }: FtrProviderContext) { created_at: response.body.created_at, updated_at: response.body.updated_at, execution_status: response.body.execution_status, + ...(response.body.next_run ? { next_run: response.body.next_run } : {}), + ...(response.body.last_run ? { last_run: response.body.last_run } : {}), }); expect(Date.parse(response.body.created_at)).to.be.greaterThan(0); expect(Date.parse(response.body.updated_at)).to.be.greaterThan(0); expect(Date.parse(response.body.updated_at)).to.eql(Date.parse(response.body.created_at)); - + if (response.body.next_run) { + expect(Date.parse(response.body.next_run)).to.be.greaterThan(0); + } expect(typeof response.body.scheduled_task_id).to.be('string'); const taskRecord = await getScheduledTask(response.body.scheduled_task_id); expect(taskRecord.type).to.eql('task'); @@ -190,8 +194,14 @@ export default function createAlertTests({ getService }: FtrProviderContext) { created_at: response.body.created_at, updated_at: response.body.updated_at, execution_status: response.body.execution_status, + ...(response.body.next_run ? { next_run: response.body.next_run } : {}), + ...(response.body.last_run ? { last_run: response.body.last_run } : {}), }); + if (response.body.next_run) { + expect(Date.parse(response.body.next_run)).to.be.greaterThan(0); + } + const esResponse = await es.get>( { index: '.kibana', @@ -485,11 +495,15 @@ export default function createAlertTests({ getService }: FtrProviderContext) { createdAt: response.body.createdAt, updatedAt: response.body.updatedAt, executionStatus: response.body.executionStatus, + ...(response.body.next_run ? { next_run: response.body.next_run } : {}), + ...(response.body.last_run ? { last_run: response.body.last_run } : {}), }); expect(Date.parse(response.body.createdAt)).to.be.greaterThan(0); expect(Date.parse(response.body.updatedAt)).to.be.greaterThan(0); expect(Date.parse(response.body.updatedAt)).to.eql(Date.parse(response.body.createdAt)); - + if (response.body.next_run) { + expect(Date.parse(response.body.next_run)).to.be.greaterThan(0); + } expect(typeof response.body.scheduledTaskId).to.be('string'); const taskRecord = await getScheduledTask(response.body.scheduledTaskId); expect(taskRecord.type).to.eql('task'); diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/find.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/find.ts index 23dcc1abaea44..d10518aca575f 100644 --- a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/find.ts +++ b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/find.ts @@ -75,6 +75,8 @@ const findTestUtils = ( created_at: match.created_at, updated_at: match.updated_at, execution_status: match.execution_status, + ...(match.next_run ? { next_run: match.next_run } : {}), + ...(match.last_run ? { last_run: match.last_run } : {}), ...(describeType === 'internal' ? { monitoring: match.monitoring, @@ -142,13 +144,13 @@ const findTestUtils = ( const response = await supertest.get( `${getUrlPrefix(Spaces.space1.id)}/${ describeType === 'public' ? 'api' : 'internal' - }/alerting/rules/_find?filter=alert.attributes.monitoring.execution.calculated_metrics.success_ratio>50` + }/alerting/rules/_find?filter=alert.attributes.monitoring.run.calculated_metrics.success_ratio>50` ); expect(response.status).to.eql(describeType === 'internal' ? 200 : 400); if (describeType === 'public') { expect(response.body.message).to.eql( - 'Error find rules: Filter is not supported on this field alert.attributes.monitoring.execution.calculated_metrics.success_ratio' + 'Error find rules: Filter is not supported on this field alert.attributes.monitoring.run.calculated_metrics.success_ratio' ); } }); @@ -159,13 +161,13 @@ const findTestUtils = ( const response = await supertest.get( `${getUrlPrefix(Spaces.space1.id)}/${ describeType === 'public' ? 'api' : 'internal' - }/alerting/rules/_find?sort_field=monitoring.execution.calculated_metrics.success_ratio` + }/alerting/rules/_find?sort_field=monitoring.run.calculated_metrics.success_ratio` ); expect(response.status).to.eql(describeType === 'internal' ? 200 : 400); if (describeType === 'public') { expect(response.body.message).to.eql( - 'Error find rules: Sort is not supported on this field monitoring.execution.calculated_metrics.success_ratio' + 'Error find rules: Sort is not supported on this field monitoring.run.calculated_metrics.success_ratio' ); } }); @@ -176,13 +178,13 @@ const findTestUtils = ( const response = await supertest.get( `${getUrlPrefix(Spaces.space1.id)}/${ describeType === 'public' ? 'api' : 'internal' - }/alerting/rules/_find?search_fields=monitoring.execution.calculated_metrics.success_ratio&search=50` + }/alerting/rules/_find?search_fields=monitoring.run.calculated_metrics.success_ratio&search=50` ); expect(response.status).to.eql(describeType === 'internal' ? 200 : 400); if (describeType === 'public') { expect(response.body.message).to.eql( - 'Error find rules: Search field monitoring.execution.calculated_metrics.success_ratio not supported' + 'Error find rules: Search field monitoring.run.calculated_metrics.success_ratio not supported' ); } }); @@ -325,6 +327,8 @@ export default function createFindTests({ getService }: FtrProviderContext) { createdAt: match.createdAt, updatedAt: match.updatedAt, executionStatus: match.executionStatus, + ...(match.nextRun ? { nextRun: match.nextRun } : {}), + ...(match.lastRun ? { lastRun: match.lastRun } : {}), }); expect(Date.parse(match.createdAt)).to.be.greaterThan(0); expect(Date.parse(match.updatedAt)).to.be.greaterThan(0); diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/get.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/get.ts index 7a94198c4ea57..c91467f698dc1 100644 --- a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/get.ts +++ b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/get.ts @@ -54,6 +54,8 @@ const getTestUtils = ( created_at: response.body.created_at, updated_at: response.body.updated_at, execution_status: response.body.execution_status, + ...(response.body.next_run ? { next_run: response.body.next_run } : {}), + ...(response.body.last_run ? { last_run: response.body.last_run } : {}), ...(describeType === 'internal' ? { monitoring: response.body.monitoring, @@ -64,6 +66,9 @@ const getTestUtils = ( }); expect(Date.parse(response.body.created_at)).to.be.greaterThan(0); expect(Date.parse(response.body.updated_at)).to.be.greaterThan(0); + if (response.body.next_run) { + expect(Date.parse(response.body.next_run)).to.be.greaterThan(0); + } }); it(`shouldn't find alert from another space`, async () => { @@ -149,9 +154,14 @@ export default function createGetTests({ getService }: FtrProviderContext) { createdAt: response.body.createdAt, updatedAt: response.body.updatedAt, executionStatus: response.body.executionStatus, + ...(response.body.nextRun ? { nextRun: response.body.nextRun } : {}), + ...(response.body.lastRun ? { lastRun: response.body.lastRun } : {}), }); expect(Date.parse(response.body.createdAt)).to.be.greaterThan(0); expect(Date.parse(response.body.updatedAt)).to.be.greaterThan(0); + if (response.body.nextRun) { + expect(Date.parse(response.body.nextRun)).to.be.greaterThan(0); + } }); }); }); diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/migrations.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/migrations.ts index 9edb414e39c77..7765c266f5a74 100644 --- a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/migrations.ts +++ b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/migrations.ts @@ -521,5 +521,65 @@ export default function createGetTests({ getService }: FtrProviderContext) { expect(response.statusCode).to.eql(200); expect(response.body._source?.alert?.params?.esQuery).to.eql('{"query":}'); }); + + it('8.6.0 migrates executionStatus and monitoring', async () => { + const response = await es.get<{ alert: RawRule }>( + { + index: '.kibana', + id: 'alert:8370ffd2-f2db-49dc-9741-92c657189b9b', + }, + { meta: true } + ); + const alert = response.body._source?.alert; + + expect(alert?.monitoring).to.eql({ + run: { + history: [ + { + duration: 60000, + success: true, + timestamp: '2022-08-24T19:05:49.817Z', + }, + ], + calculated_metrics: { + success_ratio: 1, + p50: 0, + p95: 60000, + p99: 60000, + }, + last_run: { + timestamp: '2022-08-24T19:05:49.817Z', + metrics: { + duration: 60000, + }, + }, + }, + }); + + expect(alert?.lastRun).to.eql({ + outcome: 'succeeded', + outcomeMsg: null, + warning: null, + alertsCount: {}, + }); + + expect(alert?.nextRun).to.eql(undefined); + }); + + it('8.6 migrates executionStatus warnings and errors', async () => { + const response = await es.get<{ alert: RawRule }>( + { + index: '.kibana', + id: 'alert:c87707ac-7328-47f7-b212-2cb40a4fc9b9', + }, + { meta: true } + ); + + const alert = response.body._source?.alert; + + expect(alert?.lastRun?.outcome).to.eql('warning'); + expect(alert?.lastRun?.warning).to.eql('warning reason'); + expect(alert?.lastRun?.outcomeMsg).to.eql('warning message'); + }); }); } diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/monitoring.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/monitoring.ts index c08a28b3c3ca3..38506eb54a4bc 100644 --- a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/monitoring.ts +++ b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/monitoring.ts @@ -35,9 +35,9 @@ export default function monitoringAlertTests({ getService }: FtrProviderContext) ); expect(getResponse.status).to.eql(200); - expect(getResponse.body.monitoring.execution.history.length).to.be(1); - expect(getResponse.body.monitoring.execution.history[0].success).to.be(true); - expect(getResponse.body.monitoring.execution.calculated_metrics.success_ratio).to.be(1); + expect(getResponse.body.monitoring.run.history.length).to.be(1); + expect(getResponse.body.monitoring.run.history[0].success).to.be(true); + expect(getResponse.body.monitoring.run.calculated_metrics.success_ratio).to.be(1); }); it('should return an accurate history for multiple success', async () => { @@ -56,11 +56,11 @@ export default function monitoringAlertTests({ getService }: FtrProviderContext) ); expect(getResponse.status).to.eql(200); - expect(getResponse.body.monitoring.execution.history.length).to.be(3); - expect(getResponse.body.monitoring.execution.history[0].success).to.be(true); - expect(getResponse.body.monitoring.execution.history[1].success).to.be(true); - expect(getResponse.body.monitoring.execution.history[2].success).to.be(true); - expect(getResponse.body.monitoring.execution.calculated_metrics.success_ratio).to.be(1); + expect(getResponse.body.monitoring.run.history.length).to.be(3); + expect(getResponse.body.monitoring.run.history[0].success).to.be(true); + expect(getResponse.body.monitoring.run.history[1].success).to.be(true); + expect(getResponse.body.monitoring.run.history[2].success).to.be(true); + expect(getResponse.body.monitoring.run.calculated_metrics.success_ratio).to.be(1); }); it('should return an accurate history for some successes and some failures', async () => { @@ -88,13 +88,13 @@ export default function monitoringAlertTests({ getService }: FtrProviderContext) ); expect(getResponse.status).to.eql(200); - expect(getResponse.body.monitoring.execution.history.length).to.be(5); - expect(getResponse.body.monitoring.execution.history[0].success).to.be(true); - expect(getResponse.body.monitoring.execution.history[1].success).to.be(true); - expect(getResponse.body.monitoring.execution.history[2].success).to.be(true); - expect(getResponse.body.monitoring.execution.history[3].success).to.be(false); - expect(getResponse.body.monitoring.execution.history[4].success).to.be(false); - expect(getResponse.body.monitoring.execution.calculated_metrics.success_ratio).to.be(0.6); + expect(getResponse.body.monitoring.run.history.length).to.be(5); + expect(getResponse.body.monitoring.run.history[0].success).to.be(true); + expect(getResponse.body.monitoring.run.history[1].success).to.be(true); + expect(getResponse.body.monitoring.run.history[2].success).to.be(true); + expect(getResponse.body.monitoring.run.history[3].success).to.be(false); + expect(getResponse.body.monitoring.run.history[4].success).to.be(false); + expect(getResponse.body.monitoring.run.calculated_metrics.success_ratio).to.be(0.6); }); it('should populate rule objects with the calculated percentiles', async () => { @@ -118,7 +118,7 @@ export default function monitoringAlertTests({ getService }: FtrProviderContext) ); expect(getResponse.status).to.eql(200); - getResponse.body.monitoring.execution.history.forEach((history: any) => { + getResponse.body.monitoring.run.history.forEach((history: any) => { expect(history.duration).to.be.a('number'); }); }); @@ -135,13 +135,13 @@ export default function monitoringAlertTests({ getService }: FtrProviderContext) `${getUrlPrefix(Spaces.space1.id)}/internal/alerting/rule/${id}` ); expect(getResponse.status).to.eql(200); - if (getResponse.body.monitoring.execution.history.length >= count) { + if (getResponse.body.monitoring.run.history.length >= count) { attempts = 0; return true; } // eslint-disable-next-line no-console console.log( - `found ${getResponse.body.monitoring.execution.history.length} and looking for ${count}, waiting 3s then retrying` + `found ${getResponse.body.monitoring.run.history.length} and looking for ${count}, waiting 3s then retrying` ); await delay(3000); return waitForExecutionCount(count, id); diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/update.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/update.ts index c5a9c93d45e81..4c740b3be9b97 100644 --- a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/update.ts +++ b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/update.ts @@ -63,12 +63,17 @@ export default function createUpdateTests({ getService }: FtrProviderContext) { created_at: response.body.created_at, updated_at: response.body.updated_at, execution_status: response.body.execution_status, + ...(response.body.next_run ? { next_run: response.body.next_run } : {}), + ...(response.body.last_run ? { last_run: response.body.last_run } : {}), }); expect(Date.parse(response.body.created_at)).to.be.greaterThan(0); expect(Date.parse(response.body.updated_at)).to.be.greaterThan(0); expect(Date.parse(response.body.updated_at)).to.be.greaterThan( Date.parse(response.body.created_at) ); + if (response.body.next_run) { + expect(Date.parse(response.body.next_run)).to.be.greaterThan(0); + } response = await supertest.get( `${getUrlPrefix( @@ -163,12 +168,17 @@ export default function createUpdateTests({ getService }: FtrProviderContext) { createdAt: response.body.createdAt, updatedAt: response.body.updatedAt, executionStatus: response.body.executionStatus, + ...(response.body.nextRun ? { nextRun: response.body.nextRun } : {}), + ...(response.body.lastRun ? { lastRun: response.body.lastRun } : {}), }); expect(Date.parse(response.body.createdAt)).to.be.greaterThan(0); expect(Date.parse(response.body.updatedAt)).to.be.greaterThan(0); expect(Date.parse(response.body.updatedAt)).to.be.greaterThan( Date.parse(response.body.createdAt) ); + if (response.body.nextRun) { + expect(Date.parse(response.body.nextRun)).to.be.greaterThan(0); + } // Ensure AAD isn't broken await checkAAD({ diff --git a/x-pack/test/functional/es_archives/alerts/data.json b/x-pack/test/functional/es_archives/alerts/data.json index 0129bb2f4729c..8d7be46b6c749 100644 --- a/x-pack/test/functional/es_archives/alerts/data.json +++ b/x-pack/test/functional/es_archives/alerts/data.json @@ -1232,3 +1232,123 @@ } } } + +{ + "type": "doc", + "value": { + "id": "alert:8370ffd2-f2db-49dc-9741-92c657189b9b", + "index": ".kibana_1", + "source": { + "alert": { + "alertTypeId": "example.always-firing", + "apiKey": null, + "apiKeyOwner": null, + "consumer": "alerts", + "createdAt": "2022-08-24T19:02:30.889Z", + "createdBy": "elastic", + "enabled": false, + "muteAll": false, + "mutedInstanceIds": [], + "name": "Test rule migration with successful execution status and monitoring", + "params": {}, + "schedule": { + "interval": "1m" + }, + "scheduledTaskId": null, + "tags": [], + "throttle": null, + "updatedBy": "elastic", + "isSnoozedUntil": "2022-08-24T19:05:49.817Z", + "executionStatus": { + "status": "ok", + "lastExecutionDate": "2022-08-24T19:05:49.817Z", + "lastDuration": 60000 + }, + "monitoring": { + "execution": { + "history": [{ + "duration": 60000, + "success": true, + "timestamp": "2022-08-24T19:05:49.817Z" + }], + "calculated_metrics": { + "success_ratio": 1, + "p50": 0, + "p95": 60000, + "p99": 60000 + } + } + } + }, + "migrationVersion": { + "alert": "8.0.1" + }, + "references": [ + ], + "type": "alert", + "updated_at": "2022-11-01T19:05:50.159Z" + } + } +} + +{ + "type": "doc", + "value": { + "id": "alert:c87707ac-7328-47f7-b212-2cb40a4fc9b9", + "index": ".kibana_1", + "source": { + "alert": { + "alertTypeId": "example.always-firing", + "apiKey": null, + "apiKeyOwner": null, + "consumer": "alerts", + "createdAt": "2022-08-24T19:02:30.889Z", + "createdBy": "elastic", + "enabled": false, + "muteAll": false, + "mutedInstanceIds": [], + "name": "Test rule migration with warning execution status", + "params": {}, + "schedule": { + "interval": "1m" + }, + "scheduledTaskId": null, + "tags": [], + "throttle": null, + "updatedBy": "elastic", + "isSnoozedUntil": "2022-08-24T19:05:49.817Z", + "executionStatus": { + "status": "warning", + "lastExecutionDate": "2022-08-24T19:05:49.817Z", + "lastDuration": 60000, + "warning": { + "reason": "warning reason", + "message": "warning message" + } + }, + "monitoring": { + "execution": { + "history": [{ + "duration": 60000, + "success": true, + "timestamp": "2022-08-24T19:05:49.817Z" + }], + "calculated_metrics": { + "success_ratio": 1, + "p50": 0, + "p95": 60000, + "p99": 60000 + } + } + } + }, + "migrationVersion": { + "alert": "8.0.1" + }, + "references": [ + ], + "type": "alert", + "updated_at": "2022-11-01T19:05:50.159Z" + } + } +} diff --git a/x-pack/test/functional/es_archives/alerts/mappings.json b/x-pack/test/functional/es_archives/alerts/mappings.json index 0da2b51499517..2e64004634fa6 100644 --- a/x-pack/test/functional/es_archives/alerts/mappings.json +++ b/x-pack/test/functional/es_archives/alerts/mappings.json @@ -175,6 +175,79 @@ }, "isSnoozedUntil": { "type": "date" + }, + "monitoring": { + "properties": { + "execution": { + "properties": { + "history": { + "properties": { + "duration": { + "type": "long" + }, + "success": { + "type": "boolean" + }, + "timestamp": { + "type": "date" + } + } + }, + "calculated_metrics": { + "properties": { + "p50": { + "type": "long" + }, + "p95": { + "type": "long" + }, + "p99": { + "type": "long" + }, + "success_ratio": { + "type": "float" + } + } + } + } + } + } + }, + "executionStatus": { + "properties": { + "numberOfTriggeredActions": { + "type": "long" + }, + "status": { + "type": "keyword" + }, + "lastExecutionDate": { + "type": "date" + }, + "lastDuration": { + "type": "long" + }, + "error": { + "properties": { + "reason": { + "type": "keyword" + }, + "message": { + "type": "keyword" + } + } + }, + "warning": { + "properties": { + "reason": { + "type": "keyword" + }, + "message": { + "type": "keyword" + } + } + } + } } } }, diff --git a/x-pack/test/functional/es_archives/security_solution/legacy_actions/data.json b/x-pack/test/functional/es_archives/security_solution/legacy_actions/data.json index 25aa371e02144..f0c883c6b3756 100644 --- a/x-pack/test/functional/es_archives/security_solution/legacy_actions/data.json +++ b/x-pack/test/functional/es_archives/security_solution/legacy_actions/data.json @@ -776,7 +776,7 @@ "scheduledTaskId" : null, "legacyId" : "29ba2fa0-b076-11ec-bb3f-1f063f8e06cf", "monitoring" : { - "execution" : { + "run" : { "history" : [ { "duration" : 111,