From 1dd3687710ad5fa3e0d1b35fae556d08ded7c02d Mon Sep 17 00:00:00 2001 From: Xavier Mouligneau Date: Tue, 13 Sep 2022 14:17:32 -0400 Subject: [PATCH 01/27] re-structure migrations code --- .../server/saved_objects/migrations.ts | 1082 ----------------- .../saved_objects/migrations/7.10/index.ts | 96 ++ .../saved_objects/migrations/7.11/index.ts | 174 +++ .../saved_objects/migrations/7.13/index.ts | 86 ++ .../saved_objects/migrations/7.14/index.ts | 46 + .../saved_objects/migrations/7.15/index.ts | 110 ++ .../saved_objects/migrations/7.16/index.ts | 155 +++ .../saved_objects/migrations/8.0/index.ts | 124 ++ .../saved_objects/migrations/8.2/index.ts | 42 + .../saved_objects/migrations/8.3/index.ts | 91 ++ .../saved_objects/migrations/8.4/index.ts | 31 + .../saved_objects/migrations/8.5/index.ts | 64 + .../saved_objects/migrations/constants.ts | 12 + .../index.test.ts} | 3 + .../server/saved_objects/migrations/index.ts | 134 ++ .../server/saved_objects/migrations/types.ts | 18 + .../server/saved_objects/migrations/utils.ts | 57 + 17 files changed, 1243 insertions(+), 1082 deletions(-) delete mode 100644 x-pack/plugins/alerting/server/saved_objects/migrations.ts create mode 100644 x-pack/plugins/alerting/server/saved_objects/migrations/7.10/index.ts create mode 100644 x-pack/plugins/alerting/server/saved_objects/migrations/7.11/index.ts create mode 100644 x-pack/plugins/alerting/server/saved_objects/migrations/7.13/index.ts create mode 100644 x-pack/plugins/alerting/server/saved_objects/migrations/7.14/index.ts create mode 100644 x-pack/plugins/alerting/server/saved_objects/migrations/7.15/index.ts create mode 100644 x-pack/plugins/alerting/server/saved_objects/migrations/7.16/index.ts create mode 100644 x-pack/plugins/alerting/server/saved_objects/migrations/8.0/index.ts create mode 100644 x-pack/plugins/alerting/server/saved_objects/migrations/8.2/index.ts create mode 100644 x-pack/plugins/alerting/server/saved_objects/migrations/8.3/index.ts create mode 100644 x-pack/plugins/alerting/server/saved_objects/migrations/8.4/index.ts create mode 100644 x-pack/plugins/alerting/server/saved_objects/migrations/8.5/index.ts create mode 100644 x-pack/plugins/alerting/server/saved_objects/migrations/constants.ts rename x-pack/plugins/alerting/server/saved_objects/{migrations.test.ts => migrations/index.test.ts} (99%) create mode 100644 x-pack/plugins/alerting/server/saved_objects/migrations/index.ts create mode 100644 x-pack/plugins/alerting/server/saved_objects/migrations/types.ts create mode 100644 x-pack/plugins/alerting/server/saved_objects/migrations/utils.ts diff --git a/x-pack/plugins/alerting/server/saved_objects/migrations.ts b/x-pack/plugins/alerting/server/saved_objects/migrations.ts deleted file mode 100644 index 20ebb7b4eb8e9..0000000000000 --- a/x-pack/plugins/alerting/server/saved_objects/migrations.ts +++ /dev/null @@ -1,1082 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { isRuleType, ruleTypeMappings } from '@kbn/securitysolution-rules'; -import { isString } from 'lodash/fp'; -import { omit, pick } from 'lodash'; -import moment from 'moment-timezone'; -import { gte } from 'semver'; -import { - LogMeta, - SavedObjectMigrationMap, - SavedObjectUnsanitizedDoc, - SavedObjectMigrationFn, - SavedObjectMigrationContext, - SavedObjectAttributes, - SavedObjectAttribute, - SavedObjectReference, -} from '@kbn/core/server'; -import { EncryptedSavedObjectsPluginSetup } from '@kbn/encrypted-saved-objects-plugin/server'; -import type { IsMigrationNeededPredicate } from '@kbn/encrypted-saved-objects-plugin/server'; -import { MigrateFunctionsObject, MigrateFunction } from '@kbn/kibana-utils-plugin/common'; -import { mergeSavedObjectMigrationMaps } from '@kbn/core/server'; -import { isSerializedSearchSource, SerializedSearchSourceFields } from '@kbn/data-plugin/common'; -import { extractRefsFromGeoContainmentAlert } from './geo_containment/migrations'; -import { RawRule, RawRuleAction, RawRuleExecutionStatus } from '../types'; -import { getMappedParams } from '../rules_client/lib/mapped_params_utils'; - -const SIEM_APP_ID = 'securitySolution'; -const SIEM_SERVER_APP_ID = 'siem'; -const MINIMUM_SS_MIGRATION_VERSION = '8.3.0'; -export const LEGACY_LAST_MODIFIED_VERSION = 'pre-7.10.0'; -export const FILEBEAT_7X_INDICATOR_PATH = 'threatintel.indicator'; - -interface AlertLogMeta extends LogMeta { - migrations: { alertDocument: SavedObjectUnsanitizedDoc }; -} - -type AlertMigration = ( - doc: SavedObjectUnsanitizedDoc, - context: SavedObjectMigrationContext -) => SavedObjectUnsanitizedDoc; - -function createEsoMigration( - encryptedSavedObjects: EncryptedSavedObjectsPluginSetup, - isMigrationNeededPredicate: IsMigrationNeededPredicate, - migrationFunc: AlertMigration -) { - return encryptedSavedObjects.createMigration({ - isMigrationNeededPredicate, - migration: migrationFunc, - shouldMigrateIfDecryptionFails: true, // shouldMigrateIfDecryptionFails flag that applies the migration to undecrypted document if decryption fails - }); -} - -const SUPPORT_INCIDENTS_ACTION_TYPES = ['.servicenow', '.jira', '.resilient']; - -export const isAnyActionSupportIncidents = (doc: SavedObjectUnsanitizedDoc): boolean => - doc.attributes.actions.some((action) => - SUPPORT_INCIDENTS_ACTION_TYPES.includes(action.actionTypeId) - ); - -// Deprecated in 8.0 -export const isSiemSignalsRuleType = (doc: SavedObjectUnsanitizedDoc): boolean => - doc.attributes.alertTypeId === 'siem.signals'; - -export const isEsQueryRuleType = (doc: SavedObjectUnsanitizedDoc) => - doc.attributes.alertTypeId === '.es-query'; - -export const isDetectionEngineAADRuleType = (doc: SavedObjectUnsanitizedDoc): boolean => - (Object.values(ruleTypeMappings) as string[]).includes(doc.attributes.alertTypeId); - -/** - * Returns true if the alert type is that of "siem.notifications" which is a legacy notification system that was deprecated in 7.16.0 - * in favor of using the newer alerting notifications system. - * @param doc The saved object alert type document - * @returns true if this is a legacy "siem.notifications" rule, otherwise false - * @deprecated Once we are confident all rules relying on side-car actions SO's have been migrated to SO references we should remove this function - */ -export const isSecuritySolutionLegacyNotification = ( - doc: SavedObjectUnsanitizedDoc -): boolean => doc.attributes.alertTypeId === 'siem.notifications'; - -export function getMigrations( - encryptedSavedObjects: EncryptedSavedObjectsPluginSetup, - searchSourceMigrations: MigrateFunctionsObject, - isPreconfigured: (connectorId: string) => boolean -): SavedObjectMigrationMap { - const migrationWhenRBACWasIntroduced = createEsoMigration( - encryptedSavedObjects, - // migrate all documents in 7.10 in order to add the "meta" RBAC field - (doc): doc is SavedObjectUnsanitizedDoc => true, - pipeMigrations( - markAsLegacyAndChangeConsumer, - setAlertIdAsDefaultDedupkeyOnPagerDutyActions, - initializeExecutionStatus - ) - ); - - const migrationAlertUpdatedAtAndNotifyWhen = createEsoMigration( - encryptedSavedObjects, - // migrate all documents in 7.11 in order to add the "updatedAt" and "notifyWhen" fields - (doc): doc is SavedObjectUnsanitizedDoc => true, - pipeMigrations(setAlertUpdatedAtDate, setNotifyWhen) - ); - - const migrationActions7112 = createEsoMigration( - encryptedSavedObjects, - (doc): doc is SavedObjectUnsanitizedDoc => isAnyActionSupportIncidents(doc), - pipeMigrations(restructureConnectorsThatSupportIncident) - ); - - const migrationSecurityRules713 = createEsoMigration( - encryptedSavedObjects, - (doc): doc is SavedObjectUnsanitizedDoc => isSiemSignalsRuleType(doc), - pipeMigrations(removeNullsFromSecurityRules) - ); - - const migrationSecurityRules714 = createEsoMigration( - encryptedSavedObjects, - (doc): doc is SavedObjectUnsanitizedDoc => isSiemSignalsRuleType(doc), - pipeMigrations(removeNullAuthorFromSecurityRules) - ); - - const migrationSecurityRules715 = createEsoMigration( - encryptedSavedObjects, - (doc): doc is SavedObjectUnsanitizedDoc => isSiemSignalsRuleType(doc), - pipeMigrations(addExceptionListsToReferences) - ); - - const migrateRules716 = createEsoMigration( - encryptedSavedObjects, - (doc): doc is SavedObjectUnsanitizedDoc => true, - pipeMigrations( - setLegacyId, - getRemovePreconfiguredConnectorsFromReferencesFn(isPreconfigured), - addRuleIdsToLegacyNotificationReferences, - extractRefsFromGeoContainmentAlert - ) - ); - - const migrationRules800 = createEsoMigration( - encryptedSavedObjects, - (doc: SavedObjectUnsanitizedDoc): doc is SavedObjectUnsanitizedDoc => true, - pipeMigrations( - addThreatIndicatorPathToThreatMatchRules, - addSecuritySolutionAADRuleTypes, - fixInventoryThresholdGroupId - ) - ); - - const migrationRules801 = createEsoMigration( - encryptedSavedObjects, - (doc: SavedObjectUnsanitizedDoc): doc is SavedObjectUnsanitizedDoc => true, - pipeMigrations(addSecuritySolutionAADRuleTypeTags) - ); - - const migrationRules820 = createEsoMigration( - encryptedSavedObjects, - (doc: SavedObjectUnsanitizedDoc): doc is SavedObjectUnsanitizedDoc => true, - pipeMigrations(addMappedParams) - ); - - const migrationRules830 = createEsoMigration( - encryptedSavedObjects, - (doc: SavedObjectUnsanitizedDoc): doc is SavedObjectUnsanitizedDoc => true, - pipeMigrations(addSearchType, removeInternalTags, convertSnoozes) - ); - - const migrationRules841 = createEsoMigration( - encryptedSavedObjects, - (doc: SavedObjectUnsanitizedDoc): doc is SavedObjectUnsanitizedDoc => true, - pipeMigrations(removeIsSnoozedUntil) - ); - - const migrationRules850 = createEsoMigration( - encryptedSavedObjects, - (doc): doc is SavedObjectUnsanitizedDoc => isEsQueryRuleType(doc), - pipeMigrations(stripOutRuntimeFieldsInOldESQuery) - ); - - return mergeSavedObjectMigrationMaps( - { - '7.10.0': executeMigrationWithErrorHandling(migrationWhenRBACWasIntroduced, '7.10.0'), - '7.11.0': executeMigrationWithErrorHandling(migrationAlertUpdatedAtAndNotifyWhen, '7.11.0'), - '7.11.2': executeMigrationWithErrorHandling(migrationActions7112, '7.11.2'), - '7.13.0': executeMigrationWithErrorHandling(migrationSecurityRules713, '7.13.0'), - '7.14.1': executeMigrationWithErrorHandling(migrationSecurityRules714, '7.14.1'), - '7.15.0': executeMigrationWithErrorHandling(migrationSecurityRules715, '7.15.0'), - '7.16.0': executeMigrationWithErrorHandling(migrateRules716, '7.16.0'), - '8.0.0': executeMigrationWithErrorHandling(migrationRules800, '8.0.0'), - '8.0.1': executeMigrationWithErrorHandling(migrationRules801, '8.0.1'), - '8.2.0': executeMigrationWithErrorHandling(migrationRules820, '8.2.0'), - '8.3.0': executeMigrationWithErrorHandling(migrationRules830, '8.3.0'), - '8.4.1': executeMigrationWithErrorHandling(migrationRules841, '8.4.1'), - '8.5.0': executeMigrationWithErrorHandling(migrationRules850, '8.5.0'), - }, - getSearchSourceMigrations(encryptedSavedObjects, searchSourceMigrations) - ); -} - -function executeMigrationWithErrorHandling( - migrationFunc: SavedObjectMigrationFn, - version: string -) { - return (doc: SavedObjectUnsanitizedDoc, context: SavedObjectMigrationContext) => { - try { - return migrationFunc(doc, context); - } catch (ex) { - context.log.error( - `encryptedSavedObject ${version} migration failed for alert ${doc.id} with error: ${ex.message}`, - { - migrations: { - alertDocument: doc, - }, - } - ); - throw ex; - } - }; -} - -const setAlertUpdatedAtDate = ( - doc: SavedObjectUnsanitizedDoc -): SavedObjectUnsanitizedDoc => { - const updatedAt = doc.updated_at || doc.attributes.createdAt; - return { - ...doc, - attributes: { - ...doc.attributes, - updatedAt, - }, - }; -}; - -const setNotifyWhen = ( - doc: SavedObjectUnsanitizedDoc -): SavedObjectUnsanitizedDoc => { - const notifyWhen = doc.attributes.throttle ? 'onThrottleInterval' : 'onActiveAlert'; - return { - ...doc, - attributes: { - ...doc.attributes, - notifyWhen, - }, - }; -}; - -const consumersToChange: Map = new Map( - Object.entries({ - alerting: 'alerts', - metrics: 'infrastructure', - [SIEM_APP_ID]: SIEM_SERVER_APP_ID, - }) -); - -function markAsLegacyAndChangeConsumer( - doc: SavedObjectUnsanitizedDoc -): SavedObjectUnsanitizedDoc { - const { - attributes: { consumer }, - } = doc; - return { - ...doc, - attributes: { - ...doc.attributes, - consumer: consumersToChange.get(consumer) ?? consumer, - // mark any alert predating 7.10 as a legacy alert - meta: { - versionApiKeyLastmodified: LEGACY_LAST_MODIFIED_VERSION, - }, - }, - }; -} - -function setAlertIdAsDefaultDedupkeyOnPagerDutyActions( - doc: SavedObjectUnsanitizedDoc -): SavedObjectUnsanitizedDoc { - const { attributes } = doc; - return { - ...doc, - attributes: { - ...attributes, - ...(attributes.actions - ? { - actions: attributes.actions.map((action) => { - if (action.actionTypeId !== '.pagerduty' || action.params.eventAction === 'trigger') { - return action; - } - return { - ...action, - params: { - ...action.params, - dedupKey: action.params.dedupKey ?? '{{alertId}}', - }, - }; - }), - } - : {}), - }, - }; -} - -function initializeExecutionStatus( - doc: SavedObjectUnsanitizedDoc -): SavedObjectUnsanitizedDoc { - const { attributes } = doc; - return { - ...doc, - attributes: { - ...attributes, - executionStatus: { - status: 'pending', - lastExecutionDate: new Date().toISOString(), - error: null, - } as RawRuleExecutionStatus, - }, - }; -} - -function isEmptyObject(obj: {}) { - for (const attr in obj) { - if (Object.prototype.hasOwnProperty.call(obj, attr)) { - return false; - } - } - return true; -} - -function restructureConnectorsThatSupportIncident( - doc: SavedObjectUnsanitizedDoc -): SavedObjectUnsanitizedDoc { - const { actions } = doc.attributes; - const newActions = actions.reduce((acc, action) => { - if ( - ['.servicenow', '.jira', '.resilient'].includes(action.actionTypeId) && - action.params.subAction === 'pushToService' - ) { - // Future developer, we needed to do that because when we created this migration - // we forget to think about user already using 7.11.0 and having an incident attribute build the right way - // IMPORTANT -> if you change this code please do the same inside of this file - // x-pack/plugins/alerting/server/saved_objects/migrations.ts - const subActionParamsIncident = - (action.params?.subActionParams as SavedObjectAttributes)?.incident ?? null; - if (subActionParamsIncident != null && !isEmptyObject(subActionParamsIncident)) { - return [...acc, action]; - } - if (action.actionTypeId === '.servicenow') { - const { - title, - comments, - comment, - description, - severity, - urgency, - impact, - short_description: shortDescription, - } = action.params.subActionParams as { - title: string; - description?: string; - severity?: string; - urgency?: string; - impact?: string; - comment?: string; - comments?: Array<{ commentId: string; comment: string }>; - short_description?: string; - }; - return [ - ...acc, - { - ...action, - params: { - subAction: 'pushToService', - subActionParams: { - incident: { - short_description: shortDescription ?? title, - description, - severity, - urgency, - impact, - }, - comments: [ - ...(comments ?? []), - ...(comment != null ? [{ commentId: '1', comment }] : []), - ], - }, - }, - }, - ] as RawRuleAction[]; - } else if (action.actionTypeId === '.jira') { - const { title, comments, description, issueType, priority, labels, parent, summary } = - action.params.subActionParams as { - title: string; - description: string; - issueType: string; - priority?: string; - labels?: string[]; - parent?: string; - comments?: unknown[]; - summary?: string; - }; - return [ - ...acc, - { - ...action, - params: { - subAction: 'pushToService', - subActionParams: { - incident: { - summary: summary ?? title, - description, - issueType, - priority, - labels, - parent, - }, - comments, - }, - }, - }, - ] as RawRuleAction[]; - } else if (action.actionTypeId === '.resilient') { - const { title, comments, description, incidentTypes, severityCode, name } = action.params - .subActionParams as { - title: string; - description: string; - incidentTypes?: number[]; - severityCode?: number; - comments?: unknown[]; - name?: string; - }; - return [ - ...acc, - { - ...action, - params: { - subAction: 'pushToService', - subActionParams: { - incident: { - name: name ?? title, - description, - incidentTypes, - severityCode, - }, - comments, - }, - }, - }, - ] as RawRuleAction[]; - } - } - - return [...acc, action]; - }, [] as RawRuleAction[]); - - return { - ...doc, - attributes: { - ...doc.attributes, - actions: newActions, - }, - }; -} - -function convertNullToUndefined(attribute: SavedObjectAttribute) { - return attribute != null ? attribute : undefined; -} - -function removeNullsFromSecurityRules( - doc: SavedObjectUnsanitizedDoc -): SavedObjectUnsanitizedDoc { - const { - attributes: { params }, - } = doc; - return { - ...doc, - attributes: { - ...doc.attributes, - params: { - ...params, - buildingBlockType: convertNullToUndefined(params.buildingBlockType), - note: convertNullToUndefined(params.note), - index: convertNullToUndefined(params.index), - language: convertNullToUndefined(params.language), - license: convertNullToUndefined(params.license), - outputIndex: convertNullToUndefined(params.outputIndex), - savedId: convertNullToUndefined(params.savedId), - timelineId: convertNullToUndefined(params.timelineId), - timelineTitle: convertNullToUndefined(params.timelineTitle), - meta: convertNullToUndefined(params.meta), - query: convertNullToUndefined(params.query), - filters: convertNullToUndefined(params.filters), - riskScoreMapping: params.riskScoreMapping != null ? params.riskScoreMapping : [], - ruleNameOverride: convertNullToUndefined(params.ruleNameOverride), - severityMapping: params.severityMapping != null ? params.severityMapping : [], - threat: params.threat != null ? params.threat : [], - threshold: - params.threshold != null && - typeof params.threshold === 'object' && - !Array.isArray(params.threshold) - ? { - field: Array.isArray(params.threshold.field) - ? params.threshold.field - : params.threshold.field === '' || params.threshold.field == null - ? [] - : [params.threshold.field], - value: params.threshold.value, - cardinality: - params.threshold.cardinality != null ? params.threshold.cardinality : [], - } - : undefined, - timestampOverride: convertNullToUndefined(params.timestampOverride), - exceptionsList: - params.exceptionsList != null - ? params.exceptionsList - : params.exceptions_list != null - ? params.exceptions_list - : params.lists != null - ? params.lists - : [], - threatFilters: convertNullToUndefined(params.threatFilters), - machineLearningJobId: - params.machineLearningJobId == null - ? undefined - : Array.isArray(params.machineLearningJobId) - ? params.machineLearningJobId - : [params.machineLearningJobId], - }, - }, - }; -} - -/** - * The author field was introduced later and was not part of the original rules. We overlooked - * the filling in the author field as an empty array in an earlier upgrade routine from - * 'removeNullsFromSecurityRules' during the 7.13.0 upgrade. Since we don't change earlier migrations, - * but rather only move forward with the "arrow of time" we are going to upgrade and fix - * it if it is missing for anyone in 7.14.0 and above release. Earlier releases if we want to fix them, - * would have to be modified as a "7.13.1", etc... if we want to fix it there. - * @param doc The document that is not migrated and contains a "null" or "undefined" author field - * @returns The document with the author field fleshed in. - */ -function removeNullAuthorFromSecurityRules( - doc: SavedObjectUnsanitizedDoc -): SavedObjectUnsanitizedDoc { - const { - attributes: { params }, - } = doc; - return { - ...doc, - attributes: { - ...doc.attributes, - params: { - ...params, - author: params.author != null ? params.author : [], - }, - }, - }; -} - -/** - * This migrates exception list containers to saved object references on an upgrade. - * We only migrate if we find these conditions: - * - exceptionLists are an array and not null, undefined, or malformed data. - * - The exceptionList item is an object and id is a string and not null, undefined, or malformed data - * - The existing references do not already have an exceptionItem reference already found within it. - * Some of these issues could crop up during either user manual errors of modifying things, earlier migration - * issues, etc... - * @param doc The document that might have exceptionListItems to migrate - * @returns The document migrated with saved object references - */ -function addExceptionListsToReferences( - doc: SavedObjectUnsanitizedDoc -): SavedObjectUnsanitizedDoc { - const { - attributes: { - params: { exceptionsList }, - }, - references, - } = doc; - if (!Array.isArray(exceptionsList)) { - // early return if we are not an array such as being undefined or null or malformed. - return doc; - } else { - const exceptionsToTransform = removeMalformedExceptionsList(exceptionsList); - const newReferences = exceptionsToTransform.flatMap( - (exceptionItem, index) => { - const existingReferenceFound = references?.find((reference) => { - return ( - reference.id === exceptionItem.id && - ((reference.type === 'exception-list' && exceptionItem.namespace_type === 'single') || - (reference.type === 'exception-list-agnostic' && - exceptionItem.namespace_type === 'agnostic')) - ); - }); - if (existingReferenceFound) { - // skip if the reference already exists for some uncommon reason so we do not add an additional one. - // This enables us to be idempotent and you can run this migration multiple times and get the same output. - return []; - } else { - return [ - { - name: `param:exceptionsList_${index}`, - id: String(exceptionItem.id), - type: - exceptionItem.namespace_type === 'agnostic' - ? 'exception-list-agnostic' - : 'exception-list', - }, - ]; - } - } - ); - if (references == null && newReferences.length === 0) { - // Avoid adding an empty references array if the existing saved object never had one to begin with - return doc; - } else { - return { ...doc, references: [...(references ?? []), ...newReferences] }; - } - } -} - -/** - * This will do a flatMap reduce where we only return exceptionsLists and their items if: - * - exceptionLists are an array and not null, undefined, or malformed data. - * - The exceptionList item is an object and id is a string and not null, undefined, or malformed data - * - * Some of these issues could crop up during either user manual errors of modifying things, earlier migration - * issues, etc... - * @param exceptionsList The list of exceptions - * @returns The exception lists if they are a valid enough shape - */ -function removeMalformedExceptionsList( - exceptionsList: SavedObjectAttribute -): SavedObjectAttributes[] { - if (!Array.isArray(exceptionsList)) { - // early return if we are not an array such as being undefined or null or malformed. - return []; - } else { - return exceptionsList.flatMap((exceptionItem) => { - if (!(exceptionItem instanceof Object) || !isString(exceptionItem.id)) { - // return early if we are not an object such as being undefined or null or malformed - // or the exceptionItem.id is not a string from being malformed - return []; - } else { - return [exceptionItem]; - } - }); - } -} - -/** - * This migrates rule_id's within the legacy siem.notification to saved object references on an upgrade. - * We only migrate if we find these conditions: - * - ruleAlertId is a string and not null, undefined, or malformed data. - * - The existing references do not already have a ruleAlertId found within it. - * Some of these issues could crop up during either user manual errors of modifying things, earlier migration - * issues, etc... so we are safer to check them as possibilities - * @deprecated Once we are confident all rules relying on side-car actions SO's have been migrated to SO references we should remove this function - * @param doc The document that might have "ruleAlertId" to migrate into the references - * @returns The document migrated with saved object references - */ -function addRuleIdsToLegacyNotificationReferences( - doc: SavedObjectUnsanitizedDoc -): SavedObjectUnsanitizedDoc { - const { - attributes: { - params: { ruleAlertId }, - }, - references, - } = doc; - if (!isSecuritySolutionLegacyNotification(doc) || !isString(ruleAlertId)) { - // early return if we are not a string or if we are not a security solution notification saved object. - return doc; - } else { - const existingReferences = references ?? []; - const existingReferenceFound = existingReferences.find((reference) => { - return reference.id === ruleAlertId && reference.type === 'alert'; - }); - if (existingReferenceFound) { - // skip this if the references already exists for some uncommon reason so we do not add an additional one. - return doc; - } else { - const savedObjectReference: SavedObjectReference = { - id: ruleAlertId, - name: 'param:alert_0', - type: 'alert', - }; - const newReferences = [...existingReferences, savedObjectReference]; - return { ...doc, references: newReferences }; - } - } -} - -function setLegacyId(doc: SavedObjectUnsanitizedDoc): SavedObjectUnsanitizedDoc { - const { id } = doc; - return { - ...doc, - attributes: { - ...doc.attributes, - legacyId: id, - }, - }; -} - -function addSecuritySolutionAADRuleTypes( - doc: SavedObjectUnsanitizedDoc -): SavedObjectUnsanitizedDoc { - const ruleType = doc.attributes.params.type; - return isSiemSignalsRuleType(doc) && isRuleType(ruleType) - ? { - ...doc, - attributes: { - ...doc.attributes, - alertTypeId: ruleTypeMappings[ruleType], - enabled: false, - params: { - ...doc.attributes.params, - outputIndex: '', - }, - }, - } - : doc; -} - -function addSearchType(doc: SavedObjectUnsanitizedDoc) { - const searchType = doc.attributes.params.searchType; - - return isEsQueryRuleType(doc) && !searchType - ? { - ...doc, - attributes: { - ...doc.attributes, - params: { - ...doc.attributes.params, - searchType: 'esQuery', - }, - }, - } - : doc; -} - -function addSecuritySolutionAADRuleTypeTags( - doc: SavedObjectUnsanitizedDoc -): SavedObjectUnsanitizedDoc { - const ruleType = doc.attributes.params.type; - return isDetectionEngineAADRuleType(doc) && isRuleType(ruleType) - ? { - ...doc, - attributes: { - ...doc.attributes, - // If the rule is disabled at this point, then the rule has not been re-enabled after - // running the 8.0.0 migrations. If `doc.attributes.scheduledTaskId` exists, then the - // rule was enabled prior to running the migration. Thus we know we should add the - // tag to indicate it was auto-disabled. - tags: - !doc.attributes.enabled && doc.attributes.scheduledTaskId - ? [...(doc.attributes.tags ?? []), 'auto_disabled_8.0'] - : doc.attributes.tags ?? [], - }, - } - : doc; -} - -function stripOutRuntimeFieldsInOldESQuery( - doc: SavedObjectUnsanitizedDoc, - context: SavedObjectMigrationContext -): SavedObjectUnsanitizedDoc { - const isESDSLrule = - isEsQueryRuleType(doc) && !isSerializedSearchSource(doc.attributes.params.searchConfiguration); - - if (isESDSLrule) { - try { - const parsedQuery = JSON.parse(doc.attributes.params.esQuery as string); - // parsing and restringifying will cause us to lose the formatting so we only do so if this rule has - // fields other than `query` which is the only valid field at this stage - const hasFieldsOtherThanQuery = Object.keys(parsedQuery).some((key) => key !== 'query'); - return hasFieldsOtherThanQuery - ? { - ...doc, - attributes: { - ...doc.attributes, - params: { - ...doc.attributes.params, - esQuery: JSON.stringify(pick(parsedQuery, 'query'), null, 4), - }, - }, - } - : doc; - } catch (err) { - // Instead of failing the upgrade when an unparsable rule is encountered, we log that the rule caouldn't be migrated and - // as a result legacy parameters might cause the rule to behave differently if it is, in fact, still running at all - context.log.error( - `unable to migrate and remove legacy runtime fields in rule ${doc.id} due to invalid query: "${doc.attributes.params.esQuery}" - query must be JSON`, - { - migrations: { - alertDocument: doc, - }, - } - ); - } - } - return doc; -} - -function addThreatIndicatorPathToThreatMatchRules( - doc: SavedObjectUnsanitizedDoc -): SavedObjectUnsanitizedDoc { - return isSiemSignalsRuleType(doc) && - doc.attributes.params?.type === 'threat_match' && - !doc.attributes.params.threatIndicatorPath - ? { - ...doc, - attributes: { - ...doc.attributes, - params: { - ...doc.attributes.params, - threatIndicatorPath: FILEBEAT_7X_INDICATOR_PATH, - }, - }, - } - : doc; -} - -function getRemovePreconfiguredConnectorsFromReferencesFn( - isPreconfigured: (connectorId: string) => boolean -) { - return (doc: SavedObjectUnsanitizedDoc) => { - return removePreconfiguredConnectorsFromReferences(doc, isPreconfigured); - }; -} - -function removePreconfiguredConnectorsFromReferences( - doc: SavedObjectUnsanitizedDoc, - isPreconfigured: (connectorId: string) => boolean -): SavedObjectUnsanitizedDoc { - const { - attributes: { actions }, - references, - } = doc; - - // Look for connector references - const connectorReferences = (references ?? []).filter((ref: SavedObjectReference) => - ref.name.startsWith('action_') - ); - if (connectorReferences.length > 0) { - const restReferences = (references ?? []).filter( - (ref: SavedObjectReference) => !ref.name.startsWith('action_') - ); - - const updatedConnectorReferences: SavedObjectReference[] = []; - const updatedActions: RawRule['actions'] = []; - - // For each connector reference, check if connector is preconfigured - // If yes, we need to remove from the references array and update - // the corresponding action so it directly references the preconfigured connector id - connectorReferences.forEach((connectorRef: SavedObjectReference) => { - // Look for the corresponding entry in the actions array - const correspondingAction = getCorrespondingAction(actions, connectorRef.name); - if (correspondingAction) { - if (isPreconfigured(connectorRef.id)) { - updatedActions.push({ - ...correspondingAction, - actionRef: `preconfigured:${connectorRef.id}`, - }); - } else { - updatedActions.push(correspondingAction); - updatedConnectorReferences.push(connectorRef); - } - } else { - // Couldn't find the matching action, leave as is - updatedConnectorReferences.push(connectorRef); - } - }); - - return { - ...doc, - attributes: { - ...doc.attributes, - actions: [...updatedActions], - }, - references: [...updatedConnectorReferences, ...restReferences], - }; - } - return doc; -} - -// This fixes an issue whereby metrics.alert.inventory.threshold rules had the -// group for actions incorrectly spelt as metrics.invenotry_threshold.fired vs metrics.inventory_threshold.fired -function fixInventoryThresholdGroupId( - doc: SavedObjectUnsanitizedDoc -): SavedObjectUnsanitizedDoc { - if (doc.attributes.alertTypeId === 'metrics.alert.inventory.threshold') { - const { - attributes: { actions }, - } = doc; - - const updatedActions = actions - ? actions.map((action) => { - // Wrong spelling - if (action.group === 'metrics.invenotry_threshold.fired') { - return { - ...action, - group: 'metrics.inventory_threshold.fired', - }; - } else { - return action; - } - }) - : []; - - return { - ...doc, - attributes: { - ...doc.attributes, - actions: updatedActions, - }, - }; - } else { - return doc; - } -} - -function addMappedParams( - doc: SavedObjectUnsanitizedDoc -): SavedObjectUnsanitizedDoc { - const { - attributes: { params }, - } = doc; - - const mappedParams = getMappedParams(params); - - if (Object.keys(mappedParams).length) { - return { - ...doc, - attributes: { - ...doc.attributes, - mapped_params: mappedParams, - }, - }; - } - - return doc; -} - -function convertSnoozes( - doc: SavedObjectUnsanitizedDoc -): SavedObjectUnsanitizedDoc { - const { - attributes: { snoozeEndTime }, - } = doc; - - return { - ...doc, - attributes: { - ...(omit(doc.attributes, ['snoozeEndTime']) as RawRule), - snoozeSchedule: snoozeEndTime - ? [ - { - duration: Date.parse(snoozeEndTime as string) - Date.now(), - rRule: { - dtstart: new Date().toISOString(), - tzid: moment.tz.guess(), - count: 1, - }, - }, - ] - : [], - }, - }; -} - -function getCorrespondingAction( - actions: SavedObjectAttribute, - connectorRef: string -): RawRuleAction | null { - if (!Array.isArray(actions)) { - return null; - } else { - return actions.find( - (action) => (action as RawRuleAction)?.actionRef === connectorRef - ) as RawRuleAction; - } -} -/** - * removes internal tags(starts with '__internal') from Security Solution rules - * @param doc rule to be migrated - * @returns migrated rule if it's Security Solution rule or unchanged if not - */ -function removeInternalTags( - doc: SavedObjectUnsanitizedDoc -): SavedObjectUnsanitizedDoc { - if (!isDetectionEngineAADRuleType(doc)) { - return doc; - } - - const { - attributes: { tags }, - } = doc; - - const filteredTags = (tags ?? []).filter((tag) => !tag.startsWith('__internal_')); - - return { - ...doc, - attributes: { - ...doc.attributes, - tags: filteredTags, - }, - }; -} - -function pipeMigrations(...migrations: AlertMigration[]): AlertMigration { - return (doc: SavedObjectUnsanitizedDoc, context: SavedObjectMigrationContext) => - migrations.reduce((migratedDoc, nextMigration) => nextMigration(migratedDoc, context), doc); -} - -function mapSearchSourceMigrationFunc( - migrateSerializedSearchSourceFields: MigrateFunction -): MigrateFunction { - return (doc) => { - const _doc = doc as { attributes: RawRule }; - - const serializedSearchSource = _doc.attributes.params.searchConfiguration; - - if (isSerializedSearchSource(serializedSearchSource)) { - return { - ..._doc, - attributes: { - ..._doc.attributes, - params: { - ..._doc.attributes.params, - searchConfiguration: migrateSerializedSearchSourceFields(serializedSearchSource), - }, - }, - }; - } - return _doc; - }; -} - -/** - * This creates a migration map that applies search source migrations to legacy es query rules. - * It doesn't modify existing migrations. The following migrations will occur at minimum version of 8.3+. - */ -function getSearchSourceMigrations( - encryptedSavedObjects: EncryptedSavedObjectsPluginSetup, - searchSourceMigrations: MigrateFunctionsObject -) { - const filteredMigrations: SavedObjectMigrationMap = {}; - for (const versionKey in searchSourceMigrations) { - if (gte(versionKey, MINIMUM_SS_MIGRATION_VERSION)) { - const migrateSearchSource = mapSearchSourceMigrationFunc( - searchSourceMigrations[versionKey] - ) as unknown as AlertMigration; - - filteredMigrations[versionKey] = executeMigrationWithErrorHandling( - createEsoMigration( - encryptedSavedObjects, - (doc: SavedObjectUnsanitizedDoc): doc is SavedObjectUnsanitizedDoc => - isEsQueryRuleType(doc), - pipeMigrations(migrateSearchSource) - ), - versionKey - ); - } - } - return filteredMigrations; -} - -function removeIsSnoozedUntil( - doc: SavedObjectUnsanitizedDoc -): SavedObjectUnsanitizedDoc { - return { - ...doc, - attributes: { - ...(omit(doc.attributes, ['isSnoozedUntil']) as RawRule), - }, - }; -} diff --git a/x-pack/plugins/alerting/server/saved_objects/migrations/7.10/index.ts b/x-pack/plugins/alerting/server/saved_objects/migrations/7.10/index.ts new file mode 100644 index 0000000000000..9e5ebb2e8382f --- /dev/null +++ b/x-pack/plugins/alerting/server/saved_objects/migrations/7.10/index.ts @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SavedObjectUnsanitizedDoc } from "@kbn/core-saved-objects-server"; +import { EncryptedSavedObjectsPluginSetup } from '@kbn/encrypted-saved-objects-plugin/server'; +import { RawRule, RawRuleExecutionStatus } from "../../../types"; +import { LEGACY_LAST_MODIFIED_VERSION, SIEM_APP_ID, SIEM_SERVER_APP_ID } from "../constants"; +import { createEsoMigration, pipeMigrations } from "../utils"; + +const consumersToChange: Map = new Map( + Object.entries({ + alerting: 'alerts', + metrics: 'infrastructure', + [SIEM_APP_ID]: SIEM_SERVER_APP_ID, + }) +); + +function markAsLegacyAndChangeConsumer( + doc: SavedObjectUnsanitizedDoc +): SavedObjectUnsanitizedDoc { + const { + attributes: { consumer }, + } = doc; + return { + ...doc, + attributes: { + ...doc.attributes, + consumer: consumersToChange.get(consumer) ?? consumer, + // mark any alert predating 7.10 as a legacy alert + meta: { + versionApiKeyLastmodified: LEGACY_LAST_MODIFIED_VERSION, + }, + }, + }; +} + +function setAlertIdAsDefaultDedupkeyOnPagerDutyActions( + doc: SavedObjectUnsanitizedDoc +): SavedObjectUnsanitizedDoc { + const { attributes } = doc; + return { + ...doc, + attributes: { + ...attributes, + ...(attributes.actions + ? { + actions: attributes.actions.map((action) => { + if (action.actionTypeId !== '.pagerduty' || action.params.eventAction === 'trigger') { + return action; + } + return { + ...action, + params: { + ...action.params, + dedupKey: action.params.dedupKey ?? '{{alertId}}', + }, + }; + }), + } + : {}), + }, + }; +} + +function initializeExecutionStatus( + doc: SavedObjectUnsanitizedDoc +): SavedObjectUnsanitizedDoc { + const { attributes } = doc; + return { + ...doc, + attributes: { + ...attributes, + executionStatus: { + status: 'pending', + lastExecutionDate: new Date().toISOString(), + error: null, + } as RawRuleExecutionStatus, + }, + }; +} + +export const getMigrations_7_10_0 = (encryptedSavedObjects: EncryptedSavedObjectsPluginSetup) => createEsoMigration( + encryptedSavedObjects, + // migrate all documents in 7.10 in order to add the "meta" RBAC field + (doc): doc is SavedObjectUnsanitizedDoc => true, + pipeMigrations( + markAsLegacyAndChangeConsumer, + setAlertIdAsDefaultDedupkeyOnPagerDutyActions, + initializeExecutionStatus + ) +); + diff --git a/x-pack/plugins/alerting/server/saved_objects/migrations/7.11/index.ts b/x-pack/plugins/alerting/server/saved_objects/migrations/7.11/index.ts new file mode 100644 index 0000000000000..d43941d275192 --- /dev/null +++ b/x-pack/plugins/alerting/server/saved_objects/migrations/7.11/index.ts @@ -0,0 +1,174 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SavedObjectAttributes } from "@kbn/core-saved-objects-common"; +import { SavedObjectUnsanitizedDoc } from "@kbn/core-saved-objects-server"; +import { EncryptedSavedObjectsPluginSetup } from '@kbn/encrypted-saved-objects-plugin/server'; +import { RawRule, RawRuleAction } from "../../../types"; +import { createEsoMigration, pipeMigrations } from "../utils"; + +const SUPPORT_INCIDENTS_ACTION_TYPES = ['.servicenow', '.jira', '.resilient']; +export const isAnyActionSupportIncidents = (doc: SavedObjectUnsanitizedDoc): boolean => + doc.attributes.actions.some((action) => + SUPPORT_INCIDENTS_ACTION_TYPES.includes(action.actionTypeId) + ); + +function isEmptyObject(obj: {}) { + for (const attr in obj) { + if (Object.prototype.hasOwnProperty.call(obj, attr)) { + return false; + } + } + return true; +} + +function restructureConnectorsThatSupportIncident( + doc: SavedObjectUnsanitizedDoc +): SavedObjectUnsanitizedDoc { + const { actions } = doc.attributes; + const newActions = actions.reduce((acc, action) => { + if ( + ['.servicenow', '.jira', '.resilient'].includes(action.actionTypeId) && + action.params.subAction === 'pushToService' + ) { + // Future developer, we needed to do that because when we created this migration + // we forget to think about user already using 7.11.0 and having an incident attribute build the right way + // IMPORTANT -> if you change this code please do the same inside of this file + // x-pack/plugins/alerting/server/saved_objects/migrations.ts + const subActionParamsIncident = + (action.params?.subActionParams as SavedObjectAttributes)?.incident ?? null; + if (subActionParamsIncident != null && !isEmptyObject(subActionParamsIncident)) { + return [...acc, action]; + } + if (action.actionTypeId === '.servicenow') { + const { + title, + comments, + comment, + description, + severity, + urgency, + impact, + short_description: shortDescription, + } = action.params.subActionParams as { + title: string; + description?: string; + severity?: string; + urgency?: string; + impact?: string; + comment?: string; + comments?: Array<{ commentId: string; comment: string }>; + short_description?: string; + }; + return [ + ...acc, + { + ...action, + params: { + subAction: 'pushToService', + subActionParams: { + incident: { + short_description: shortDescription ?? title, + description, + severity, + urgency, + impact, + }, + comments: [ + ...(comments ?? []), + ...(comment != null ? [{ commentId: '1', comment }] : []), + ], + }, + }, + }, + ] as RawRuleAction[]; + } else if (action.actionTypeId === '.jira') { + const { title, comments, description, issueType, priority, labels, parent, summary } = + action.params.subActionParams as { + title: string; + description: string; + issueType: string; + priority?: string; + labels?: string[]; + parent?: string; + comments?: unknown[]; + summary?: string; + }; + return [ + ...acc, + { + ...action, + params: { + subAction: 'pushToService', + subActionParams: { + incident: { + summary: summary ?? title, + description, + issueType, + priority, + labels, + parent, + }, + comments, + }, + }, + }, + ] as RawRuleAction[]; + } else if (action.actionTypeId === '.resilient') { + const { title, comments, description, incidentTypes, severityCode, name } = action.params + .subActionParams as { + title: string; + description: string; + incidentTypes?: number[]; + severityCode?: number; + comments?: unknown[]; + name?: string; + }; + return [ + ...acc, + { + ...action, + params: { + subAction: 'pushToService', + subActionParams: { + incident: { + name: name ?? title, + description, + incidentTypes, + severityCode, + }, + comments, + }, + }, + }, + ] as RawRuleAction[]; + } + } + + return [...acc, action]; + }, [] as RawRuleAction[]); + + return { + ...doc, + attributes: { + ...doc.attributes, + actions: newActions, + }, + }; +} + +export const getMigrations_7_11_0 = (encryptedSavedObjects: EncryptedSavedObjectsPluginSetup) => createEsoMigration( + encryptedSavedObjects, + (doc): doc is SavedObjectUnsanitizedDoc => isAnyActionSupportIncidents(doc), + pipeMigrations(restructureConnectorsThatSupportIncident) +); + +export const getMigrations_7_11_2 = (encryptedSavedObjects: EncryptedSavedObjectsPluginSetup) => createEsoMigration( + encryptedSavedObjects, + (doc): doc is SavedObjectUnsanitizedDoc => isAnyActionSupportIncidents(doc), + pipeMigrations(restructureConnectorsThatSupportIncident) +); diff --git a/x-pack/plugins/alerting/server/saved_objects/migrations/7.13/index.ts b/x-pack/plugins/alerting/server/saved_objects/migrations/7.13/index.ts new file mode 100644 index 0000000000000..8e0b826af5a70 --- /dev/null +++ b/x-pack/plugins/alerting/server/saved_objects/migrations/7.13/index.ts @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SavedObjectAttribute } from "@kbn/core-saved-objects-common"; +import { SavedObjectUnsanitizedDoc } from "@kbn/core-saved-objects-server"; +import { EncryptedSavedObjectsPluginSetup } from '@kbn/encrypted-saved-objects-plugin/server'; +import { RawRule } from "../../../types"; +import { createEsoMigration, isSiemSignalsRuleType, pipeMigrations } from "../utils"; + +function convertNullToUndefined(attribute: SavedObjectAttribute) { + return attribute != null ? attribute : undefined; +} + +function removeNullsFromSecurityRules( + doc: SavedObjectUnsanitizedDoc +): SavedObjectUnsanitizedDoc { + const { + attributes: { params }, + } = doc; + return { + ...doc, + attributes: { + ...doc.attributes, + params: { + ...params, + buildingBlockType: convertNullToUndefined(params.buildingBlockType), + note: convertNullToUndefined(params.note), + index: convertNullToUndefined(params.index), + language: convertNullToUndefined(params.language), + license: convertNullToUndefined(params.license), + outputIndex: convertNullToUndefined(params.outputIndex), + savedId: convertNullToUndefined(params.savedId), + timelineId: convertNullToUndefined(params.timelineId), + timelineTitle: convertNullToUndefined(params.timelineTitle), + meta: convertNullToUndefined(params.meta), + query: convertNullToUndefined(params.query), + filters: convertNullToUndefined(params.filters), + riskScoreMapping: params.riskScoreMapping != null ? params.riskScoreMapping : [], + ruleNameOverride: convertNullToUndefined(params.ruleNameOverride), + severityMapping: params.severityMapping != null ? params.severityMapping : [], + threat: params.threat != null ? params.threat : [], + threshold: + params.threshold != null && + typeof params.threshold === 'object' && + !Array.isArray(params.threshold) + ? { + field: Array.isArray(params.threshold.field) + ? params.threshold.field + : params.threshold.field === '' || params.threshold.field == null + ? [] + : [params.threshold.field], + value: params.threshold.value, + cardinality: + params.threshold.cardinality != null ? params.threshold.cardinality : [], + } + : undefined, + timestampOverride: convertNullToUndefined(params.timestampOverride), + exceptionsList: + params.exceptionsList != null + ? params.exceptionsList + : params.exceptions_list != null + ? params.exceptions_list + : params.lists != null + ? params.lists + : [], + threatFilters: convertNullToUndefined(params.threatFilters), + machineLearningJobId: + params.machineLearningJobId == null + ? undefined + : Array.isArray(params.machineLearningJobId) + ? params.machineLearningJobId + : [params.machineLearningJobId], + }, + }, + }; +} + +export const getMigrations_7_13_0 = (encryptedSavedObjects: EncryptedSavedObjectsPluginSetup) => createEsoMigration( + encryptedSavedObjects, + (doc): doc is SavedObjectUnsanitizedDoc => isSiemSignalsRuleType(doc), + pipeMigrations(removeNullsFromSecurityRules) +); diff --git a/x-pack/plugins/alerting/server/saved_objects/migrations/7.14/index.ts b/x-pack/plugins/alerting/server/saved_objects/migrations/7.14/index.ts new file mode 100644 index 0000000000000..c6068e7ebe8c1 --- /dev/null +++ b/x-pack/plugins/alerting/server/saved_objects/migrations/7.14/index.ts @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SavedObjectUnsanitizedDoc } from "@kbn/core-saved-objects-server"; +import { EncryptedSavedObjectsPluginSetup } from '@kbn/encrypted-saved-objects-plugin/server'; +import { RawRule } from "../../../types"; +import { createEsoMigration, isSiemSignalsRuleType, pipeMigrations } from "../utils"; + + +/** + * The author field was introduced later and was not part of the original rules. We overlooked + * the filling in the author field as an empty array in an earlier upgrade routine from + * 'removeNullsFromSecurityRules' during the 7.13.0 upgrade. Since we don't change earlier migrations, + * but rather only move forward with the "arrow of time" we are going to upgrade and fix + * it if it is missing for anyone in 7.14.0 and above release. Earlier releases if we want to fix them, + * would have to be modified as a "7.13.1", etc... if we want to fix it there. + * @param doc The document that is not migrated and contains a "null" or "undefined" author field + * @returns The document with the author field fleshed in. + */ +function removeNullAuthorFromSecurityRules( + doc: SavedObjectUnsanitizedDoc +): SavedObjectUnsanitizedDoc { + const { + attributes: { params }, + } = doc; + return { + ...doc, + attributes: { + ...doc.attributes, + params: { + ...params, + author: params.author != null ? params.author : [], + }, + }, + }; +} + +export const getMigrations_7_14_0 = (encryptedSavedObjects: EncryptedSavedObjectsPluginSetup) => createEsoMigration( + encryptedSavedObjects, + (doc): doc is SavedObjectUnsanitizedDoc => isSiemSignalsRuleType(doc), + pipeMigrations(removeNullAuthorFromSecurityRules) +); diff --git a/x-pack/plugins/alerting/server/saved_objects/migrations/7.15/index.ts b/x-pack/plugins/alerting/server/saved_objects/migrations/7.15/index.ts new file mode 100644 index 0000000000000..a206af8f04343 --- /dev/null +++ b/x-pack/plugins/alerting/server/saved_objects/migrations/7.15/index.ts @@ -0,0 +1,110 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SavedObjectAttribute, SavedObjectAttributes, SavedObjectReference } from "@kbn/core-saved-objects-common"; +import { SavedObjectUnsanitizedDoc } from "@kbn/core-saved-objects-server"; +import { EncryptedSavedObjectsPluginSetup } from '@kbn/encrypted-saved-objects-plugin/server'; +import { isString } from 'lodash/fp'; +import { RawRule } from "../../../types"; +import { createEsoMigration, isSiemSignalsRuleType, pipeMigrations } from "../utils"; + +/** + * This will do a flatMap reduce where we only return exceptionsLists and their items if: + * - exceptionLists are an array and not null, undefined, or malformed data. + * - The exceptionList item is an object and id is a string and not null, undefined, or malformed data + * + * Some of these issues could crop up during either user manual errors of modifying things, earlier migration + * issues, etc... + * @param exceptionsList The list of exceptions + * @returns The exception lists if they are a valid enough shape + */ +function removeMalformedExceptionsList( + exceptionsList: SavedObjectAttribute +): SavedObjectAttributes[] { + if (!Array.isArray(exceptionsList)) { + // early return if we are not an array such as being undefined or null or malformed. + return []; + } else { + return exceptionsList.flatMap((exceptionItem) => { + if (!(exceptionItem instanceof Object) || !isString(exceptionItem.id)) { + // return early if we are not an object such as being undefined or null or malformed + // or the exceptionItem.id is not a string from being malformed + return []; + } else { + return [exceptionItem]; + } + }); + } +} + +/** + * This migrates exception list containers to saved object references on an upgrade. + * We only migrate if we find these conditions: + * - exceptionLists are an array and not null, undefined, or malformed data. + * - The exceptionList item is an object and id is a string and not null, undefined, or malformed data + * - The existing references do not already have an exceptionItem reference already found within it. + * Some of these issues could crop up during either user manual errors of modifying things, earlier migration + * issues, etc... + * @param doc The document that might have exceptionListItems to migrate + * @returns The document migrated with saved object references + */ +function addExceptionListsToReferences( + doc: SavedObjectUnsanitizedDoc +): SavedObjectUnsanitizedDoc { + const { + attributes: { + params: { exceptionsList }, + }, + references, + } = doc; + if (!Array.isArray(exceptionsList)) { + // early return if we are not an array such as being undefined or null or malformed. + return doc; + } else { + const exceptionsToTransform = removeMalformedExceptionsList(exceptionsList); + const newReferences = exceptionsToTransform.flatMap( + (exceptionItem, index) => { + const existingReferenceFound = references?.find((reference) => { + return ( + reference.id === exceptionItem.id && + ((reference.type === 'exception-list' && exceptionItem.namespace_type === 'single') || + (reference.type === 'exception-list-agnostic' && + exceptionItem.namespace_type === 'agnostic')) + ); + }); + if (existingReferenceFound) { + // skip if the reference already exists for some uncommon reason so we do not add an additional one. + // This enables us to be idempotent and you can run this migration multiple times and get the same output. + return []; + } else { + return [ + { + name: `param:exceptionsList_${index}`, + id: String(exceptionItem.id), + type: + exceptionItem.namespace_type === 'agnostic' + ? 'exception-list-agnostic' + : 'exception-list', + }, + ]; + } + } + ); + if (references == null && newReferences.length === 0) { + // Avoid adding an empty references array if the existing saved object never had one to begin with + return doc; + } else { + return { ...doc, references: [...(references ?? []), ...newReferences] }; + } + } +} + +export const getMigrations_7_15_0 = (encryptedSavedObjects: EncryptedSavedObjectsPluginSetup) => createEsoMigration( + encryptedSavedObjects, + (doc): doc is SavedObjectUnsanitizedDoc => isSiemSignalsRuleType(doc), + pipeMigrations(addExceptionListsToReferences) +); diff --git a/x-pack/plugins/alerting/server/saved_objects/migrations/7.16/index.ts b/x-pack/plugins/alerting/server/saved_objects/migrations/7.16/index.ts new file mode 100644 index 0000000000000..d45eaba6a599d --- /dev/null +++ b/x-pack/plugins/alerting/server/saved_objects/migrations/7.16/index.ts @@ -0,0 +1,155 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SavedObjectAttribute, SavedObjectReference } from "@kbn/core-saved-objects-common"; +import { SavedObjectUnsanitizedDoc } from "@kbn/core-saved-objects-server"; +import { EncryptedSavedObjectsPluginSetup } from '@kbn/encrypted-saved-objects-plugin/server'; +import { isString } from 'lodash/fp'; +import { RawRule, RawRuleAction } from "../../../types"; +import { extractRefsFromGeoContainmentAlert } from "../../geo_containment/migrations"; +import { createEsoMigration, isSecuritySolutionLegacyNotification, pipeMigrations } from "../utils"; + +function setLegacyId(doc: SavedObjectUnsanitizedDoc): SavedObjectUnsanitizedDoc { + const { id } = doc; + return { + ...doc, + attributes: { + ...doc.attributes, + legacyId: id, + }, + }; +} + +function getRemovePreconfiguredConnectorsFromReferencesFn( + isPreconfigured: (connectorId: string) => boolean +) { + return (doc: SavedObjectUnsanitizedDoc) => { + return removePreconfiguredConnectorsFromReferences(doc, isPreconfigured); + }; +} + +function getCorrespondingAction( + actions: SavedObjectAttribute, + connectorRef: string +): RawRuleAction | null { + if (!Array.isArray(actions)) { + return null; + } else { + return actions.find( + (action) => (action as RawRuleAction)?.actionRef === connectorRef + ) as RawRuleAction; + } +} + +function removePreconfiguredConnectorsFromReferences( + doc: SavedObjectUnsanitizedDoc, + isPreconfigured: (connectorId: string) => boolean +): SavedObjectUnsanitizedDoc { + const { + attributes: { actions }, + references, + } = doc; + + // Look for connector references + const connectorReferences = (references ?? []).filter((ref: SavedObjectReference) => + ref.name.startsWith('action_') + ); + if (connectorReferences.length > 0) { + const restReferences = (references ?? []).filter( + (ref: SavedObjectReference) => !ref.name.startsWith('action_') + ); + + const updatedConnectorReferences: SavedObjectReference[] = []; + const updatedActions: RawRule['actions'] = []; + + // For each connector reference, check if connector is preconfigured + // If yes, we need to remove from the references array and update + // the corresponding action so it directly references the preconfigured connector id + connectorReferences.forEach((connectorRef: SavedObjectReference) => { + // Look for the corresponding entry in the actions array + const correspondingAction = getCorrespondingAction(actions, connectorRef.name); + if (correspondingAction) { + if (isPreconfigured(connectorRef.id)) { + updatedActions.push({ + ...correspondingAction, + actionRef: `preconfigured:${connectorRef.id}`, + }); + } else { + updatedActions.push(correspondingAction); + updatedConnectorReferences.push(connectorRef); + } + } else { + // Couldn't find the matching action, leave as is + updatedConnectorReferences.push(connectorRef); + } + }); + + return { + ...doc, + attributes: { + ...doc.attributes, + actions: [...updatedActions], + }, + references: [...updatedConnectorReferences, ...restReferences], + }; + } + return doc; +} + +/** + * This migrates rule_id's within the legacy siem.notification to saved object references on an upgrade. + * We only migrate if we find these conditions: + * - ruleAlertId is a string and not null, undefined, or malformed data. + * - The existing references do not already have a ruleAlertId found within it. + * Some of these issues could crop up during either user manual errors of modifying things, earlier migration + * issues, etc... so we are safer to check them as possibilities + * @deprecated Once we are confident all rules relying on side-car actions SO's have been migrated to SO references we should remove this function + * @param doc The document that might have "ruleAlertId" to migrate into the references + * @returns The document migrated with saved object references + */ +function addRuleIdsToLegacyNotificationReferences( + doc: SavedObjectUnsanitizedDoc +): SavedObjectUnsanitizedDoc { + const { + attributes: { + params: { ruleAlertId }, + }, + references, + } = doc; + if (!isSecuritySolutionLegacyNotification(doc) || !isString(ruleAlertId)) { + // early return if we are not a string or if we are not a security solution notification saved object. + return doc; + } else { + const existingReferences = references ?? []; + const existingReferenceFound = existingReferences.find((reference) => { + return reference.id === ruleAlertId && reference.type === 'alert'; + }); + if (existingReferenceFound) { + // skip this if the references already exists for some uncommon reason so we do not add an additional one. + return doc; + } else { + const savedObjectReference: SavedObjectReference = { + id: ruleAlertId, + name: 'param:alert_0', + type: 'alert', + }; + const newReferences = [...existingReferences, savedObjectReference]; + return { ...doc, references: newReferences }; + } + } +} + +export const getMigrations_7_16_0 = (encryptedSavedObjects: EncryptedSavedObjectsPluginSetup, isPreconfigured: (connectorId: string) => boolean) => createEsoMigration( + encryptedSavedObjects, + (doc): doc is SavedObjectUnsanitizedDoc => true, + pipeMigrations( + setLegacyId, + getRemovePreconfiguredConnectorsFromReferencesFn(isPreconfigured), + addRuleIdsToLegacyNotificationReferences, + extractRefsFromGeoContainmentAlert + ) +); diff --git a/x-pack/plugins/alerting/server/saved_objects/migrations/8.0/index.ts b/x-pack/plugins/alerting/server/saved_objects/migrations/8.0/index.ts new file mode 100644 index 0000000000000..bf9313bdec4e6 --- /dev/null +++ b/x-pack/plugins/alerting/server/saved_objects/migrations/8.0/index.ts @@ -0,0 +1,124 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SavedObjectUnsanitizedDoc } from "@kbn/core-saved-objects-server"; +import { EncryptedSavedObjectsPluginSetup } from '@kbn/encrypted-saved-objects-plugin/server'; +import { isRuleType, ruleTypeMappings } from "@kbn/securitysolution-rules"; +import { RawRule } from "../../../types"; +import { FILEBEAT_7X_INDICATOR_PATH } from "../constants"; +import { createEsoMigration, isDetectionEngineAADRuleType, isSiemSignalsRuleType, pipeMigrations } from "../utils"; + + +function addThreatIndicatorPathToThreatMatchRules( + doc: SavedObjectUnsanitizedDoc +): SavedObjectUnsanitizedDoc < RawRule > { + return isSiemSignalsRuleType(doc) && + doc.attributes.params?.type === 'threat_match' && + !doc.attributes.params.threatIndicatorPath + ? { + ...doc, + attributes: { + ...doc.attributes, + params: { + ...doc.attributes.params, + threatIndicatorPath: FILEBEAT_7X_INDICATOR_PATH, + }, + }, + } + : doc; +} + +function addSecuritySolutionAADRuleTypes( + doc: SavedObjectUnsanitizedDoc +): SavedObjectUnsanitizedDoc { + const ruleType = doc.attributes.params.type; + return isSiemSignalsRuleType(doc) && isRuleType(ruleType) + ? { + ...doc, + attributes: { + ...doc.attributes, + alertTypeId: ruleTypeMappings[ruleType], + enabled: false, + params: { + ...doc.attributes.params, + outputIndex: '', + }, + }, + } + : doc; +} + +function addSecuritySolutionAADRuleTypeTags( + doc: SavedObjectUnsanitizedDoc +): SavedObjectUnsanitizedDoc { + const ruleType = doc.attributes.params.type; + return isDetectionEngineAADRuleType(doc) && isRuleType(ruleType) + ? { + ...doc, + attributes: { + ...doc.attributes, + // If the rule is disabled at this point, then the rule has not been re-enabled after + // running the 8.0.0 migrations. If `doc.attributes.scheduledTaskId` exists, then the + // rule was enabled prior to running the migration. Thus we know we should add the + // tag to indicate it was auto-disabled. + tags: + !doc.attributes.enabled && doc.attributes.scheduledTaskId + ? [...(doc.attributes.tags ?? []), 'auto_disabled_8.0'] + : doc.attributes.tags ?? [], + }, + } + : doc; +} + + +// This fixes an issue whereby metrics.alert.inventory.threshold rules had the +// group for actions incorrectly spelt as metrics.invenotry_threshold.fired vs metrics.inventory_threshold.fired +function fixInventoryThresholdGroupId( + doc: SavedObjectUnsanitizedDoc +): SavedObjectUnsanitizedDoc { + if (doc.attributes.alertTypeId === 'metrics.alert.inventory.threshold') { + const { + attributes: { actions }, + } = doc; + + const updatedActions = actions + ? actions.map((action) => { + // Wrong spelling + if (action.group === 'metrics.invenotry_threshold.fired') { + return { + ...action, + group: 'metrics.inventory_threshold.fired', + }; + } else { + return action; + } + }) + : []; + + return { + ...doc, + attributes: { + ...doc.attributes, + actions: updatedActions, + }, + }; + } else { + return doc; + } +} + +export const getMigrations_8_0_0 = (encryptedSavedObjects: EncryptedSavedObjectsPluginSetup) => createEsoMigration( + encryptedSavedObjects, + (doc: SavedObjectUnsanitizedDoc): doc is SavedObjectUnsanitizedDoc => true, + pipeMigrations(addThreatIndicatorPathToThreatMatchRules, addSecuritySolutionAADRuleTypes, fixInventoryThresholdGroupId) +); + +export const getMigrations_8_0_1 = (encryptedSavedObjects: EncryptedSavedObjectsPluginSetup) => createEsoMigration( + encryptedSavedObjects, + (doc: SavedObjectUnsanitizedDoc): doc is SavedObjectUnsanitizedDoc => true, + pipeMigrations(addSecuritySolutionAADRuleTypeTags) +); diff --git a/x-pack/plugins/alerting/server/saved_objects/migrations/8.2/index.ts b/x-pack/plugins/alerting/server/saved_objects/migrations/8.2/index.ts new file mode 100644 index 0000000000000..b232c5f1bf790 --- /dev/null +++ b/x-pack/plugins/alerting/server/saved_objects/migrations/8.2/index.ts @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + + +import { SavedObjectUnsanitizedDoc } from "@kbn/core-saved-objects-server"; +import { EncryptedSavedObjectsPluginSetup } from '@kbn/encrypted-saved-objects-plugin/server'; +import { getMappedParams } from "../../../rules_client/lib/mapped_params_utils"; +import { RawRule } from "../../../types"; +import { createEsoMigration, pipeMigrations } from "../utils"; + +function addMappedParams( + doc: SavedObjectUnsanitizedDoc +): SavedObjectUnsanitizedDoc { + const { + attributes: { params }, + } = doc; + + const mappedParams = getMappedParams(params); + + if (Object.keys(mappedParams).length) { + return { + ...doc, + attributes: { + ...doc.attributes, + mapped_params: mappedParams, + }, + }; + } + + return doc; +} + +export const getMigrations_8_2_0 = (encryptedSavedObjects: EncryptedSavedObjectsPluginSetup) => createEsoMigration( + encryptedSavedObjects, + (doc: SavedObjectUnsanitizedDoc): doc is SavedObjectUnsanitizedDoc => true, + pipeMigrations(addMappedParams) +); + diff --git a/x-pack/plugins/alerting/server/saved_objects/migrations/8.3/index.ts b/x-pack/plugins/alerting/server/saved_objects/migrations/8.3/index.ts new file mode 100644 index 0000000000000..a55038a4ff9e8 --- /dev/null +++ b/x-pack/plugins/alerting/server/saved_objects/migrations/8.3/index.ts @@ -0,0 +1,91 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SavedObjectUnsanitizedDoc } from "@kbn/core-saved-objects-server"; +import { EncryptedSavedObjectsPluginSetup } from '@kbn/encrypted-saved-objects-plugin/server'; +import { omit } from "lodash"; +import moment from 'moment-timezone'; +import { RawRule } from "../../../types"; +import { createEsoMigration, isDetectionEngineAADRuleType, isEsQueryRuleType, pipeMigrations } from "../utils"; + + +function addSearchType(doc: SavedObjectUnsanitizedDoc) { + const searchType = doc.attributes.params.searchType; + + return isEsQueryRuleType(doc) && !searchType + ? { + ...doc, + attributes: { + ...doc.attributes, + params: { + ...doc.attributes.params, + searchType: 'esQuery', + }, + }, + } + : doc; +} + +/** + * removes internal tags(starts with '__internal') from Security Solution rules + * @param doc rule to be migrated + * @returns migrated rule if it's Security Solution rule or unchanged if not + */ +function removeInternalTags( + doc: SavedObjectUnsanitizedDoc +): SavedObjectUnsanitizedDoc { + if (!isDetectionEngineAADRuleType(doc)) { + return doc; + } + + const { + attributes: { tags }, + } = doc; + + const filteredTags = (tags ?? []).filter((tag) => !tag.startsWith('__internal_')); + + return { + ...doc, + attributes: { + ...doc.attributes, + tags: filteredTags, + }, + }; +} + +function convertSnoozes( + doc: SavedObjectUnsanitizedDoc +): SavedObjectUnsanitizedDoc { + const { + attributes: { snoozeEndTime }, + } = doc; + + return { + ...doc, + attributes: { + ...(omit(doc.attributes, ['snoozeEndTime']) as RawRule), + snoozeSchedule: snoozeEndTime + ? [ + { + duration: Date.parse(snoozeEndTime as string) - Date.now(), + rRule: { + dtstart: new Date().toISOString(), + tzid: moment.tz.guess(), + count: 1, + }, + }, + ] + : [], + }, + }; +} + +export const getMigrations_8_3_0 = (encryptedSavedObjects: EncryptedSavedObjectsPluginSetup) => createEsoMigration( + encryptedSavedObjects, + (doc: SavedObjectUnsanitizedDoc): doc is SavedObjectUnsanitizedDoc => true, + pipeMigrations(addSearchType, removeInternalTags, convertSnoozes) +); diff --git a/x-pack/plugins/alerting/server/saved_objects/migrations/8.4/index.ts b/x-pack/plugins/alerting/server/saved_objects/migrations/8.4/index.ts new file mode 100644 index 0000000000000..a8b908650c807 --- /dev/null +++ b/x-pack/plugins/alerting/server/saved_objects/migrations/8.4/index.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SavedObjectUnsanitizedDoc } from "@kbn/core-saved-objects-server"; +import { EncryptedSavedObjectsPluginSetup } from '@kbn/encrypted-saved-objects-plugin/server'; +import { omit } from "lodash"; +import { RawRule } from "../../../types"; +import { createEsoMigration, pipeMigrations } from "../utils"; + + + +function removeIsSnoozedUntil( + doc: SavedObjectUnsanitizedDoc +): SavedObjectUnsanitizedDoc { + return { + ...doc, + attributes: { + ...(omit(doc.attributes, ['isSnoozedUntil']) as RawRule), + }, + }; +} + +export const getMigrations_8_4_1 = (encryptedSavedObjects: EncryptedSavedObjectsPluginSetup) => createEsoMigration( + encryptedSavedObjects, + (doc: SavedObjectUnsanitizedDoc): doc is SavedObjectUnsanitizedDoc => true, + pipeMigrations(removeIsSnoozedUntil) +); diff --git a/x-pack/plugins/alerting/server/saved_objects/migrations/8.5/index.ts b/x-pack/plugins/alerting/server/saved_objects/migrations/8.5/index.ts new file mode 100644 index 0000000000000..1faae1b4ea08a --- /dev/null +++ b/x-pack/plugins/alerting/server/saved_objects/migrations/8.5/index.ts @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SavedObjectMigrationContext, SavedObjectUnsanitizedDoc } from "@kbn/core-saved-objects-server"; +import { EncryptedSavedObjectsPluginSetup } from '@kbn/encrypted-saved-objects-plugin/server'; +import { isSerializedSearchSource } from '@kbn/data-plugin/common'; +import { pick } from "lodash"; +import { RawRule } from "../../../types"; +import { createEsoMigration, isEsQueryRuleType, pipeMigrations } from "../utils"; +import { AlertLogMeta } from "../types"; + + + +function stripOutRuntimeFieldsInOldESQuery( + doc: SavedObjectUnsanitizedDoc, + context: SavedObjectMigrationContext +): SavedObjectUnsanitizedDoc { + const isESDSLrule = + isEsQueryRuleType(doc) && !isSerializedSearchSource(doc.attributes.params.searchConfiguration); + + if (isESDSLrule) { + try { + const parsedQuery = JSON.parse(doc.attributes.params.esQuery as string); + // parsing and restringifying will cause us to lose the formatting so we only do so if this rule has + // fields other than `query` which is the only valid field at this stage + const hasFieldsOtherThanQuery = Object.keys(parsedQuery).some((key) => key !== 'query'); + return hasFieldsOtherThanQuery + ? { + ...doc, + attributes: { + ...doc.attributes, + params: { + ...doc.attributes.params, + esQuery: JSON.stringify(pick(parsedQuery, 'query'), null, 4), + }, + }, + } + : doc; + } catch (err) { + // Instead of failing the upgrade when an unparsable rule is encountered, we log that the rule caouldn't be migrated and + // as a result legacy parameters might cause the rule to behave differently if it is, in fact, still running at all + context.log.error( + `unable to migrate and remove legacy runtime fields in rule ${doc.id} due to invalid query: "${doc.attributes.params.esQuery}" - query must be JSON`, + { + migrations: { + alertDocument: doc, + }, + } + ); + } + } + return doc; +} + + +export const getMigrations_8_5_0 = (encryptedSavedObjects: EncryptedSavedObjectsPluginSetup) => createEsoMigration( + encryptedSavedObjects, + (doc): doc is SavedObjectUnsanitizedDoc => isEsQueryRuleType(doc), + pipeMigrations(stripOutRuntimeFieldsInOldESQuery) +); diff --git a/x-pack/plugins/alerting/server/saved_objects/migrations/constants.ts b/x-pack/plugins/alerting/server/saved_objects/migrations/constants.ts new file mode 100644 index 0000000000000..849021f3c58c3 --- /dev/null +++ b/x-pack/plugins/alerting/server/saved_objects/migrations/constants.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const SIEM_APP_ID = 'securitySolution'; +export const SIEM_SERVER_APP_ID = 'siem'; +export const MINIMUM_SS_MIGRATION_VERSION = '8.3.0'; +export const LEGACY_LAST_MODIFIED_VERSION = 'pre-7.10.0'; +export const FILEBEAT_7X_INDICATOR_PATH = 'threatintel.indicator'; diff --git a/x-pack/plugins/alerting/server/saved_objects/migrations.test.ts b/x-pack/plugins/alerting/server/saved_objects/migrations/index.test.ts similarity index 99% rename from x-pack/plugins/alerting/server/saved_objects/migrations.test.ts rename to x-pack/plugins/alerting/server/saved_objects/migrations/index.test.ts index be9bee7634990..0f2bbdf94d28d 100644 --- a/x-pack/plugins/alerting/server/saved_objects/migrations.test.ts +++ b/x-pack/plugins/alerting/server/saved_objects/migrations/index.test.ts @@ -24,6 +24,7 @@ describe('successful migrations', () => { jest.resetAllMocks(); encryptedSavedObjectsSetup.createMigration.mockImplementation(({ migration }) => migration); }); + describe('7.10.0', () => { test('marks alerts as legacy', () => { const migration710 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)['7.10.0']; @@ -2736,6 +2737,7 @@ describe('handles errors during migrations', () => { }); }); + function getUpdatedAt(): string { const updatedAt = new Date(); updatedAt.setHours(updatedAt.getHours() + 2); @@ -2782,3 +2784,4 @@ function getMockData( type: 'alert', }; } + diff --git a/x-pack/plugins/alerting/server/saved_objects/migrations/index.ts b/x-pack/plugins/alerting/server/saved_objects/migrations/index.ts new file mode 100644 index 0000000000000..da44774b34423 --- /dev/null +++ b/x-pack/plugins/alerting/server/saved_objects/migrations/index.ts @@ -0,0 +1,134 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { gte } from 'semver'; +import { + SavedObjectMigrationMap, + SavedObjectUnsanitizedDoc, + SavedObjectMigrationFn, + SavedObjectMigrationContext, +} from '@kbn/core/server'; +import { EncryptedSavedObjectsPluginSetup } from '@kbn/encrypted-saved-objects-plugin/server'; +import { MigrateFunctionsObject, MigrateFunction } from '@kbn/kibana-utils-plugin/common'; +import { mergeSavedObjectMigrationMaps } from '@kbn/core/server'; +import { isSerializedSearchSource, SerializedSearchSourceFields } from '@kbn/data-plugin/common'; +import { RawRule } from '../../types'; +import { getMigrations_7_10_0 } from './7.10'; +import { getMigrations_7_11_0, getMigrations_7_11_2 } from './7.11'; +import { getMigrations_7_13_0 } from './7.13'; +import { getMigrations_7_14_0 } from './7.14'; +import { getMigrations_7_15_0 } from './7.15'; +import { getMigrations_7_16_0 } from './7.16'; +import { getMigrations_8_0_0, getMigrations_8_0_1 } from './8.0'; +import { getMigrations_8_2_0 } from './8.2'; +import { getMigrations_8_3_0 } from './8.3'; +import { getMigrations_8_4_1 } from './8.4'; +import { getMigrations_8_5_0 } from './8.5'; +import { AlertLogMeta, AlertMigration } from './types'; +import { MINIMUM_SS_MIGRATION_VERSION } from './constants'; +import { createEsoMigration, isEsQueryRuleType, pipeMigrations } from './utils'; + + +export function getMigrations( + encryptedSavedObjects: EncryptedSavedObjectsPluginSetup, + searchSourceMigrations: MigrateFunctionsObject, + isPreconfigured: (connectorId: string) => boolean +): SavedObjectMigrationMap { + return mergeSavedObjectMigrationMaps( + { + '7.10.0': executeMigrationWithErrorHandling(getMigrations_7_10_0(encryptedSavedObjects), '7.10.0'), + '7.11.0': executeMigrationWithErrorHandling(getMigrations_7_11_0(encryptedSavedObjects), '7.11.0'), + '7.11.2': executeMigrationWithErrorHandling(getMigrations_7_11_2(encryptedSavedObjects), '7.11.2'), + '7.13.0': executeMigrationWithErrorHandling(getMigrations_7_13_0(encryptedSavedObjects), '7.13.0'), + '7.14.1': executeMigrationWithErrorHandling(getMigrations_7_14_0(encryptedSavedObjects), '7.14.1'), + '7.15.0': executeMigrationWithErrorHandling(getMigrations_7_15_0(encryptedSavedObjects), '7.15.0'), + '7.16.0': executeMigrationWithErrorHandling(getMigrations_7_16_0(encryptedSavedObjects, isPreconfigured), '7.16.0'), + '8.0.0': executeMigrationWithErrorHandling(getMigrations_8_0_0(encryptedSavedObjects), '8.0.0'), + '8.0.1': executeMigrationWithErrorHandling(getMigrations_8_0_1(encryptedSavedObjects), '8.0.1'), + '8.2.0': executeMigrationWithErrorHandling(getMigrations_8_2_0(encryptedSavedObjects), '8.2.0'), + '8.3.0': executeMigrationWithErrorHandling(getMigrations_8_3_0(encryptedSavedObjects), '8.3.0'), + '8.4.1': executeMigrationWithErrorHandling(getMigrations_8_4_1(encryptedSavedObjects), '8.4.1'), + '8.5.0': executeMigrationWithErrorHandling(getMigrations_8_5_0(encryptedSavedObjects), '8.5.0'), + }, + getSearchSourceMigrations(encryptedSavedObjects, searchSourceMigrations) + ); +} + +function executeMigrationWithErrorHandling( + migrationFunc: SavedObjectMigrationFn, + version: string +) { + return (doc: SavedObjectUnsanitizedDoc, context: SavedObjectMigrationContext) => { + try { + return migrationFunc(doc, context); + } catch (ex) { + context.log.error( + `encryptedSavedObject ${version} migration failed for alert ${doc.id} with error: ${ex.message}`, + { + migrations: { + alertDocument: doc, + }, + } + ); + throw ex; + } + }; +} + +function mapSearchSourceMigrationFunc( + migrateSerializedSearchSourceFields: MigrateFunction +): MigrateFunction { + return (doc) => { + const _doc = doc as { attributes: RawRule }; + + const serializedSearchSource = _doc.attributes.params.searchConfiguration; + + if (isSerializedSearchSource(serializedSearchSource)) { + return { + ..._doc, + attributes: { + ..._doc.attributes, + params: { + ..._doc.attributes.params, + searchConfiguration: migrateSerializedSearchSourceFields(serializedSearchSource), + }, + }, + }; + } + return _doc; + }; +} + +/** + * This creates a migration map that applies search source migrations to legacy es query rules. + * It doesn't modify existing migrations. The following migrations will occur at minimum version of 8.3+. + */ +function getSearchSourceMigrations( + encryptedSavedObjects: EncryptedSavedObjectsPluginSetup, + searchSourceMigrations: MigrateFunctionsObject +) { + const filteredMigrations: SavedObjectMigrationMap = {}; + for (const versionKey in searchSourceMigrations) { + if (gte(versionKey, MINIMUM_SS_MIGRATION_VERSION)) { + const migrateSearchSource = mapSearchSourceMigrationFunc( + searchSourceMigrations[versionKey] + ) as unknown as AlertMigration; + + filteredMigrations[versionKey] = executeMigrationWithErrorHandling( + createEsoMigration( + encryptedSavedObjects, + (doc: SavedObjectUnsanitizedDoc): doc is SavedObjectUnsanitizedDoc => + isEsQueryRuleType(doc), + pipeMigrations(migrateSearchSource) + ), + versionKey + ); + } + } + return filteredMigrations; +} + diff --git a/x-pack/plugins/alerting/server/saved_objects/migrations/types.ts b/x-pack/plugins/alerting/server/saved_objects/migrations/types.ts new file mode 100644 index 0000000000000..6d657e168187e --- /dev/null +++ b/x-pack/plugins/alerting/server/saved_objects/migrations/types.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { LogMeta, SavedObjectMigrationContext, SavedObjectUnsanitizedDoc } from '@kbn/core/server'; +import { RawRule } from '../../types'; + +export interface AlertLogMeta extends LogMeta { + migrations: { alertDocument: SavedObjectUnsanitizedDoc }; +} + +export type AlertMigration = ( + doc: SavedObjectUnsanitizedDoc, + context: SavedObjectMigrationContext +) => SavedObjectUnsanitizedDoc; diff --git a/x-pack/plugins/alerting/server/saved_objects/migrations/utils.ts b/x-pack/plugins/alerting/server/saved_objects/migrations/utils.ts new file mode 100644 index 0000000000000..039aa7364335d --- /dev/null +++ b/x-pack/plugins/alerting/server/saved_objects/migrations/utils.ts @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ruleTypeMappings } from '@kbn/securitysolution-rules'; +import { SavedObjectMigrationContext, SavedObjectUnsanitizedDoc } from "@kbn/core-saved-objects-server"; +import { EncryptedSavedObjectsPluginSetup } from '@kbn/encrypted-saved-objects-plugin/server'; +import type { IsMigrationNeededPredicate } from '@kbn/encrypted-saved-objects-plugin/server'; + +import { RawRule } from "../../types"; + +type AlertMigration = ( + doc: SavedObjectUnsanitizedDoc, + context: SavedObjectMigrationContext +) => SavedObjectUnsanitizedDoc; + +export function createEsoMigration( + encryptedSavedObjects: EncryptedSavedObjectsPluginSetup, + isMigrationNeededPredicate: IsMigrationNeededPredicate, + migrationFunc: AlertMigration +) { + return encryptedSavedObjects.createMigration({ + isMigrationNeededPredicate, + migration: migrationFunc, + shouldMigrateIfDecryptionFails: true, // shouldMigrateIfDecryptionFails flag that applies the migration to undecrypted document if decryption fails + }); +} + +export function pipeMigrations(...migrations: AlertMigration[]): AlertMigration { + return (doc: SavedObjectUnsanitizedDoc, context: SavedObjectMigrationContext) => + migrations.reduce((migratedDoc, nextMigration) => nextMigration(migratedDoc, context), doc); +} + + +// Deprecated in 8.0 +export const isSiemSignalsRuleType = (doc: SavedObjectUnsanitizedDoc): boolean => + doc.attributes.alertTypeId === 'siem.signals'; + +export const isEsQueryRuleType = (doc: SavedObjectUnsanitizedDoc) => + doc.attributes.alertTypeId === '.es-query'; + +export const isDetectionEngineAADRuleType = (doc: SavedObjectUnsanitizedDoc): boolean => + (Object.values(ruleTypeMappings) as string[]).includes(doc.attributes.alertTypeId); + +/** + * Returns true if the alert type is that of "siem.notifications" which is a legacy notification system that was deprecated in 7.16.0 + * in favor of using the newer alerting notifications system. + * @param doc The saved object alert type document + * @returns true if this is a legacy "siem.notifications" rule, otherwise false + * @deprecated Once we are confident all rules relying on side-car actions SO's have been migrated to SO references we should remove this function + */ +export const isSecuritySolutionLegacyNotification = ( + doc: SavedObjectUnsanitizedDoc +): boolean => doc.attributes.alertTypeId === 'siem.notifications'; From d1ca8f0a5e74768b1bbb1b2e127b5ba64a98c332 Mon Sep 17 00:00:00 2001 From: Xavier Mouligneau Date: Tue, 13 Sep 2022 14:18:52 -0400 Subject: [PATCH 02/27] change mappings to match new design --- .../alerting/server/saved_objects/mappings.ts | 146 +++++++++++++----- 1 file changed, 109 insertions(+), 37 deletions(-) diff --git a/x-pack/plugins/alerting/server/saved_objects/mappings.ts b/x-pack/plugins/alerting/server/saved_objects/mappings.ts index f33dbdd788534..d341952af15d5 100644 --- a/x-pack/plugins/alerting/server/saved_objects/mappings.ts +++ b/x-pack/plugins/alerting/server/saved_objects/mappings.ts @@ -114,7 +114,8 @@ export const alertMappings: SavedObjectsTypeMappingDefinition = { }, monitoring: { properties: { - execution: { + //TODO migrations + run: { properties: { history: { properties: { @@ -127,6 +128,9 @@ export const alertMappings: SavedObjectsTypeMappingDefinition = { timestamp: { type: 'date', }, + outcome: { + type: 'keyword', + }, }, }, calculated_metrics: { @@ -145,46 +149,76 @@ export const alertMappings: SavedObjectsTypeMappingDefinition = { }, }, }, + last_run: { + properties: { + timestamp: { + type: 'date', + }, + metrics: { + properties: { + duration: { + type: 'long', + }, + total_search_duration_ms: { + type: 'long', + }, + total_indexing_duration_ms: { + type: 'long', + }, + total_alerts_detected: { + type: 'float', + }, + total_alerts_created: { + type: 'float', + }, + gap_duration_s: { + type: 'float', + }, + }, + }, + }, + } }, }, }, }, - executionStatus: { - properties: { - numberOfTriggeredActions: { - type: 'long', - }, - status: { - type: 'keyword', - }, - lastExecutionDate: { - type: 'date', - }, - lastDuration: { - type: 'long', - }, - error: { - properties: { - reason: { - type: 'keyword', - }, - message: { - type: 'keyword', - }, - }, - }, - warning: { - properties: { - reason: { - type: 'keyword', - }, - message: { - type: 'keyword', - }, - }, - }, - }, - }, + // TODO Need to be deprecated by last_run + // executionStatus: { + // properties: { + // numberOfTriggeredActions: { + // type: 'long', + // }, + // status: { + // type: 'keyword', + // }, + // lastExecutionDate: { + // type: 'date', + // }, + // lastDuration: { + // type: 'long', + // }, + // error: { + // properties: { + // reason: { + // type: 'keyword', + // }, + // message: { + // type: 'keyword', + // }, + // }, + // }, + // warning: { + // properties: { + // reason: { + // type: 'keyword', + // }, + // message: { + // type: 'keyword', + // }, + // }, + // }, + // }, + // }, snoozeSchedule: { type: 'nested', properties: { @@ -255,5 +289,43 @@ export const alertMappings: SavedObjectsTypeMappingDefinition = { }, }, }, + running: { + type: 'boolean', + }, + next_run: { + type: 'date', + }, + last_run: { + properties: { + outcome: { + type: 'keyword', + }, + outcome_order: { + type: 'float', + }, + warning: { + type: 'text', + }, + outcome_msg: { + type: 'text', + }, + alerts_count: { + properties: { + active: { + type: 'float', + }, + new: { + type: 'float', + }, + recovered: { + type: 'float', + }, + ignored: { + type: 'float', + }, + }, + }, + }, + }, }, }; From 0b74e72706a8e4f4ebac0bdfe97241641f5e0536 Mon Sep 17 00:00:00 2001 From: Xavier Mouligneau Date: Wed, 14 Sep 2022 17:57:23 -0400 Subject: [PATCH 03/27] wip --- .../alerting/server/saved_objects/mappings.ts | 82 +++++++++---------- .../saved_objects/migrations/8.6/index.ts | 13 +++ 2 files changed, 54 insertions(+), 41 deletions(-) create mode 100644 x-pack/plugins/alerting/server/saved_objects/migrations/8.6/index.ts diff --git a/x-pack/plugins/alerting/server/saved_objects/mappings.ts b/x-pack/plugins/alerting/server/saved_objects/mappings.ts index d341952af15d5..0c8fd5ab65d88 100644 --- a/x-pack/plugins/alerting/server/saved_objects/mappings.ts +++ b/x-pack/plugins/alerting/server/saved_objects/mappings.ts @@ -114,7 +114,7 @@ export const alertMappings: SavedObjectsTypeMappingDefinition = { }, monitoring: { properties: { - //TODO migrations + // TODO migrations run: { properties: { history: { @@ -177,48 +177,11 @@ export const alertMappings: SavedObjectsTypeMappingDefinition = { }, }, }, - } + }, }, }, }, }, - // TODO Need to be deprecated by last_run - // executionStatus: { - // properties: { - // numberOfTriggeredActions: { - // type: 'long', - // }, - // status: { - // type: 'keyword', - // }, - // lastExecutionDate: { - // type: 'date', - // }, - // lastDuration: { - // type: 'long', - // }, - // error: { - // properties: { - // reason: { - // type: 'keyword', - // }, - // message: { - // type: 'keyword', - // }, - // }, - // }, - // warning: { - // properties: { - // reason: { - // type: 'keyword', - // }, - // message: { - // type: 'keyword', - // }, - // }, - // }, - // }, - // }, snoozeSchedule: { type: 'nested', properties: { @@ -295,6 +258,43 @@ export const alertMappings: SavedObjectsTypeMappingDefinition = { next_run: { type: 'date', }, + // Deprecated, if you need to add new property please do it in `last_run` + executionStatus: { + properties: { + numberOfTriggeredActions: { + type: 'long', + }, + status: { + type: 'keyword', + }, + lastExecutionDate: { + type: 'date', + }, + lastDuration: { + type: 'long', + }, + error: { + properties: { + reason: { + type: 'keyword', + }, + message: { + type: 'keyword', + }, + }, + }, + warning: { + properties: { + reason: { + type: 'keyword', + }, + message: { + type: 'keyword', + }, + }, + }, + }, + }, last_run: { properties: { outcome: { @@ -320,8 +320,8 @@ export const alertMappings: SavedObjectsTypeMappingDefinition = { recovered: { type: 'float', }, - ignored: { - type: 'float', + ignored: { + type: 'float', }, }, }, diff --git a/x-pack/plugins/alerting/server/saved_objects/migrations/8.6/index.ts b/x-pack/plugins/alerting/server/saved_objects/migrations/8.6/index.ts new file mode 100644 index 0000000000000..140b19267c4b5 --- /dev/null +++ b/x-pack/plugins/alerting/server/saved_objects/migrations/8.6/index.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SavedObjectMigrationContext, SavedObjectUnsanitizedDoc } from "@kbn/core-saved-objects-server"; +import { EncryptedSavedObjectsPluginSetup } from '@kbn/encrypted-saved-objects-plugin/server'; +import { isSerializedSearchSource } from '@kbn/data-plugin/common'; +import { pick } from "lodash"; +import { RawRule } from "../../../types"; +import { createEsoMigration, isEsQueryRuleType, pipeMigrations } from "../utils"; From bbfc35a1bf9152f7415e596e49f7b90bb1651820 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Fri, 16 Sep 2022 17:24:14 +0000 Subject: [PATCH 04/27] [CI] Auto-commit changed files from 'node scripts/precommit_hook.js --ref HEAD~1..HEAD --fix' --- .../alerting/server/saved_objects/migrations/8.6/index.ts | 7 ------- 1 file changed, 7 deletions(-) diff --git a/x-pack/plugins/alerting/server/saved_objects/migrations/8.6/index.ts b/x-pack/plugins/alerting/server/saved_objects/migrations/8.6/index.ts index 140b19267c4b5..1fec1c76430eb 100644 --- a/x-pack/plugins/alerting/server/saved_objects/migrations/8.6/index.ts +++ b/x-pack/plugins/alerting/server/saved_objects/migrations/8.6/index.ts @@ -4,10 +4,3 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - -import { SavedObjectMigrationContext, SavedObjectUnsanitizedDoc } from "@kbn/core-saved-objects-server"; -import { EncryptedSavedObjectsPluginSetup } from '@kbn/encrypted-saved-objects-plugin/server'; -import { isSerializedSearchSource } from '@kbn/data-plugin/common'; -import { pick } from "lodash"; -import { RawRule } from "../../../types"; -import { createEsoMigration, isEsQueryRuleType, pipeMigrations } from "../utils"; From 5579627d8e3e6bd51cd1782d1abe6f3c0d3924ac Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Fri, 16 Sep 2022 17:50:55 +0000 Subject: [PATCH 05/27] [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' --- .../saved_objects/migrations/7.10/index.ts | 56 ++++---- .../saved_objects/migrations/7.11/index.ts | 44 +++--- .../saved_objects/migrations/7.13/index.ts | 49 +++---- .../saved_objects/migrations/7.14/index.ts | 18 +-- .../saved_objects/migrations/7.15/index.ts | 23 ++-- .../saved_objects/migrations/7.16/index.ts | 34 ++--- .../saved_objects/migrations/8.0/index.ts | 127 ++++++++++-------- .../saved_objects/migrations/8.2/index.ts | 21 ++- .../saved_objects/migrations/8.3/index.ts | 57 ++++---- .../saved_objects/migrations/8.4/index.ts | 21 ++- .../saved_objects/migrations/8.5/index.ts | 43 +++--- .../saved_objects/migrations/index.test.ts | 2 - .../server/saved_objects/migrations/index.ts | 67 ++++++--- .../server/saved_objects/migrations/utils.ts | 8 +- 14 files changed, 316 insertions(+), 254 deletions(-) diff --git a/x-pack/plugins/alerting/server/saved_objects/migrations/7.10/index.ts b/x-pack/plugins/alerting/server/saved_objects/migrations/7.10/index.ts index 9e5ebb2e8382f..766d019b2de23 100644 --- a/x-pack/plugins/alerting/server/saved_objects/migrations/7.10/index.ts +++ b/x-pack/plugins/alerting/server/saved_objects/migrations/7.10/index.ts @@ -5,11 +5,11 @@ * 2.0. */ -import { SavedObjectUnsanitizedDoc } from "@kbn/core-saved-objects-server"; +import { SavedObjectUnsanitizedDoc } from '@kbn/core-saved-objects-server'; import { EncryptedSavedObjectsPluginSetup } from '@kbn/encrypted-saved-objects-plugin/server'; -import { RawRule, RawRuleExecutionStatus } from "../../../types"; -import { LEGACY_LAST_MODIFIED_VERSION, SIEM_APP_ID, SIEM_SERVER_APP_ID } from "../constants"; -import { createEsoMigration, pipeMigrations } from "../utils"; +import { RawRule, RawRuleExecutionStatus } from '../../../types'; +import { LEGACY_LAST_MODIFIED_VERSION, SIEM_APP_ID, SIEM_SERVER_APP_ID } from '../constants'; +import { createEsoMigration, pipeMigrations } from '../utils'; const consumersToChange: Map = new Map( Object.entries({ @@ -48,19 +48,19 @@ function setAlertIdAsDefaultDedupkeyOnPagerDutyActions( ...attributes, ...(attributes.actions ? { - actions: attributes.actions.map((action) => { - if (action.actionTypeId !== '.pagerduty' || action.params.eventAction === 'trigger') { - return action; - } - return { - ...action, - params: { - ...action.params, - dedupKey: action.params.dedupKey ?? '{{alertId}}', - }, - }; - }), - } + actions: attributes.actions.map((action) => { + if (action.actionTypeId !== '.pagerduty' || action.params.eventAction === 'trigger') { + return action; + } + return { + ...action, + params: { + ...action.params, + dedupKey: action.params.dedupKey ?? '{{alertId}}', + }, + }; + }), + } : {}), }, }; @@ -83,14 +83,14 @@ function initializeExecutionStatus( }; } -export const getMigrations_7_10_0 = (encryptedSavedObjects: EncryptedSavedObjectsPluginSetup) => createEsoMigration( - encryptedSavedObjects, - // migrate all documents in 7.10 in order to add the "meta" RBAC field - (doc): doc is SavedObjectUnsanitizedDoc => true, - pipeMigrations( - markAsLegacyAndChangeConsumer, - setAlertIdAsDefaultDedupkeyOnPagerDutyActions, - initializeExecutionStatus - ) -); - +export const getMigrations_7_10_0 = (encryptedSavedObjects: EncryptedSavedObjectsPluginSetup) => + createEsoMigration( + encryptedSavedObjects, + // migrate all documents in 7.10 in order to add the "meta" RBAC field + (doc): doc is SavedObjectUnsanitizedDoc => true, + pipeMigrations( + markAsLegacyAndChangeConsumer, + setAlertIdAsDefaultDedupkeyOnPagerDutyActions, + initializeExecutionStatus + ) + ); diff --git a/x-pack/plugins/alerting/server/saved_objects/migrations/7.11/index.ts b/x-pack/plugins/alerting/server/saved_objects/migrations/7.11/index.ts index d43941d275192..52a95b1fa3ec3 100644 --- a/x-pack/plugins/alerting/server/saved_objects/migrations/7.11/index.ts +++ b/x-pack/plugins/alerting/server/saved_objects/migrations/7.11/index.ts @@ -5,11 +5,11 @@ * 2.0. */ -import { SavedObjectAttributes } from "@kbn/core-saved-objects-common"; -import { SavedObjectUnsanitizedDoc } from "@kbn/core-saved-objects-server"; +import { SavedObjectAttributes } from '@kbn/core-saved-objects-common'; +import { SavedObjectUnsanitizedDoc } from '@kbn/core-saved-objects-server'; import { EncryptedSavedObjectsPluginSetup } from '@kbn/encrypted-saved-objects-plugin/server'; -import { RawRule, RawRuleAction } from "../../../types"; -import { createEsoMigration, pipeMigrations } from "../utils"; +import { RawRule, RawRuleAction } from '../../../types'; +import { createEsoMigration, pipeMigrations } from '../utils'; const SUPPORT_INCIDENTS_ACTION_TYPES = ['.servicenow', '.jira', '.resilient']; export const isAnyActionSupportIncidents = (doc: SavedObjectUnsanitizedDoc): boolean => @@ -121,13 +121,13 @@ function restructureConnectorsThatSupportIncident( } else if (action.actionTypeId === '.resilient') { const { title, comments, description, incidentTypes, severityCode, name } = action.params .subActionParams as { - title: string; - description: string; - incidentTypes?: number[]; - severityCode?: number; - comments?: unknown[]; - name?: string; - }; + title: string; + description: string; + incidentTypes?: number[]; + severityCode?: number; + comments?: unknown[]; + name?: string; + }; return [ ...acc, { @@ -161,14 +161,16 @@ function restructureConnectorsThatSupportIncident( }; } -export const getMigrations_7_11_0 = (encryptedSavedObjects: EncryptedSavedObjectsPluginSetup) => createEsoMigration( - encryptedSavedObjects, - (doc): doc is SavedObjectUnsanitizedDoc => isAnyActionSupportIncidents(doc), - pipeMigrations(restructureConnectorsThatSupportIncident) -); +export const getMigrations_7_11_0 = (encryptedSavedObjects: EncryptedSavedObjectsPluginSetup) => + createEsoMigration( + encryptedSavedObjects, + (doc): doc is SavedObjectUnsanitizedDoc => isAnyActionSupportIncidents(doc), + pipeMigrations(restructureConnectorsThatSupportIncident) + ); -export const getMigrations_7_11_2 = (encryptedSavedObjects: EncryptedSavedObjectsPluginSetup) => createEsoMigration( - encryptedSavedObjects, - (doc): doc is SavedObjectUnsanitizedDoc => isAnyActionSupportIncidents(doc), - pipeMigrations(restructureConnectorsThatSupportIncident) -); +export const getMigrations_7_11_2 = (encryptedSavedObjects: EncryptedSavedObjectsPluginSetup) => + createEsoMigration( + encryptedSavedObjects, + (doc): doc is SavedObjectUnsanitizedDoc => isAnyActionSupportIncidents(doc), + pipeMigrations(restructureConnectorsThatSupportIncident) + ); diff --git a/x-pack/plugins/alerting/server/saved_objects/migrations/7.13/index.ts b/x-pack/plugins/alerting/server/saved_objects/migrations/7.13/index.ts index 8e0b826af5a70..391abdd4432dd 100644 --- a/x-pack/plugins/alerting/server/saved_objects/migrations/7.13/index.ts +++ b/x-pack/plugins/alerting/server/saved_objects/migrations/7.13/index.ts @@ -5,11 +5,11 @@ * 2.0. */ -import { SavedObjectAttribute } from "@kbn/core-saved-objects-common"; -import { SavedObjectUnsanitizedDoc } from "@kbn/core-saved-objects-server"; +import { SavedObjectAttribute } from '@kbn/core-saved-objects-common'; +import { SavedObjectUnsanitizedDoc } from '@kbn/core-saved-objects-server'; import { EncryptedSavedObjectsPluginSetup } from '@kbn/encrypted-saved-objects-plugin/server'; -import { RawRule } from "../../../types"; -import { createEsoMigration, isSiemSignalsRuleType, pipeMigrations } from "../utils"; +import { RawRule } from '../../../types'; +import { createEsoMigration, isSiemSignalsRuleType, pipeMigrations } from '../utils'; function convertNullToUndefined(attribute: SavedObjectAttribute) { return attribute != null ? attribute : undefined; @@ -45,42 +45,43 @@ function removeNullsFromSecurityRules( threat: params.threat != null ? params.threat : [], threshold: params.threshold != null && - typeof params.threshold === 'object' && - !Array.isArray(params.threshold) + typeof params.threshold === 'object' && + !Array.isArray(params.threshold) ? { - field: Array.isArray(params.threshold.field) - ? params.threshold.field - : params.threshold.field === '' || params.threshold.field == null + field: Array.isArray(params.threshold.field) + ? params.threshold.field + : params.threshold.field === '' || params.threshold.field == null ? [] : [params.threshold.field], - value: params.threshold.value, - cardinality: - params.threshold.cardinality != null ? params.threshold.cardinality : [], - } + value: params.threshold.value, + cardinality: + params.threshold.cardinality != null ? params.threshold.cardinality : [], + } : undefined, timestampOverride: convertNullToUndefined(params.timestampOverride), exceptionsList: params.exceptionsList != null ? params.exceptionsList : params.exceptions_list != null - ? params.exceptions_list - : params.lists != null - ? params.lists - : [], + ? params.exceptions_list + : params.lists != null + ? params.lists + : [], threatFilters: convertNullToUndefined(params.threatFilters), machineLearningJobId: params.machineLearningJobId == null ? undefined : Array.isArray(params.machineLearningJobId) - ? params.machineLearningJobId - : [params.machineLearningJobId], + ? params.machineLearningJobId + : [params.machineLearningJobId], }, }, }; } -export const getMigrations_7_13_0 = (encryptedSavedObjects: EncryptedSavedObjectsPluginSetup) => createEsoMigration( - encryptedSavedObjects, - (doc): doc is SavedObjectUnsanitizedDoc => isSiemSignalsRuleType(doc), - pipeMigrations(removeNullsFromSecurityRules) -); +export const getMigrations_7_13_0 = (encryptedSavedObjects: EncryptedSavedObjectsPluginSetup) => + createEsoMigration( + encryptedSavedObjects, + (doc): doc is SavedObjectUnsanitizedDoc => isSiemSignalsRuleType(doc), + pipeMigrations(removeNullsFromSecurityRules) + ); diff --git a/x-pack/plugins/alerting/server/saved_objects/migrations/7.14/index.ts b/x-pack/plugins/alerting/server/saved_objects/migrations/7.14/index.ts index c6068e7ebe8c1..10703255ec6a5 100644 --- a/x-pack/plugins/alerting/server/saved_objects/migrations/7.14/index.ts +++ b/x-pack/plugins/alerting/server/saved_objects/migrations/7.14/index.ts @@ -5,11 +5,10 @@ * 2.0. */ -import { SavedObjectUnsanitizedDoc } from "@kbn/core-saved-objects-server"; +import { SavedObjectUnsanitizedDoc } from '@kbn/core-saved-objects-server'; import { EncryptedSavedObjectsPluginSetup } from '@kbn/encrypted-saved-objects-plugin/server'; -import { RawRule } from "../../../types"; -import { createEsoMigration, isSiemSignalsRuleType, pipeMigrations } from "../utils"; - +import { RawRule } from '../../../types'; +import { createEsoMigration, isSiemSignalsRuleType, pipeMigrations } from '../utils'; /** * The author field was introduced later and was not part of the original rules. We overlooked @@ -39,8 +38,9 @@ function removeNullAuthorFromSecurityRules( }; } -export const getMigrations_7_14_0 = (encryptedSavedObjects: EncryptedSavedObjectsPluginSetup) => createEsoMigration( - encryptedSavedObjects, - (doc): doc is SavedObjectUnsanitizedDoc => isSiemSignalsRuleType(doc), - pipeMigrations(removeNullAuthorFromSecurityRules) -); +export const getMigrations_7_14_0 = (encryptedSavedObjects: EncryptedSavedObjectsPluginSetup) => + createEsoMigration( + encryptedSavedObjects, + (doc): doc is SavedObjectUnsanitizedDoc => isSiemSignalsRuleType(doc), + pipeMigrations(removeNullAuthorFromSecurityRules) + ); diff --git a/x-pack/plugins/alerting/server/saved_objects/migrations/7.15/index.ts b/x-pack/plugins/alerting/server/saved_objects/migrations/7.15/index.ts index a206af8f04343..33e9a0f01b289 100644 --- a/x-pack/plugins/alerting/server/saved_objects/migrations/7.15/index.ts +++ b/x-pack/plugins/alerting/server/saved_objects/migrations/7.15/index.ts @@ -5,12 +5,16 @@ * 2.0. */ -import { SavedObjectAttribute, SavedObjectAttributes, SavedObjectReference } from "@kbn/core-saved-objects-common"; -import { SavedObjectUnsanitizedDoc } from "@kbn/core-saved-objects-server"; +import { + SavedObjectAttribute, + SavedObjectAttributes, + SavedObjectReference, +} from '@kbn/core-saved-objects-common'; +import { SavedObjectUnsanitizedDoc } from '@kbn/core-saved-objects-server'; import { EncryptedSavedObjectsPluginSetup } from '@kbn/encrypted-saved-objects-plugin/server'; import { isString } from 'lodash/fp'; -import { RawRule } from "../../../types"; -import { createEsoMigration, isSiemSignalsRuleType, pipeMigrations } from "../utils"; +import { RawRule } from '../../../types'; +import { createEsoMigration, isSiemSignalsRuleType, pipeMigrations } from '../utils'; /** * This will do a flatMap reduce where we only return exceptionsLists and their items if: @@ -103,8 +107,9 @@ function addExceptionListsToReferences( } } -export const getMigrations_7_15_0 = (encryptedSavedObjects: EncryptedSavedObjectsPluginSetup) => createEsoMigration( - encryptedSavedObjects, - (doc): doc is SavedObjectUnsanitizedDoc => isSiemSignalsRuleType(doc), - pipeMigrations(addExceptionListsToReferences) -); +export const getMigrations_7_15_0 = (encryptedSavedObjects: EncryptedSavedObjectsPluginSetup) => + createEsoMigration( + encryptedSavedObjects, + (doc): doc is SavedObjectUnsanitizedDoc => isSiemSignalsRuleType(doc), + pipeMigrations(addExceptionListsToReferences) + ); diff --git a/x-pack/plugins/alerting/server/saved_objects/migrations/7.16/index.ts b/x-pack/plugins/alerting/server/saved_objects/migrations/7.16/index.ts index d45eaba6a599d..71dcb16ca3c96 100644 --- a/x-pack/plugins/alerting/server/saved_objects/migrations/7.16/index.ts +++ b/x-pack/plugins/alerting/server/saved_objects/migrations/7.16/index.ts @@ -5,13 +5,13 @@ * 2.0. */ -import { SavedObjectAttribute, SavedObjectReference } from "@kbn/core-saved-objects-common"; -import { SavedObjectUnsanitizedDoc } from "@kbn/core-saved-objects-server"; +import { SavedObjectAttribute, SavedObjectReference } from '@kbn/core-saved-objects-common'; +import { SavedObjectUnsanitizedDoc } from '@kbn/core-saved-objects-server'; import { EncryptedSavedObjectsPluginSetup } from '@kbn/encrypted-saved-objects-plugin/server'; import { isString } from 'lodash/fp'; -import { RawRule, RawRuleAction } from "../../../types"; -import { extractRefsFromGeoContainmentAlert } from "../../geo_containment/migrations"; -import { createEsoMigration, isSecuritySolutionLegacyNotification, pipeMigrations } from "../utils"; +import { RawRule, RawRuleAction } from '../../../types'; +import { extractRefsFromGeoContainmentAlert } from '../../geo_containment/migrations'; +import { createEsoMigration, isSecuritySolutionLegacyNotification, pipeMigrations } from '../utils'; function setLegacyId(doc: SavedObjectUnsanitizedDoc): SavedObjectUnsanitizedDoc { const { id } = doc; @@ -143,13 +143,17 @@ function addRuleIdsToLegacyNotificationReferences( } } -export const getMigrations_7_16_0 = (encryptedSavedObjects: EncryptedSavedObjectsPluginSetup, isPreconfigured: (connectorId: string) => boolean) => createEsoMigration( - encryptedSavedObjects, - (doc): doc is SavedObjectUnsanitizedDoc => true, - pipeMigrations( - setLegacyId, - getRemovePreconfiguredConnectorsFromReferencesFn(isPreconfigured), - addRuleIdsToLegacyNotificationReferences, - extractRefsFromGeoContainmentAlert - ) -); +export const getMigrations_7_16_0 = ( + encryptedSavedObjects: EncryptedSavedObjectsPluginSetup, + isPreconfigured: (connectorId: string) => boolean +) => + createEsoMigration( + encryptedSavedObjects, + (doc): doc is SavedObjectUnsanitizedDoc => true, + pipeMigrations( + setLegacyId, + getRemovePreconfiguredConnectorsFromReferencesFn(isPreconfigured), + addRuleIdsToLegacyNotificationReferences, + extractRefsFromGeoContainmentAlert + ) + ); diff --git a/x-pack/plugins/alerting/server/saved_objects/migrations/8.0/index.ts b/x-pack/plugins/alerting/server/saved_objects/migrations/8.0/index.ts index bf9313bdec4e6..dcf9426ebab55 100644 --- a/x-pack/plugins/alerting/server/saved_objects/migrations/8.0/index.ts +++ b/x-pack/plugins/alerting/server/saved_objects/migrations/8.0/index.ts @@ -5,30 +5,34 @@ * 2.0. */ -import { SavedObjectUnsanitizedDoc } from "@kbn/core-saved-objects-server"; +import { SavedObjectUnsanitizedDoc } from '@kbn/core-saved-objects-server'; import { EncryptedSavedObjectsPluginSetup } from '@kbn/encrypted-saved-objects-plugin/server'; -import { isRuleType, ruleTypeMappings } from "@kbn/securitysolution-rules"; -import { RawRule } from "../../../types"; -import { FILEBEAT_7X_INDICATOR_PATH } from "../constants"; -import { createEsoMigration, isDetectionEngineAADRuleType, isSiemSignalsRuleType, pipeMigrations } from "../utils"; - +import { isRuleType, ruleTypeMappings } from '@kbn/securitysolution-rules'; +import { RawRule } from '../../../types'; +import { FILEBEAT_7X_INDICATOR_PATH } from '../constants'; +import { + createEsoMigration, + isDetectionEngineAADRuleType, + isSiemSignalsRuleType, + pipeMigrations, +} from '../utils'; function addThreatIndicatorPathToThreatMatchRules( doc: SavedObjectUnsanitizedDoc -): SavedObjectUnsanitizedDoc < RawRule > { +): SavedObjectUnsanitizedDoc { return isSiemSignalsRuleType(doc) && doc.attributes.params?.type === 'threat_match' && !doc.attributes.params.threatIndicatorPath ? { - ...doc, - attributes: { - ...doc.attributes, - params: { - ...doc.attributes.params, - threatIndicatorPath: FILEBEAT_7X_INDICATOR_PATH, + ...doc, + attributes: { + ...doc.attributes, + params: { + ...doc.attributes.params, + threatIndicatorPath: FILEBEAT_7X_INDICATOR_PATH, + }, }, - }, - } + } : doc; } @@ -38,17 +42,17 @@ function addSecuritySolutionAADRuleTypes( const ruleType = doc.attributes.params.type; return isSiemSignalsRuleType(doc) && isRuleType(ruleType) ? { - ...doc, - attributes: { - ...doc.attributes, - alertTypeId: ruleTypeMappings[ruleType], - enabled: false, - params: { - ...doc.attributes.params, - outputIndex: '', + ...doc, + attributes: { + ...doc.attributes, + alertTypeId: ruleTypeMappings[ruleType], + enabled: false, + params: { + ...doc.attributes.params, + outputIndex: '', + }, }, - }, - } + } : doc; } @@ -58,23 +62,22 @@ function addSecuritySolutionAADRuleTypeTags( const ruleType = doc.attributes.params.type; return isDetectionEngineAADRuleType(doc) && isRuleType(ruleType) ? { - ...doc, - attributes: { - ...doc.attributes, - // If the rule is disabled at this point, then the rule has not been re-enabled after - // running the 8.0.0 migrations. If `doc.attributes.scheduledTaskId` exists, then the - // rule was enabled prior to running the migration. Thus we know we should add the - // tag to indicate it was auto-disabled. - tags: - !doc.attributes.enabled && doc.attributes.scheduledTaskId - ? [...(doc.attributes.tags ?? []), 'auto_disabled_8.0'] - : doc.attributes.tags ?? [], - }, - } + ...doc, + attributes: { + ...doc.attributes, + // If the rule is disabled at this point, then the rule has not been re-enabled after + // running the 8.0.0 migrations. If `doc.attributes.scheduledTaskId` exists, then the + // rule was enabled prior to running the migration. Thus we know we should add the + // tag to indicate it was auto-disabled. + tags: + !doc.attributes.enabled && doc.attributes.scheduledTaskId + ? [...(doc.attributes.tags ?? []), 'auto_disabled_8.0'] + : doc.attributes.tags ?? [], + }, + } : doc; } - // This fixes an issue whereby metrics.alert.inventory.threshold rules had the // group for actions incorrectly spelt as metrics.invenotry_threshold.fired vs metrics.inventory_threshold.fired function fixInventoryThresholdGroupId( @@ -87,16 +90,16 @@ function fixInventoryThresholdGroupId( const updatedActions = actions ? actions.map((action) => { - // Wrong spelling - if (action.group === 'metrics.invenotry_threshold.fired') { - return { - ...action, - group: 'metrics.inventory_threshold.fired', - }; - } else { - return action; - } - }) + // Wrong spelling + if (action.group === 'metrics.invenotry_threshold.fired') { + return { + ...action, + group: 'metrics.inventory_threshold.fired', + }; + } else { + return action; + } + }) : []; return { @@ -111,14 +114,20 @@ function fixInventoryThresholdGroupId( } } -export const getMigrations_8_0_0 = (encryptedSavedObjects: EncryptedSavedObjectsPluginSetup) => createEsoMigration( - encryptedSavedObjects, - (doc: SavedObjectUnsanitizedDoc): doc is SavedObjectUnsanitizedDoc => true, - pipeMigrations(addThreatIndicatorPathToThreatMatchRules, addSecuritySolutionAADRuleTypes, fixInventoryThresholdGroupId) -); +export const getMigrations_8_0_0 = (encryptedSavedObjects: EncryptedSavedObjectsPluginSetup) => + createEsoMigration( + encryptedSavedObjects, + (doc: SavedObjectUnsanitizedDoc): doc is SavedObjectUnsanitizedDoc => true, + pipeMigrations( + addThreatIndicatorPathToThreatMatchRules, + addSecuritySolutionAADRuleTypes, + fixInventoryThresholdGroupId + ) + ); -export const getMigrations_8_0_1 = (encryptedSavedObjects: EncryptedSavedObjectsPluginSetup) => createEsoMigration( - encryptedSavedObjects, - (doc: SavedObjectUnsanitizedDoc): doc is SavedObjectUnsanitizedDoc => true, - pipeMigrations(addSecuritySolutionAADRuleTypeTags) -); +export const getMigrations_8_0_1 = (encryptedSavedObjects: EncryptedSavedObjectsPluginSetup) => + createEsoMigration( + encryptedSavedObjects, + (doc: SavedObjectUnsanitizedDoc): doc is SavedObjectUnsanitizedDoc => true, + pipeMigrations(addSecuritySolutionAADRuleTypeTags) + ); diff --git a/x-pack/plugins/alerting/server/saved_objects/migrations/8.2/index.ts b/x-pack/plugins/alerting/server/saved_objects/migrations/8.2/index.ts index b232c5f1bf790..5bb3c5b41cb5f 100644 --- a/x-pack/plugins/alerting/server/saved_objects/migrations/8.2/index.ts +++ b/x-pack/plugins/alerting/server/saved_objects/migrations/8.2/index.ts @@ -5,12 +5,11 @@ * 2.0. */ - -import { SavedObjectUnsanitizedDoc } from "@kbn/core-saved-objects-server"; +import { SavedObjectUnsanitizedDoc } from '@kbn/core-saved-objects-server'; import { EncryptedSavedObjectsPluginSetup } from '@kbn/encrypted-saved-objects-plugin/server'; -import { getMappedParams } from "../../../rules_client/lib/mapped_params_utils"; -import { RawRule } from "../../../types"; -import { createEsoMigration, pipeMigrations } from "../utils"; +import { getMappedParams } from '../../../rules_client/lib/mapped_params_utils'; +import { RawRule } from '../../../types'; +import { createEsoMigration, pipeMigrations } from '../utils'; function addMappedParams( doc: SavedObjectUnsanitizedDoc @@ -34,9 +33,9 @@ function addMappedParams( return doc; } -export const getMigrations_8_2_0 = (encryptedSavedObjects: EncryptedSavedObjectsPluginSetup) => createEsoMigration( - encryptedSavedObjects, - (doc: SavedObjectUnsanitizedDoc): doc is SavedObjectUnsanitizedDoc => true, - pipeMigrations(addMappedParams) -); - +export const getMigrations_8_2_0 = (encryptedSavedObjects: EncryptedSavedObjectsPluginSetup) => + createEsoMigration( + encryptedSavedObjects, + (doc: SavedObjectUnsanitizedDoc): doc is SavedObjectUnsanitizedDoc => true, + pipeMigrations(addMappedParams) + ); diff --git a/x-pack/plugins/alerting/server/saved_objects/migrations/8.3/index.ts b/x-pack/plugins/alerting/server/saved_objects/migrations/8.3/index.ts index a55038a4ff9e8..49070a05bf571 100644 --- a/x-pack/plugins/alerting/server/saved_objects/migrations/8.3/index.ts +++ b/x-pack/plugins/alerting/server/saved_objects/migrations/8.3/index.ts @@ -5,28 +5,32 @@ * 2.0. */ -import { SavedObjectUnsanitizedDoc } from "@kbn/core-saved-objects-server"; +import { SavedObjectUnsanitizedDoc } from '@kbn/core-saved-objects-server'; import { EncryptedSavedObjectsPluginSetup } from '@kbn/encrypted-saved-objects-plugin/server'; -import { omit } from "lodash"; +import { omit } from 'lodash'; import moment from 'moment-timezone'; -import { RawRule } from "../../../types"; -import { createEsoMigration, isDetectionEngineAADRuleType, isEsQueryRuleType, pipeMigrations } from "../utils"; - +import { RawRule } from '../../../types'; +import { + createEsoMigration, + isDetectionEngineAADRuleType, + isEsQueryRuleType, + pipeMigrations, +} from '../utils'; function addSearchType(doc: SavedObjectUnsanitizedDoc) { const searchType = doc.attributes.params.searchType; return isEsQueryRuleType(doc) && !searchType ? { - ...doc, - attributes: { - ...doc.attributes, - params: { - ...doc.attributes.params, - searchType: 'esQuery', + ...doc, + attributes: { + ...doc.attributes, + params: { + ...doc.attributes.params, + searchType: 'esQuery', + }, }, - }, - } + } : doc; } @@ -70,22 +74,23 @@ function convertSnoozes( ...(omit(doc.attributes, ['snoozeEndTime']) as RawRule), snoozeSchedule: snoozeEndTime ? [ - { - duration: Date.parse(snoozeEndTime as string) - Date.now(), - rRule: { - dtstart: new Date().toISOString(), - tzid: moment.tz.guess(), - count: 1, + { + duration: Date.parse(snoozeEndTime as string) - Date.now(), + rRule: { + dtstart: new Date().toISOString(), + tzid: moment.tz.guess(), + count: 1, + }, }, - }, - ] + ] : [], }, }; } -export const getMigrations_8_3_0 = (encryptedSavedObjects: EncryptedSavedObjectsPluginSetup) => createEsoMigration( - encryptedSavedObjects, - (doc: SavedObjectUnsanitizedDoc): doc is SavedObjectUnsanitizedDoc => true, - pipeMigrations(addSearchType, removeInternalTags, convertSnoozes) -); +export const getMigrations_8_3_0 = (encryptedSavedObjects: EncryptedSavedObjectsPluginSetup) => + createEsoMigration( + encryptedSavedObjects, + (doc: SavedObjectUnsanitizedDoc): doc is SavedObjectUnsanitizedDoc => true, + pipeMigrations(addSearchType, removeInternalTags, convertSnoozes) + ); diff --git a/x-pack/plugins/alerting/server/saved_objects/migrations/8.4/index.ts b/x-pack/plugins/alerting/server/saved_objects/migrations/8.4/index.ts index a8b908650c807..629d33e3c12da 100644 --- a/x-pack/plugins/alerting/server/saved_objects/migrations/8.4/index.ts +++ b/x-pack/plugins/alerting/server/saved_objects/migrations/8.4/index.ts @@ -5,13 +5,11 @@ * 2.0. */ -import { SavedObjectUnsanitizedDoc } from "@kbn/core-saved-objects-server"; +import { SavedObjectUnsanitizedDoc } from '@kbn/core-saved-objects-server'; import { EncryptedSavedObjectsPluginSetup } from '@kbn/encrypted-saved-objects-plugin/server'; -import { omit } from "lodash"; -import { RawRule } from "../../../types"; -import { createEsoMigration, pipeMigrations } from "../utils"; - - +import { omit } from 'lodash'; +import { RawRule } from '../../../types'; +import { createEsoMigration, pipeMigrations } from '../utils'; function removeIsSnoozedUntil( doc: SavedObjectUnsanitizedDoc @@ -24,8 +22,9 @@ function removeIsSnoozedUntil( }; } -export const getMigrations_8_4_1 = (encryptedSavedObjects: EncryptedSavedObjectsPluginSetup) => createEsoMigration( - encryptedSavedObjects, - (doc: SavedObjectUnsanitizedDoc): doc is SavedObjectUnsanitizedDoc => true, - pipeMigrations(removeIsSnoozedUntil) -); +export const getMigrations_8_4_1 = (encryptedSavedObjects: EncryptedSavedObjectsPluginSetup) => + createEsoMigration( + encryptedSavedObjects, + (doc: SavedObjectUnsanitizedDoc): doc is SavedObjectUnsanitizedDoc => true, + pipeMigrations(removeIsSnoozedUntil) + ); diff --git a/x-pack/plugins/alerting/server/saved_objects/migrations/8.5/index.ts b/x-pack/plugins/alerting/server/saved_objects/migrations/8.5/index.ts index 1faae1b4ea08a..6acc2e4f63e65 100644 --- a/x-pack/plugins/alerting/server/saved_objects/migrations/8.5/index.ts +++ b/x-pack/plugins/alerting/server/saved_objects/migrations/8.5/index.ts @@ -5,15 +5,16 @@ * 2.0. */ -import { SavedObjectMigrationContext, SavedObjectUnsanitizedDoc } from "@kbn/core-saved-objects-server"; +import { + SavedObjectMigrationContext, + SavedObjectUnsanitizedDoc, +} from '@kbn/core-saved-objects-server'; import { EncryptedSavedObjectsPluginSetup } from '@kbn/encrypted-saved-objects-plugin/server'; import { isSerializedSearchSource } from '@kbn/data-plugin/common'; -import { pick } from "lodash"; -import { RawRule } from "../../../types"; -import { createEsoMigration, isEsQueryRuleType, pipeMigrations } from "../utils"; -import { AlertLogMeta } from "../types"; - - +import { pick } from 'lodash'; +import { RawRule } from '../../../types'; +import { createEsoMigration, isEsQueryRuleType, pipeMigrations } from '../utils'; +import { AlertLogMeta } from '../types'; function stripOutRuntimeFieldsInOldESQuery( doc: SavedObjectUnsanitizedDoc, @@ -30,15 +31,15 @@ function stripOutRuntimeFieldsInOldESQuery( const hasFieldsOtherThanQuery = Object.keys(parsedQuery).some((key) => key !== 'query'); return hasFieldsOtherThanQuery ? { - ...doc, - attributes: { - ...doc.attributes, - params: { - ...doc.attributes.params, - esQuery: JSON.stringify(pick(parsedQuery, 'query'), null, 4), + ...doc, + attributes: { + ...doc.attributes, + params: { + ...doc.attributes.params, + esQuery: JSON.stringify(pick(parsedQuery, 'query'), null, 4), + }, }, - }, - } + } : doc; } catch (err) { // Instead of failing the upgrade when an unparsable rule is encountered, we log that the rule caouldn't be migrated and @@ -56,9 +57,9 @@ function stripOutRuntimeFieldsInOldESQuery( return doc; } - -export const getMigrations_8_5_0 = (encryptedSavedObjects: EncryptedSavedObjectsPluginSetup) => createEsoMigration( - encryptedSavedObjects, - (doc): doc is SavedObjectUnsanitizedDoc => isEsQueryRuleType(doc), - pipeMigrations(stripOutRuntimeFieldsInOldESQuery) -); +export const getMigrations_8_5_0 = (encryptedSavedObjects: EncryptedSavedObjectsPluginSetup) => + createEsoMigration( + encryptedSavedObjects, + (doc): doc is SavedObjectUnsanitizedDoc => isEsQueryRuleType(doc), + pipeMigrations(stripOutRuntimeFieldsInOldESQuery) + ); diff --git a/x-pack/plugins/alerting/server/saved_objects/migrations/index.test.ts b/x-pack/plugins/alerting/server/saved_objects/migrations/index.test.ts index 0f2bbdf94d28d..56a0698415c50 100644 --- a/x-pack/plugins/alerting/server/saved_objects/migrations/index.test.ts +++ b/x-pack/plugins/alerting/server/saved_objects/migrations/index.test.ts @@ -2737,7 +2737,6 @@ describe('handles errors during migrations', () => { }); }); - function getUpdatedAt(): string { const updatedAt = new Date(); updatedAt.setHours(updatedAt.getHours() + 2); @@ -2784,4 +2783,3 @@ function getMockData( type: 'alert', }; } - diff --git a/x-pack/plugins/alerting/server/saved_objects/migrations/index.ts b/x-pack/plugins/alerting/server/saved_objects/migrations/index.ts index da44774b34423..018b78cf5fb60 100644 --- a/x-pack/plugins/alerting/server/saved_objects/migrations/index.ts +++ b/x-pack/plugins/alerting/server/saved_objects/migrations/index.ts @@ -32,7 +32,6 @@ import { AlertLogMeta, AlertMigration } from './types'; import { MINIMUM_SS_MIGRATION_VERSION } from './constants'; import { createEsoMigration, isEsQueryRuleType, pipeMigrations } from './utils'; - export function getMigrations( encryptedSavedObjects: EncryptedSavedObjectsPluginSetup, searchSourceMigrations: MigrateFunctionsObject, @@ -40,19 +39,58 @@ export function getMigrations( ): SavedObjectMigrationMap { return mergeSavedObjectMigrationMaps( { - '7.10.0': executeMigrationWithErrorHandling(getMigrations_7_10_0(encryptedSavedObjects), '7.10.0'), - '7.11.0': executeMigrationWithErrorHandling(getMigrations_7_11_0(encryptedSavedObjects), '7.11.0'), - '7.11.2': executeMigrationWithErrorHandling(getMigrations_7_11_2(encryptedSavedObjects), '7.11.2'), - '7.13.0': executeMigrationWithErrorHandling(getMigrations_7_13_0(encryptedSavedObjects), '7.13.0'), - '7.14.1': executeMigrationWithErrorHandling(getMigrations_7_14_0(encryptedSavedObjects), '7.14.1'), - '7.15.0': executeMigrationWithErrorHandling(getMigrations_7_15_0(encryptedSavedObjects), '7.15.0'), - '7.16.0': executeMigrationWithErrorHandling(getMigrations_7_16_0(encryptedSavedObjects, isPreconfigured), '7.16.0'), - '8.0.0': executeMigrationWithErrorHandling(getMigrations_8_0_0(encryptedSavedObjects), '8.0.0'), - '8.0.1': executeMigrationWithErrorHandling(getMigrations_8_0_1(encryptedSavedObjects), '8.0.1'), - '8.2.0': executeMigrationWithErrorHandling(getMigrations_8_2_0(encryptedSavedObjects), '8.2.0'), - '8.3.0': executeMigrationWithErrorHandling(getMigrations_8_3_0(encryptedSavedObjects), '8.3.0'), - '8.4.1': executeMigrationWithErrorHandling(getMigrations_8_4_1(encryptedSavedObjects), '8.4.1'), - '8.5.0': executeMigrationWithErrorHandling(getMigrations_8_5_0(encryptedSavedObjects), '8.5.0'), + '7.10.0': executeMigrationWithErrorHandling( + getMigrations_7_10_0(encryptedSavedObjects), + '7.10.0' + ), + '7.11.0': executeMigrationWithErrorHandling( + getMigrations_7_11_0(encryptedSavedObjects), + '7.11.0' + ), + '7.11.2': executeMigrationWithErrorHandling( + getMigrations_7_11_2(encryptedSavedObjects), + '7.11.2' + ), + '7.13.0': executeMigrationWithErrorHandling( + getMigrations_7_13_0(encryptedSavedObjects), + '7.13.0' + ), + '7.14.1': executeMigrationWithErrorHandling( + getMigrations_7_14_0(encryptedSavedObjects), + '7.14.1' + ), + '7.15.0': executeMigrationWithErrorHandling( + getMigrations_7_15_0(encryptedSavedObjects), + '7.15.0' + ), + '7.16.0': executeMigrationWithErrorHandling( + getMigrations_7_16_0(encryptedSavedObjects, isPreconfigured), + '7.16.0' + ), + '8.0.0': executeMigrationWithErrorHandling( + getMigrations_8_0_0(encryptedSavedObjects), + '8.0.0' + ), + '8.0.1': executeMigrationWithErrorHandling( + getMigrations_8_0_1(encryptedSavedObjects), + '8.0.1' + ), + '8.2.0': executeMigrationWithErrorHandling( + getMigrations_8_2_0(encryptedSavedObjects), + '8.2.0' + ), + '8.3.0': executeMigrationWithErrorHandling( + getMigrations_8_3_0(encryptedSavedObjects), + '8.3.0' + ), + '8.4.1': executeMigrationWithErrorHandling( + getMigrations_8_4_1(encryptedSavedObjects), + '8.4.1' + ), + '8.5.0': executeMigrationWithErrorHandling( + getMigrations_8_5_0(encryptedSavedObjects), + '8.5.0' + ), }, getSearchSourceMigrations(encryptedSavedObjects, searchSourceMigrations) ); @@ -131,4 +169,3 @@ function getSearchSourceMigrations( } return filteredMigrations; } - diff --git a/x-pack/plugins/alerting/server/saved_objects/migrations/utils.ts b/x-pack/plugins/alerting/server/saved_objects/migrations/utils.ts index 039aa7364335d..c96d0cfa5d7fc 100644 --- a/x-pack/plugins/alerting/server/saved_objects/migrations/utils.ts +++ b/x-pack/plugins/alerting/server/saved_objects/migrations/utils.ts @@ -6,11 +6,14 @@ */ import { ruleTypeMappings } from '@kbn/securitysolution-rules'; -import { SavedObjectMigrationContext, SavedObjectUnsanitizedDoc } from "@kbn/core-saved-objects-server"; +import { + SavedObjectMigrationContext, + SavedObjectUnsanitizedDoc, +} from '@kbn/core-saved-objects-server'; import { EncryptedSavedObjectsPluginSetup } from '@kbn/encrypted-saved-objects-plugin/server'; import type { IsMigrationNeededPredicate } from '@kbn/encrypted-saved-objects-plugin/server'; -import { RawRule } from "../../types"; +import { RawRule } from '../../types'; type AlertMigration = ( doc: SavedObjectUnsanitizedDoc, @@ -34,7 +37,6 @@ export function pipeMigrations(...migrations: AlertMigration[]): AlertMigration migrations.reduce((migratedDoc, nextMigration) => nextMigration(migratedDoc, context), doc); } - // Deprecated in 8.0 export const isSiemSignalsRuleType = (doc: SavedObjectUnsanitizedDoc): boolean => doc.attributes.alertTypeId === 'siem.signals'; From 1707107dd4d3077e0448c70dbe2f450f8ed2d366 Mon Sep 17 00:00:00 2001 From: Xavier Mouligneau Date: Fri, 28 Oct 2022 11:08:45 -0400 Subject: [PATCH 06/27] wip to add a monitoring service --- x-pack/plugins/alerting/common/rule.ts | 30 +++- .../plugins/alerting/server/lib/monitoring.ts | 4 +- .../monitoring/rule_monitoring_client.ts | 128 ++++++++++++++++++ .../server/task_runner/task_runner.ts | 43 ++---- x-pack/plugins/alerting/server/types.ts | 9 ++ 5 files changed, 170 insertions(+), 44 deletions(-) create mode 100644 x-pack/plugins/alerting/server/monitoring/rule_monitoring_client.ts diff --git a/x-pack/plugins/alerting/common/rule.ts b/x-pack/plugins/alerting/common/rule.ts index c6c598cf8aebc..3232ffec17448 100644 --- a/x-pack/plugins/alerting/common/rule.ts +++ b/x-pack/plugins/alerting/common/rule.ts @@ -174,16 +174,32 @@ export interface RuleMonitoringHistory extends SavedObjectAttributes { success: boolean; timestamp: number; duration?: number; + outcome?: string; // TODO create enum +} + +interface RuleMonitoringLastRun extends SavedObjectAttributes { + timestamp: string; + metrics: { + duration: number; + total_search_duration_ms: number | null; + total_indexing_duration_ms: number | null; + total_alerts_detected: number | null; + total_alerts_created: number | null; + gap_duration_s: number | null; + }; +} + +interface RuleMonitoringCalculatedMetrics extends SavedObjectAttributes { + p50?: number; + p95?: number; + p99?: number; + success_ratio: number; } export interface RuleMonitoring extends SavedObjectAttributes { - execution: { + run: { history: RuleMonitoringHistory[]; - calculated_metrics: { - p50?: number; - p95?: number; - p99?: number; - success_ratio: number; - }; + calculated_metrics: RuleMonitoringCalculatedMetrics; + last_run: RuleMonitoringLastRun; }; } diff --git a/x-pack/plugins/alerting/server/lib/monitoring.ts b/x-pack/plugins/alerting/server/lib/monitoring.ts index d817b10225ee0..6377505b26ae4 100644 --- a/x-pack/plugins/alerting/server/lib/monitoring.ts +++ b/x-pack/plugins/alerting/server/lib/monitoring.ts @@ -9,12 +9,12 @@ import stats from 'stats-lite'; import { RuleMonitoring } from '../types'; export const getExecutionSuccessRatio = (ruleMonitoring: RuleMonitoring) => { - const { history } = ruleMonitoring.execution; + const { history } = ruleMonitoring.run; return history.filter(({ success }) => success).length / history.length; }; export const getExecutionDurationPercentiles = (ruleMonitoring: RuleMonitoring) => { - const durationSamples = ruleMonitoring.execution.history.reduce((duration, history) => { + const durationSamples = ruleMonitoring.run.history.reduce((duration, history) => { if (typeof history.duration === 'number') { return [...duration, history.duration]; } diff --git a/x-pack/plugins/alerting/server/monitoring/rule_monitoring_client.ts b/x-pack/plugins/alerting/server/monitoring/rule_monitoring_client.ts new file mode 100644 index 0000000000000..64d3bb014af34 --- /dev/null +++ b/x-pack/plugins/alerting/server/monitoring/rule_monitoring_client.ts @@ -0,0 +1,128 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import stats from 'stats-lite'; +import { RuleMonitoring, RuleMonitoringHistory } from '../types'; + + + + +export class RuleMonitoringService { + private monitoring: RuleMonitoring = { + run: { + history: [], + calculated_metrics: { + success_ratio: 0, + }, + last_run: { + timestamp: new Date().toISOString(), + metrics: { + duration: 0, + total_search_duration_ms: null, + total_indexing_duration_ms: null, + total_alerts_detected: null, + total_alerts_created: null, + gap_duration_s: null, + }, + }, + }, + }; + + public setLastRunMetricsDuration(duration: number) { + this.monitoring.run.last_run.metrics.duration = duration; + } + + public setMonitoring(monitoringFromSO: RuleMonitoring | undefined) { + if (monitoringFromSO) { + this.monitoring = monitoringFromSO; + } + } + + public getMonitoring() { + return this.monitoring; + } + + public addHistory(duration: number | undefined, hasError: boolean = true) { + const date = new Date() + const monitoringHistory: RuleMonitoringHistory = { + success: true, + timestamp: date.getTime(), + }; + if (null != duration) { + monitoringHistory.duration = duration; + } + if (hasError) { + monitoringHistory.success = false; + } + this.monitoring.run.last_run.timestamp = date.toISOString(); + this.monitoring.run.history.push(monitoringHistory); + this.monitoring.run.calculated_metrics = { + success_ratio: this.buildExecutionSuccessRatio(), + ...this.buildExecutionDurationPercentiles(), + }; + } + + public getLastRunMetricsSetters() { + return { + setLastRunMetricsTotalSearchDurationMs: this.setLastRunMetricsTotalSearchDurationMs, + setLastRunMetricsTotalIndexingDurationMs: this.setLastRunMetricsTotalIndexingDurationMs, + setLastRunMetricsTotalAlertDetected: this.setLastRunMetricsTotalAlertDetected, + setLastRunMetricsTotalAlertCreated: this.setLastRunMetricsTotalAlertCreated, + setLastRunMetricsGapDurationS: this.setLastRunMetricsGapDurationS, + } + + } + + + + private setLastRunMetricsTotalSearchDurationMs(totalSearchDurationMs: number) { + this.monitoring.run.last_run.metrics.total_search_duration_ms = totalSearchDurationMs; + } + + private setLastRunMetricsTotalIndexingDurationMs(totalIndexingDurationMs: number) { + this.monitoring.run.last_run.metrics.total_indexing_duration_ms = totalIndexingDurationMs; + } + + private setLastRunMetricsTotalAlertDetected(totalAlertDetected: number) { + this.monitoring.run.last_run.metrics.total_alerts_detected = totalAlertDetected; + } + + private setLastRunMetricsTotalAlertCreated(totalAlertCreated: number) { + this.monitoring.run.last_run.metrics.total_alerts_created = totalAlertCreated; + } + + private setLastRunMetricsGapDurationS(gapDurationS: number) { + this.monitoring.run.last_run.metrics.gap_duration_s = gapDurationS; + } + + + + private buildExecutionSuccessRatio() { + const { history } = this.monitoring.run; + return history.filter(({ success }) => success).length / history.length; + } + + private buildExecutionDurationPercentiles = () => { + const { history } = this.monitoring.run; + const durationSamples = history.reduce((duration, historyItem) => { + if (typeof historyItem.duration === 'number') { + return [...duration, historyItem.duration]; + } + return duration; + }, []); + + if (durationSamples.length) { + return { + p50: stats.percentile(durationSamples as number[], 0.5), + p95: stats.percentile(durationSamples as number[], 0.95), + p99: stats.percentile(durationSamples as number[], 0.99), + }; + } + + return {}; + }; +} diff --git a/x-pack/plugins/alerting/server/task_runner/task_runner.ts b/x-pack/plugins/alerting/server/task_runner/task_runner.ts index 0c6bbfbbd9866..9717e0300faf8 100644 --- a/x-pack/plugins/alerting/server/task_runner/task_runner.ts +++ b/x-pack/plugins/alerting/server/task_runner/task_runner.ts @@ -33,12 +33,10 @@ import { RawRule, RawRuleExecutionStatus, RuleMonitoring, - RuleMonitoringHistory, RuleTaskState, RuleTypeRegistry, } from '../types'; import { asErr, asOk, map, resolveErr, Result } from '../lib/result_type'; -import { getExecutionDurationPercentiles, getExecutionSuccessRatio } from '../lib/monitoring'; import { taskInstanceToAlertTaskInstance } from './alert_task_instance'; import { isAlertSavedObjectNotFoundError, isEsUnavailableError } from '../lib/is_alerting_error'; import { partiallyUpdateAlert } from '../saved_objects'; @@ -69,19 +67,11 @@ import { logAlerts } from './log_alerts'; import { scheduleActionsForAlerts } from './schedule_actions_for_alerts'; import { getPublicAlertFactory } from '../alert/create_alert_factory'; import { TaskRunnerTimer, TaskRunnerTimerSpan } from './task_runner_timer'; +import { RuleMonitoringService } from '../monitoring/rule_monitoring_client'; const FALLBACK_RETRY_INTERVAL = '5m'; const CONNECTIVITY_RETRY_INTERVAL = '5m'; -export const getDefaultRuleMonitoring = (): RuleMonitoring => ({ - execution: { - history: [], - calculated_metrics: { - success_ratio: 0, - }, - }, -}); - interface StackTraceLog { message: ElasticsearchError; stackTrace?: string; @@ -120,6 +110,7 @@ export class TaskRunner< private searchAbortController: AbortController; private cancelled: boolean; private stackTraceLog: StackTraceLog | null; + private ruleMonitoring: RuleMonitoringService; constructor( ruleType: NormalizedRuleType< @@ -152,6 +143,7 @@ export class TaskRunner< this.timer = new TaskRunnerTimer({ logger: this.logger }); this.alertingEventLogger = new AlertingEventLogger(this.context.eventLogger); this.stackTraceLog = null; + this.ruleMonitoring = new RuleMonitoringService(); } private getExecutionHandler( @@ -365,6 +357,7 @@ export class TaskRunner< alertFactory: getPublicAlertFactory(alertFactory), shouldWriteAlerts: () => this.shouldLogAndScheduleActionsForAlerts(), shouldStopExecution: () => this.cancelled, + ruleMonitoringService: this.ruleMonitoring.getLastRunMetricsSetters(), }, params, state: ruleTypeState as RuleState, @@ -585,11 +578,9 @@ export class TaskRunner< private async processRunResults({ runDate, stateWithMetrics, - monitoring, }: { runDate: Date; stateWithMetrics: Result; - monitoring: RuleMonitoring; }) { const { params: { alertId: ruleId, spaceId }, @@ -624,11 +615,6 @@ export class TaskRunner< ); } - const monitoringHistory: RuleMonitoringHistory = { - success: true, - timestamp: +new Date(), - }; - // set start and duration based on event log const { start, duration } = this.alertingEventLogger.getStartAndDuration(); if (null != start) { @@ -636,20 +622,10 @@ export class TaskRunner< } if (null != duration) { executionStatus.lastDuration = nanosToMillis(duration); - monitoringHistory.duration = executionStatus.lastDuration; } // if executionStatus indicates an error, fill in fields in - // event from it - if (executionStatus.error) { - monitoringHistory.success = false; - } - - monitoring.execution.history.push(monitoringHistory); - monitoring.execution.calculated_metrics = { - success_ratio: getExecutionSuccessRatio(monitoring), - ...getExecutionDurationPercentiles(monitoring), - }; + this.ruleMonitoring.addHistory(executionStatus.lastDuration, executionStatus.error != null) if (!this.cancelled) { this.inMemoryMetrics.increment(IN_MEMORY_METRICS.RULE_EXECUTIONS); @@ -663,7 +639,7 @@ export class TaskRunner< ); await this.updateRuleSavedObject(ruleId, namespace, { executionStatus: ruleExecutionStatusToRaw(executionStatus), - monitoring, + monitoring: this.ruleMonitoring.getMonitoring(), }); } @@ -687,15 +663,13 @@ export class TaskRunner< } let stateWithMetrics: Result; - let monitoring: RuleMonitoring = getDefaultRuleMonitoring(); let schedule: Result; try { const preparedResult = await this.timer.runWithTimer( TaskRunnerTimerSpan.PrepareRule, async () => this.prepareToRun() ); - - monitoring = preparedResult.rule.monitoring ?? getDefaultRuleMonitoring(); + this.ruleMonitoring.setMonitoring(preparedResult.rule.monitoring) stateWithMetrics = asOk(await this.runRule(preparedResult)); @@ -713,7 +687,6 @@ export class TaskRunner< this.processRunResults({ runDate, stateWithMetrics, - monitoring, }) ); @@ -783,7 +756,7 @@ export class TaskRunner< return { interval: retryInterval }; }), - monitoring, + monitoring: this.ruleMonitoring.getMonitoring(), }; } diff --git a/x-pack/plugins/alerting/server/types.ts b/x-pack/plugins/alerting/server/types.ts index f1917a079a26d..d29e41e93212b 100644 --- a/x-pack/plugins/alerting/server/types.ts +++ b/x-pack/plugins/alerting/server/types.ts @@ -81,6 +81,7 @@ export interface RuleExecutorServices< alertFactory: PublicAlertFactory; shouldWriteAlerts: () => boolean; shouldStopExecution: () => boolean; + ruleMonitoringService: RuleMonitoringService; } export interface RuleExecutorOptions< @@ -280,3 +281,11 @@ export interface InvalidatePendingApiKey { export type RuleTypeRegistry = PublicMethodsOf; export type RulesClientApi = PublicMethodsOf; + +export interface RuleMonitoringService { + setLastRunMetricsTotalSearchDurationMs: (totalSearchDurationMs: number) => void; + setLastRunMetricsTotalIndexingDurationMs: (totalIndexingDurationMs: number) => void; + setLastRunMetricsTotalAlertDetected: (totalAlertDetected: number) => void; + setLastRunMetricsTotalAlertCreated: (totalAlertCreated: number) => void; + setLastRunMetricsGapDurationS: (gapDurationS: number) => void; +} From c8824266fcc36a3ec52fd542d7784a7036ef8082 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Fri, 28 Oct 2022 15:52:40 +0000 Subject: [PATCH 07/27] [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' --- .../server/monitoring/rule_monitoring_client.ts | 12 ++---------- .../alerting/server/task_runner/task_runner.ts | 4 ++-- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/x-pack/plugins/alerting/server/monitoring/rule_monitoring_client.ts b/x-pack/plugins/alerting/server/monitoring/rule_monitoring_client.ts index 64d3bb014af34..f743089d20159 100644 --- a/x-pack/plugins/alerting/server/monitoring/rule_monitoring_client.ts +++ b/x-pack/plugins/alerting/server/monitoring/rule_monitoring_client.ts @@ -8,9 +8,6 @@ import stats from 'stats-lite'; import { RuleMonitoring, RuleMonitoringHistory } from '../types'; - - - export class RuleMonitoringService { private monitoring: RuleMonitoring = { run: { @@ -47,7 +44,7 @@ export class RuleMonitoringService { } public addHistory(duration: number | undefined, hasError: boolean = true) { - const date = new Date() + const date = new Date(); const monitoringHistory: RuleMonitoringHistory = { success: true, timestamp: date.getTime(), @@ -73,12 +70,9 @@ export class RuleMonitoringService { setLastRunMetricsTotalAlertDetected: this.setLastRunMetricsTotalAlertDetected, setLastRunMetricsTotalAlertCreated: this.setLastRunMetricsTotalAlertCreated, setLastRunMetricsGapDurationS: this.setLastRunMetricsGapDurationS, - } - + }; } - - private setLastRunMetricsTotalSearchDurationMs(totalSearchDurationMs: number) { this.monitoring.run.last_run.metrics.total_search_duration_ms = totalSearchDurationMs; } @@ -99,8 +93,6 @@ export class RuleMonitoringService { this.monitoring.run.last_run.metrics.gap_duration_s = gapDurationS; } - - private buildExecutionSuccessRatio() { const { history } = this.monitoring.run; return history.filter(({ success }) => success).length / history.length; diff --git a/x-pack/plugins/alerting/server/task_runner/task_runner.ts b/x-pack/plugins/alerting/server/task_runner/task_runner.ts index 9717e0300faf8..5f794a5c821f7 100644 --- a/x-pack/plugins/alerting/server/task_runner/task_runner.ts +++ b/x-pack/plugins/alerting/server/task_runner/task_runner.ts @@ -625,7 +625,7 @@ export class TaskRunner< } // if executionStatus indicates an error, fill in fields in - this.ruleMonitoring.addHistory(executionStatus.lastDuration, executionStatus.error != null) + this.ruleMonitoring.addHistory(executionStatus.lastDuration, executionStatus.error != null); if (!this.cancelled) { this.inMemoryMetrics.increment(IN_MEMORY_METRICS.RULE_EXECUTIONS); @@ -669,7 +669,7 @@ export class TaskRunner< TaskRunnerTimerSpan.PrepareRule, async () => this.prepareToRun() ); - this.ruleMonitoring.setMonitoring(preparedResult.rule.monitoring) + this.ruleMonitoring.setMonitoring(preparedResult.rule.monitoring); stateWithMetrics = asOk(await this.runRule(preparedResult)); From 0fe91eb6df91251d6f8e2c85ce63a9c153acc3d2 Mon Sep 17 00:00:00 2001 From: Jiawei Wu Date: Tue, 1 Nov 2022 15:26:43 -0400 Subject: [PATCH 08/27] jiawei work on backend of rule --- x-pack/plugins/alerting/common/rule.ts | 51 +++++++---- .../public/lib/common_transformations.ts | 42 ++++++++- x-pack/plugins/alerting/server/lib/index.ts | 5 ++ .../alerting/server/lib/last_run_status.ts | 87 +++++++++++++++++++ .../plugins/alerting/server/lib/monitoring.ts | 52 +++++++---- .../plugins/alerting/server/lib/next_run.ts | 13 +++ .../monitoring/rule_monitoring_client.ts | 24 ++--- .../alerting/server/routes/aggregate_rules.ts | 2 + .../alerting/server/routes/create_rule.ts | 5 ++ .../alerting/server/routes/get_rule.ts | 6 +- .../alerting/server/routes/lib/index.ts | 2 +- .../server/routes/lib/rewrite_request_case.ts | 2 +- .../server/routes/lib/rewrite_rule.ts | 15 +++- .../alerting/server/routes/resolve_rule.ts | 6 +- .../alerting/server/routes/update_rule.ts | 5 ++ .../server/rules_client/rules_client.ts | 64 ++++++++++++-- .../alerting/server/saved_objects/index.ts | 8 +- .../alerting/server/saved_objects/mappings.ts | 11 ++- .../saved_objects/migrations/8.6/index.ts | 85 ++++++++++++++++++ .../server/saved_objects/migrations/index.ts | 2 + .../alerting/server/task_runner/fixtures.ts | 4 +- .../server/task_runner/rule_loader.ts | 4 +- .../server/task_runner/task_runner.ts | 73 ++++++++++++++-- x-pack/plugins/alerting/server/types.ts | 3 + 24 files changed, 491 insertions(+), 80 deletions(-) create mode 100644 x-pack/plugins/alerting/server/lib/last_run_status.ts create mode 100644 x-pack/plugins/alerting/server/lib/next_run.ts diff --git a/x-pack/plugins/alerting/common/rule.ts b/x-pack/plugins/alerting/common/rule.ts index 3232ffec17448..30b35b7043ea7 100644 --- a/x-pack/plugins/alerting/common/rule.ts +++ b/x-pack/plugins/alerting/common/rule.ts @@ -32,6 +32,9 @@ export const RuleExecutionStatusValues = [ ] as const; export type RuleExecutionStatuses = typeof RuleExecutionStatusValues[number]; +export const RuleLastRunOutcomeValues = ['succeeded', 'warning', 'failed'] as const; +export type RuleLastRunOutcomes = typeof RuleLastRunOutcomeValues[number]; + export enum RuleExecutionStatusErrorReasons { Read = 'read', Decrypt = 'decrypt', @@ -76,12 +79,25 @@ export interface RuleAction { export interface RuleAggregations { alertExecutionStatus: { [status: string]: number }; + ruleLastRunOutcome: { [status: string]: number }; ruleEnabledStatus: { enabled: number; disabled: number }; ruleMutedStatus: { muted: number; unmuted: number }; ruleSnoozedStatus: { snoozed: number }; ruleTags: string[]; } +export interface RuleLastRun { + outcome: RuleLastRunOutcomes; + warning?: RuleExecutionStatusErrorReasons | RuleExecutionStatusWarningReasons | null; + outcomeMsg?: string | null; + alertsCount: { + active?: number | null; + new?: number | null; + recovered?: number | null; + ignored?: number | null; + }; +} + export interface MappedParamsProperties { risk_score?: number; severity?: string; @@ -116,6 +132,9 @@ export interface Rule { snoozeSchedule?: RuleSnooze; // Remove ? when this parameter is made available in the public API activeSnoozes?: string[]; isSnoozedUntil?: Date | null; + lastRun?: RuleLastRun | null; + nextRun?: Date | null; + running: boolean; } export type SanitizedRule = Omit, 'apiKey'>; @@ -174,29 +193,31 @@ export interface RuleMonitoringHistory extends SavedObjectAttributes { success: boolean; timestamp: number; duration?: number; - outcome?: string; // TODO create enum + outcome?: RuleLastRunOutcomes; } -interface RuleMonitoringLastRun extends SavedObjectAttributes { - timestamp: string; - metrics: { - duration: number; - total_search_duration_ms: number | null; - total_indexing_duration_ms: number | null; - total_alerts_detected: number | null; - total_alerts_created: number | null; - gap_duration_s: number | null; - }; -} - -interface RuleMonitoringCalculatedMetrics extends SavedObjectAttributes { +export interface RuleMonitoringCalculatedMetrics extends SavedObjectAttributes { p50?: number; p95?: number; p99?: number; success_ratio: number; } -export interface RuleMonitoring extends SavedObjectAttributes { +export interface RuleMonitoringLastRunMetrics { + duration: number; + total_search_duration_ms?: number | null; + total_indexing_duration_ms?: number | null; + total_alerts_detected?: number | null; + total_alerts_created?: number | null; + gap_duration_s?: number | null; +} + +export interface RuleMonitoringLastRun { + timestamp: string; + metrics: RuleMonitoringLastRunMetrics; +} + +export interface RuleMonitoring { run: { history: RuleMonitoringHistory[]; calculated_metrics: RuleMonitoringCalculatedMetrics; diff --git a/x-pack/plugins/alerting/public/lib/common_transformations.ts b/x-pack/plugins/alerting/public/lib/common_transformations.ts index 1b306aae0ae2f..b58dd8922fd7b 100644 --- a/x-pack/plugins/alerting/public/lib/common_transformations.ts +++ b/x-pack/plugins/alerting/public/lib/common_transformations.ts @@ -5,7 +5,14 @@ * 2.0. */ import { AsApiContract } from '@kbn/actions-plugin/common'; -import { RuleExecutionStatus, Rule, RuleAction, RuleType } from '../../common'; +import { + RuleExecutionStatus, + RuleMonitoring, + Rule, + RuleLastRun, + RuleAction, + RuleType, +} from '../../common'; function transformAction(input: AsApiContract): RuleAction { const { connector_type_id: actionTypeId, ...rest } = input; @@ -27,6 +34,31 @@ function transformExecutionStatus(input: ApiRuleExecutionStatus): RuleExecutionS }; } +function transformMonitoring(input: RuleMonitoring): RuleMonitoring { + const { run } = input; + const { last_run: lastRun, ...rest } = run; + const { timestamp, ...restLastRun } = lastRun; + + return { + run: { + last_run: { + timestamp: new Date(input.run.last_run.timestamp), + ...restLastRun, + }, + ...rest, + }, + }; +} + +function transformLastRun(input: AsApiContract): RuleLastRun { + const { outcome_msg: outcomeMsg, alerts_count: alertsCount, ...rest } = input; + return { + outcomeMsg, + alertsCount, + ...rest, + }; +} + // AsApiContract does not deal with object properties that also // need snake -> camel conversion, Dates, are renamed, etc, so we do by hand export type ApiRule = Omit< @@ -37,6 +69,7 @@ export type ApiRule = Omit< | 'updated_at' | 'alert_type_id' | 'muted_instance_ids' + | 'last_run' > & { execution_status: ApiRuleExecutionStatus; actions: Array>; @@ -44,6 +77,7 @@ export type ApiRule = Omit< updated_at: string; rule_type_id: string; muted_alert_ids: string[]; + last_run?: AsApiContract; }; export function transformRule(input: ApiRule): Rule { @@ -61,6 +95,9 @@ export function transformRule(input: ApiRule): Rule { scheduled_task_id: scheduledTaskId, execution_status: executionStatusAPI, actions: actionsAPI, + next_run: nextRun, + last_run: lastRun, + monitoring: monitoring, ...rest } = input; @@ -78,6 +115,9 @@ export function transformRule(input: ApiRule): Rule { executionStatus: transformExecutionStatus(executionStatusAPI), actions: actionsAPI ? actionsAPI.map((action) => transformAction(action)) : [], scheduledTaskId, + ...(nextRun ? { nextRun: new Date(nextRun) } : {}), + ...(monitoring ? { monitoring: transformMonitoring(monitoring) } : {}), + ...(lastRun ? { lastRun: transformLastRun(lastRun) } : {}), ...rest, }; } diff --git a/x-pack/plugins/alerting/server/lib/index.ts b/x-pack/plugins/alerting/server/lib/index.ts index 825a9863badbb..5ea4207068a7f 100644 --- a/x-pack/plugins/alerting/server/lib/index.ts +++ b/x-pack/plugins/alerting/server/lib/index.ts @@ -25,6 +25,11 @@ export { ruleExecutionStatusToRaw, ruleExecutionStatusFromRaw, } from './rule_execution_status'; +export { lastRunFromState, lastRunFromError, lastRunToRaw } from './last_run_status'; +export { + updateMonitoring, +} from './monitoring'; +export { getNextRunDate, getNextRunString } from './next_run'; export { processAlerts } from './process_alerts'; export { createWrappedScopedClusterClientFactory } from './wrap_scoped_cluster_client'; export { isRuleSnoozed, getRuleSnoozeEndTime } from './is_rule_snoozed'; diff --git a/x-pack/plugins/alerting/server/lib/last_run_status.ts b/x-pack/plugins/alerting/server/lib/last_run_status.ts new file mode 100644 index 0000000000000..ce2e94c145429 --- /dev/null +++ b/x-pack/plugins/alerting/server/lib/last_run_status.ts @@ -0,0 +1,87 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { RuleTaskStateAndMetrics } from '../task_runner/types'; +import { getReasonFromError } from './error_with_reason'; +import { getEsErrorMessage } from './errors'; +import { ActionsCompletion } from '../../common'; +import { + RuleLastRunOutcomeValues, + RuleLastRunOutcomes, + RuleExecutionStatusWarningReasons, + RawRuleLastRun, + RuleLastRun, +} from '../types'; +import { translations } from '../constants/translations'; +import { RuleRunMetrics } from './rule_run_metrics_store'; + +export interface ILastRun { + lastRun: RuleLastRun; + metrics: RuleRunMetrics | null; +} + +export const lastRunFromState = (stateWithMetrics: RuleTaskStateAndMetrics): ILastRun => { + const { metrics } = stateWithMetrics; + let outcome: RuleLastRunOutcomes = RuleLastRunOutcomeValues[0]; + // Check for warning states + let warning = null; + let outcomeMsg = null; + + // We only have a single warning field so prioritizing the alert circuit breaker over the actions circuit breaker + if (metrics.hasReachedAlertLimit) { + outcome = RuleLastRunOutcomeValues[1]; + warning = RuleExecutionStatusWarningReasons.MAX_ALERTS; + outcomeMsg = translations.taskRunner.warning.maxAlerts; + } else if (metrics.triggeredActionsStatus === ActionsCompletion.PARTIAL) { + outcome = RuleLastRunOutcomeValues[1]; + warning = RuleExecutionStatusWarningReasons.MAX_EXECUTABLE_ACTIONS; + outcomeMsg = translations.taskRunner.warning.maxExecutableActions; + } + + return { + lastRun: { + outcome, + alertsCount: { + active: metrics.numberOfActiveAlerts, + new: metrics.numberOfNewAlerts, + recovered: metrics.numberOfRecoveredAlerts, + ignored: 0, + }, + ...(warning ? { warning } : {}), + ...(outcomeMsg ? { outcome_msg: outcomeMsg } : {}), + }, + metrics, + }; +}; + +export const lastRunFromError = (error: Error): ILastRun => { + return { + lastRun: { + outcome: RuleLastRunOutcomeValues[2], + warning: getReasonFromError(error), + outcomeMsg: getEsErrorMessage(error), + alertsCount: {}, + }, + metrics: null, + }; +}; + +export const lastRunToRaw = (lastRun: ILastRun['lastRun']): RawRuleLastRun => { + const { warning, alertsCount, outcomeMsg } = lastRun; + + return { + ...lastRun, + alertsCount: { + active: alertsCount.active || 0, + new: alertsCount.new || 0, + recovered: alertsCount.recovered || 0, + ignored: alertsCount.ignored || 0, + }, + warning: warning ?? null, + outcomeMsg: outcomeMsg ?? null, + }; +}; diff --git a/x-pack/plugins/alerting/server/lib/monitoring.ts b/x-pack/plugins/alerting/server/lib/monitoring.ts index 6377505b26ae4..be46803bd3e5d 100644 --- a/x-pack/plugins/alerting/server/lib/monitoring.ts +++ b/x-pack/plugins/alerting/server/lib/monitoring.ts @@ -5,29 +5,43 @@ * 2.0. */ -import stats from 'stats-lite'; import { RuleMonitoring } from '../types'; -export const getExecutionSuccessRatio = (ruleMonitoring: RuleMonitoring) => { - const { history } = ruleMonitoring.run; - return history.filter(({ success }) => success).length / history.length; +export const INITIAL_METRICS = { + duration: 0, + total_search_duration_ms: null, + total_indexing_duration_ms: null, + total_alerts_detected: null, + total_alerts_created: null, + gap_duration_s: null, }; -export const getExecutionDurationPercentiles = (ruleMonitoring: RuleMonitoring) => { - const durationSamples = ruleMonitoring.run.history.reduce((duration, history) => { - if (typeof history.duration === 'number') { - return [...duration, history.duration]; - } - return duration; - }, []); - if (durationSamples.length) { - return { - p50: stats.percentile(durationSamples as number[], 0.5), - p95: stats.percentile(durationSamples as number[], 0.95), - p99: stats.percentile(durationSamples as number[], 0.99), - }; - } +// Immutably updates the monitoring object with timestamp and duration. +// Used when converting from and between raw monitoring object +export const updateMonitoring = ({ + monitoring, + timestamp, + duration, +}: { + monitoring: RuleMonitoring; + timestamp: string; + duration: number; +}) => { + const { run } = monitoring; + const { last_run: lastRun, ...rest } = run; + const { metrics = INITIAL_METRICS } = lastRun; - return {}; + return { + run: { + last_run: { + timestamp, + metrics: { + ...metrics, + duration, + }, + }, + ...rest, + }, + }; }; diff --git a/x-pack/plugins/alerting/server/lib/next_run.ts b/x-pack/plugins/alerting/server/lib/next_run.ts new file mode 100644 index 0000000000000..4c40549cf4d04 --- /dev/null +++ b/x-pack/plugins/alerting/server/lib/next_run.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import moment from 'moment'; +import { parseDuration } from '../../common'; + +export const getNextRunDate = (interval: string) => moment().add(parseDuration(interval), 'ms'); + +export const getNextRunString = (interval: string) => getNextRunDate(interval).toISOString(); diff --git a/x-pack/plugins/alerting/server/monitoring/rule_monitoring_client.ts b/x-pack/plugins/alerting/server/monitoring/rule_monitoring_client.ts index f743089d20159..ef0da102c3100 100644 --- a/x-pack/plugins/alerting/server/monitoring/rule_monitoring_client.ts +++ b/x-pack/plugins/alerting/server/monitoring/rule_monitoring_client.ts @@ -6,6 +6,7 @@ */ import stats from 'stats-lite'; +import { INITIAL_METRICS } from '../lib/monitoring'; import { RuleMonitoring, RuleMonitoringHistory } from '../types'; export class RuleMonitoringService { @@ -17,14 +18,7 @@ export class RuleMonitoringService { }, last_run: { timestamp: new Date().toISOString(), - metrics: { - duration: 0, - total_search_duration_ms: null, - total_indexing_duration_ms: null, - total_alerts_detected: null, - total_alerts_created: null, - gap_duration_s: null, - }, + metrics: INITIAL_METRICS, }, }, }; @@ -43,8 +37,8 @@ export class RuleMonitoringService { return this.monitoring; } - public addHistory(duration: number | undefined, hasError: boolean = true) { - const date = new Date(); + public addHistory({ duration, hasError = true, runDate }: {duration: number | undefined; hasError: boolean; runDate: Date}) { + const date = runDate ?? new Date(); const monitoringHistory: RuleMonitoringHistory = { success: true, timestamp: date.getTime(), @@ -65,11 +59,11 @@ export class RuleMonitoringService { public getLastRunMetricsSetters() { return { - setLastRunMetricsTotalSearchDurationMs: this.setLastRunMetricsTotalSearchDurationMs, - setLastRunMetricsTotalIndexingDurationMs: this.setLastRunMetricsTotalIndexingDurationMs, - setLastRunMetricsTotalAlertDetected: this.setLastRunMetricsTotalAlertDetected, - setLastRunMetricsTotalAlertCreated: this.setLastRunMetricsTotalAlertCreated, - setLastRunMetricsGapDurationS: this.setLastRunMetricsGapDurationS, + setLastRunMetricsTotalSearchDurationMs: this.setLastRunMetricsTotalSearchDurationMs.bind(this), + setLastRunMetricsTotalIndexingDurationMs: this.setLastRunMetricsTotalIndexingDurationMs.bind(this), + setLastRunMetricsTotalAlertDetected: this.setLastRunMetricsTotalAlertDetected.bind(this), + setLastRunMetricsTotalAlertCreated: this.setLastRunMetricsTotalAlertCreated.bind(this), + setLastRunMetricsGapDurationS: this.setLastRunMetricsGapDurationS.bind(this), }; } diff --git a/x-pack/plugins/alerting/server/routes/aggregate_rules.ts b/x-pack/plugins/alerting/server/routes/aggregate_rules.ts index c48c74fc28754..acab5ca75d2e0 100644 --- a/x-pack/plugins/alerting/server/routes/aggregate_rules.ts +++ b/x-pack/plugins/alerting/server/routes/aggregate_rules.ts @@ -47,6 +47,7 @@ const rewriteQueryReq: RewriteRequestCase = ({ }); const rewriteBodyRes: RewriteResponseCase = ({ alertExecutionStatus, + ruleLastRunOutcome, ruleEnabledStatus, ruleMutedStatus, ruleSnoozedStatus, @@ -55,6 +56,7 @@ const rewriteBodyRes: RewriteResponseCase = ({ }) => ({ ...rest, rule_execution_status: alertExecutionStatus, + rule_last_run_outcome: ruleLastRunOutcome, rule_enabled_status: ruleEnabledStatus, rule_muted_status: ruleMutedStatus, rule_snoozed_status: ruleSnoozedStatus, diff --git a/x-pack/plugins/alerting/server/routes/create_rule.ts b/x-pack/plugins/alerting/server/routes/create_rule.ts index 442162ae21cbb..1b114dc54d26b 100644 --- a/x-pack/plugins/alerting/server/routes/create_rule.ts +++ b/x-pack/plugins/alerting/server/routes/create_rule.ts @@ -14,6 +14,7 @@ import { handleDisabledApiKeysError, verifyAccessAndContext, countUsageOfPredefinedIds, + rewriteRuleLastRun, } from './lib'; import { SanitizedRule, @@ -68,6 +69,8 @@ const rewriteBodyRes: RewriteResponseCase> = ({ muteAll, mutedInstanceIds, snoozeSchedule, + lastRun, + nextRun, executionStatus: { lastExecutionDate, lastDuration, ...executionStatus }, ...rest }) => ({ @@ -94,6 +97,8 @@ const rewriteBodyRes: RewriteResponseCase> = ({ params, connector_type_id: actionTypeId, })), + ...(lastRun ? { last_run: rewriteRuleLastRun(lastRun) } : {}), + ...(nextRun ? { next_run: nextRun } : {}), }); export const createRuleRoute = ({ router, licenseState, usageCounter }: RouteOptions) => { diff --git a/x-pack/plugins/alerting/server/routes/get_rule.ts b/x-pack/plugins/alerting/server/routes/get_rule.ts index f7834ec52b9ed..0b529b35fa466 100644 --- a/x-pack/plugins/alerting/server/routes/get_rule.ts +++ b/x-pack/plugins/alerting/server/routes/get_rule.ts @@ -9,7 +9,7 @@ import { omit } from 'lodash'; import { schema } from '@kbn/config-schema'; import { IRouter } from '@kbn/core/server'; import { ILicenseState } from '../lib'; -import { verifyAccessAndContext, RewriteResponseCase } from './lib'; +import { verifyAccessAndContext, RewriteResponseCase, rewriteRuleLastRun } from './lib'; import { RuleTypeParams, AlertingRequestHandlerContext, @@ -37,6 +37,8 @@ const rewriteBodyRes: RewriteResponseCase> = ({ scheduledTaskId, snoozeSchedule, isSnoozedUntil, + lastRun, + nextRun, ...rest }) => ({ ...rest, @@ -63,6 +65,8 @@ const rewriteBodyRes: RewriteResponseCase> = ({ params, connector_type_id: actionTypeId, })), + ...(lastRun ? { last_run: rewriteRuleLastRun(lastRun) } : {}), + ...(nextRun ? { next_run: nextRun } : {}), }); interface BuildGetRulesRouteParams { diff --git a/x-pack/plugins/alerting/server/routes/lib/index.ts b/x-pack/plugins/alerting/server/routes/lib/index.ts index e772f091bb059..9c71d29f0988c 100644 --- a/x-pack/plugins/alerting/server/routes/lib/index.ts +++ b/x-pack/plugins/alerting/server/routes/lib/index.ts @@ -18,4 +18,4 @@ export type { } from './rewrite_request_case'; export { verifyAccessAndContext } from './verify_access_and_context'; export { countUsageOfPredefinedIds } from './count_usage_of_predefined_ids'; -export { rewriteRule } from './rewrite_rule'; +export { rewriteRule, rewriteRuleLastRun } from './rewrite_rule'; diff --git a/x-pack/plugins/alerting/server/routes/lib/rewrite_request_case.ts b/x-pack/plugins/alerting/server/routes/lib/rewrite_request_case.ts index 2874f9567231b..4eb352d2b2b8c 100644 --- a/x-pack/plugins/alerting/server/routes/lib/rewrite_request_case.ts +++ b/x-pack/plugins/alerting/server/routes/lib/rewrite_request_case.ts @@ -24,7 +24,7 @@ type RenameAlertToRule = K extends `alertTypeId` export type AsApiContract< T, - ComplexPropertyKeys = `actions` | `executionStatus`, + ComplexPropertyKeys = `actions` | `executionStatus` | 'lastRun', OpaquePropertyKeys = `params` > = T extends Array ? Array> diff --git a/x-pack/plugins/alerting/server/routes/lib/rewrite_rule.ts b/x-pack/plugins/alerting/server/routes/lib/rewrite_rule.ts index 4b3ab65e3e8e4..270db64812b28 100644 --- a/x-pack/plugins/alerting/server/routes/lib/rewrite_rule.ts +++ b/x-pack/plugins/alerting/server/routes/lib/rewrite_rule.ts @@ -6,7 +6,16 @@ */ import { omit } from 'lodash'; -import { RuleTypeParams, SanitizedRule } from '../../types'; +import { RuleTypeParams, SanitizedRule, RuleLastRun } from '../../types'; + +export const rewriteRuleLastRun = (lastRun: RuleLastRun) => { + const { outcomeMsg, alertsCount, ...rest } = lastRun; + return { + alerts_count: alertsCount, + outcome_msg: outcomeMsg, + ...rest, + }; +}; export const rewriteRule = ({ alertTypeId, @@ -24,6 +33,8 @@ export const rewriteRule = ({ snoozeSchedule, isSnoozedUntil, activeSnoozes, + lastRun, + nextRun, ...rest }: SanitizedRule & { activeSnoozes?: string[] }) => ({ ...rest, @@ -51,4 +62,6 @@ export const rewriteRule = ({ params, connector_type_id: actionTypeId, })), + ...(lastRun ? { last_run: rewriteRuleLastRun(lastRun) } : {}), + ...(nextRun ? { next_run: nextRun } : {}), }); diff --git a/x-pack/plugins/alerting/server/routes/resolve_rule.ts b/x-pack/plugins/alerting/server/routes/resolve_rule.ts index b3576c0c5ed44..48d2253a03fc9 100644 --- a/x-pack/plugins/alerting/server/routes/resolve_rule.ts +++ b/x-pack/plugins/alerting/server/routes/resolve_rule.ts @@ -9,7 +9,7 @@ import { omit } from 'lodash'; import { schema } from '@kbn/config-schema'; import { IRouter } from '@kbn/core/server'; import { ILicenseState } from '../lib'; -import { verifyAccessAndContext, RewriteResponseCase } from './lib'; +import { verifyAccessAndContext, RewriteResponseCase, rewriteRuleLastRun } from './lib'; import { RuleTypeParams, AlertingRequestHandlerContext, @@ -34,6 +34,8 @@ const rewriteBodyRes: RewriteResponseCase> executionStatus, actions, scheduledTaskId, + lastRun, + nextRun, ...rest }) => ({ ...rest, @@ -58,6 +60,8 @@ const rewriteBodyRes: RewriteResponseCase> params, connector_type_id: actionTypeId, })), + ...(lastRun ? { last_run: rewriteRuleLastRun(lastRun) } : {}), + ...(nextRun ? { next_run: nextRun } : {}), }); export const resolveRuleRoute = ( diff --git a/x-pack/plugins/alerting/server/routes/update_rule.ts b/x-pack/plugins/alerting/server/routes/update_rule.ts index 1faddd66c8f0e..2b7f6b3c98b39 100644 --- a/x-pack/plugins/alerting/server/routes/update_rule.ts +++ b/x-pack/plugins/alerting/server/routes/update_rule.ts @@ -15,6 +15,7 @@ import { RewriteResponseCase, RewriteRequestCase, handleDisabledApiKeysError, + rewriteRuleLastRun, } from './lib'; import { RuleTypeParams, @@ -72,6 +73,8 @@ const rewriteBodyRes: RewriteResponseCase> = ({ executionStatus, snoozeSchedule, isSnoozedUntil, + lastRun, + nextRun, ...rest }) => ({ ...rest, @@ -106,6 +109,8 @@ const rewriteBodyRes: RewriteResponseCase> = ({ })), } : {}), + ...(lastRun ? { last_run: rewriteRuleLastRun(lastRun) } : {}), + ...(nextRun ? { next_run: nextRun } : {}), }); export const updateRuleRoute = ( diff --git a/x-pack/plugins/alerting/server/rules_client/rules_client.ts b/x-pack/plugins/alerting/server/rules_client/rules_client.ts index 777cf340b53e7..5abf8d498c9cc 100644 --- a/x-pack/plugins/alerting/server/rules_client/rules_client.ts +++ b/x-pack/plugins/alerting/server/rules_client/rules_client.ts @@ -65,6 +65,7 @@ import { RuleTaskState, AlertSummary, RuleExecutionStatusValues, + RuleLastRunOutcomeValues, RuleNotifyWhenType, RuleTypeParams, ResolvedSanitizedRule, @@ -74,6 +75,7 @@ import { RuleSnooze, RuleSnoozeSchedule, RawAlertInstance as RawAlert, + RawRuleMonitoring, } from '../types'; import { validateRuleTypeParams, @@ -83,6 +85,10 @@ import { convertRuleIdsToKueryNode, getRuleSnoozeEndTime, convertEsSortToEventLogSort, + getDefaultRawRuleMonitoring, + updateMonitoring, + convertMonitoringFromRawAndVerify, + getNextRunString, } from '../lib'; import { taskInstanceToAlertTaskInstance } from '../task_runner/alert_task_instance'; import { RegistryRuleType, UntypedNormalizedRuleType } from '../rule_type_registry'; @@ -113,7 +119,6 @@ import { getRuleExecutionStatusPending } from '../lib/rule_execution_status'; import { Alert } from '../alert'; import { EVENT_LOG_ACTIONS } from '../plugin'; import { createAlertEventLogRecordObject } from '../lib/create_alert_event_log_record_object'; -import { getDefaultRuleMonitoring } from '../task_runner/task_runner'; import { getMappedParams, getModifiedField, @@ -153,6 +158,12 @@ export interface RuleAggregation { doc_count: number; }>; }; + outcome: { + buckets: Array<{ + key: string; + doc_count: number; + }>; + }; muted: { buckets: Array<{ key: number; @@ -343,6 +354,7 @@ interface IndexType { export interface AggregateResult { alertExecutionStatus: { [status: string]: number }; + ruleLastRunOutcome: { [status: string]: number }; ruleEnabledStatus?: { enabled: number; disabled: number }; ruleMutedStatus?: { muted: number; unmuted: number }; ruleSnoozedStatus?: { snoozed: number }; @@ -372,6 +384,9 @@ export interface CreateOptions { | 'executionStatus' | 'snoozeSchedule' | 'isSnoozedUntil' + | 'lastRun' + | 'nextRun' + | 'running' > & { actions: NormalizedAlertAction[] }; options?: { id?: string; @@ -586,6 +601,7 @@ export class RulesClient { } = await this.extractReferences(ruleType, data.actions, validatedAlertTypeParams); const createTime = Date.now(); + const lastRunTimestamp = new Date(); const legacyId = Semver.lt(this.kibanaVersion, '8.0.0') ? id : null; const notifyWhen = getRuleNotifyWhenType(data.notifyWhen, data.throttle); @@ -603,8 +619,9 @@ export class RulesClient { muteAll: false, mutedInstanceIds: [], notifyWhen, - executionStatus: getRuleExecutionStatusPending(new Date().toISOString()), - monitoring: getDefaultRuleMonitoring(), + running: false, + executionStatus: getRuleExecutionStatusPending(lastRunTimestamp.toISOString()), + monitoring: getDefaultRawRuleMonitoring(lastRunTimestamp.toISOString()), }; const mappedParams = getMappedParams(updatedParams); @@ -1358,6 +1375,9 @@ export class RulesClient { status: { terms: { field: 'alert.attributes.executionStatus.status' }, }, + outcome: { + terms: { field: 'alert.attributes.lastRun.outcome' }, + }, enabled: { terms: { field: 'alert.attributes.enabled' }, }, @@ -1388,6 +1408,7 @@ export class RulesClient { // Return a placeholder with all zeroes const placeholder: AggregateResult = { alertExecutionStatus: {}, + ruleLastRunOutcome: {}, ruleEnabledStatus: { enabled: 0, disabled: 0, @@ -1412,11 +1433,21 @@ export class RulesClient { }) ); + const ruleLastRunOutcome = resp.aggregations.outcome.buckets.map( + ({ key, doc_count: docCount }) => ({ + [key]: docCount, + }) + ); + const ret: AggregateResult = { alertExecutionStatus: alertExecutionStatus.reduce( (acc, curr: { [status: string]: number }) => Object.assign(acc, curr), {} ), + ruleLastRunOutcome: ruleLastRunOutcome.reduce( + (acc, curr: { [status: string]: number }) => Object.assign(acc, curr), + {} + ), }; // Fill missing keys with zeroes @@ -1425,6 +1456,11 @@ export class RulesClient { ret.alertExecutionStatus[key] = 0; } } + for (const key of RuleLastRunOutcomeValues) { + if (!ret.ruleLastRunOutcome.hasOwnProperty(key)) { + ret.ruleLastRunOutcome[key] = 0; + } + } const enabledBuckets = resp.aggregations.enabled.buckets; ret.ruleEnabledStatus = { @@ -2525,17 +2561,29 @@ export class RulesClient { if (attributes.enabled === false) { const username = await this.getUserName(); + const now = new Date(); + + const schedule = attributes.schedule as IntervalSchedule; const updateAttributes = this.updateMeta({ ...attributes, ...(!existingApiKey && (await this.createNewAPIKeySet({ attributes, username }))), + ...(attributes.monitoring && { + monitoring: updateMonitoring({ + monitoring: attributes.monitoring, + timestamp: now.toISOString(), + duration: 0, + }), + }), + running: false, + nextRun: getNextRunString(schedule.interval), enabled: true, updatedBy: username, - updatedAt: new Date().toISOString(), + updatedAt: now.toISOString(), executionStatus: { status: 'pending', lastDuration: 0, - lastExecutionDate: new Date().toISOString(), + lastExecutionDate: now.toISOString(), error: null, warning: null, }, @@ -3361,6 +3409,8 @@ export class RulesClient { scheduledTaskId, params, executionStatus, + monitoring, + nextRun, schedule, actions, snoozeSchedule, @@ -3412,6 +3462,10 @@ export class RulesClient { ...(executionStatus ? { executionStatus: ruleExecutionStatusFromRaw(this.logger, id, executionStatus) } : {}), + ...(monitoring + ? { monitoring: convertMonitoringFromRawAndVerify(this.logger, id, monitoring) } + : {}), + ...(nextRun ? { nextRun: new Date(nextRun) } : {}), }; return includeLegacyId diff --git a/x-pack/plugins/alerting/server/saved_objects/index.ts b/x-pack/plugins/alerting/server/saved_objects/index.ts index 31f6f96d67c8b..d8d53f4978d55 100644 --- a/x-pack/plugins/alerting/server/saved_objects/index.ts +++ b/x-pack/plugins/alerting/server/saved_objects/index.ts @@ -36,6 +36,9 @@ export const AlertAttributesExcludedFromAAD = [ 'snoozeEndTime', // field removed in 8.2, but must be retained in case an rule created/updated in 8.2 is being migrated 'snoozeSchedule', 'isSnoozedUntil', + 'lastRun', + 'nextRun', + 'running', ]; // useful for Pick which is a @@ -52,7 +55,10 @@ export type AlertAttributesExcludedFromAADType = | 'monitoring' | 'snoozeEndTime' | 'snoozeSchedule' - | 'isSnoozedUntil'; + | 'isSnoozedUntil' + | 'lastRun' + | 'nextRun' + | 'running'; export function setupSavedObjects( savedObjects: SavedObjectsServiceSetup, diff --git a/x-pack/plugins/alerting/server/saved_objects/mappings.ts b/x-pack/plugins/alerting/server/saved_objects/mappings.ts index 0c8fd5ab65d88..dd50d14b6c622 100644 --- a/x-pack/plugins/alerting/server/saved_objects/mappings.ts +++ b/x-pack/plugins/alerting/server/saved_objects/mappings.ts @@ -114,7 +114,6 @@ export const alertMappings: SavedObjectsTypeMappingDefinition = { }, monitoring: { properties: { - // TODO migrations run: { properties: { history: { @@ -255,7 +254,7 @@ export const alertMappings: SavedObjectsTypeMappingDefinition = { running: { type: 'boolean', }, - next_run: { + nextRun: { type: 'date', }, // Deprecated, if you need to add new property please do it in `last_run` @@ -295,21 +294,21 @@ export const alertMappings: SavedObjectsTypeMappingDefinition = { }, }, }, - last_run: { + lastRun: { properties: { outcome: { type: 'keyword', }, - outcome_order: { + outcomeOrder: { type: 'float', }, warning: { type: 'text', }, - outcome_msg: { + outcomeMsg: { type: 'text', }, - alerts_count: { + alertsCount: { properties: { active: { type: 'float', diff --git a/x-pack/plugins/alerting/server/saved_objects/migrations/8.6/index.ts b/x-pack/plugins/alerting/server/saved_objects/migrations/8.6/index.ts index 1fec1c76430eb..836ff6983513b 100644 --- a/x-pack/plugins/alerting/server/saved_objects/migrations/8.6/index.ts +++ b/x-pack/plugins/alerting/server/saved_objects/migrations/8.6/index.ts @@ -4,3 +4,88 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ + +import { SavedObjectUnsanitizedDoc } from '@kbn/core-saved-objects-server'; +import { EncryptedSavedObjectsPluginSetup } from '@kbn/encrypted-saved-objects-plugin/server'; +import { createEsoMigration, pipeMigrations } from '../utils'; +import { RawRule, RuleLastRunOutcomeValues } from '../../../types'; + +const succeededStatus = ['ok', 'active', 'succeeded']; +const warningStatus = ['warning']; +const failedStatus = ['error', 'failed']; + +const getLastRun = (attributes: RawRule) => { + const { executionStatus } = attributes; + const { status, warning, error } = executionStatus; + + let outcome; + if (succeededStatus.includes(status)) { + outcome = RuleLastRunOutcomeValues[0]; + } else if (warningStatus.includes(status) || warning) { + outcome = RuleLastRunOutcomeValues[1]; + } else if (failedStatus.includes(status) || error) { + outcome = RuleLastRunOutcomeValues[2]; + } + + // Don't set last run if status is unknown or pending, let the + // task runner do it instead + if (!outcome) { + return null; + } + + return { + outcome, + outcomeMsg: warning?.message || error?.message || null, + warning: warning?.reason || null, + alertsCount: {}, + }; +}; + +const getMonitoring = (attributes: RawRule) => { + const { executionStatus, monitoring } = attributes; + if (!monitoring) { + return null; + } + + // Question: Do we want to backfill the history? + const { lastExecutionDate, lastDuration } = executionStatus; + + return { + run: { + ...monitoring.run, + last_run: { + timestamp: lastExecutionDate, + metrics: { + duration: lastDuration || 0, + }, + }, + }, + }; +}; + +function migrateLastRun( + doc: SavedObjectUnsanitizedDoc +): SavedObjectUnsanitizedDoc { + const { attributes } = doc; + const lastRun = getLastRun(attributes); + const monitoring = getMonitoring(attributes); + + // Question: Do we want to migrate running and next_rule? It might be + // very unreliable since we don't know for sure (long running rules) when the rule finishes running + // and therefore next_run would be incorrect. + return { + ...doc, + attributes: { + ...attributes, + ...(lastRun ? { lastRun } : {}), + ...(monitoring ? { monitoring } : {}), + }, + }; +} + +export const getMigrations860 = (encryptedSavedObjects: EncryptedSavedObjectsPluginSetup) => + createEsoMigration( + encryptedSavedObjects, + (doc: SavedObjectUnsanitizedDoc): doc is SavedObjectUnsanitizedDoc => true, + pipeMigrations(migrateLastRun) + ); diff --git a/x-pack/plugins/alerting/server/saved_objects/migrations/index.ts b/x-pack/plugins/alerting/server/saved_objects/migrations/index.ts index a3a06614ec4c5..b44e6e4c09c6e 100644 --- a/x-pack/plugins/alerting/server/saved_objects/migrations/index.ts +++ b/x-pack/plugins/alerting/server/saved_objects/migrations/index.ts @@ -28,6 +28,7 @@ import { getMigrations820 } from './8.2'; import { getMigrations830 } from './8.3'; import { getMigrations841 } from './8.4'; import { getMigrations850 } from './8.5'; +import { getMigrations860 } from './8.6'; import { AlertLogMeta, AlertMigration } from './types'; import { MINIMUM_SS_MIGRATION_VERSION } from './constants'; import { createEsoMigration, isEsQueryRuleType, pipeMigrations } from './utils'; @@ -75,6 +76,7 @@ export function getMigrations( '8.3.0': executeMigrationWithErrorHandling(getMigrations830(encryptedSavedObjects), '8.3.0'), '8.4.1': executeMigrationWithErrorHandling(getMigrations841(encryptedSavedObjects), '8.4.1'), '8.5.0': executeMigrationWithErrorHandling(getMigrations850(encryptedSavedObjects), '8.5.0'), + '8.6.0': executeMigrationWithErrorHandling(getMigrations860(encryptedSavedObjects), '8.6.0'), }, getSearchSourceMigrations(encryptedSavedObjects, searchSourceMigrations) ); diff --git a/x-pack/plugins/alerting/server/task_runner/fixtures.ts b/x-pack/plugins/alerting/server/task_runner/fixtures.ts index 0a90456dbdad6..cb8da20372d5e 100644 --- a/x-pack/plugins/alerting/server/task_runner/fixtures.ts +++ b/x-pack/plugins/alerting/server/task_runner/fixtures.ts @@ -67,7 +67,7 @@ export const generateSavedObjectParams = ({ '1', { monitoring: { - execution: { + run: { calculated_metrics: { success_ratio: 1, }, @@ -217,7 +217,7 @@ export const generateRunnerResult = ({ }: GeneratorParams = {}) => { return { monitoring: { - execution: { + run: { calculated_metrics: { success_ratio: successRatio, }, diff --git a/x-pack/plugins/alerting/server/task_runner/rule_loader.ts b/x-pack/plugins/alerting/server/task_runner/rule_loader.ts index 75fcff9f4510f..8d4b00a54e094 100644 --- a/x-pack/plugins/alerting/server/task_runner/rule_loader.ts +++ b/x-pack/plugins/alerting/server/task_runner/rule_loader.ts @@ -79,9 +79,9 @@ export async function loadRule(params: LoadRulePa } if (rule.monitoring) { - if (rule.monitoring.execution.history.length >= MONITORING_HISTORY_LIMIT) { + if (rule.monitoring.run.history.length >= MONITORING_HISTORY_LIMIT) { // Remove the first (oldest) record - rule.monitoring.execution.history.shift(); + rule.monitoring.run.history.shift(); } } diff --git a/x-pack/plugins/alerting/server/task_runner/task_runner.ts b/x-pack/plugins/alerting/server/task_runner/task_runner.ts index 5f794a5c821f7..244252260561c 100644 --- a/x-pack/plugins/alerting/server/task_runner/task_runner.ts +++ b/x-pack/plugins/alerting/server/task_runner/task_runner.ts @@ -23,6 +23,8 @@ import { ruleExecutionStatusToRaw, isRuleSnoozed, processAlerts, + lastRunFromError, + getNextRunString, } from '../lib'; import { Rule, @@ -35,8 +37,9 @@ import { RuleMonitoring, RuleTaskState, RuleTypeRegistry, + RawRuleLastRun, } from '../types'; -import { asErr, asOk, map, resolveErr, Result } from '../lib/result_type'; +import { asErr, asOk, isOk, map, resolveErr, Result } from '../lib/result_type'; import { taskInstanceToAlertTaskInstance } from './alert_task_instance'; import { isAlertSavedObjectNotFoundError, isEsUnavailableError } from '../lib/is_alerting_error'; import { partiallyUpdateAlert } from '../saved_objects'; @@ -68,6 +71,7 @@ import { scheduleActionsForAlerts } from './schedule_actions_for_alerts'; import { getPublicAlertFactory } from '../alert/create_alert_factory'; import { TaskRunnerTimer, TaskRunnerTimerSpan } from './task_runner_timer'; import { RuleMonitoringService } from '../monitoring/rule_monitoring_client'; +import { ILastRun, lastRunFromState, lastRunToRaw } from '../lib/last_run_status'; const FALLBACK_RETRY_INTERVAL = '5m'; const CONNECTIVITY_RETRY_INTERVAL = '5m'; @@ -190,10 +194,14 @@ export class TaskRunner< private async updateRuleSavedObject( ruleId: string, namespace: string | undefined, - attributes: { executionStatus?: RawRuleExecutionStatus; monitoring?: RuleMonitoring } + attributes: { + executionStatus?: RawRuleExecutionStatus; + monitoring?: RuleMonitoring; + nextRun?: string | null; + lastRun?: RawRuleLastRun | null; + running?: boolean; } ) { const client = this.context.internalSavedObjectsRepository; - try { await partiallyUpdateAlert(client, ruleId, attributes, { ignore404: true, @@ -576,9 +584,11 @@ export class TaskRunner< } private async processRunResults({ + nextRun, runDate, stateWithMetrics, }: { + nextRun: string| null; runDate: Date; stateWithMetrics: Result; }) { @@ -588,7 +598,8 @@ export class TaskRunner< const namespace = this.context.spaceIdToNamespace(spaceId); - const { status: executionStatus, metrics: executionMetrics } = map< + // Getting executionStatus for backwards compatibility + const { status: executionStatus } = map< RuleTaskStateAndMetrics, ElasticsearchError, IExecutionStatusAndMetrics @@ -598,16 +609,34 @@ export class TaskRunner< (err: ElasticsearchError) => executionStatusFromError(err, runDate) ); + // New consolidated statuses for lastRun + const { lastRun, metrics: executionMetrics } = map< + RuleTaskStateAndMetrics, + ElasticsearchError, + ILastRun + >( + stateWithMetrics, + (ruleRunStateWithMetrics) => lastRunFromState(ruleRunStateWithMetrics), + (err: ElasticsearchError) => lastRunFromError(err) + ); + if (apm.currentTransaction) { if (executionStatus.status === 'ok' || executionStatus.status === 'active') { apm.currentTransaction.setOutcome('success'); } else if (executionStatus.status === 'error' || executionStatus.status === 'unknown') { apm.currentTransaction.setOutcome('failure'); + } else if (lastRun.outcome === 'succeeded') { + apm.currentTransaction.setOutcome('success'); + } else if (lastRun.outcome === 'failed') { + apm.currentTransaction.setOutcome('failure'); } } this.logger.debug( - `ruleRunStatus for ${this.ruleType.id}:${ruleId}: ${JSON.stringify(executionStatus)}` + `deprecated ruleRunStatus for ${this.ruleType.id}:${ruleId}: ${JSON.stringify(executionStatus)}` + ); + this.logger.debug( + `ruleRunStatus for ${this.ruleType.id}:${ruleId}: ${JSON.stringify(lastRun)}` ); if (executionMetrics) { this.logger.debug( @@ -625,20 +654,30 @@ export class TaskRunner< } // if executionStatus indicates an error, fill in fields in - this.ruleMonitoring.addHistory(executionStatus.lastDuration, executionStatus.error != null); + this.ruleMonitoring.addHistory({ + duration: executionStatus.lastDuration, + hasError: executionStatus.error != null, + runDate + }); if (!this.cancelled) { this.inMemoryMetrics.increment(IN_MEMORY_METRICS.RULE_EXECUTIONS); - if (executionStatus.error) { + if (lastRun.outcome === 'failed') { + this.inMemoryMetrics.increment(IN_MEMORY_METRICS.RULE_FAILURES); + } else if (executionStatus.error) { this.inMemoryMetrics.increment(IN_MEMORY_METRICS.RULE_FAILURES); } this.logger.debug( `Updating rule task for ${this.ruleType.id} rule with id ${ruleId} - ${JSON.stringify( executionStatus + )} - ${JSON.stringify( + lastRun )}` ); await this.updateRuleSavedObject(ruleId, namespace, { executionStatus: ruleExecutionStatusToRaw(executionStatus), + nextRun, + lastRun: lastRunToRaw(lastRun), monitoring: this.ruleMonitoring.getMonitoring(), }); } @@ -681,10 +720,13 @@ export class TaskRunner< schedule = asErr(err); } + const nextRunString = isOk(schedule) ? getNextRunString(schedule.value.interval) : null; + const { executionStatus, executionMetrics } = await this.timer.runWithTimer( TaskRunnerTimerSpan.ProcessRuleRun, async () => this.processRunResults({ + nextRun: nextRunString, runDate, stateWithMetrics, }) @@ -770,6 +812,7 @@ export class TaskRunner< // Write event log entry const { params: { alertId: ruleId, spaceId, consumer }, + schedule: taskSchedule, } = this.taskInstance; const namespace = this.context.spaceIdToNamespace(spaceId); @@ -790,13 +833,16 @@ export class TaskRunner< this.inMemoryMetrics.increment(IN_MEMORY_METRICS.RULE_TIMEOUTS); + const nextRunString = taskSchedule ? getNextRunString(taskSchedule.interval) : null; + const outcomeMsg = `${this.ruleType.id}:${ruleId}: execution cancelled due to timeout - exceeded rule type timeout of ${this.ruleType.ruleTaskTimeout}`; + const date = new Date(); // Update the rule saved object with execution status const executionStatus: RuleExecutionStatus = { - lastExecutionDate: new Date(), + lastExecutionDate: date, status: 'error', error: { reason: RuleExecutionStatusErrorReasons.Timeout, - message: `${this.ruleType.id}:${ruleId}: execution cancelled due to timeout - exceeded rule type timeout of ${this.ruleType.ruleTaskTimeout}`, + message: outcomeMsg, }, }; this.logger.debug( @@ -804,6 +850,15 @@ export class TaskRunner< ); await this.updateRuleSavedObject(ruleId, namespace, { executionStatus: ruleExecutionStatusToRaw(executionStatus), + lastRun: { + outcome: 'failed', + warning: RuleExecutionStatusErrorReasons.Timeout, + outcomeMsg, + alertsCount: {}, + }, + monitoring: this.ruleMonitoring.getMonitoring(), + running: false, + nextRun: nextRunString && new Date(nextRunString).getTime() > date.getTime() ? nextRunString : null, }); } } diff --git a/x-pack/plugins/alerting/server/types.ts b/x-pack/plugins/alerting/server/types.ts index 5a4c9d0e79db6..1b9c97074aa71 100644 --- a/x-pack/plugins/alerting/server/types.ts +++ b/x-pack/plugins/alerting/server/types.ts @@ -43,6 +43,7 @@ import { RuleMonitoring, MappedParams, RuleSnooze, + RuleLastRun, } from '../common'; import { PublicAlertFactory } from './alert/create_alert_factory'; export type WithoutQueryAndParams = Pick>; @@ -289,3 +290,5 @@ export interface RuleMonitoringService { setLastRunMetricsTotalAlertCreated: (totalAlertCreated: number) => void; setLastRunMetricsGapDurationS: (gapDurationS: number) => void; } + +export interface RawRuleLastRun extends SavedObjectAttributes, RuleLastRun {} From 6b4531ff9b51f88dd1abe7f91e56f96f60bac212 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 1 Nov 2022 19:34:41 +0000 Subject: [PATCH 09/27] [CI] Auto-commit changed files from 'node scripts/precommit_hook.js --ref HEAD~1..HEAD --fix' --- x-pack/plugins/alerting/server/lib/index.ts | 4 +--- .../plugins/alerting/server/lib/monitoring.ts | 1 - .../monitoring/rule_monitoring_client.ts | 16 +++++++++++++--- .../alerting/server/task_runner/task_runner.ts | 18 ++++++++++-------- 4 files changed, 24 insertions(+), 15 deletions(-) diff --git a/x-pack/plugins/alerting/server/lib/index.ts b/x-pack/plugins/alerting/server/lib/index.ts index 5ea4207068a7f..64e28ce990c8d 100644 --- a/x-pack/plugins/alerting/server/lib/index.ts +++ b/x-pack/plugins/alerting/server/lib/index.ts @@ -26,9 +26,7 @@ export { ruleExecutionStatusFromRaw, } from './rule_execution_status'; export { lastRunFromState, lastRunFromError, lastRunToRaw } from './last_run_status'; -export { - updateMonitoring, -} from './monitoring'; +export { updateMonitoring } from './monitoring'; export { getNextRunDate, getNextRunString } from './next_run'; export { processAlerts } from './process_alerts'; export { createWrappedScopedClusterClientFactory } from './wrap_scoped_cluster_client'; diff --git a/x-pack/plugins/alerting/server/lib/monitoring.ts b/x-pack/plugins/alerting/server/lib/monitoring.ts index be46803bd3e5d..d794380c545a1 100644 --- a/x-pack/plugins/alerting/server/lib/monitoring.ts +++ b/x-pack/plugins/alerting/server/lib/monitoring.ts @@ -16,7 +16,6 @@ export const INITIAL_METRICS = { gap_duration_s: null, }; - // Immutably updates the monitoring object with timestamp and duration. // Used when converting from and between raw monitoring object export const updateMonitoring = ({ diff --git a/x-pack/plugins/alerting/server/monitoring/rule_monitoring_client.ts b/x-pack/plugins/alerting/server/monitoring/rule_monitoring_client.ts index ef0da102c3100..332cdc4396c85 100644 --- a/x-pack/plugins/alerting/server/monitoring/rule_monitoring_client.ts +++ b/x-pack/plugins/alerting/server/monitoring/rule_monitoring_client.ts @@ -37,7 +37,15 @@ export class RuleMonitoringService { return this.monitoring; } - public addHistory({ duration, hasError = true, runDate }: {duration: number | undefined; hasError: boolean; runDate: Date}) { + public addHistory({ + duration, + hasError = true, + runDate, + }: { + duration: number | undefined; + hasError: boolean; + runDate: Date; + }) { const date = runDate ?? new Date(); const monitoringHistory: RuleMonitoringHistory = { success: true, @@ -59,8 +67,10 @@ export class RuleMonitoringService { public getLastRunMetricsSetters() { return { - setLastRunMetricsTotalSearchDurationMs: this.setLastRunMetricsTotalSearchDurationMs.bind(this), - setLastRunMetricsTotalIndexingDurationMs: this.setLastRunMetricsTotalIndexingDurationMs.bind(this), + setLastRunMetricsTotalSearchDurationMs: + this.setLastRunMetricsTotalSearchDurationMs.bind(this), + setLastRunMetricsTotalIndexingDurationMs: + this.setLastRunMetricsTotalIndexingDurationMs.bind(this), setLastRunMetricsTotalAlertDetected: this.setLastRunMetricsTotalAlertDetected.bind(this), setLastRunMetricsTotalAlertCreated: this.setLastRunMetricsTotalAlertCreated.bind(this), setLastRunMetricsGapDurationS: this.setLastRunMetricsGapDurationS.bind(this), diff --git a/x-pack/plugins/alerting/server/task_runner/task_runner.ts b/x-pack/plugins/alerting/server/task_runner/task_runner.ts index 244252260561c..f118460a8aa74 100644 --- a/x-pack/plugins/alerting/server/task_runner/task_runner.ts +++ b/x-pack/plugins/alerting/server/task_runner/task_runner.ts @@ -199,7 +199,8 @@ export class TaskRunner< monitoring?: RuleMonitoring; nextRun?: string | null; lastRun?: RawRuleLastRun | null; - running?: boolean; } + running?: boolean; + } ) { const client = this.context.internalSavedObjectsRepository; try { @@ -588,7 +589,7 @@ export class TaskRunner< runDate, stateWithMetrics, }: { - nextRun: string| null; + nextRun: string | null; runDate: Date; stateWithMetrics: Result; }) { @@ -633,7 +634,9 @@ export class TaskRunner< } this.logger.debug( - `deprecated ruleRunStatus for ${this.ruleType.id}:${ruleId}: ${JSON.stringify(executionStatus)}` + `deprecated ruleRunStatus for ${this.ruleType.id}:${ruleId}: ${JSON.stringify( + executionStatus + )}` ); this.logger.debug( `ruleRunStatus for ${this.ruleType.id}:${ruleId}: ${JSON.stringify(lastRun)}` @@ -657,7 +660,7 @@ export class TaskRunner< this.ruleMonitoring.addHistory({ duration: executionStatus.lastDuration, hasError: executionStatus.error != null, - runDate + runDate, }); if (!this.cancelled) { @@ -670,9 +673,7 @@ export class TaskRunner< this.logger.debug( `Updating rule task for ${this.ruleType.id} rule with id ${ruleId} - ${JSON.stringify( executionStatus - )} - ${JSON.stringify( - lastRun - )}` + )} - ${JSON.stringify(lastRun)}` ); await this.updateRuleSavedObject(ruleId, namespace, { executionStatus: ruleExecutionStatusToRaw(executionStatus), @@ -858,7 +859,8 @@ export class TaskRunner< }, monitoring: this.ruleMonitoring.getMonitoring(), running: false, - nextRun: nextRunString && new Date(nextRunString).getTime() > date.getTime() ? nextRunString : null, + nextRun: + nextRunString && new Date(nextRunString).getTime() > date.getTime() ? nextRunString : null, }); } } From 6430691c287b7f1feb81a4d20d95b83ec2b14ceb Mon Sep 17 00:00:00 2001 From: Jiawei Wu Date: Tue, 1 Nov 2022 14:28:51 -0700 Subject: [PATCH 10/27] Fix some of the missing functions used in rules_client --- x-pack/plugins/alerting/common/rule.ts | 4 +- x-pack/plugins/alerting/server/lib/index.ts | 6 ++- .../plugins/alerting/server/lib/monitoring.ts | 43 ++++++++++++++++++- .../monitoring/rule_monitoring_client.ts | 17 ++------ .../server/rules_client/rules_client.ts | 7 ++- .../server/task_runner/task_runner.ts | 8 ++-- x-pack/plugins/alerting/server/types.ts | 6 ++- 7 files changed, 63 insertions(+), 28 deletions(-) diff --git a/x-pack/plugins/alerting/common/rule.ts b/x-pack/plugins/alerting/common/rule.ts index 30b35b7043ea7..8ea4de66d98ca 100644 --- a/x-pack/plugins/alerting/common/rule.ts +++ b/x-pack/plugins/alerting/common/rule.ts @@ -203,7 +203,7 @@ export interface RuleMonitoringCalculatedMetrics extends SavedObjectAttributes { success_ratio: number; } -export interface RuleMonitoringLastRunMetrics { +export interface RuleMonitoringLastRunMetrics extends SavedObjectAttributes { duration: number; total_search_duration_ms?: number | null; total_indexing_duration_ms?: number | null; @@ -212,7 +212,7 @@ export interface RuleMonitoringLastRunMetrics { gap_duration_s?: number | null; } -export interface RuleMonitoringLastRun { +export interface RuleMonitoringLastRun extends SavedObjectAttributes { timestamp: string; metrics: RuleMonitoringLastRunMetrics; } diff --git a/x-pack/plugins/alerting/server/lib/index.ts b/x-pack/plugins/alerting/server/lib/index.ts index 64e28ce990c8d..00081903d9506 100644 --- a/x-pack/plugins/alerting/server/lib/index.ts +++ b/x-pack/plugins/alerting/server/lib/index.ts @@ -26,7 +26,11 @@ export { ruleExecutionStatusFromRaw, } from './rule_execution_status'; export { lastRunFromState, lastRunFromError, lastRunToRaw } from './last_run_status'; -export { updateMonitoring } from './monitoring'; +export { + updateMonitoring, + getDefaultMonitoring, + convertMonitoringFromRawAndVerify, +} from './monitoring'; export { getNextRunDate, getNextRunString } from './next_run'; export { processAlerts } from './process_alerts'; export { createWrappedScopedClusterClientFactory } from './wrap_scoped_cluster_client'; diff --git a/x-pack/plugins/alerting/server/lib/monitoring.ts b/x-pack/plugins/alerting/server/lib/monitoring.ts index d794380c545a1..b9f42e3bb8175 100644 --- a/x-pack/plugins/alerting/server/lib/monitoring.ts +++ b/x-pack/plugins/alerting/server/lib/monitoring.ts @@ -4,8 +4,8 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - -import { RuleMonitoring } from '../types'; +import { Logger } from '@kbn/core/server'; +import { RuleMonitoring, RawRuleMonitoring } from '../types'; export const INITIAL_METRICS = { duration: 0, @@ -16,6 +16,21 @@ export const INITIAL_METRICS = { gap_duration_s: null, }; +export const getDefaultMonitoring = (timestamp: string) => { + return { + run: { + history: [], + calculated_metrics: { + success_ratio: 0, + }, + last_run: { + timestamp, + metrics: INITIAL_METRICS, + }, + }, + }; +}; + // Immutably updates the monitoring object with timestamp and duration. // Used when converting from and between raw monitoring object export const updateMonitoring = ({ @@ -44,3 +59,27 @@ export const updateMonitoring = ({ }, }; }; + +export const convertMonitoringFromRawAndVerify = ( + logger: Logger, + ruleId: string, + monitoring: RawRuleMonitoring +): RuleMonitoring | undefined => { + if (!monitoring) { + return undefined; + } + + const lastRunDate = monitoring.run.last_run.timestamp; + + let parsedDateMillis = lastRunDate ? Date.parse(lastRunDate) : Date.now(); + if (isNaN(parsedDateMillis)) { + logger.debug(`invalid monitoring last_run.timestamp "${lastRunDate}" in raw rule ${ruleId}`); + parsedDateMillis = Date.now(); + } + + return updateMonitoring({ + monitoring, + timestamp: new Date(parsedDateMillis).toISOString(), + duration: monitoring.run.last_run.metrics.duration || 0, + }); +}; diff --git a/x-pack/plugins/alerting/server/monitoring/rule_monitoring_client.ts b/x-pack/plugins/alerting/server/monitoring/rule_monitoring_client.ts index 332cdc4396c85..d1726452c7696 100644 --- a/x-pack/plugins/alerting/server/monitoring/rule_monitoring_client.ts +++ b/x-pack/plugins/alerting/server/monitoring/rule_monitoring_client.ts @@ -6,22 +6,11 @@ */ import stats from 'stats-lite'; -import { INITIAL_METRICS } from '../lib/monitoring'; +import { getDefaultMonitoring } from '../lib/monitoring'; import { RuleMonitoring, RuleMonitoringHistory } from '../types'; export class RuleMonitoringService { - private monitoring: RuleMonitoring = { - run: { - history: [], - calculated_metrics: { - success_ratio: 0, - }, - last_run: { - timestamp: new Date().toISOString(), - metrics: INITIAL_METRICS, - }, - }, - }; + private monitoring: RuleMonitoring = getDefaultMonitoring(new Date().toISOString()); public setLastRunMetricsDuration(duration: number) { this.monitoring.run.last_run.metrics.duration = duration; @@ -33,7 +22,7 @@ export class RuleMonitoringService { } } - public getMonitoring() { + public getMonitoring(): RuleMonitoring { return this.monitoring; } diff --git a/x-pack/plugins/alerting/server/rules_client/rules_client.ts b/x-pack/plugins/alerting/server/rules_client/rules_client.ts index 83da238dede2e..3062f9544f615 100644 --- a/x-pack/plugins/alerting/server/rules_client/rules_client.ts +++ b/x-pack/plugins/alerting/server/rules_client/rules_client.ts @@ -75,7 +75,6 @@ import { RuleSnooze, RuleSnoozeSchedule, RawAlertInstance as RawAlert, - RawRuleMonitoring, } from '../types'; import { validateRuleTypeParams, @@ -85,7 +84,7 @@ import { convertRuleIdsToKueryNode, getRuleSnoozeEndTime, convertEsSortToEventLogSort, - getDefaultRawRuleMonitoring, + getDefaultMonitoring, updateMonitoring, convertMonitoringFromRawAndVerify, getNextRunString, @@ -627,7 +626,7 @@ export class RulesClient { notifyWhen, running: false, executionStatus: getRuleExecutionStatusPending(lastRunTimestamp.toISOString()), - monitoring: getDefaultRawRuleMonitoring(lastRunTimestamp.toISOString()), + monitoring: getDefaultMonitoring(lastRunTimestamp.toISOString()), }; const mappedParams = getMappedParams(updatedParams); @@ -2652,7 +2651,7 @@ export class RulesClient { ...attributes, ...(!existingApiKey && (await this.createNewAPIKeySet({ attributes, username }))), ...(attributes.monitoring && { - monitoring: updateMonitoring({ + monitoring: updateMonitoring({ monitoring: attributes.monitoring, timestamp: now.toISOString(), duration: 0, diff --git a/x-pack/plugins/alerting/server/task_runner/task_runner.ts b/x-pack/plugins/alerting/server/task_runner/task_runner.ts index f118460a8aa74..8d3d2b0d2a3fe 100644 --- a/x-pack/plugins/alerting/server/task_runner/task_runner.ts +++ b/x-pack/plugins/alerting/server/task_runner/task_runner.ts @@ -34,7 +34,7 @@ import { RawAlertInstance, RawRule, RawRuleExecutionStatus, - RuleMonitoring, + RawRuleMonitoring, RuleTaskState, RuleTypeRegistry, RawRuleLastRun, @@ -196,7 +196,7 @@ export class TaskRunner< namespace: string | undefined, attributes: { executionStatus?: RawRuleExecutionStatus; - monitoring?: RuleMonitoring; + monitoring?: RawRuleMonitoring; nextRun?: string | null; lastRun?: RawRuleLastRun | null; running?: boolean; @@ -679,7 +679,7 @@ export class TaskRunner< executionStatus: ruleExecutionStatusToRaw(executionStatus), nextRun, lastRun: lastRunToRaw(lastRun), - monitoring: this.ruleMonitoring.getMonitoring(), + monitoring: this.ruleMonitoring.getMonitoring() as RawRuleMonitoring, }); } @@ -857,7 +857,7 @@ export class TaskRunner< outcomeMsg, alertsCount: {}, }, - monitoring: this.ruleMonitoring.getMonitoring(), + monitoring: this.ruleMonitoring.getMonitoring() as RawRuleMonitoring, running: false, nextRun: nextRunString && new Date(nextRunString).getTime() > date.getTime() ? nextRunString : null, diff --git a/x-pack/plugins/alerting/server/types.ts b/x-pack/plugins/alerting/server/types.ts index 1b9c97074aa71..d7f2b446954bc 100644 --- a/x-pack/plugins/alerting/server/types.ts +++ b/x-pack/plugins/alerting/server/types.ts @@ -251,9 +251,12 @@ export interface RawRule extends SavedObjectAttributes { mutedInstanceIds: string[]; meta?: RuleMeta; executionStatus: RawRuleExecutionStatus; - monitoring?: RuleMonitoring; + monitoring?: RawRuleMonitoring; snoozeSchedule?: RuleSnooze; // Remove ? when this parameter is made available in the public API isSnoozedUntil?: string | null; + lastRun?: RawRuleLastRun | null; + nextRun?: string | null; + running: boolean; } export interface AlertingPlugin { @@ -292,3 +295,4 @@ export interface RuleMonitoringService { } export interface RawRuleLastRun extends SavedObjectAttributes, RuleLastRun {} +export interface RawRuleMonitoring extends SavedObjectAttributes, RuleMonitoring {} From 4c57e090784dea596f33488307fc5bff905e3584 Mon Sep 17 00:00:00 2001 From: Jiawei Wu Date: Wed, 2 Nov 2022 17:21:00 -0700 Subject: [PATCH 11/27] Fix types and unit tests --- .../plugins/alerting/public/alert_api.test.ts | 3 + .../public/lib/common_transformations.test.ts | 4 ++ .../public/lib/common_transformations.ts | 2 +- .../lib/alert_summary_from_event_log.test.ts | 1 + .../alerting/server/lib/monitoring.test.ts | 10 +-- .../plugins/alerting/server/lib/monitoring.ts | 22 +++++- .../monitoring/rule_monitoring_client.ts | 20 +----- .../server/routes/aggregate_rules.test.ts | 25 +++++++ .../server/routes/bulk_edit_rules.test.ts | 1 + .../server/routes/create_rule.test.ts | 2 + .../alerting/server/routes/get_rule.test.ts | 2 + .../server/routes/legacy/aggregate.test.ts | 15 ++++ .../server/routes/legacy/create.test.ts | 1 + .../alerting/server/routes/legacy/get.test.ts | 1 + .../server/routes/resolve_rule.test.ts | 2 + .../rules_client/tests/aggregate.test.ts | 18 +++++ .../server/rules_client/tests/create.test.ts | 68 +++++++++++++++---- .../server/rules_client/tests/enable.test.ts | 6 ++ .../tests/get_action_error_log.test.ts | 1 + .../tests/get_alert_summary.test.ts | 1 + .../tests/get_execution_log.test.ts | 1 + .../task_runner/alert_task_instance.test.ts | 1 + .../alerting/server/task_runner/fixtures.ts | 5 +- .../server/task_runner/rule_loader.test.ts | 6 +- .../server/task_runner/task_runner.test.ts | 10 +-- 25 files changed, 181 insertions(+), 47 deletions(-) diff --git a/x-pack/plugins/alerting/public/alert_api.test.ts b/x-pack/plugins/alerting/public/alert_api.test.ts index 2f09643e499e3..8ee76f61421f5 100644 --- a/x-pack/plugins/alerting/public/alert_api.test.ts +++ b/x-pack/plugins/alerting/public/alert_api.test.ts @@ -131,6 +131,7 @@ describe('loadRule', () => { "params": Object { "x": 42, }, + "running": false, "schedule": Object { "interval": "1s", }, @@ -289,6 +290,7 @@ function getApiRule() { last_execution_date: RuleExecuteDate.toISOString(), last_duration: 1194, }, + running: false, }; } @@ -330,5 +332,6 @@ function getRule(): Rule<{ x: number }> { lastExecutionDate: RuleExecuteDate, lastDuration: 1194, }, + running: false, }; } diff --git a/x-pack/plugins/alerting/public/lib/common_transformations.test.ts b/x-pack/plugins/alerting/public/lib/common_transformations.test.ts index 51d24538b449e..b9e9e27ba6f92 100644 --- a/x-pack/plugins/alerting/public/lib/common_transformations.test.ts +++ b/x-pack/plugins/alerting/public/lib/common_transformations.test.ts @@ -54,6 +54,7 @@ describe('common_transformations', () => { message: 'this is just a test', }, }, + running: false, }; expect(transformRule(apiRule)).toMatchInlineSnapshot(` Object { @@ -105,6 +106,7 @@ describe('common_transformations', () => { ], }, }, + "running": false, "schedule": Object { "interval": "1s", }, @@ -152,6 +154,7 @@ describe('common_transformations', () => { last_execution_date: dateExecuted.toISOString(), status: 'error', }, + running: false, }; expect(transformRule(apiRule)).toMatchInlineSnapshot(` Object { @@ -184,6 +187,7 @@ describe('common_transformations', () => { "name": "some-name", "notifyWhen": "onActiveAlert", "params": Object {}, + "running": false, "schedule": Object { "interval": "1s", }, diff --git a/x-pack/plugins/alerting/public/lib/common_transformations.ts b/x-pack/plugins/alerting/public/lib/common_transformations.ts index b58dd8922fd7b..d68bcaec29ecb 100644 --- a/x-pack/plugins/alerting/public/lib/common_transformations.ts +++ b/x-pack/plugins/alerting/public/lib/common_transformations.ts @@ -42,7 +42,7 @@ function transformMonitoring(input: RuleMonitoring): RuleMonitoring { return { run: { last_run: { - timestamp: new Date(input.run.last_run.timestamp), + timestamp: input.run.last_run.timestamp, ...restLastRun, }, ...rest, diff --git a/x-pack/plugins/alerting/server/lib/alert_summary_from_event_log.test.ts b/x-pack/plugins/alerting/server/lib/alert_summary_from_event_log.test.ts index 56a862f2ad6ca..1a22006fe2115 100644 --- a/x-pack/plugins/alerting/server/lib/alert_summary_from_event_log.test.ts +++ b/x-pack/plugins/alerting/server/lib/alert_summary_from_event_log.test.ts @@ -730,4 +730,5 @@ const BaseRule: SanitizedRule<{ bar: boolean }> = { status: 'unknown', lastExecutionDate: new Date('2020-08-20T19:23:38Z'), }, + running: false, }; diff --git a/x-pack/plugins/alerting/server/lib/monitoring.test.ts b/x-pack/plugins/alerting/server/lib/monitoring.test.ts index eeafdfcff1cbe..270b270dc0de7 100644 --- a/x-pack/plugins/alerting/server/lib/monitoring.test.ts +++ b/x-pack/plugins/alerting/server/lib/monitoring.test.ts @@ -42,7 +42,7 @@ const mockHistory = [ ]; const mockRuleMonitoring = { - execution: { + run: { history: mockHistory, calculated_metrics: { success_ratio: 0, @@ -52,7 +52,7 @@ const mockRuleMonitoring = { describe('getExecutionDurationPercentiles', () => { it('Calculates the percentile given partly undefined durations', () => { - const percentiles = getExecutionDurationPercentiles(mockRuleMonitoring); + const percentiles = getExecutionDurationPercentiles(mockRuleMonitoring.run.history); expect(percentiles.p50).toEqual(250); expect(percentiles.p95).toEqual(500); expect(percentiles.p99).toEqual(500); @@ -66,13 +66,13 @@ describe('getExecutionDurationPercentiles', () => { const newMockRuleMonitoring = { ...mockRuleMonitoring, - execution: { - ...mockRuleMonitoring.execution, + run: { + ...mockRuleMonitoring.run, history: nullDurationHistory, }, } as RuleMonitoring; - const percentiles = getExecutionDurationPercentiles(newMockRuleMonitoring); + const percentiles = getExecutionDurationPercentiles(newMockRuleMonitoring.run.history); expect(Object.keys(percentiles).length).toEqual(0); }); }); diff --git a/x-pack/plugins/alerting/server/lib/monitoring.ts b/x-pack/plugins/alerting/server/lib/monitoring.ts index b9f42e3bb8175..4c65c7aac5962 100644 --- a/x-pack/plugins/alerting/server/lib/monitoring.ts +++ b/x-pack/plugins/alerting/server/lib/monitoring.ts @@ -5,7 +5,8 @@ * 2.0. */ import { Logger } from '@kbn/core/server'; -import { RuleMonitoring, RawRuleMonitoring } from '../types'; +import stats from 'stats-lite'; +import { RuleMonitoring, RawRuleMonitoring, RuleMonitoringHistory } from '../types'; export const INITIAL_METRICS = { duration: 0, @@ -31,6 +32,25 @@ export const getDefaultMonitoring = (timestamp: string) => { }; }; +export const getExecutionDurationPercentiles = (history: RuleMonitoringHistory[]) => { + const durationSamples = history.reduce((duration, historyItem) => { + if (typeof historyItem.duration === 'number') { + return [...duration, historyItem.duration]; + } + return duration; + }, []); + + if (durationSamples.length) { + return { + p50: stats.percentile(durationSamples as number[], 0.5), + p95: stats.percentile(durationSamples as number[], 0.95), + p99: stats.percentile(durationSamples as number[], 0.99), + }; + } + + return {}; +}; + // Immutably updates the monitoring object with timestamp and duration. // Used when converting from and between raw monitoring object export const updateMonitoring = ({ diff --git a/x-pack/plugins/alerting/server/monitoring/rule_monitoring_client.ts b/x-pack/plugins/alerting/server/monitoring/rule_monitoring_client.ts index d1726452c7696..60b026bcefffb 100644 --- a/x-pack/plugins/alerting/server/monitoring/rule_monitoring_client.ts +++ b/x-pack/plugins/alerting/server/monitoring/rule_monitoring_client.ts @@ -5,8 +5,7 @@ * 2.0. */ -import stats from 'stats-lite'; -import { getDefaultMonitoring } from '../lib/monitoring'; +import { getDefaultMonitoring, getExecutionDurationPercentiles } from '../lib/monitoring'; import { RuleMonitoring, RuleMonitoringHistory } from '../types'; export class RuleMonitoringService { @@ -93,21 +92,6 @@ export class RuleMonitoringService { private buildExecutionDurationPercentiles = () => { const { history } = this.monitoring.run; - const durationSamples = history.reduce((duration, historyItem) => { - if (typeof historyItem.duration === 'number') { - return [...duration, historyItem.duration]; - } - return duration; - }, []); - - if (durationSamples.length) { - return { - p50: stats.percentile(durationSamples as number[], 0.5), - p95: stats.percentile(durationSamples as number[], 0.95), - p99: stats.percentile(durationSamples as number[], 0.99), - }; - } - - return {}; + return getExecutionDurationPercentiles(history); }; } diff --git a/x-pack/plugins/alerting/server/routes/aggregate_rules.test.ts b/x-pack/plugins/alerting/server/routes/aggregate_rules.test.ts index 8c24b457df565..26210e9bed285 100644 --- a/x-pack/plugins/alerting/server/routes/aggregate_rules.test.ts +++ b/x-pack/plugins/alerting/server/routes/aggregate_rules.test.ts @@ -49,6 +49,11 @@ describe('aggregateRulesRoute', () => { pending: 1, unknown: 0, }, + ruleLastRunOutcome: { + succeeded: 1, + failed: 2, + warning: 3, + }, ruleEnabledStatus: { disabled: 1, enabled: 40, @@ -88,6 +93,11 @@ describe('aggregateRulesRoute', () => { "pending": 1, "unknown": 0, }, + "rule_last_run_outcome": Object { + "failed": 2, + "succeeded": 1, + "warning": 3, + }, "rule_muted_status": Object { "muted": 2, "unmuted": 39, @@ -128,6 +138,11 @@ describe('aggregateRulesRoute', () => { pending: 1, unknown: 0, }, + rule_last_run_outcome: { + succeeded: 1, + failed: 2, + warning: 3, + }, rule_muted_status: { muted: 2, unmuted: 39, @@ -156,6 +171,11 @@ describe('aggregateRulesRoute', () => { pending: 1, unknown: 0, }, + ruleLastRunOutcome: { + succeeded: 2, + failed: 4, + warning: 6, + }, }); const [context, req, res] = mockHandlerArguments( @@ -209,6 +229,11 @@ describe('aggregateRulesRoute', () => { pending: 1, unknown: 0, }, + ruleLastRunOutcome: { + succeeded: 2, + failed: 4, + warning: 6, + }, }; rulesClient.aggregate.mockResolvedValueOnce(aggregateResult); const [, handler] = router.get.mock.calls[0]; diff --git a/x-pack/plugins/alerting/server/routes/bulk_edit_rules.test.ts b/x-pack/plugins/alerting/server/routes/bulk_edit_rules.test.ts index b70e4734ab4ff..e2ffa68c6d303 100644 --- a/x-pack/plugins/alerting/server/routes/bulk_edit_rules.test.ts +++ b/x-pack/plugins/alerting/server/routes/bulk_edit_rules.test.ts @@ -58,6 +58,7 @@ describe('bulkEditInternalRulesRoute', () => { status: 'unknown', lastExecutionDate: new Date('2020-08-20T19:23:38Z'), }, + running: false, }; const mockedAlerts: Array> = [mockedAlert]; diff --git a/x-pack/plugins/alerting/server/routes/create_rule.test.ts b/x-pack/plugins/alerting/server/routes/create_rule.test.ts index cbb7965cab081..2f317e1e36d0a 100644 --- a/x-pack/plugins/alerting/server/routes/create_rule.test.ts +++ b/x-pack/plugins/alerting/server/routes/create_rule.test.ts @@ -67,6 +67,7 @@ describe('createRuleRoute', () => { status: 'unknown', lastExecutionDate: new Date('2020-08-20T19:23:38Z'), }, + running: false, }; const ruleToCreate: AsApiContract['data']> = { @@ -102,6 +103,7 @@ describe('createRuleRoute', () => { connector_type_id: 'test', }, ], + running: false, }; it('creates a rule with proper parameters', async () => { diff --git a/x-pack/plugins/alerting/server/routes/get_rule.test.ts b/x-pack/plugins/alerting/server/routes/get_rule.test.ts index 065cd567f1f69..9ed9db182960d 100644 --- a/x-pack/plugins/alerting/server/routes/get_rule.test.ts +++ b/x-pack/plugins/alerting/server/routes/get_rule.test.ts @@ -61,6 +61,7 @@ describe('getRuleRoute', () => { status: 'unknown', lastExecutionDate: new Date('2020-08-20T19:23:38Z'), }, + running: false, }; const getResult: AsApiContract> = { @@ -87,6 +88,7 @@ describe('getRuleRoute', () => { connector_type_id: mockedAlert.actions[0].actionTypeId, }, ], + running: false, }; it('gets a rule with proper parameters', async () => { diff --git a/x-pack/plugins/alerting/server/routes/legacy/aggregate.test.ts b/x-pack/plugins/alerting/server/routes/legacy/aggregate.test.ts index a2fad24b24c0f..1fc3ee17ab51e 100644 --- a/x-pack/plugins/alerting/server/routes/legacy/aggregate.test.ts +++ b/x-pack/plugins/alerting/server/routes/legacy/aggregate.test.ts @@ -54,6 +54,11 @@ describe('aggregateAlertRoute', () => { pending: 1, unknown: 0, }, + ruleLastRunOutcome: { + succeeded: 1, + failed: 2, + warning: 3, + }, }; rulesClient.aggregate.mockResolvedValueOnce(aggregateResult); @@ -77,6 +82,11 @@ describe('aggregateAlertRoute', () => { "pending": 1, "unknown": 0, }, + "ruleLastRunOutcome": Object { + "failed": 2, + "succeeded": 1, + "warning": 3, + }, }, } `); @@ -113,6 +123,11 @@ describe('aggregateAlertRoute', () => { pending: 1, unknown: 0, }, + ruleLastRunOutcome: { + succeeded: 1, + failed: 2, + warning: 3, + }, }); const [context, req, res] = mockHandlerArguments( diff --git a/x-pack/plugins/alerting/server/routes/legacy/create.test.ts b/x-pack/plugins/alerting/server/routes/legacy/create.test.ts index 60b299be6db97..1a1c4d7b83759 100644 --- a/x-pack/plugins/alerting/server/routes/legacy/create.test.ts +++ b/x-pack/plugins/alerting/server/routes/legacy/create.test.ts @@ -80,6 +80,7 @@ describe('createAlertRoute', () => { status: 'unknown', lastExecutionDate: new Date('2020-08-20T19:23:38Z'), }, + running: false, }; it('creates an alert with proper parameters', async () => { diff --git a/x-pack/plugins/alerting/server/routes/legacy/get.test.ts b/x-pack/plugins/alerting/server/routes/legacy/get.test.ts index 6860140ce2540..1a5edb8cd6215 100644 --- a/x-pack/plugins/alerting/server/routes/legacy/get.test.ts +++ b/x-pack/plugins/alerting/server/routes/legacy/get.test.ts @@ -66,6 +66,7 @@ describe('getAlertRoute', () => { status: 'unknown', lastExecutionDate: new Date('2020-08-20T19:23:38Z'), }, + running: false, }; it('gets an alert with proper parameters', async () => { diff --git a/x-pack/plugins/alerting/server/routes/resolve_rule.test.ts b/x-pack/plugins/alerting/server/routes/resolve_rule.test.ts index 656874b5cf332..ce6350080f5d8 100644 --- a/x-pack/plugins/alerting/server/routes/resolve_rule.test.ts +++ b/x-pack/plugins/alerting/server/routes/resolve_rule.test.ts @@ -63,6 +63,7 @@ describe('resolveRuleRoute', () => { }, outcome: 'aliasMatch', alias_target_id: '2', + running: false, }; const resolveResult: AsApiContract> = { @@ -100,6 +101,7 @@ describe('resolveRuleRoute', () => { }, ], outcome: 'aliasMatch', + running: false, }; it('resolves a rule with proper parameters', async () => { diff --git a/x-pack/plugins/alerting/server/rules_client/tests/aggregate.test.ts b/x-pack/plugins/alerting/server/rules_client/tests/aggregate.test.ts index 2826ba0284b90..494af8f668bfb 100644 --- a/x-pack/plugins/alerting/server/rules_client/tests/aggregate.test.ts +++ b/x-pack/plugins/alerting/server/rules_client/tests/aggregate.test.ts @@ -90,6 +90,13 @@ describe('aggregate()', () => { { key: 'warning', doc_count: 1 }, ], }, + outcome: { + buckets: [ + { key: 'succeeded', doc_count: 2 }, + { key: 'failed', doc_count: 4 }, + { key: 'warning', doc_count: 6 }, + ], + }, enabled: { buckets: [ { key: 0, key_as_string: '0', doc_count: 2 }, @@ -165,6 +172,11 @@ describe('aggregate()', () => { "disabled": 2, "enabled": 28, }, + "ruleLastRunOutcome": Object { + "failed": 4, + "succeeded": 2, + "warning": 6, + }, "ruleMutedStatus": Object { "muted": 3, "unmuted": 27, @@ -191,6 +203,9 @@ describe('aggregate()', () => { status: { terms: { field: 'alert.attributes.executionStatus.status' }, }, + outcome: { + terms: { field: 'alert.attributes.lastRun.outcome' }, + }, enabled: { terms: { field: 'alert.attributes.enabled' }, }, @@ -246,6 +261,9 @@ describe('aggregate()', () => { status: { terms: { field: 'alert.attributes.executionStatus.status' }, }, + outcome: { + terms: { field: 'alert.attributes.lastRun.outcome' }, + }, enabled: { terms: { field: 'alert.attributes.enabled' }, }, diff --git a/x-pack/plugins/alerting/server/rules_client/tests/create.test.ts b/x-pack/plugins/alerting/server/rules_client/tests/create.test.ts index f5192bf6cbe65..c52ef44d3e64f 100644 --- a/x-pack/plugins/alerting/server/rules_client/tests/create.test.ts +++ b/x-pack/plugins/alerting/server/rules_client/tests/create.test.ts @@ -19,7 +19,7 @@ import { TaskStatus } from '@kbn/task-manager-plugin/server'; import { auditLoggerMock } from '@kbn/security-plugin/server/audit/mocks'; import { getBeforeSetup, setGlobalDate } from './lib'; import { RecoveredActionGroup } from '../../../common'; -import { getDefaultRuleMonitoring } from '../../task_runner/task_runner'; +import { getDefaultMonitoring } from '../../lib/monitoring'; import { bulkMarkApiKeysForInvalidation } from '../../invalidate_pending_api_keys/bulk_mark_api_keys_for_invalidation'; jest.mock('../../invalidate_pending_api_keys/bulk_mark_api_keys_for_invalidation', () => ({ @@ -421,11 +421,22 @@ describe('create()', () => { "versionApiKeyLastmodified": "v8.0.0", }, "monitoring": Object { - "execution": Object { + "run": Object { "calculated_metrics": Object { "success_ratio": 0, }, "history": Array [], + "last_run": Object { + "metrics": Object { + "duration": 0, + "gap_duration_s": null, + "total_alerts_created": null, + "total_alerts_detected": null, + "total_indexing_duration_ms": null, + "total_search_duration_ms": null, + }, + "timestamp": "2019-02-12T21:01:22.479Z", + }, }, }, "muteAll": false, @@ -435,6 +446,7 @@ describe('create()', () => { "params": Object { "bar": true, }, + "running": false, "schedule": Object { "interval": "1m", }, @@ -628,11 +640,22 @@ describe('create()', () => { "versionApiKeyLastmodified": "v7.10.0", }, "monitoring": Object { - "execution": Object { + "run": Object { "calculated_metrics": Object { "success_ratio": 0, }, "history": Array [], + "last_run": Object { + "metrics": Object { + "duration": 0, + "gap_duration_s": null, + "total_alerts_created": null, + "total_alerts_detected": null, + "total_indexing_duration_ms": null, + "total_search_duration_ms": null, + }, + "timestamp": "2019-02-12T21:01:22.479Z", + }, }, }, "muteAll": false, @@ -642,6 +665,7 @@ describe('create()', () => { "params": Object { "bar": true, }, + "running": false, "schedule": Object { "interval": "1m", }, @@ -1056,7 +1080,7 @@ describe('create()', () => { status: 'pending', warning: null, }, - monitoring: getDefaultRuleMonitoring(), + monitoring: getDefaultMonitoring('2019-02-12T21:01:22.479Z'), meta: { versionApiKeyLastmodified: kibanaVersion }, muteAll: false, snoozeSchedule: [], @@ -1069,6 +1093,7 @@ describe('create()', () => { throttle: null, updatedAt: '2019-02-12T21:01:22.479Z', updatedBy: 'elastic', + running: false, }, { id: 'mock-saved-object-id', @@ -1255,7 +1280,7 @@ describe('create()', () => { status: 'pending', warning: null, }, - monitoring: getDefaultRuleMonitoring(), + monitoring: getDefaultMonitoring('2019-02-12T21:01:22.479Z'), meta: { versionApiKeyLastmodified: kibanaVersion }, muteAll: false, snoozeSchedule: [], @@ -1268,6 +1293,7 @@ describe('create()', () => { throttle: null, updatedAt: '2019-02-12T21:01:22.479Z', updatedBy: 'elastic', + running: false, }, { id: 'mock-saved-object-id', @@ -1423,7 +1449,7 @@ describe('create()', () => { status: 'pending', warning: null, }, - monitoring: getDefaultRuleMonitoring(), + monitoring: getDefaultMonitoring('2019-02-12T21:01:22.479Z'), meta: { versionApiKeyLastmodified: kibanaVersion }, muteAll: false, snoozeSchedule: [], @@ -1436,6 +1462,7 @@ describe('create()', () => { throttle: null, updatedAt: '2019-02-12T21:01:22.479Z', updatedBy: 'elastic', + running: false, }, { id: 'mock-saved-object-id', @@ -1601,7 +1628,8 @@ describe('create()', () => { error: null, warning: null, }, - monitoring: getDefaultRuleMonitoring(), + monitoring: getDefaultMonitoring('2019-02-12T21:01:22.479Z'), + running: false, }, { id: 'mock-saved-object-id', @@ -1733,7 +1761,8 @@ describe('create()', () => { error: null, warning: null, }, - monitoring: getDefaultRuleMonitoring(), + monitoring: getDefaultMonitoring('2019-02-12T21:01:22.479Z'), + running: false, }, { id: 'mock-saved-object-id', @@ -1865,7 +1894,8 @@ describe('create()', () => { error: null, warning: null, }, - monitoring: getDefaultRuleMonitoring(), + monitoring: getDefaultMonitoring('2019-02-12T21:01:22.479Z'), + running: false, }, { id: 'mock-saved-object-id', @@ -2013,13 +2043,25 @@ describe('create()', () => { warning: null, }, monitoring: { - execution: { + run: { history: [], calculated_metrics: { success_ratio: 0, }, + last_run: { + timestamp: '2019-02-12T21:01:22.479Z', + metrics: { + duration: 0, + gap_duration_s: null, + total_alerts_created: null, + total_alerts_detected: null, + total_indexing_duration_ms: null, + total_search_duration_ms: null, + }, + }, }, }, + running: false, mapped_params: { risk_score: 42, severity: '20-low', @@ -2375,7 +2417,8 @@ describe('create()', () => { error: null, warning: null, }, - monitoring: getDefaultRuleMonitoring(), + monitoring: getDefaultMonitoring('2019-02-12T21:01:22.479Z'), + running: false, }, { id: 'mock-saved-object-id', @@ -2477,7 +2520,8 @@ describe('create()', () => { error: null, warning: null, }, - monitoring: getDefaultRuleMonitoring(), + monitoring: getDefaultMonitoring('2019-02-12T21:01:22.479Z'), + running: false, }, { id: 'mock-saved-object-id', diff --git a/x-pack/plugins/alerting/server/rules_client/tests/enable.test.ts b/x-pack/plugins/alerting/server/rules_client/tests/enable.test.ts index 1c96241b15800..4cebe9ce0d6be 100644 --- a/x-pack/plugins/alerting/server/rules_client/tests/enable.test.ts +++ b/x-pack/plugins/alerting/server/rules_client/tests/enable.test.ts @@ -234,6 +234,8 @@ describe('enable()', () => { error: null, warning: null, }, + running: false, + nextRun: '2019-02-12T21:01:32.479Z', }, { version: '123', @@ -290,6 +292,8 @@ describe('enable()', () => { error: null, warning: null, }, + running: false, + nextRun: '2019-02-12T21:01:32.479Z', }, { version: '123', @@ -356,6 +360,8 @@ describe('enable()', () => { error: null, warning: null, }, + running: false, + nextRun: '2019-02-12T21:01:32.479Z', }, { version: '123', diff --git a/x-pack/plugins/alerting/server/rules_client/tests/get_action_error_log.test.ts b/x-pack/plugins/alerting/server/rules_client/tests/get_action_error_log.test.ts index 6b635abe5d7f0..24d2430a32b29 100644 --- a/x-pack/plugins/alerting/server/rules_client/tests/get_action_error_log.test.ts +++ b/x-pack/plugins/alerting/server/rules_client/tests/get_action_error_log.test.ts @@ -89,6 +89,7 @@ const BaseRuleSavedObject: SavedObject = { error: null, warning: null, }, + running: false, }, references: [], }; diff --git a/x-pack/plugins/alerting/server/rules_client/tests/get_alert_summary.test.ts b/x-pack/plugins/alerting/server/rules_client/tests/get_alert_summary.test.ts index 4aa7ae40f8782..beef428f7ba26 100644 --- a/x-pack/plugins/alerting/server/rules_client/tests/get_alert_summary.test.ts +++ b/x-pack/plugins/alerting/server/rules_client/tests/get_alert_summary.test.ts @@ -94,6 +94,7 @@ const BaseRuleSavedObject: SavedObject = { error: null, warning: null, }, + running: false, }, references: [], }; diff --git a/x-pack/plugins/alerting/server/rules_client/tests/get_execution_log.test.ts b/x-pack/plugins/alerting/server/rules_client/tests/get_execution_log.test.ts index ffc40cd705abd..248a84700d4a0 100644 --- a/x-pack/plugins/alerting/server/rules_client/tests/get_execution_log.test.ts +++ b/x-pack/plugins/alerting/server/rules_client/tests/get_execution_log.test.ts @@ -91,6 +91,7 @@ const BaseRuleSavedObject: SavedObject = { error: null, warning: null, }, + running: false, }, references: [], }; diff --git a/x-pack/plugins/alerting/server/task_runner/alert_task_instance.test.ts b/x-pack/plugins/alerting/server/task_runner/alert_task_instance.test.ts index 3faea1a4d2261..ffda3656e69df 100644 --- a/x-pack/plugins/alerting/server/task_runner/alert_task_instance.test.ts +++ b/x-pack/plugins/alerting/server/task_runner/alert_task_instance.test.ts @@ -37,6 +37,7 @@ const alert: SanitizedRule<{ status: 'unknown', lastExecutionDate: new Date('2020-08-20T19:23:38Z'), }, + running: false, }; describe('Alert Task Instance', () => { diff --git a/x-pack/plugins/alerting/server/task_runner/fixtures.ts b/x-pack/plugins/alerting/server/task_runner/fixtures.ts index cb8da20372d5e..48ca43c617061 100644 --- a/x-pack/plugins/alerting/server/task_runner/fixtures.ts +++ b/x-pack/plugins/alerting/server/task_runner/fixtures.ts @@ -7,7 +7,7 @@ import { TaskStatus } from '@kbn/task-manager-plugin/server'; import { Rule, RuleTypeParams, RecoveredActionGroup } from '../../common'; -import { getDefaultRuleMonitoring } from './task_runner'; +import { getDefaultMonitoring } from '../lib/monitoring'; import { UntypedNormalizedRuleType } from '../rule_type_registry'; import { EVENT_LOG_ACTIONS } from '../plugin'; @@ -155,7 +155,8 @@ export const mockedRuleTypeSavedObject: Rule = { status: 'unknown', lastExecutionDate: new Date('2020-08-20T19:23:38Z'), }, - monitoring: getDefaultRuleMonitoring(), + monitoring: getDefaultMonitoring('2020-08-20T19:23:38Z'), + running: false, }; export const mockTaskInstance = () => ({ diff --git a/x-pack/plugins/alerting/server/task_runner/rule_loader.test.ts b/x-pack/plugins/alerting/server/task_runner/rule_loader.test.ts index 96815743daaef..1cbe2bb2ae3b9 100644 --- a/x-pack/plugins/alerting/server/task_runner/rule_loader.test.ts +++ b/x-pack/plugins/alerting/server/task_runner/rule_loader.test.ts @@ -79,7 +79,7 @@ describe('rule_loader', () => { expect(result.rule.alertTypeId).toBe(ruleTypeId); expect(result.rule.name).toBe(ruleName); expect(result.rule.params).toBe(ruleParams); - expect(result.rule.monitoring?.execution.history.length).toBe(MONITORING_HISTORY_LIMIT - 1); + expect(result.rule.monitoring?.run.history.length).toBe(MONITORING_HISTORY_LIMIT - 1); }); test('without API key, any execution history, or validator', async () => { @@ -102,7 +102,7 @@ describe('rule_loader', () => { expect(result.rule.alertTypeId).toBe(ruleTypeId); expect(result.rule.name).toBe(ruleName); expect(result.rule.params).toBe(ruleParams); - expect(result.rule.monitoring?.execution.history.length).toBe(0); + expect(result.rule.monitoring?.run.history.length).toBe(0); }); }); @@ -348,7 +348,7 @@ function getTaskRunnerContext(ruleParameters: unknown, historyElements: number) alertTypeId: ruleTypeId, params: ruleParameters, monitoring: { - execution: { + run: { history: new Array(historyElements), }, }, diff --git a/x-pack/plugins/alerting/server/task_runner/task_runner.test.ts b/x-pack/plugins/alerting/server/task_runner/task_runner.test.ts index 20310fdae01f7..84e8f0366f2cb 100644 --- a/x-pack/plugins/alerting/server/task_runner/task_runner.test.ts +++ b/x-pack/plugins/alerting/server/task_runner/task_runner.test.ts @@ -197,8 +197,8 @@ describe('Task Runner', () => { taskRunnerFactoryInitializerParams.executionContext.withContext.mockImplementation((ctx, fn) => fn() ); - mockedRuleTypeSavedObject.monitoring!.execution.history = []; - mockedRuleTypeSavedObject.monitoring!.execution.calculated_metrics.success_ratio = 0; + mockedRuleTypeSavedObject.monitoring!.run.history = []; + mockedRuleTypeSavedObject.monitoring!.run.calculated_metrics.success_ratio = 0; alertingEventLogger.getStartAndDuration.mockImplementation(() => ({ start: new Date() })); (AlertingEventLogger as jest.Mock).mockImplementation(() => alertingEventLogger); @@ -1549,7 +1549,7 @@ describe('Task Runner', () => { expect(ex.toString()).toEqual(`Error: Saved object [alert/1] not found`); const executeRuleDebugLogger = logger.debug.mock.calls[3][0]; expect(executeRuleDebugLogger as string).toMatchInlineSnapshot( - `"Executing Rule foo:test:1 has resulted in Error: Saved object [alert/1] not found"` + `"Updating rule task for test rule with id 1 - {\\"lastExecutionDate\\":\\"1970-01-01T00:00:00.000Z\\",\\"status\\":\\"error\\",\\"error\\":{\\"reason\\":\\"read\\",\\"message\\":\\"Saved object [alert/1] not found\\"}} - {\\"outcome\\":\\"failed\\",\\"warning\\":\\"read\\",\\"outcomeMsg\\":\\"Saved object [alert/1] not found\\",\\"alertsCount\\":{}}"` ); expect(logger.error).not.toHaveBeenCalled(); expect(logger.warn).toHaveBeenCalledTimes(1); @@ -1631,7 +1631,7 @@ describe('Task Runner', () => { expect(ex.toString()).toEqual(`Error: Saved object [alert/1] not found`); const ruleExecuteDebugLog = logger.debug.mock.calls[3][0]; expect(ruleExecuteDebugLog as string).toMatchInlineSnapshot( - `"Executing Rule test space:test:1 has resulted in Error: Saved object [alert/1] not found"` + `"Updating rule task for test rule with id 1 - {\\"lastExecutionDate\\":\\"1970-01-01T00:00:00.000Z\\",\\"status\\":\\"error\\",\\"error\\":{\\"reason\\":\\"read\\",\\"message\\":\\"Saved object [alert/1] not found\\"}} - {\\"outcome\\":\\"failed\\",\\"warning\\":\\"read\\",\\"outcomeMsg\\":\\"Saved object [alert/1] not found\\",\\"alertsCount\\":{}}"` ); expect(logger.error).not.toHaveBeenCalled(); expect(logger.warn).toHaveBeenCalledTimes(1); @@ -2198,7 +2198,7 @@ describe('Task Runner', () => { await taskRunner.run(); } const runnerResult = await taskRunner.run(); - expect(runnerResult.monitoring?.execution.history.length).toBe(200); + expect(runnerResult.monitoring?.run.history.length).toBe(200); }); test('Actions circuit breaker kicked in, should set status as warning and log a message in event log', async () => { From 2151d4ff97241f0947c510d6ec008592111f6c35 Mon Sep 17 00:00:00 2001 From: Jiawei Wu Date: Thu, 3 Nov 2022 14:45:45 -0700 Subject: [PATCH 12/27] Removing running and fix associated tests --- x-pack/plugins/alerting/common/rule.ts | 1 - x-pack/plugins/alerting/public/alert_api.test.ts | 3 --- .../public/lib/common_transformations.test.ts | 4 ---- .../server/lib/alert_summary_from_event_log.test.ts | 1 - .../alerting/server/routes/bulk_edit_rules.test.ts | 1 - .../alerting/server/routes/create_rule.test.ts | 2 -- .../plugins/alerting/server/routes/get_rule.test.ts | 2 -- .../alerting/server/routes/legacy/create.test.ts | 1 - .../plugins/alerting/server/routes/legacy/get.test.ts | 1 - .../alerting/server/routes/resolve_rule.test.ts | 2 -- .../alerting/server/rules_client/rules_client.ts | 3 --- .../alerting/server/rules_client/tests/create.test.ts | 11 ----------- .../alerting/server/rules_client/tests/enable.test.ts | 3 --- .../rules_client/tests/get_action_error_log.test.ts | 1 - .../rules_client/tests/get_alert_summary.test.ts | 1 - .../rules_client/tests/get_execution_log.test.ts | 1 - .../plugins/alerting/server/saved_objects/mappings.ts | 3 --- .../server/task_runner/alert_task_instance.test.ts | 1 - .../plugins/alerting/server/task_runner/fixtures.ts | 1 - .../alerting/server/task_runner/task_runner.ts | 2 -- x-pack/plugins/alerting/server/types.ts | 1 - 21 files changed, 46 deletions(-) diff --git a/x-pack/plugins/alerting/common/rule.ts b/x-pack/plugins/alerting/common/rule.ts index 8ea4de66d98ca..04cdc8c71c059 100644 --- a/x-pack/plugins/alerting/common/rule.ts +++ b/x-pack/plugins/alerting/common/rule.ts @@ -134,7 +134,6 @@ export interface Rule { isSnoozedUntil?: Date | null; lastRun?: RuleLastRun | null; nextRun?: Date | null; - running: boolean; } export type SanitizedRule = Omit, 'apiKey'>; diff --git a/x-pack/plugins/alerting/public/alert_api.test.ts b/x-pack/plugins/alerting/public/alert_api.test.ts index 8ee76f61421f5..2f09643e499e3 100644 --- a/x-pack/plugins/alerting/public/alert_api.test.ts +++ b/x-pack/plugins/alerting/public/alert_api.test.ts @@ -131,7 +131,6 @@ describe('loadRule', () => { "params": Object { "x": 42, }, - "running": false, "schedule": Object { "interval": "1s", }, @@ -290,7 +289,6 @@ function getApiRule() { last_execution_date: RuleExecuteDate.toISOString(), last_duration: 1194, }, - running: false, }; } @@ -332,6 +330,5 @@ function getRule(): Rule<{ x: number }> { lastExecutionDate: RuleExecuteDate, lastDuration: 1194, }, - running: false, }; } diff --git a/x-pack/plugins/alerting/public/lib/common_transformations.test.ts b/x-pack/plugins/alerting/public/lib/common_transformations.test.ts index b9e9e27ba6f92..51d24538b449e 100644 --- a/x-pack/plugins/alerting/public/lib/common_transformations.test.ts +++ b/x-pack/plugins/alerting/public/lib/common_transformations.test.ts @@ -54,7 +54,6 @@ describe('common_transformations', () => { message: 'this is just a test', }, }, - running: false, }; expect(transformRule(apiRule)).toMatchInlineSnapshot(` Object { @@ -106,7 +105,6 @@ describe('common_transformations', () => { ], }, }, - "running": false, "schedule": Object { "interval": "1s", }, @@ -154,7 +152,6 @@ describe('common_transformations', () => { last_execution_date: dateExecuted.toISOString(), status: 'error', }, - running: false, }; expect(transformRule(apiRule)).toMatchInlineSnapshot(` Object { @@ -187,7 +184,6 @@ describe('common_transformations', () => { "name": "some-name", "notifyWhen": "onActiveAlert", "params": Object {}, - "running": false, "schedule": Object { "interval": "1s", }, diff --git a/x-pack/plugins/alerting/server/lib/alert_summary_from_event_log.test.ts b/x-pack/plugins/alerting/server/lib/alert_summary_from_event_log.test.ts index 1a22006fe2115..56a862f2ad6ca 100644 --- a/x-pack/plugins/alerting/server/lib/alert_summary_from_event_log.test.ts +++ b/x-pack/plugins/alerting/server/lib/alert_summary_from_event_log.test.ts @@ -730,5 +730,4 @@ const BaseRule: SanitizedRule<{ bar: boolean }> = { status: 'unknown', lastExecutionDate: new Date('2020-08-20T19:23:38Z'), }, - running: false, }; diff --git a/x-pack/plugins/alerting/server/routes/bulk_edit_rules.test.ts b/x-pack/plugins/alerting/server/routes/bulk_edit_rules.test.ts index e2ffa68c6d303..b70e4734ab4ff 100644 --- a/x-pack/plugins/alerting/server/routes/bulk_edit_rules.test.ts +++ b/x-pack/plugins/alerting/server/routes/bulk_edit_rules.test.ts @@ -58,7 +58,6 @@ describe('bulkEditInternalRulesRoute', () => { status: 'unknown', lastExecutionDate: new Date('2020-08-20T19:23:38Z'), }, - running: false, }; const mockedAlerts: Array> = [mockedAlert]; diff --git a/x-pack/plugins/alerting/server/routes/create_rule.test.ts b/x-pack/plugins/alerting/server/routes/create_rule.test.ts index 2f317e1e36d0a..cbb7965cab081 100644 --- a/x-pack/plugins/alerting/server/routes/create_rule.test.ts +++ b/x-pack/plugins/alerting/server/routes/create_rule.test.ts @@ -67,7 +67,6 @@ describe('createRuleRoute', () => { status: 'unknown', lastExecutionDate: new Date('2020-08-20T19:23:38Z'), }, - running: false, }; const ruleToCreate: AsApiContract['data']> = { @@ -103,7 +102,6 @@ describe('createRuleRoute', () => { connector_type_id: 'test', }, ], - running: false, }; it('creates a rule with proper parameters', async () => { diff --git a/x-pack/plugins/alerting/server/routes/get_rule.test.ts b/x-pack/plugins/alerting/server/routes/get_rule.test.ts index 9ed9db182960d..065cd567f1f69 100644 --- a/x-pack/plugins/alerting/server/routes/get_rule.test.ts +++ b/x-pack/plugins/alerting/server/routes/get_rule.test.ts @@ -61,7 +61,6 @@ describe('getRuleRoute', () => { status: 'unknown', lastExecutionDate: new Date('2020-08-20T19:23:38Z'), }, - running: false, }; const getResult: AsApiContract> = { @@ -88,7 +87,6 @@ describe('getRuleRoute', () => { connector_type_id: mockedAlert.actions[0].actionTypeId, }, ], - running: false, }; it('gets a rule with proper parameters', async () => { diff --git a/x-pack/plugins/alerting/server/routes/legacy/create.test.ts b/x-pack/plugins/alerting/server/routes/legacy/create.test.ts index 1a1c4d7b83759..60b299be6db97 100644 --- a/x-pack/plugins/alerting/server/routes/legacy/create.test.ts +++ b/x-pack/plugins/alerting/server/routes/legacy/create.test.ts @@ -80,7 +80,6 @@ describe('createAlertRoute', () => { status: 'unknown', lastExecutionDate: new Date('2020-08-20T19:23:38Z'), }, - running: false, }; it('creates an alert with proper parameters', async () => { diff --git a/x-pack/plugins/alerting/server/routes/legacy/get.test.ts b/x-pack/plugins/alerting/server/routes/legacy/get.test.ts index 1a5edb8cd6215..6860140ce2540 100644 --- a/x-pack/plugins/alerting/server/routes/legacy/get.test.ts +++ b/x-pack/plugins/alerting/server/routes/legacy/get.test.ts @@ -66,7 +66,6 @@ describe('getAlertRoute', () => { status: 'unknown', lastExecutionDate: new Date('2020-08-20T19:23:38Z'), }, - running: false, }; it('gets an alert with proper parameters', async () => { diff --git a/x-pack/plugins/alerting/server/routes/resolve_rule.test.ts b/x-pack/plugins/alerting/server/routes/resolve_rule.test.ts index ce6350080f5d8..656874b5cf332 100644 --- a/x-pack/plugins/alerting/server/routes/resolve_rule.test.ts +++ b/x-pack/plugins/alerting/server/routes/resolve_rule.test.ts @@ -63,7 +63,6 @@ describe('resolveRuleRoute', () => { }, outcome: 'aliasMatch', alias_target_id: '2', - running: false, }; const resolveResult: AsApiContract> = { @@ -101,7 +100,6 @@ describe('resolveRuleRoute', () => { }, ], outcome: 'aliasMatch', - running: false, }; it('resolves a rule with proper parameters', async () => { diff --git a/x-pack/plugins/alerting/server/rules_client/rules_client.ts b/x-pack/plugins/alerting/server/rules_client/rules_client.ts index 3062f9544f615..b018f41994709 100644 --- a/x-pack/plugins/alerting/server/rules_client/rules_client.ts +++ b/x-pack/plugins/alerting/server/rules_client/rules_client.ts @@ -385,7 +385,6 @@ export interface CreateOptions { | 'isSnoozedUntil' | 'lastRun' | 'nextRun' - | 'running' > & { actions: NormalizedAlertAction[] }; options?: { id?: string; @@ -624,7 +623,6 @@ export class RulesClient { muteAll: false, mutedInstanceIds: [], notifyWhen, - running: false, executionStatus: getRuleExecutionStatusPending(lastRunTimestamp.toISOString()), monitoring: getDefaultMonitoring(lastRunTimestamp.toISOString()), }; @@ -2657,7 +2655,6 @@ export class RulesClient { duration: 0, }), }), - running: false, nextRun: getNextRunString(schedule.interval), enabled: true, updatedBy: username, diff --git a/x-pack/plugins/alerting/server/rules_client/tests/create.test.ts b/x-pack/plugins/alerting/server/rules_client/tests/create.test.ts index c52ef44d3e64f..639234391f4bc 100644 --- a/x-pack/plugins/alerting/server/rules_client/tests/create.test.ts +++ b/x-pack/plugins/alerting/server/rules_client/tests/create.test.ts @@ -446,7 +446,6 @@ describe('create()', () => { "params": Object { "bar": true, }, - "running": false, "schedule": Object { "interval": "1m", }, @@ -665,7 +664,6 @@ describe('create()', () => { "params": Object { "bar": true, }, - "running": false, "schedule": Object { "interval": "1m", }, @@ -1093,7 +1091,6 @@ describe('create()', () => { throttle: null, updatedAt: '2019-02-12T21:01:22.479Z', updatedBy: 'elastic', - running: false, }, { id: 'mock-saved-object-id', @@ -1293,7 +1290,6 @@ describe('create()', () => { throttle: null, updatedAt: '2019-02-12T21:01:22.479Z', updatedBy: 'elastic', - running: false, }, { id: 'mock-saved-object-id', @@ -1462,7 +1458,6 @@ describe('create()', () => { throttle: null, updatedAt: '2019-02-12T21:01:22.479Z', updatedBy: 'elastic', - running: false, }, { id: 'mock-saved-object-id', @@ -1629,7 +1624,6 @@ describe('create()', () => { warning: null, }, monitoring: getDefaultMonitoring('2019-02-12T21:01:22.479Z'), - running: false, }, { id: 'mock-saved-object-id', @@ -1762,7 +1756,6 @@ describe('create()', () => { warning: null, }, monitoring: getDefaultMonitoring('2019-02-12T21:01:22.479Z'), - running: false, }, { id: 'mock-saved-object-id', @@ -1895,7 +1888,6 @@ describe('create()', () => { warning: null, }, monitoring: getDefaultMonitoring('2019-02-12T21:01:22.479Z'), - running: false, }, { id: 'mock-saved-object-id', @@ -2061,7 +2053,6 @@ describe('create()', () => { }, }, }, - running: false, mapped_params: { risk_score: 42, severity: '20-low', @@ -2418,7 +2409,6 @@ describe('create()', () => { warning: null, }, monitoring: getDefaultMonitoring('2019-02-12T21:01:22.479Z'), - running: false, }, { id: 'mock-saved-object-id', @@ -2521,7 +2511,6 @@ describe('create()', () => { warning: null, }, monitoring: getDefaultMonitoring('2019-02-12T21:01:22.479Z'), - running: false, }, { id: 'mock-saved-object-id', diff --git a/x-pack/plugins/alerting/server/rules_client/tests/enable.test.ts b/x-pack/plugins/alerting/server/rules_client/tests/enable.test.ts index 4cebe9ce0d6be..9cb6c356edaed 100644 --- a/x-pack/plugins/alerting/server/rules_client/tests/enable.test.ts +++ b/x-pack/plugins/alerting/server/rules_client/tests/enable.test.ts @@ -234,7 +234,6 @@ describe('enable()', () => { error: null, warning: null, }, - running: false, nextRun: '2019-02-12T21:01:32.479Z', }, { @@ -292,7 +291,6 @@ describe('enable()', () => { error: null, warning: null, }, - running: false, nextRun: '2019-02-12T21:01:32.479Z', }, { @@ -360,7 +358,6 @@ describe('enable()', () => { error: null, warning: null, }, - running: false, nextRun: '2019-02-12T21:01:32.479Z', }, { diff --git a/x-pack/plugins/alerting/server/rules_client/tests/get_action_error_log.test.ts b/x-pack/plugins/alerting/server/rules_client/tests/get_action_error_log.test.ts index 24d2430a32b29..6b635abe5d7f0 100644 --- a/x-pack/plugins/alerting/server/rules_client/tests/get_action_error_log.test.ts +++ b/x-pack/plugins/alerting/server/rules_client/tests/get_action_error_log.test.ts @@ -89,7 +89,6 @@ const BaseRuleSavedObject: SavedObject = { error: null, warning: null, }, - running: false, }, references: [], }; diff --git a/x-pack/plugins/alerting/server/rules_client/tests/get_alert_summary.test.ts b/x-pack/plugins/alerting/server/rules_client/tests/get_alert_summary.test.ts index beef428f7ba26..4aa7ae40f8782 100644 --- a/x-pack/plugins/alerting/server/rules_client/tests/get_alert_summary.test.ts +++ b/x-pack/plugins/alerting/server/rules_client/tests/get_alert_summary.test.ts @@ -94,7 +94,6 @@ const BaseRuleSavedObject: SavedObject = { error: null, warning: null, }, - running: false, }, references: [], }; diff --git a/x-pack/plugins/alerting/server/rules_client/tests/get_execution_log.test.ts b/x-pack/plugins/alerting/server/rules_client/tests/get_execution_log.test.ts index 248a84700d4a0..ffc40cd705abd 100644 --- a/x-pack/plugins/alerting/server/rules_client/tests/get_execution_log.test.ts +++ b/x-pack/plugins/alerting/server/rules_client/tests/get_execution_log.test.ts @@ -91,7 +91,6 @@ const BaseRuleSavedObject: SavedObject = { error: null, warning: null, }, - running: false, }, references: [], }; diff --git a/x-pack/plugins/alerting/server/saved_objects/mappings.ts b/x-pack/plugins/alerting/server/saved_objects/mappings.ts index dd50d14b6c622..727e70e26055e 100644 --- a/x-pack/plugins/alerting/server/saved_objects/mappings.ts +++ b/x-pack/plugins/alerting/server/saved_objects/mappings.ts @@ -251,9 +251,6 @@ export const alertMappings: SavedObjectsTypeMappingDefinition = { }, }, }, - running: { - type: 'boolean', - }, nextRun: { type: 'date', }, diff --git a/x-pack/plugins/alerting/server/task_runner/alert_task_instance.test.ts b/x-pack/plugins/alerting/server/task_runner/alert_task_instance.test.ts index ffda3656e69df..3faea1a4d2261 100644 --- a/x-pack/plugins/alerting/server/task_runner/alert_task_instance.test.ts +++ b/x-pack/plugins/alerting/server/task_runner/alert_task_instance.test.ts @@ -37,7 +37,6 @@ const alert: SanitizedRule<{ status: 'unknown', lastExecutionDate: new Date('2020-08-20T19:23:38Z'), }, - running: false, }; describe('Alert Task Instance', () => { diff --git a/x-pack/plugins/alerting/server/task_runner/fixtures.ts b/x-pack/plugins/alerting/server/task_runner/fixtures.ts index 48ca43c617061..1619a4af7d307 100644 --- a/x-pack/plugins/alerting/server/task_runner/fixtures.ts +++ b/x-pack/plugins/alerting/server/task_runner/fixtures.ts @@ -156,7 +156,6 @@ export const mockedRuleTypeSavedObject: Rule = { lastExecutionDate: new Date('2020-08-20T19:23:38Z'), }, monitoring: getDefaultMonitoring('2020-08-20T19:23:38Z'), - running: false, }; export const mockTaskInstance = () => ({ diff --git a/x-pack/plugins/alerting/server/task_runner/task_runner.ts b/x-pack/plugins/alerting/server/task_runner/task_runner.ts index 8d3d2b0d2a3fe..bf38aa8efbf15 100644 --- a/x-pack/plugins/alerting/server/task_runner/task_runner.ts +++ b/x-pack/plugins/alerting/server/task_runner/task_runner.ts @@ -199,7 +199,6 @@ export class TaskRunner< monitoring?: RawRuleMonitoring; nextRun?: string | null; lastRun?: RawRuleLastRun | null; - running?: boolean; } ) { const client = this.context.internalSavedObjectsRepository; @@ -858,7 +857,6 @@ export class TaskRunner< alertsCount: {}, }, monitoring: this.ruleMonitoring.getMonitoring() as RawRuleMonitoring, - running: false, nextRun: nextRunString && new Date(nextRunString).getTime() > date.getTime() ? nextRunString : null, }); diff --git a/x-pack/plugins/alerting/server/types.ts b/x-pack/plugins/alerting/server/types.ts index d7f2b446954bc..ff76c5cacd5ca 100644 --- a/x-pack/plugins/alerting/server/types.ts +++ b/x-pack/plugins/alerting/server/types.ts @@ -256,7 +256,6 @@ export interface RawRule extends SavedObjectAttributes { isSnoozedUntil?: string | null; lastRun?: RawRuleLastRun | null; nextRun?: string | null; - running: boolean; } export interface AlertingPlugin { From aa1977c6f861a10a2e6ba0c7863bb746fc8cff52 Mon Sep 17 00:00:00 2001 From: Jiawei Wu Date: Thu, 3 Nov 2022 14:57:56 -0700 Subject: [PATCH 13/27] Remove more references to running --- x-pack/plugins/alerting/server/saved_objects/index.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/x-pack/plugins/alerting/server/saved_objects/index.ts b/x-pack/plugins/alerting/server/saved_objects/index.ts index d8d53f4978d55..76fc730409294 100644 --- a/x-pack/plugins/alerting/server/saved_objects/index.ts +++ b/x-pack/plugins/alerting/server/saved_objects/index.ts @@ -38,7 +38,6 @@ export const AlertAttributesExcludedFromAAD = [ 'isSnoozedUntil', 'lastRun', 'nextRun', - 'running', ]; // useful for Pick which is a @@ -57,8 +56,7 @@ export type AlertAttributesExcludedFromAADType = | 'snoozeSchedule' | 'isSnoozedUntil' | 'lastRun' - | 'nextRun' - | 'running'; + | 'nextRun'; export function setupSavedObjects( savedObjects: SavedObjectsServiceSetup, From 992cccbac16b91a25208084be943b5d7215cd01a Mon Sep 17 00:00:00 2001 From: Jiawei Wu Date: Thu, 3 Nov 2022 19:46:02 -0700 Subject: [PATCH 14/27] Address comments and update task runner tests --- x-pack/plugins/alerting/common/rule.ts | 2 +- x-pack/plugins/alerting/server/lib/index.ts | 2 +- .../alerting/server/lib/last_run_status.ts | 4 +- .../plugins/alerting/server/lib/monitoring.ts | 1 - .../plugins/alerting/server/lib/next_run.ts | 14 +++- ...g_client.ts => rule_monitoring_service.ts} | 1 + .../server/rules_client/rules_client.ts | 4 +- .../server/rules_client/tests/create.test.ts | 3 - .../saved_objects/migrations/8.6/index.ts | 7 +- .../alerting/server/task_runner/fixtures.ts | 37 ++++++++++ .../server/task_runner/task_runner.test.ts | 70 ++++++++++++------- .../server/task_runner/task_runner.ts | 21 ++++-- .../task_runner/task_runner_cancel.test.ts | 32 ++++++++- x-pack/plugins/alerting/server/types.ts | 4 +- 14 files changed, 150 insertions(+), 52 deletions(-) rename x-pack/plugins/alerting/server/monitoring/{rule_monitoring_client.ts => rule_monitoring_service.ts} (98%) diff --git a/x-pack/plugins/alerting/common/rule.ts b/x-pack/plugins/alerting/common/rule.ts index 04cdc8c71c059..828ae01398a5d 100644 --- a/x-pack/plugins/alerting/common/rule.ts +++ b/x-pack/plugins/alerting/common/rule.ts @@ -203,7 +203,7 @@ export interface RuleMonitoringCalculatedMetrics extends SavedObjectAttributes { } export interface RuleMonitoringLastRunMetrics extends SavedObjectAttributes { - duration: number; + duration?: number; total_search_duration_ms?: number | null; total_indexing_duration_ms?: number | null; total_alerts_detected?: number | null; diff --git a/x-pack/plugins/alerting/server/lib/index.ts b/x-pack/plugins/alerting/server/lib/index.ts index 00081903d9506..de4c89e06f33d 100644 --- a/x-pack/plugins/alerting/server/lib/index.ts +++ b/x-pack/plugins/alerting/server/lib/index.ts @@ -31,7 +31,7 @@ export { getDefaultMonitoring, convertMonitoringFromRawAndVerify, } from './monitoring'; -export { getNextRunDate, getNextRunString } from './next_run'; +export { getNextRun } from './next_run'; export { processAlerts } from './process_alerts'; export { createWrappedScopedClusterClientFactory } from './wrap_scoped_cluster_client'; export { isRuleSnoozed, getRuleSnoozeEndTime } from './is_rule_snoozed'; diff --git a/x-pack/plugins/alerting/server/lib/last_run_status.ts b/x-pack/plugins/alerting/server/lib/last_run_status.ts index ce2e94c145429..d7027fc7c2770 100644 --- a/x-pack/plugins/alerting/server/lib/last_run_status.ts +++ b/x-pack/plugins/alerting/server/lib/last_run_status.ts @@ -45,14 +45,14 @@ export const lastRunFromState = (stateWithMetrics: RuleTaskStateAndMetrics): ILa return { lastRun: { outcome, + outcomeMsg: outcomeMsg || null, + warning: warning || null, alertsCount: { active: metrics.numberOfActiveAlerts, new: metrics.numberOfNewAlerts, recovered: metrics.numberOfRecoveredAlerts, ignored: 0, }, - ...(warning ? { warning } : {}), - ...(outcomeMsg ? { outcome_msg: outcomeMsg } : {}), }, metrics, }; diff --git a/x-pack/plugins/alerting/server/lib/monitoring.ts b/x-pack/plugins/alerting/server/lib/monitoring.ts index 4c65c7aac5962..249a770b1ba57 100644 --- a/x-pack/plugins/alerting/server/lib/monitoring.ts +++ b/x-pack/plugins/alerting/server/lib/monitoring.ts @@ -9,7 +9,6 @@ import stats from 'stats-lite'; import { RuleMonitoring, RawRuleMonitoring, RuleMonitoringHistory } from '../types'; export const INITIAL_METRICS = { - duration: 0, total_search_duration_ms: null, total_indexing_duration_ms: null, total_alerts_detected: null, diff --git a/x-pack/plugins/alerting/server/lib/next_run.ts b/x-pack/plugins/alerting/server/lib/next_run.ts index 4c40549cf4d04..8ce87e9db6883 100644 --- a/x-pack/plugins/alerting/server/lib/next_run.ts +++ b/x-pack/plugins/alerting/server/lib/next_run.ts @@ -8,6 +8,14 @@ import moment from 'moment'; import { parseDuration } from '../../common'; -export const getNextRunDate = (interval: string) => moment().add(parseDuration(interval), 'ms'); - -export const getNextRunString = (interval: string) => getNextRunDate(interval).toISOString(); +export const getNextRun = ({ + startDate, + interval, +}: { + startDate?: Date | null; + interval: string; +}) => { + return moment(startDate || new Date()) + .add(parseDuration(interval), 'ms') + .toISOString(); +}; diff --git a/x-pack/plugins/alerting/server/monitoring/rule_monitoring_client.ts b/x-pack/plugins/alerting/server/monitoring/rule_monitoring_service.ts similarity index 98% rename from x-pack/plugins/alerting/server/monitoring/rule_monitoring_client.ts rename to x-pack/plugins/alerting/server/monitoring/rule_monitoring_service.ts index 60b026bcefffb..634104d3f2970 100644 --- a/x-pack/plugins/alerting/server/monitoring/rule_monitoring_client.ts +++ b/x-pack/plugins/alerting/server/monitoring/rule_monitoring_service.ts @@ -41,6 +41,7 @@ export class RuleMonitoringService { }; if (null != duration) { monitoringHistory.duration = duration; + this.setLastRunMetricsDuration(duration); } if (hasError) { monitoringHistory.success = false; diff --git a/x-pack/plugins/alerting/server/rules_client/rules_client.ts b/x-pack/plugins/alerting/server/rules_client/rules_client.ts index b018f41994709..f689ce3765a52 100644 --- a/x-pack/plugins/alerting/server/rules_client/rules_client.ts +++ b/x-pack/plugins/alerting/server/rules_client/rules_client.ts @@ -87,7 +87,7 @@ import { getDefaultMonitoring, updateMonitoring, convertMonitoringFromRawAndVerify, - getNextRunString, + getNextRun, } from '../lib'; import { taskInstanceToAlertTaskInstance } from '../task_runner/alert_task_instance'; import { RegistryRuleType, UntypedNormalizedRuleType } from '../rule_type_registry'; @@ -2655,7 +2655,7 @@ export class RulesClient { duration: 0, }), }), - nextRun: getNextRunString(schedule.interval), + nextRun: getNextRun({ interval: schedule.interval }), enabled: true, updatedBy: username, updatedAt: now.toISOString(), diff --git a/x-pack/plugins/alerting/server/rules_client/tests/create.test.ts b/x-pack/plugins/alerting/server/rules_client/tests/create.test.ts index 639234391f4bc..aad483f09fe9e 100644 --- a/x-pack/plugins/alerting/server/rules_client/tests/create.test.ts +++ b/x-pack/plugins/alerting/server/rules_client/tests/create.test.ts @@ -428,7 +428,6 @@ describe('create()', () => { "history": Array [], "last_run": Object { "metrics": Object { - "duration": 0, "gap_duration_s": null, "total_alerts_created": null, "total_alerts_detected": null, @@ -646,7 +645,6 @@ describe('create()', () => { "history": Array [], "last_run": Object { "metrics": Object { - "duration": 0, "gap_duration_s": null, "total_alerts_created": null, "total_alerts_detected": null, @@ -2043,7 +2041,6 @@ describe('create()', () => { last_run: { timestamp: '2019-02-12T21:01:22.479Z', metrics: { - duration: 0, gap_duration_s: null, total_alerts_created: null, total_alerts_detected: null, diff --git a/x-pack/plugins/alerting/server/saved_objects/migrations/8.6/index.ts b/x-pack/plugins/alerting/server/saved_objects/migrations/8.6/index.ts index 836ff6983513b..00033182230fb 100644 --- a/x-pack/plugins/alerting/server/saved_objects/migrations/8.6/index.ts +++ b/x-pack/plugins/alerting/server/saved_objects/migrations/8.6/index.ts @@ -50,13 +50,16 @@ const getMonitoring = (attributes: RawRule) => { // Question: Do we want to backfill the history? const { lastExecutionDate, lastDuration } = executionStatus; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const monitoringExecution = (monitoring as any).execution; + return { run: { - ...monitoring.run, + ...monitoringExecution, last_run: { timestamp: lastExecutionDate, metrics: { - duration: lastDuration || 0, + ...(lastDuration ? { duration: lastDuration } : {}), }, }, }, diff --git a/x-pack/plugins/alerting/server/task_runner/fixtures.ts b/x-pack/plugins/alerting/server/task_runner/fixtures.ts index 1619a4af7d307..4304d8f1956e2 100644 --- a/x-pack/plugins/alerting/server/task_runner/fixtures.ts +++ b/x-pack/plugins/alerting/server/task_runner/fixtures.ts @@ -58,10 +58,14 @@ export const generateSavedObjectParams = ({ error = null, warning = null, status = 'ok', + outcome = 'succeeded', + alertsCount, }: { error?: null | { reason: string; message: string }; warning?: null | { reason: string; message: string }; status?: string; + outcome?: string; + alertsCount?: Record; }) => [ 'alert', '1', @@ -77,6 +81,16 @@ export const generateSavedObjectParams = ({ timestamp: 0, }, ], + last_run: { + timestamp: '1970-01-01T00:00:00.000Z', + metrics: { + gap_duration_s: null, + total_alerts_created: null, + total_alerts_detected: null, + total_indexing_duration_ms: null, + total_search_duration_ms: null, + }, + }, }, }, executionStatus: { @@ -86,6 +100,19 @@ export const generateSavedObjectParams = ({ status, warning, }, + lastRun: { + outcome, + outcomeMsg: error?.message || warning?.message || null, + warning: error?.reason || warning?.reason || null, + alertsCount: { + active: 0, + ignored: 0, + new: 0, + recovered: 0, + ...(alertsCount || {}), + }, + }, + nextRun: '1970-01-01T00:00:10.000Z', }, { refresh: false, namespace: undefined }, ]; @@ -223,6 +250,16 @@ export const generateRunnerResult = ({ }, // @ts-ignore history: history.map((success) => ({ success, timestamp: 0 })), + last_run: { + metrics: { + gap_duration_s: null, + total_alerts_created: null, + total_alerts_detected: null, + total_indexing_duration_ms: null, + total_search_duration_ms: null, + }, + timestamp: '1970-01-01T00:00:00.000Z', + }, }, }, schedule: { diff --git a/x-pack/plugins/alerting/server/task_runner/task_runner.test.ts b/x-pack/plugins/alerting/server/task_runner/task_runner.test.ts index 84e8f0366f2cb..0d1872bbbfa3a 100644 --- a/x-pack/plugins/alerting/server/task_runner/task_runner.test.ts +++ b/x-pack/plugins/alerting/server/task_runner/task_runner.test.ts @@ -255,14 +255,14 @@ describe('Task Runner', () => { expect(call.services.scopedClusterClient).toBeTruthy(); expect(call.services).toBeTruthy(); - expect(logger.debug).toHaveBeenCalledTimes(4); + expect(logger.debug).toHaveBeenCalledTimes(5); expect(logger.debug).nthCalledWith(1, 'executing rule test:1 at 1970-01-01T00:00:00.000Z'); expect(logger.debug).nthCalledWith( 2, - 'ruleRunStatus for test:1: {"lastExecutionDate":"1970-01-01T00:00:00.000Z","status":"ok"}' + 'deprecated ruleRunStatus for test:1: {"lastExecutionDate":"1970-01-01T00:00:00.000Z","status":"ok"}' ); expect(logger.debug).nthCalledWith( - 3, + 4, 'ruleRunMetrics for test:1: {"numSearches":3,"totalSearchDurationMs":23423,"esSearchDurationMs":33,"numberOfTriggeredActions":0,"numberOfGeneratedActions":0,"numberOfActiveAlerts":0,"numberOfRecoveredAlerts":0,"numberOfNewAlerts":0,"hasReachedAlertLimit":false,"triggeredActionsStatus":"complete"}' ); @@ -330,7 +330,7 @@ describe('Task Runner', () => { expect(enqueueFunction).toHaveBeenCalledTimes(1); expect(enqueueFunction).toHaveBeenCalledWith(generateEnqueueFunctionInput(inputIsArray)); - expect(logger.debug).toHaveBeenCalledTimes(5); + expect(logger.debug).toHaveBeenCalledTimes(6); expect(logger.debug).nthCalledWith(1, 'executing rule test:1 at 1970-01-01T00:00:00.000Z'); expect(logger.debug).nthCalledWith( 2, @@ -338,10 +338,10 @@ describe('Task Runner', () => { ); expect(logger.debug).nthCalledWith( 3, - 'ruleRunStatus for test:1: {"lastExecutionDate":"1970-01-01T00:00:00.000Z","status":"active"}' + 'deprecated ruleRunStatus for test:1: {"lastExecutionDate":"1970-01-01T00:00:00.000Z","status":"active"}' ); expect(logger.debug).nthCalledWith( - 4, + 5, 'ruleRunMetrics for test:1: {"numSearches":3,"totalSearchDurationMs":23423,"esSearchDurationMs":33,"numberOfTriggeredActions":1,"numberOfGeneratedActions":1,"numberOfActiveAlerts":1,"numberOfRecoveredAlerts":0,"numberOfNewAlerts":1,"hasReachedAlertLimit":false,"triggeredActionsStatus":"complete"}' ); @@ -408,7 +408,7 @@ describe('Task Runner', () => { await taskRunner.run(); expect(actionsClient.ephemeralEnqueuedExecution).toHaveBeenCalledTimes(0); - expect(logger.debug).toHaveBeenCalledTimes(6); + expect(logger.debug).toHaveBeenCalledTimes(7); expect(logger.debug).nthCalledWith(1, 'executing rule test:1 at 1970-01-01T00:00:00.000Z'); expect(logger.debug).nthCalledWith( 2, @@ -420,10 +420,10 @@ describe('Task Runner', () => { ); expect(logger.debug).nthCalledWith( 4, - 'ruleRunStatus for test:1: {"lastExecutionDate":"1970-01-01T00:00:00.000Z","status":"active"}' + 'deprecated ruleRunStatus for test:1: {"lastExecutionDate":"1970-01-01T00:00:00.000Z","status":"active"}' ); expect(logger.debug).nthCalledWith( - 5, + 6, 'ruleRunMetrics for test:1: {"numSearches":3,"totalSearchDurationMs":23423,"esSearchDurationMs":33,"numberOfTriggeredActions":0,"numberOfGeneratedActions":0,"numberOfActiveAlerts":1,"numberOfRecoveredAlerts":0,"numberOfNewAlerts":1,"hasReachedAlertLimit":false,"triggeredActionsStatus":"complete"}' ); @@ -582,7 +582,7 @@ describe('Task Runner', () => { await taskRunner.run(); expect(enqueueFunction).toHaveBeenCalledTimes(1); - expect(logger.debug).toHaveBeenCalledTimes(6); + expect(logger.debug).toHaveBeenCalledTimes(7); expect(logger.debug).nthCalledWith(1, 'executing rule test:1 at 1970-01-01T00:00:00.000Z'); expect(logger.debug).nthCalledWith( 2, @@ -594,10 +594,10 @@ describe('Task Runner', () => { ); expect(logger.debug).nthCalledWith( 4, - 'ruleRunStatus for test:1: {"lastExecutionDate":"1970-01-01T00:00:00.000Z","status":"active"}' + 'deprecated ruleRunStatus for test:1: {"lastExecutionDate":"1970-01-01T00:00:00.000Z","status":"active"}' ); expect(logger.debug).nthCalledWith( - 5, + 6, 'ruleRunMetrics for test:1: {"numSearches":3,"totalSearchDurationMs":23423,"esSearchDurationMs":33,"numberOfTriggeredActions":1,"numberOfGeneratedActions":1,"numberOfActiveAlerts":2,"numberOfRecoveredAlerts":0,"numberOfNewAlerts":2,"hasReachedAlertLimit":false,"triggeredActionsStatus":"complete"}' ); expect(mockUsageCounter.incrementCounter).not.toHaveBeenCalled(); @@ -705,7 +705,7 @@ describe('Task Runner', () => { encryptedSavedObjectsClient.getDecryptedAsInternalUser.mockResolvedValue(SAVED_OBJECT); await taskRunner.run(); expect(enqueueFunction).toHaveBeenCalledTimes(1); - expect(logger.debug).toHaveBeenCalledTimes(6); + expect(logger.debug).toHaveBeenCalledTimes(7); expect(logger.debug).nthCalledWith( 3, `skipping scheduling of actions for '2' in rule test:1: '${RULE_NAME}': rule is muted` @@ -1013,7 +1013,7 @@ describe('Task Runner', () => { generateAlertInstance({ id: 1, duration: MOCK_DURATION, start: DATE_1969 }) ); - expect(logger.debug).toHaveBeenCalledTimes(6); + expect(logger.debug).toHaveBeenCalledTimes(7); expect(logger.debug).nthCalledWith(1, 'executing rule test:1 at 1970-01-01T00:00:00.000Z'); expect(logger.debug).nthCalledWith( 2, @@ -1025,10 +1025,10 @@ describe('Task Runner', () => { ); expect(logger.debug).nthCalledWith( 4, - 'ruleRunStatus for test:1: {"lastExecutionDate":"1970-01-01T00:00:00.000Z","status":"active"}' + 'deprecated ruleRunStatus for test:1: {"lastExecutionDate":"1970-01-01T00:00:00.000Z","status":"active"}' ); expect(logger.debug).nthCalledWith( - 5, + 6, 'ruleRunMetrics for test:1: {"numSearches":3,"totalSearchDurationMs":23423,"esSearchDurationMs":33,"numberOfTriggeredActions":2,"numberOfGeneratedActions":2,"numberOfActiveAlerts":1,"numberOfRecoveredAlerts":1,"numberOfNewAlerts":0,"hasReachedAlertLimit":false,"triggeredActionsStatus":"complete"}' ); @@ -1138,10 +1138,10 @@ describe('Task Runner', () => { ); expect(logger.debug).nthCalledWith( 4, - `ruleRunStatus for test:1: {"lastExecutionDate":"1970-01-01T00:00:00.000Z","status":"active"}` + `deprecated ruleRunStatus for test:1: {"lastExecutionDate":"1970-01-01T00:00:00.000Z","status":"active"}` ); expect(logger.debug).nthCalledWith( - 5, + 6, `ruleRunMetrics for test:1: {"numSearches":3,"totalSearchDurationMs":23423,"esSearchDurationMs":33,"numberOfTriggeredActions":2,"numberOfGeneratedActions":2,"numberOfActiveAlerts":1,"numberOfRecoveredAlerts":1,"numberOfNewAlerts":0,"hasReachedAlertLimit":false,"triggeredActionsStatus":"complete"}` ); @@ -2083,14 +2083,14 @@ describe('Task Runner', () => { expect(call.services.scopedClusterClient).toBeTruthy(); expect(call.services).toBeTruthy(); - expect(logger.debug).toHaveBeenCalledTimes(4); + expect(logger.debug).toHaveBeenCalledTimes(5); expect(logger.debug).nthCalledWith(1, 'executing rule test:1 at 1970-01-01T00:00:00.000Z'); expect(logger.debug).nthCalledWith( 2, - 'ruleRunStatus for test:1: {"lastExecutionDate":"1970-01-01T00:00:00.000Z","status":"ok"}' + 'deprecated ruleRunStatus for test:1: {"lastExecutionDate":"1970-01-01T00:00:00.000Z","status":"ok"}' ); expect(logger.debug).nthCalledWith( - 3, + 4, 'ruleRunMetrics for test:1: {"numSearches":3,"totalSearchDurationMs":23423,"esSearchDurationMs":33,"numberOfTriggeredActions":0,"numberOfGeneratedActions":0,"numberOfActiveAlerts":0,"numberOfRecoveredAlerts":0,"numberOfNewAlerts":0,"hasReachedAlertLimit":false,"triggeredActionsStatus":"complete"}' ); @@ -2282,7 +2282,17 @@ describe('Task Runner', () => { expect( taskRunnerFactoryInitializerParams.internalSavedObjectsRepository.update - ).toHaveBeenCalledWith(...generateSavedObjectParams({ status: 'warning', warning })); + ).toHaveBeenCalledWith( + ...generateSavedObjectParams({ + status: 'warning', + outcome: 'warning', + warning, + alertsCount: { + active: 1, + new: 1, + }, + }) + ); expect(runnerResult).toEqual( generateRunnerResult({ @@ -2305,7 +2315,7 @@ describe('Task Runner', () => { }) ); - expect(logger.debug).toHaveBeenCalledTimes(6); + expect(logger.debug).toHaveBeenCalledTimes(7); expect(logger.debug).nthCalledWith( 3, @@ -2435,7 +2445,17 @@ describe('Task Runner', () => { expect( taskRunnerFactoryInitializerParams.internalSavedObjectsRepository.update - ).toHaveBeenCalledWith(...generateSavedObjectParams({ status: 'warning', warning })); + ).toHaveBeenCalledWith( + ...generateSavedObjectParams({ + status: 'warning', + outcome: 'warning', + warning, + alertsCount: { + active: 2, + new: 2, + }, + }) + ); expect(runnerResult).toEqual( generateRunnerResult({ @@ -2470,7 +2490,7 @@ describe('Task Runner', () => { }) ); - expect(logger.debug).toHaveBeenCalledTimes(6); + expect(logger.debug).toHaveBeenCalledTimes(7); expect(logger.debug).nthCalledWith( 3, diff --git a/x-pack/plugins/alerting/server/task_runner/task_runner.ts b/x-pack/plugins/alerting/server/task_runner/task_runner.ts index bf38aa8efbf15..aa69d1102f6b1 100644 --- a/x-pack/plugins/alerting/server/task_runner/task_runner.ts +++ b/x-pack/plugins/alerting/server/task_runner/task_runner.ts @@ -24,7 +24,7 @@ import { isRuleSnoozed, processAlerts, lastRunFromError, - getNextRunString, + getNextRun, } from '../lib'; import { Rule, @@ -70,7 +70,7 @@ import { logAlerts } from './log_alerts'; import { scheduleActionsForAlerts } from './schedule_actions_for_alerts'; import { getPublicAlertFactory } from '../alert/create_alert_factory'; import { TaskRunnerTimer, TaskRunnerTimerSpan } from './task_runner_timer'; -import { RuleMonitoringService } from '../monitoring/rule_monitoring_client'; +import { RuleMonitoringService } from '../monitoring/rule_monitoring_service'; import { ILastRun, lastRunFromState, lastRunToRaw } from '../lib/last_run_status'; const FALLBACK_RETRY_INTERVAL = '5m'; @@ -720,13 +720,16 @@ export class TaskRunner< schedule = asErr(err); } - const nextRunString = isOk(schedule) ? getNextRunString(schedule.value.interval) : null; + let nextRun: string | null = null; + if (isOk(schedule)) { + nextRun = getNextRun({ startDate: startedAt, interval: schedule.value.interval }); + } const { executionStatus, executionMetrics } = await this.timer.runWithTimer( TaskRunnerTimerSpan.ProcessRuleRun, async () => this.processRunResults({ - nextRun: nextRunString, + nextRun, runDate, stateWithMetrics, }) @@ -813,6 +816,7 @@ export class TaskRunner< const { params: { alertId: ruleId, spaceId, consumer }, schedule: taskSchedule, + startedAt, } = this.taskInstance; const namespace = this.context.spaceIdToNamespace(spaceId); @@ -833,7 +837,11 @@ export class TaskRunner< this.inMemoryMetrics.increment(IN_MEMORY_METRICS.RULE_TIMEOUTS); - const nextRunString = taskSchedule ? getNextRunString(taskSchedule.interval) : null; + let nextRun: string | null = null; + if (taskSchedule) { + nextRun = getNextRun({ startDate: startedAt, interval: taskSchedule.interval }); + } + const outcomeMsg = `${this.ruleType.id}:${ruleId}: execution cancelled due to timeout - exceeded rule type timeout of ${this.ruleType.ruleTaskTimeout}`; const date = new Date(); // Update the rule saved object with execution status @@ -857,8 +865,7 @@ export class TaskRunner< alertsCount: {}, }, monitoring: this.ruleMonitoring.getMonitoring() as RawRuleMonitoring, - nextRun: - nextRunString && new Date(nextRunString).getTime() > date.getTime() ? nextRunString : null, + nextRun: nextRun && new Date(nextRun).getTime() > date.getTime() ? nextRun : null, }); } } diff --git a/x-pack/plugins/alerting/server/task_runner/task_runner_cancel.test.ts b/x-pack/plugins/alerting/server/task_runner/task_runner_cancel.test.ts index 92353cb043984..33efc649add26 100644 --- a/x-pack/plugins/alerting/server/task_runner/task_runner_cancel.test.ts +++ b/x-pack/plugins/alerting/server/task_runner/task_runner_cancel.test.ts @@ -214,6 +214,32 @@ describe('Task Runner Cancel', () => { status: 'error', warning: null, }, + lastRun: { + alertsCount: {}, + outcome: 'failed', + outcomeMsg: + 'test:1: execution cancelled due to timeout - exceeded rule type timeout of 5m', + warning: 'timeout', + }, + monitoring: { + run: { + calculated_metrics: { + success_ratio: 0, + }, + history: [], + last_run: { + metrics: { + gap_duration_s: null, + total_alerts_created: null, + total_alerts_detected: null, + total_indexing_duration_ms: null, + total_search_duration_ms: null, + }, + timestamp: '1970-01-01T00:00:00.000Z', + }, + }, + }, + nextRun: '1970-01-01T00:00:10.000Z', }, { refresh: false, namespace: undefined } ); @@ -391,7 +417,7 @@ describe('Task Runner Cancel', () => { }); function testLogger() { - expect(logger.debug).toHaveBeenCalledTimes(7); + expect(logger.debug).toHaveBeenCalledTimes(8); expect(logger.debug).nthCalledWith(1, 'executing rule test:1 at 1970-01-01T00:00:00.000Z'); expect(logger.debug).nthCalledWith( 2, @@ -411,10 +437,10 @@ describe('Task Runner Cancel', () => { ); expect(logger.debug).nthCalledWith( 6, - 'ruleRunStatus for test:1: {"lastExecutionDate":"1970-01-01T00:00:00.000Z","status":"active"}' + 'deprecated ruleRunStatus for test:1: {"lastExecutionDate":"1970-01-01T00:00:00.000Z","status":"active"}' ); expect(logger.debug).nthCalledWith( - 7, + 8, 'ruleRunMetrics for test:1: {"numSearches":3,"totalSearchDurationMs":23423,"esSearchDurationMs":33,"numberOfTriggeredActions":1,"numberOfGeneratedActions":1,"numberOfActiveAlerts":1,"numberOfRecoveredAlerts":0,"numberOfNewAlerts":1,"hasReachedAlertLimit":false,"triggeredActionsStatus":"complete"}' ); } diff --git a/x-pack/plugins/alerting/server/types.ts b/x-pack/plugins/alerting/server/types.ts index ff76c5cacd5ca..eef6e49c6a56a 100644 --- a/x-pack/plugins/alerting/server/types.ts +++ b/x-pack/plugins/alerting/server/types.ts @@ -82,7 +82,7 @@ export interface RuleExecutorServices< alertFactory: PublicAlertFactory; shouldWriteAlerts: () => boolean; shouldStopExecution: () => boolean; - ruleMonitoringService: RuleMonitoringService; + ruleMonitoringService: PublicRuleMonitoringService; } export interface RuleExecutorOptions< @@ -285,7 +285,7 @@ export type RuleTypeRegistry = PublicMethodsOf; export type RulesClientApi = PublicMethodsOf; -export interface RuleMonitoringService { +export interface PublicRuleMonitoringService { setLastRunMetricsTotalSearchDurationMs: (totalSearchDurationMs: number) => void; setLastRunMetricsTotalIndexingDurationMs: (totalIndexingDurationMs: number) => void; setLastRunMetricsTotalAlertDetected: (totalAlertDetected: number) => void; From a90721764b4d746b2a3977f36bc66c2a7db043b0 Mon Sep 17 00:00:00 2001 From: Jiawei Wu Date: Mon, 7 Nov 2022 00:35:51 -0800 Subject: [PATCH 15/27] Fix and add more tests --- .../rule_monitoring_service.test.ts | 175 ++++++++++++++++++ .../monitoring/rule_monitoring_service.ts | 8 +- .../server/rules_client/rules_client.ts | 3 +- x-pack/plugins/alerting/server/types.ts | 4 +- .../common/lib/wait_for_execution_count.ts | 4 +- .../group1/tests/alerting/create.ts | 1 + .../group1/tests/alerting/find.ts | 4 + .../group1/tests/alerting/get.ts | 2 + .../group2/tests/alerting/update.ts | 10 + .../spaces_only/tests/alerting/aggregate.ts | 15 ++ .../spaces_only/tests/alerting/create.ts | 6 + .../spaces_only/tests/alerting/find.ts | 16 +- .../spaces_only/tests/alerting/get.ts | 4 + .../spaces_only/tests/alerting/monitoring.ts | 36 ++-- .../spaces_only/tests/alerting/update.ts | 4 + 15 files changed, 259 insertions(+), 33 deletions(-) create mode 100644 x-pack/plugins/alerting/server/monitoring/rule_monitoring_service.test.ts diff --git a/x-pack/plugins/alerting/server/monitoring/rule_monitoring_service.test.ts b/x-pack/plugins/alerting/server/monitoring/rule_monitoring_service.test.ts new file mode 100644 index 0000000000000..2f441671265be --- /dev/null +++ b/x-pack/plugins/alerting/server/monitoring/rule_monitoring_service.test.ts @@ -0,0 +1,175 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { RuleMonitoringService } from './rule_monitoring_service'; +import { getDefaultMonitoring } from '../lib/monitoring'; + +const mockNow = '2020-01-01T02:00:00.000Z'; + +const ONE_MINUTE = 60 * 1000; +const ONE_HOUR = 60 * ONE_MINUTE; + +describe('RuleMonitoringService', () => { + beforeAll(() => { + jest.useFakeTimers('modern'); + jest.setSystemTime(new Date(mockNow).getTime()); + }); + + afterAll(() => { + jest.useRealTimers(); + }); + + it('should initialize with default monitoring', () => { + const ruleMonitoringService = new RuleMonitoringService(); + expect(ruleMonitoringService.getMonitoring()).toEqual( + getDefaultMonitoring(new Date(mockNow).toISOString()) + ); + }); + + it('should add history', () => { + const ruleMonitoringService = new RuleMonitoringService(); + + jest.advanceTimersByTime(ONE_HOUR); + const firstRunDate = new Date(); + + ruleMonitoringService.addHistory({ + duration: ONE_MINUTE, + hasError: false, + runDate: firstRunDate, + }); + + jest.advanceTimersByTime(ONE_HOUR); + const secondRunDate = new Date(); + + ruleMonitoringService.addHistory({ + duration: 2 * ONE_MINUTE, + hasError: true, + runDate: secondRunDate, + }); + + const { run } = ruleMonitoringService.getMonitoring(); + const { history, last_run: lastRun, calculated_metrics: calculatedMetrics } = run; + const { timestamp, metrics } = lastRun; + + expect(history.length).toEqual(2); + expect(history[0]).toEqual({ + success: true, + timestamp: firstRunDate.getTime(), + duration: ONE_MINUTE, + }); + expect(history[1]).toEqual({ + success: false, + timestamp: secondRunDate.getTime(), + duration: 2 * ONE_MINUTE, + }); + + expect(timestamp).toEqual(secondRunDate.toISOString()); + expect(metrics.duration).toEqual(2 * ONE_MINUTE); + + expect(calculatedMetrics).toEqual({ success_ratio: 0.5, p50: 90000, p95: 120000, p99: 120000 }); + }); + + describe('setters', () => { + it('should set monitoring', () => { + const ruleMonitoringService = new RuleMonitoringService(); + const customMonitoring = { + run: { + history: [ + { + success: true, + duration: 100000, + timestamp: 0, + }, + ], + calculated_metrics: { + success_ratio: 1, + p50: 100, + p95: 1000, + p99: 10000, + }, + last_run: { + timestamp: mockNow, + metrics: { + duration: 100000, + }, + }, + }, + }; + ruleMonitoringService.setMonitoring(customMonitoring); + expect(ruleMonitoringService.getMonitoring()).toEqual(customMonitoring); + }); + + it('should set totalSearchDurationMs', () => { + const ruleMonitoringService = new RuleMonitoringService(); + const { setLastRunMetricsTotalSearchDurationMs } = + ruleMonitoringService.getLastRunMetricsSetters(); + setLastRunMetricsTotalSearchDurationMs(123); + + const { + run: { + last_run: { metrics }, + }, + } = ruleMonitoringService.getMonitoring(); + expect(metrics.total_search_duration_ms).toEqual(123); + }); + + it('should set totalIndexDurationMs', () => { + const ruleMonitoringService = new RuleMonitoringService(); + const { setLastRunMetricsTotalIndexingDurationMs } = + ruleMonitoringService.getLastRunMetricsSetters(); + setLastRunMetricsTotalIndexingDurationMs(234); + + const { + run: { + last_run: { metrics }, + }, + } = ruleMonitoringService.getMonitoring(); + expect(metrics.total_indexing_duration_ms).toEqual(234); + }); + + it('should set totalAlertsDetected', () => { + const ruleMonitoringService = new RuleMonitoringService(); + const { setLastRunMetricsTotalAlertDetected } = + ruleMonitoringService.getLastRunMetricsSetters(); + setLastRunMetricsTotalAlertDetected(345); + + const { + run: { + last_run: { metrics }, + }, + } = ruleMonitoringService.getMonitoring(); + expect(metrics.total_alerts_detected).toEqual(345); + }); + + it('should set totalAlertsCreated', () => { + const ruleMonitoringService = new RuleMonitoringService(); + const { setLastRunMetricsTotalAlertCreated } = + ruleMonitoringService.getLastRunMetricsSetters(); + setLastRunMetricsTotalAlertCreated(456); + + const { + run: { + last_run: { metrics }, + }, + } = ruleMonitoringService.getMonitoring(); + expect(metrics.total_alerts_created).toEqual(456); + }); + + it('should set gapDurationS', () => { + const ruleMonitoringService = new RuleMonitoringService(); + const { setLastRunMetricsGapDurationS } = ruleMonitoringService.getLastRunMetricsSetters(); + setLastRunMetricsGapDurationS(567); + + const { + run: { + last_run: { metrics }, + }, + } = ruleMonitoringService.getMonitoring(); + expect(metrics.gap_duration_s).toEqual(567); + }); + }); +}); diff --git a/x-pack/plugins/alerting/server/monitoring/rule_monitoring_service.ts b/x-pack/plugins/alerting/server/monitoring/rule_monitoring_service.ts index 634104d3f2970..7f32e4a040f64 100644 --- a/x-pack/plugins/alerting/server/monitoring/rule_monitoring_service.ts +++ b/x-pack/plugins/alerting/server/monitoring/rule_monitoring_service.ts @@ -60,8 +60,8 @@ export class RuleMonitoringService { this.setLastRunMetricsTotalSearchDurationMs.bind(this), setLastRunMetricsTotalIndexingDurationMs: this.setLastRunMetricsTotalIndexingDurationMs.bind(this), - setLastRunMetricsTotalAlertDetected: this.setLastRunMetricsTotalAlertDetected.bind(this), - setLastRunMetricsTotalAlertCreated: this.setLastRunMetricsTotalAlertCreated.bind(this), + setLastRunMetricsTotalAlertDetected: this.setLastRunMetricsTotalAlertsDetected.bind(this), + setLastRunMetricsTotalAlertCreated: this.setLastRunMetricsTotalAlertsCreated.bind(this), setLastRunMetricsGapDurationS: this.setLastRunMetricsGapDurationS.bind(this), }; } @@ -74,11 +74,11 @@ export class RuleMonitoringService { this.monitoring.run.last_run.metrics.total_indexing_duration_ms = totalIndexingDurationMs; } - private setLastRunMetricsTotalAlertDetected(totalAlertDetected: number) { + private setLastRunMetricsTotalAlertsDetected(totalAlertDetected: number) { this.monitoring.run.last_run.metrics.total_alerts_detected = totalAlertDetected; } - private setLastRunMetricsTotalAlertCreated(totalAlertCreated: number) { + private setLastRunMetricsTotalAlertsCreated(totalAlertCreated: number) { this.monitoring.run.last_run.metrics.total_alerts_created = totalAlertCreated; } diff --git a/x-pack/plugins/alerting/server/rules_client/rules_client.ts b/x-pack/plugins/alerting/server/rules_client/rules_client.ts index f689ce3765a52..4bea41bd4fdb3 100644 --- a/x-pack/plugins/alerting/server/rules_client/rules_client.ts +++ b/x-pack/plugins/alerting/server/rules_client/rules_client.ts @@ -3516,6 +3516,7 @@ export class RulesClient { snoozeSchedule, }) : null; + const includeMonitoring = monitoring && !excludeFromPublicApi; const rule = { id, notifyWhen, @@ -3541,7 +3542,7 @@ export class RulesClient { ...(executionStatus ? { executionStatus: ruleExecutionStatusFromRaw(this.logger, id, executionStatus) } : {}), - ...(monitoring + ...(includeMonitoring ? { monitoring: convertMonitoringFromRawAndVerify(this.logger, id, monitoring) } : {}), ...(nextRun ? { nextRun: new Date(nextRun) } : {}), diff --git a/x-pack/plugins/alerting/server/types.ts b/x-pack/plugins/alerting/server/types.ts index eef6e49c6a56a..b9d4f68977398 100644 --- a/x-pack/plugins/alerting/server/types.ts +++ b/x-pack/plugins/alerting/server/types.ts @@ -288,8 +288,8 @@ export type RulesClientApi = PublicMethodsOf; export interface PublicRuleMonitoringService { setLastRunMetricsTotalSearchDurationMs: (totalSearchDurationMs: number) => void; setLastRunMetricsTotalIndexingDurationMs: (totalIndexingDurationMs: number) => void; - setLastRunMetricsTotalAlertDetected: (totalAlertDetected: number) => void; - setLastRunMetricsTotalAlertCreated: (totalAlertCreated: number) => void; + setLastRunMetricsTotalAlertsDetected: (totalAlertDetected: number) => void; + setLastRunMetricsTotalAlertsCreated: (totalAlertCreated: number) => void; setLastRunMetricsGapDurationS: (gapDurationS: number) => void; } diff --git a/x-pack/test/alerting_api_integration/common/lib/wait_for_execution_count.ts b/x-pack/test/alerting_api_integration/common/lib/wait_for_execution_count.ts index 76d8a9afb7098..6ad0dab431d81 100644 --- a/x-pack/test/alerting_api_integration/common/lib/wait_for_execution_count.ts +++ b/x-pack/test/alerting_api_integration/common/lib/wait_for_execution_count.ts @@ -28,13 +28,13 @@ export function createWaitForExecutionCount( const prefix = spaceId ? getUrlPrefix(spaceId) : ''; const getResponse = await st.get(`${prefix}/internal/alerting/rule/${id}`); expect(getResponse.status).to.eql(200); - if (getResponse.body.monitoring.execution.history.length >= count) { + if (getResponse.body.monitoring.run.history.length >= count) { attempts = 0; return true; } // eslint-disable-next-line no-console console.log( - `found ${getResponse.body.monitoring.execution.history.length} and looking for ${count}, waiting 3s then retrying` + `found ${getResponse.body.monitoring.run.history.length} and looking for ${count}, waiting 3s then retrying` ); await delay(delayMs); return waitForExecutionCount(count, id); diff --git a/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/create.ts b/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/create.ts index f775b3607fade..ca67447856c10 100644 --- a/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/create.ts +++ b/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/create.ts @@ -124,6 +124,7 @@ export default function createAlertTests({ getService }: FtrProviderContext) { mute_all: false, muted_alert_ids: [], execution_status: response.body.execution_status, + monitoring: response.body.monitoring, }); expect(typeof response.body.scheduled_task_id).to.be('string'); expect(Date.parse(response.body.created_at)).to.be.greaterThan(0); diff --git a/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/find.ts b/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/find.ts index 484d2c9cdac82..b8460e84202c9 100644 --- a/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/find.ts +++ b/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/find.ts @@ -84,6 +84,8 @@ const findTestUtils = ( mute_all: false, muted_alert_ids: [], execution_status: match.execution_status, + ...(match.next_run ? { next_run: match.next_run } : {}), + ...(match.last_run ? { last_run: match.last_run } : {}), ...(describeType === 'internal' ? { monitoring: match.monitoring, @@ -291,6 +293,8 @@ const findTestUtils = ( created_at: match.created_at, updated_at: match.updated_at, execution_status: match.execution_status, + ...(match.next_run ? { next_run: match.next_run } : {}), + ...(match.last_run ? { last_run: match.last_run } : {}), ...(describeType === 'internal' ? { monitoring: match.monitoring, diff --git a/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/get.ts b/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/get.ts index 4b2afe01d7a86..7ecd699bfb7e7 100644 --- a/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/get.ts +++ b/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/get.ts @@ -81,6 +81,8 @@ const getTestUtils = ( mute_all: false, muted_alert_ids: [], execution_status: response.body.execution_status, + ...(response.next_run ? { next_run: response.next_run } : {}), + ...(response.last_run ? { last_run: response.last_run } : {}), ...(describeType === 'internal' ? { monitoring: response.body.monitoring, diff --git a/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/alerting/update.ts b/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/alerting/update.ts index c49fa62c606b6..6e9102b62c043 100644 --- a/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/alerting/update.ts +++ b/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/alerting/update.ts @@ -132,6 +132,8 @@ export default function createUpdateTests({ getService }: FtrProviderContext) { created_at: response.body.created_at, updated_at: response.body.updated_at, execution_status: response.body.execution_status, + ...(response.body.next_run ? { next_run: response.body.next_run } : {}), + ...(response.body.last_run ? { last_run: response.body.last_run } : {}), }); expect(Date.parse(response.body.created_at)).to.be.greaterThan(0); expect(Date.parse(response.body.updated_at)).to.be.greaterThan(0); @@ -216,6 +218,8 @@ export default function createUpdateTests({ getService }: FtrProviderContext) { created_at: response.body.created_at, updated_at: response.body.updated_at, execution_status: response.body.execution_status, + ...(response.body.next_run ? { next_run: response.body.next_run } : {}), + ...(response.body.last_run ? { last_run: response.body.last_run } : {}), }); expect(Date.parse(response.body.created_at)).to.be.greaterThan(0); expect(Date.parse(response.body.updated_at)).to.be.greaterThan(0); @@ -311,6 +315,8 @@ export default function createUpdateTests({ getService }: FtrProviderContext) { created_at: response.body.created_at, updated_at: response.body.updated_at, execution_status: response.body.execution_status, + ...(response.body.next_run ? { next_run: response.body.next_run } : {}), + ...(response.body.last_run ? { last_run: response.body.last_run } : {}), }); expect(Date.parse(response.body.created_at)).to.be.greaterThan(0); expect(Date.parse(response.body.updated_at)).to.be.greaterThan(0); @@ -406,6 +412,8 @@ export default function createUpdateTests({ getService }: FtrProviderContext) { created_at: response.body.created_at, updated_at: response.body.updated_at, execution_status: response.body.execution_status, + ...(response.body.next_run ? { next_run: response.body.next_run } : {}), + ...(response.body.last_run ? { last_run: response.body.last_run } : {}), }); expect(Date.parse(response.body.created_at)).to.be.greaterThan(0); expect(Date.parse(response.body.updated_at)).to.be.greaterThan(0); @@ -499,6 +507,8 @@ export default function createUpdateTests({ getService }: FtrProviderContext) { created_at: response.body.created_at, updated_at: response.body.updated_at, execution_status: response.body.execution_status, + ...(response.body.next_run ? { next_run: response.body.next_run } : {}), + ...(response.body.last_run ? { last_run: response.body.last_run } : {}), }); expect(Date.parse(response.body.created_at)).to.be.greaterThan(0); expect(Date.parse(response.body.updated_at)).to.be.greaterThan(0); diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/aggregate.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/aggregate.ts index 4424175e36953..ff24b25d89fa2 100644 --- a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/aggregate.ts +++ b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/aggregate.ts @@ -37,6 +37,11 @@ export default function createAggregateTests({ getService }: FtrProviderContext) unknown: 0, warning: 0, }, + rule_last_run_outcome: { + succeeded: 0, + warning: 0, + failed: 0, + }, rule_muted_status: { muted: 0, unmuted: 0, @@ -116,6 +121,11 @@ export default function createAggregateTests({ getService }: FtrProviderContext) unknown: 0, warning: 0, }, + rule_last_run_outcome: { + succeeded: 5, + warning: 0, + failed: 2, + }, rule_muted_status: { muted: 0, unmuted: 7, @@ -200,6 +210,11 @@ export default function createAggregateTests({ getService }: FtrProviderContext) disabled: 0, enabled: 7, }, + ruleLastRunOutcome: { + succeeded: 5, + warning: 0, + failed: 2, + }, ruleMutedStatus: { muted: 0, unmuted: 7, diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/create.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/create.ts index 7860bf15dc8e5..8b00b773e0018 100644 --- a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/create.ts +++ b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/create.ts @@ -94,6 +94,8 @@ export default function createAlertTests({ getService }: FtrProviderContext) { created_at: response.body.created_at, updated_at: response.body.updated_at, execution_status: response.body.execution_status, + ...(response.body.next_run ? { next_run: response.body.next_run } : {}), + ...(response.body.last_run ? { last_run: response.body.last_run } : {}), }); expect(Date.parse(response.body.created_at)).to.be.greaterThan(0); expect(Date.parse(response.body.updated_at)).to.be.greaterThan(0); @@ -190,6 +192,8 @@ export default function createAlertTests({ getService }: FtrProviderContext) { created_at: response.body.created_at, updated_at: response.body.updated_at, execution_status: response.body.execution_status, + ...(response.body.next_run ? { next_run: response.body.next_run } : {}), + ...(response.body.last_run ? { last_run: response.body.last_run } : {}), }); const esResponse = await es.get>( @@ -485,6 +489,8 @@ export default function createAlertTests({ getService }: FtrProviderContext) { createdAt: response.body.createdAt, updatedAt: response.body.updatedAt, executionStatus: response.body.executionStatus, + ...(response.body.next_run ? { next_run: response.body.next_run } : {}), + ...(response.body.last_run ? { last_run: response.body.last_run } : {}), }); expect(Date.parse(response.body.createdAt)).to.be.greaterThan(0); expect(Date.parse(response.body.updatedAt)).to.be.greaterThan(0); diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/find.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/find.ts index 23dcc1abaea44..0bb9129230068 100644 --- a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/find.ts +++ b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/find.ts @@ -75,6 +75,8 @@ const findTestUtils = ( created_at: match.created_at, updated_at: match.updated_at, execution_status: match.execution_status, + ...(match.next_run ? { next_run: match.next_run } : {}), + ...(match.last_run ? { last_run: match.last_run } : {}), ...(describeType === 'internal' ? { monitoring: match.monitoring, @@ -142,13 +144,13 @@ const findTestUtils = ( const response = await supertest.get( `${getUrlPrefix(Spaces.space1.id)}/${ describeType === 'public' ? 'api' : 'internal' - }/alerting/rules/_find?filter=alert.attributes.monitoring.execution.calculated_metrics.success_ratio>50` + }/alerting/rules/_find?filter=alert.attributes.monitoring.run.calculated_metrics.success_ratio>50` ); expect(response.status).to.eql(describeType === 'internal' ? 200 : 400); if (describeType === 'public') { expect(response.body.message).to.eql( - 'Error find rules: Filter is not supported on this field alert.attributes.monitoring.execution.calculated_metrics.success_ratio' + 'Error find rules: Filter is not supported on this field alert.attributes.monitoring.run.calculated_metrics.success_ratio' ); } }); @@ -159,13 +161,13 @@ const findTestUtils = ( const response = await supertest.get( `${getUrlPrefix(Spaces.space1.id)}/${ describeType === 'public' ? 'api' : 'internal' - }/alerting/rules/_find?sort_field=monitoring.execution.calculated_metrics.success_ratio` + }/alerting/rules/_find?sort_field=monitoring.run.calculated_metrics.success_ratio` ); expect(response.status).to.eql(describeType === 'internal' ? 200 : 400); if (describeType === 'public') { expect(response.body.message).to.eql( - 'Error find rules: Sort is not supported on this field monitoring.execution.calculated_metrics.success_ratio' + 'Error find rules: Sort is not supported on this field monitoring.run.calculated_metrics.success_ratio' ); } }); @@ -176,13 +178,13 @@ const findTestUtils = ( const response = await supertest.get( `${getUrlPrefix(Spaces.space1.id)}/${ describeType === 'public' ? 'api' : 'internal' - }/alerting/rules/_find?search_fields=monitoring.execution.calculated_metrics.success_ratio&search=50` + }/alerting/rules/_find?search_fields=monitoring.run.calculated_metrics.success_ratio&search=50` ); expect(response.status).to.eql(describeType === 'internal' ? 200 : 400); if (describeType === 'public') { expect(response.body.message).to.eql( - 'Error find rules: Search field monitoring.execution.calculated_metrics.success_ratio not supported' + 'Error find rules: Search field monitoring.run.calculated_metrics.success_ratio not supported' ); } }); @@ -325,6 +327,8 @@ export default function createFindTests({ getService }: FtrProviderContext) { createdAt: match.createdAt, updatedAt: match.updatedAt, executionStatus: match.executionStatus, + ...(match.next_run ? { next_run: match.next_run } : {}), + ...(match.last_run ? { last_run: match.last_run } : {}), }); expect(Date.parse(match.createdAt)).to.be.greaterThan(0); expect(Date.parse(match.updatedAt)).to.be.greaterThan(0); diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/get.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/get.ts index 7a94198c4ea57..3ed3eeb067860 100644 --- a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/get.ts +++ b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/get.ts @@ -54,6 +54,8 @@ const getTestUtils = ( created_at: response.body.created_at, updated_at: response.body.updated_at, execution_status: response.body.execution_status, + ...(response.body.next_run ? { next_run: response.body.next_run } : {}), + ...(response.body.last_run ? { last_run: response.body.last_run } : {}), ...(describeType === 'internal' ? { monitoring: response.body.monitoring, @@ -149,6 +151,8 @@ export default function createGetTests({ getService }: FtrProviderContext) { createdAt: response.body.createdAt, updatedAt: response.body.updatedAt, executionStatus: response.body.executionStatus, + ...(response.body.next_run ? { next_run: response.body.next_run } : {}), + ...(response.body.last_run ? { last_run: response.body.last_run } : {}), }); expect(Date.parse(response.body.createdAt)).to.be.greaterThan(0); expect(Date.parse(response.body.updatedAt)).to.be.greaterThan(0); diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/monitoring.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/monitoring.ts index c08a28b3c3ca3..38506eb54a4bc 100644 --- a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/monitoring.ts +++ b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/monitoring.ts @@ -35,9 +35,9 @@ export default function monitoringAlertTests({ getService }: FtrProviderContext) ); expect(getResponse.status).to.eql(200); - expect(getResponse.body.monitoring.execution.history.length).to.be(1); - expect(getResponse.body.monitoring.execution.history[0].success).to.be(true); - expect(getResponse.body.monitoring.execution.calculated_metrics.success_ratio).to.be(1); + expect(getResponse.body.monitoring.run.history.length).to.be(1); + expect(getResponse.body.monitoring.run.history[0].success).to.be(true); + expect(getResponse.body.monitoring.run.calculated_metrics.success_ratio).to.be(1); }); it('should return an accurate history for multiple success', async () => { @@ -56,11 +56,11 @@ export default function monitoringAlertTests({ getService }: FtrProviderContext) ); expect(getResponse.status).to.eql(200); - expect(getResponse.body.monitoring.execution.history.length).to.be(3); - expect(getResponse.body.monitoring.execution.history[0].success).to.be(true); - expect(getResponse.body.monitoring.execution.history[1].success).to.be(true); - expect(getResponse.body.monitoring.execution.history[2].success).to.be(true); - expect(getResponse.body.monitoring.execution.calculated_metrics.success_ratio).to.be(1); + expect(getResponse.body.monitoring.run.history.length).to.be(3); + expect(getResponse.body.monitoring.run.history[0].success).to.be(true); + expect(getResponse.body.monitoring.run.history[1].success).to.be(true); + expect(getResponse.body.monitoring.run.history[2].success).to.be(true); + expect(getResponse.body.monitoring.run.calculated_metrics.success_ratio).to.be(1); }); it('should return an accurate history for some successes and some failures', async () => { @@ -88,13 +88,13 @@ export default function monitoringAlertTests({ getService }: FtrProviderContext) ); expect(getResponse.status).to.eql(200); - expect(getResponse.body.monitoring.execution.history.length).to.be(5); - expect(getResponse.body.monitoring.execution.history[0].success).to.be(true); - expect(getResponse.body.monitoring.execution.history[1].success).to.be(true); - expect(getResponse.body.monitoring.execution.history[2].success).to.be(true); - expect(getResponse.body.monitoring.execution.history[3].success).to.be(false); - expect(getResponse.body.monitoring.execution.history[4].success).to.be(false); - expect(getResponse.body.monitoring.execution.calculated_metrics.success_ratio).to.be(0.6); + expect(getResponse.body.monitoring.run.history.length).to.be(5); + expect(getResponse.body.monitoring.run.history[0].success).to.be(true); + expect(getResponse.body.monitoring.run.history[1].success).to.be(true); + expect(getResponse.body.monitoring.run.history[2].success).to.be(true); + expect(getResponse.body.monitoring.run.history[3].success).to.be(false); + expect(getResponse.body.monitoring.run.history[4].success).to.be(false); + expect(getResponse.body.monitoring.run.calculated_metrics.success_ratio).to.be(0.6); }); it('should populate rule objects with the calculated percentiles', async () => { @@ -118,7 +118,7 @@ export default function monitoringAlertTests({ getService }: FtrProviderContext) ); expect(getResponse.status).to.eql(200); - getResponse.body.monitoring.execution.history.forEach((history: any) => { + getResponse.body.monitoring.run.history.forEach((history: any) => { expect(history.duration).to.be.a('number'); }); }); @@ -135,13 +135,13 @@ export default function monitoringAlertTests({ getService }: FtrProviderContext) `${getUrlPrefix(Spaces.space1.id)}/internal/alerting/rule/${id}` ); expect(getResponse.status).to.eql(200); - if (getResponse.body.monitoring.execution.history.length >= count) { + if (getResponse.body.monitoring.run.history.length >= count) { attempts = 0; return true; } // eslint-disable-next-line no-console console.log( - `found ${getResponse.body.monitoring.execution.history.length} and looking for ${count}, waiting 3s then retrying` + `found ${getResponse.body.monitoring.run.history.length} and looking for ${count}, waiting 3s then retrying` ); await delay(3000); return waitForExecutionCount(count, id); diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/update.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/update.ts index c5a9c93d45e81..33897ac0df062 100644 --- a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/update.ts +++ b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/update.ts @@ -63,6 +63,8 @@ export default function createUpdateTests({ getService }: FtrProviderContext) { created_at: response.body.created_at, updated_at: response.body.updated_at, execution_status: response.body.execution_status, + ...(response.body.next_run ? { next_run: response.body.next_run } : {}), + ...(response.body.last_run ? { last_run: response.body.last_run } : {}), }); expect(Date.parse(response.body.created_at)).to.be.greaterThan(0); expect(Date.parse(response.body.updated_at)).to.be.greaterThan(0); @@ -163,6 +165,8 @@ export default function createUpdateTests({ getService }: FtrProviderContext) { createdAt: response.body.createdAt, updatedAt: response.body.updatedAt, executionStatus: response.body.executionStatus, + ...(response.body.next_run ? { next_run: response.body.next_run } : {}), + ...(response.body.last_run ? { last_run: response.body.last_run } : {}), }); expect(Date.parse(response.body.createdAt)).to.be.greaterThan(0); expect(Date.parse(response.body.updatedAt)).to.be.greaterThan(0); From 74269acc4e8e987a381183a9401112ab1052240a Mon Sep 17 00:00:00 2001 From: Jiawei Wu Date: Mon, 7 Nov 2022 10:28:49 -0800 Subject: [PATCH 16/27] Migration unit and integration tests, fixed some more tests --- .../plugins/alerting/server/lib/monitoring.ts | 2 +- .../monitoring/rule_monitoring_service.ts | 8 +- .../saved_objects/migrations/8.6/index.ts | 21 ++- .../saved_objects/migrations/index.test.ts | 103 +++++++++++++++ .../group1/tests/alerting/create.ts | 3 +- .../group1/tests/alerting/get.ts | 4 +- .../spaces_only/tests/alerting/migrations.ts | 60 +++++++++ .../functional/es_archives/alerts/data.json | 120 ++++++++++++++++++ .../es_archives/alerts/mappings.json | 73 +++++++++++ 9 files changed, 379 insertions(+), 15 deletions(-) diff --git a/x-pack/plugins/alerting/server/lib/monitoring.ts b/x-pack/plugins/alerting/server/lib/monitoring.ts index 249a770b1ba57..aeb34e743f6db 100644 --- a/x-pack/plugins/alerting/server/lib/monitoring.ts +++ b/x-pack/plugins/alerting/server/lib/monitoring.ts @@ -16,7 +16,7 @@ export const INITIAL_METRICS = { gap_duration_s: null, }; -export const getDefaultMonitoring = (timestamp: string) => { +export const getDefaultMonitoring = (timestamp: string): RawRuleMonitoring => { return { run: { history: [], diff --git a/x-pack/plugins/alerting/server/monitoring/rule_monitoring_service.ts b/x-pack/plugins/alerting/server/monitoring/rule_monitoring_service.ts index 7f32e4a040f64..0043f47c51633 100644 --- a/x-pack/plugins/alerting/server/monitoring/rule_monitoring_service.ts +++ b/x-pack/plugins/alerting/server/monitoring/rule_monitoring_service.ts @@ -6,7 +6,7 @@ */ import { getDefaultMonitoring, getExecutionDurationPercentiles } from '../lib/monitoring'; -import { RuleMonitoring, RuleMonitoringHistory } from '../types'; +import { RuleMonitoring, RuleMonitoringHistory, PublicRuleMonitoringService } from '../types'; export class RuleMonitoringService { private monitoring: RuleMonitoring = getDefaultMonitoring(new Date().toISOString()); @@ -54,14 +54,14 @@ export class RuleMonitoringService { }; } - public getLastRunMetricsSetters() { + public getLastRunMetricsSetters(): PublicRuleMonitoringService { return { setLastRunMetricsTotalSearchDurationMs: this.setLastRunMetricsTotalSearchDurationMs.bind(this), setLastRunMetricsTotalIndexingDurationMs: this.setLastRunMetricsTotalIndexingDurationMs.bind(this), - setLastRunMetricsTotalAlertDetected: this.setLastRunMetricsTotalAlertsDetected.bind(this), - setLastRunMetricsTotalAlertCreated: this.setLastRunMetricsTotalAlertsCreated.bind(this), + setLastRunMetricsTotalAlertsDetected: this.setLastRunMetricsTotalAlertsDetected.bind(this), + setLastRunMetricsTotalAlertsCreated: this.setLastRunMetricsTotalAlertsCreated.bind(this), setLastRunMetricsGapDurationS: this.setLastRunMetricsGapDurationS.bind(this), }; } diff --git a/x-pack/plugins/alerting/server/saved_objects/migrations/8.6/index.ts b/x-pack/plugins/alerting/server/saved_objects/migrations/8.6/index.ts index 00033182230fb..d4535008bfbe6 100644 --- a/x-pack/plugins/alerting/server/saved_objects/migrations/8.6/index.ts +++ b/x-pack/plugins/alerting/server/saved_objects/migrations/8.6/index.ts @@ -9,6 +9,7 @@ import { SavedObjectUnsanitizedDoc } from '@kbn/core-saved-objects-server'; import { EncryptedSavedObjectsPluginSetup } from '@kbn/encrypted-saved-objects-plugin/server'; import { createEsoMigration, pipeMigrations } from '../utils'; import { RawRule, RuleLastRunOutcomeValues } from '../../../types'; +import { getDefaultMonitoring } from '../../../lib/monitoring'; const succeededStatus = ['ok', 'active', 'succeeded']; const warningStatus = ['warning']; @@ -16,7 +17,7 @@ const failedStatus = ['error', 'failed']; const getLastRun = (attributes: RawRule) => { const { executionStatus } = attributes; - const { status, warning, error } = executionStatus; + const { status, warning, error } = executionStatus || {}; let outcome; if (succeededStatus.includes(status)) { @@ -36,7 +37,7 @@ const getLastRun = (attributes: RawRule) => { return { outcome, outcomeMsg: warning?.message || error?.message || null, - warning: warning?.reason || null, + warning: warning?.reason || error?.reason || null, alertsCount: {}, }; }; @@ -44,10 +45,19 @@ const getLastRun = (attributes: RawRule) => { const getMonitoring = (attributes: RawRule) => { const { executionStatus, monitoring } = attributes; if (!monitoring) { - return null; + if (!executionStatus) { + return null; + } + + // monitoring now has data from executionStatus, therefore, we should migrate + // these fields even if monitoring doesn't exist. + const defaultMonitoring = getDefaultMonitoring(executionStatus.lastExecutionDate); + if (executionStatus.lastDuration) { + defaultMonitoring.run.last_run.metrics.duration = executionStatus.lastDuration; + } + return defaultMonitoring; } - // Question: Do we want to backfill the history? const { lastExecutionDate, lastDuration } = executionStatus; // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -73,9 +83,6 @@ function migrateLastRun( const lastRun = getLastRun(attributes); const monitoring = getMonitoring(attributes); - // Question: Do we want to migrate running and next_rule? It might be - // very unreliable since we don't know for sure (long running rules) when the rule finishes running - // and therefore next_run would be incorrect. return { ...doc, attributes: { diff --git a/x-pack/plugins/alerting/server/saved_objects/migrations/index.test.ts b/x-pack/plugins/alerting/server/saved_objects/migrations/index.test.ts index 09f466a8e9a37..147c4d1f9df82 100644 --- a/x-pack/plugins/alerting/server/saved_objects/migrations/index.test.ts +++ b/x-pack/plugins/alerting/server/saved_objects/migrations/index.test.ts @@ -2426,6 +2426,109 @@ describe('successful migrations', () => { }); }); + describe('8.6.0', () => { + test('migrates executionStatus success', () => { + const migration860 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)[ + '8.6.0' + ]; + + let ruleWithExecutionStatus = getMockData({ + executionStatus: { + status: 'ok', + lastExecutionDate: '2022-01-02T00:00:00.000Z', + lastDuration: 60000, + }, + }); + + let migratedRule = migration860(ruleWithExecutionStatus, migrationContext); + expect(migratedRule.attributes.lastRun.outcome).toEqual('succeeded'); + expect(migratedRule.attributes.lastRun.warning).toEqual(null); + ruleWithExecutionStatus = getMockData({ + executionStatus: { + status: 'active', + lastExecutionDate: '2022-01-02T00:00:00.000Z', + lastDuration: 60000, + }, + }); + + migratedRule = migration860(ruleWithExecutionStatus, migrationContext); + expect(migratedRule.attributes.lastRun.outcome).toEqual('succeeded'); + expect(migratedRule.attributes.lastRun.warning).toEqual(null); + }); + + test('migrates executionStatus warning and error', () => { + const migration860 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)[ + '8.6.0' + ]; + + let ruleWithExecutionStatus = getMockData({ + executionStatus: { + status: 'warning', + lastExecutionDate: '2022-01-02T00:00:00.000Z', + lastDuration: 60000, + warning: { + reason: 'warning reason', + message: 'warning message', + }, + }, + }); + + let migratedRule = migration860(ruleWithExecutionStatus, migrationContext); + expect(migratedRule.attributes.lastRun.outcome).toEqual('warning'); + expect(migratedRule.attributes.lastRun.outcomeMsg).toEqual('warning message'); + expect(migratedRule.attributes.lastRun.warning).toEqual('warning reason'); + + ruleWithExecutionStatus = getMockData({ + executionStatus: { + status: 'error', + lastExecutionDate: '2022-01-02T00:00:00.000Z', + lastDuration: 60000, + error: { + reason: 'failed reason', + message: 'failed message', + }, + }, + }); + + migratedRule = migration860(ruleWithExecutionStatus, migrationContext); + expect(migratedRule.attributes.lastRun.outcome).toEqual('failed'); + expect(migratedRule.attributes.lastRun.outcomeMsg).toEqual('failed message'); + expect(migratedRule.attributes.lastRun.warning).toEqual('failed reason'); + }); + + test('migrates empty monitoring', () => { + const migration860 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)[ + '8.6.0' + ]; + + const ruleWithoutMonitoring = getMockData(); + const migratedRule = migration860(ruleWithoutMonitoring, migrationContext); + + expect(migratedRule.attributes.monitoring).toBeUndefined(); + }); + + test('migrates empty monitoring when executionStatus exists', () => { + const migration860 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)[ + '8.6.0' + ]; + + const ruleWithMonitoring = getMockData({ + executionStatus: { + status: 'ok', + lastExecutionDate: '2022-01-02T00:00:00.000Z', + lastDuration: 60000, + }, + }); + const migratedRule = migration860(ruleWithMonitoring, migrationContext); + + expect(migratedRule.attributes.monitoring.run.history).toEqual([]); + expect(migratedRule.attributes.monitoring.run.last_run.timestamp).toEqual( + '2022-01-02T00:00:00.000Z' + ); + expect(migratedRule.attributes.monitoring.run.last_run.metrics.duration).toEqual(60000); + }); + }); + describe('Metrics Inventory Threshold rule', () => { test('Migrates incorrect action group spelling', () => { const migration800 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)[ diff --git a/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/create.ts b/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/create.ts index ca67447856c10..1fe600ee06d4a 100644 --- a/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/create.ts +++ b/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/create.ts @@ -124,7 +124,8 @@ export default function createAlertTests({ getService }: FtrProviderContext) { mute_all: false, muted_alert_ids: [], execution_status: response.body.execution_status, - monitoring: response.body.monitoring, + ...(response.body.next_run ? { next_run: response.body.next_run } : {}), + ...(response.body.last_run ? { last_run: response.body.last_run } : {}), }); expect(typeof response.body.scheduled_task_id).to.be('string'); expect(Date.parse(response.body.created_at)).to.be.greaterThan(0); diff --git a/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/get.ts b/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/get.ts index 7ecd699bfb7e7..8da3e6d3c5def 100644 --- a/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/get.ts +++ b/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/get.ts @@ -81,8 +81,8 @@ const getTestUtils = ( mute_all: false, muted_alert_ids: [], execution_status: response.body.execution_status, - ...(response.next_run ? { next_run: response.next_run } : {}), - ...(response.last_run ? { last_run: response.last_run } : {}), + ...(response.body.next_run ? { next_run: response.body.next_run } : {}), + ...(response.body.last_run ? { last_run: response.body.last_run } : {}), ...(describeType === 'internal' ? { monitoring: response.body.monitoring, diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/migrations.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/migrations.ts index 9edb414e39c77..7765c266f5a74 100644 --- a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/migrations.ts +++ b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/migrations.ts @@ -521,5 +521,65 @@ export default function createGetTests({ getService }: FtrProviderContext) { expect(response.statusCode).to.eql(200); expect(response.body._source?.alert?.params?.esQuery).to.eql('{"query":}'); }); + + it('8.6.0 migrates executionStatus and monitoring', async () => { + const response = await es.get<{ alert: RawRule }>( + { + index: '.kibana', + id: 'alert:8370ffd2-f2db-49dc-9741-92c657189b9b', + }, + { meta: true } + ); + const alert = response.body._source?.alert; + + expect(alert?.monitoring).to.eql({ + run: { + history: [ + { + duration: 60000, + success: true, + timestamp: '2022-08-24T19:05:49.817Z', + }, + ], + calculated_metrics: { + success_ratio: 1, + p50: 0, + p95: 60000, + p99: 60000, + }, + last_run: { + timestamp: '2022-08-24T19:05:49.817Z', + metrics: { + duration: 60000, + }, + }, + }, + }); + + expect(alert?.lastRun).to.eql({ + outcome: 'succeeded', + outcomeMsg: null, + warning: null, + alertsCount: {}, + }); + + expect(alert?.nextRun).to.eql(undefined); + }); + + it('8.6 migrates executionStatus warnings and errors', async () => { + const response = await es.get<{ alert: RawRule }>( + { + index: '.kibana', + id: 'alert:c87707ac-7328-47f7-b212-2cb40a4fc9b9', + }, + { meta: true } + ); + + const alert = response.body._source?.alert; + + expect(alert?.lastRun?.outcome).to.eql('warning'); + expect(alert?.lastRun?.warning).to.eql('warning reason'); + expect(alert?.lastRun?.outcomeMsg).to.eql('warning message'); + }); }); } diff --git a/x-pack/test/functional/es_archives/alerts/data.json b/x-pack/test/functional/es_archives/alerts/data.json index 0129bb2f4729c..8d7be46b6c749 100644 --- a/x-pack/test/functional/es_archives/alerts/data.json +++ b/x-pack/test/functional/es_archives/alerts/data.json @@ -1232,3 +1232,123 @@ } } } + +{ + "type": "doc", + "value": { + "id": "alert:8370ffd2-f2db-49dc-9741-92c657189b9b", + "index": ".kibana_1", + "source": { + "alert": { + "alertTypeId": "example.always-firing", + "apiKey": null, + "apiKeyOwner": null, + "consumer": "alerts", + "createdAt": "2022-08-24T19:02:30.889Z", + "createdBy": "elastic", + "enabled": false, + "muteAll": false, + "mutedInstanceIds": [], + "name": "Test rule migration with successful execution status and monitoring", + "params": {}, + "schedule": { + "interval": "1m" + }, + "scheduledTaskId": null, + "tags": [], + "throttle": null, + "updatedBy": "elastic", + "isSnoozedUntil": "2022-08-24T19:05:49.817Z", + "executionStatus": { + "status": "ok", + "lastExecutionDate": "2022-08-24T19:05:49.817Z", + "lastDuration": 60000 + }, + "monitoring": { + "execution": { + "history": [{ + "duration": 60000, + "success": true, + "timestamp": "2022-08-24T19:05:49.817Z" + }], + "calculated_metrics": { + "success_ratio": 1, + "p50": 0, + "p95": 60000, + "p99": 60000 + } + } + } + }, + "migrationVersion": { + "alert": "8.0.1" + }, + "references": [ + ], + "type": "alert", + "updated_at": "2022-11-01T19:05:50.159Z" + } + } +} + +{ + "type": "doc", + "value": { + "id": "alert:c87707ac-7328-47f7-b212-2cb40a4fc9b9", + "index": ".kibana_1", + "source": { + "alert": { + "alertTypeId": "example.always-firing", + "apiKey": null, + "apiKeyOwner": null, + "consumer": "alerts", + "createdAt": "2022-08-24T19:02:30.889Z", + "createdBy": "elastic", + "enabled": false, + "muteAll": false, + "mutedInstanceIds": [], + "name": "Test rule migration with warning execution status", + "params": {}, + "schedule": { + "interval": "1m" + }, + "scheduledTaskId": null, + "tags": [], + "throttle": null, + "updatedBy": "elastic", + "isSnoozedUntil": "2022-08-24T19:05:49.817Z", + "executionStatus": { + "status": "warning", + "lastExecutionDate": "2022-08-24T19:05:49.817Z", + "lastDuration": 60000, + "warning": { + "reason": "warning reason", + "message": "warning message" + } + }, + "monitoring": { + "execution": { + "history": [{ + "duration": 60000, + "success": true, + "timestamp": "2022-08-24T19:05:49.817Z" + }], + "calculated_metrics": { + "success_ratio": 1, + "p50": 0, + "p95": 60000, + "p99": 60000 + } + } + } + }, + "migrationVersion": { + "alert": "8.0.1" + }, + "references": [ + ], + "type": "alert", + "updated_at": "2022-11-01T19:05:50.159Z" + } + } +} diff --git a/x-pack/test/functional/es_archives/alerts/mappings.json b/x-pack/test/functional/es_archives/alerts/mappings.json index 0da2b51499517..2e64004634fa6 100644 --- a/x-pack/test/functional/es_archives/alerts/mappings.json +++ b/x-pack/test/functional/es_archives/alerts/mappings.json @@ -175,6 +175,79 @@ }, "isSnoozedUntil": { "type": "date" + }, + "monitoring": { + "properties": { + "execution": { + "properties": { + "history": { + "properties": { + "duration": { + "type": "long" + }, + "success": { + "type": "boolean" + }, + "timestamp": { + "type": "date" + } + } + }, + "calculated_metrics": { + "properties": { + "p50": { + "type": "long" + }, + "p95": { + "type": "long" + }, + "p99": { + "type": "long" + }, + "success_ratio": { + "type": "float" + } + } + } + } + } + } + }, + "executionStatus": { + "properties": { + "numberOfTriggeredActions": { + "type": "long" + }, + "status": { + "type": "keyword" + }, + "lastExecutionDate": { + "type": "date" + }, + "lastDuration": { + "type": "long" + }, + "error": { + "properties": { + "reason": { + "type": "keyword" + }, + "message": { + "type": "keyword" + } + } + }, + "warning": { + "properties": { + "reason": { + "type": "keyword" + }, + "message": { + "type": "keyword" + } + } + } + } } } }, From be1feb3faea42ad46d315aaaebecb90fb9723424 Mon Sep 17 00:00:00 2001 From: Jiawei Wu Date: Mon, 7 Nov 2022 10:57:43 -0800 Subject: [PATCH 17/27] Fix types --- .../server/monitoring/rule_monitoring_service.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/alerting/server/monitoring/rule_monitoring_service.test.ts b/x-pack/plugins/alerting/server/monitoring/rule_monitoring_service.test.ts index 2f441671265be..54fc6552308b5 100644 --- a/x-pack/plugins/alerting/server/monitoring/rule_monitoring_service.test.ts +++ b/x-pack/plugins/alerting/server/monitoring/rule_monitoring_service.test.ts @@ -133,9 +133,9 @@ describe('RuleMonitoringService', () => { it('should set totalAlertsDetected', () => { const ruleMonitoringService = new RuleMonitoringService(); - const { setLastRunMetricsTotalAlertDetected } = + const { setLastRunMetricsTotalAlertsDetected } = ruleMonitoringService.getLastRunMetricsSetters(); - setLastRunMetricsTotalAlertDetected(345); + setLastRunMetricsTotalAlertsDetected(345); const { run: { @@ -147,9 +147,9 @@ describe('RuleMonitoringService', () => { it('should set totalAlertsCreated', () => { const ruleMonitoringService = new RuleMonitoringService(); - const { setLastRunMetricsTotalAlertCreated } = + const { setLastRunMetricsTotalAlertsCreated } = ruleMonitoringService.getLastRunMetricsSetters(); - setLastRunMetricsTotalAlertCreated(456); + setLastRunMetricsTotalAlertsCreated(456); const { run: { From d404fd860d0f53a64d87ed9aa47a152b8c0be3ab Mon Sep 17 00:00:00 2001 From: Jiawei Wu Date: Tue, 8 Nov 2022 00:11:38 -0800 Subject: [PATCH 18/27] Fix tests and types --- x-pack/plugins/alerting/server/mocks.ts | 16 +++++++++++++++- .../server/utils/rule_executor.test_helpers.ts | 3 ++- .../spaces_only/tests/alerting/find.ts | 4 ++-- .../security_solution/legacy_actions/data.json | 2 +- 4 files changed, 20 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/alerting/server/mocks.ts b/x-pack/plugins/alerting/server/mocks.ts index 0a99489822d38..67876537c3d3e 100644 --- a/x-pack/plugins/alerting/server/mocks.ts +++ b/x-pack/plugins/alerting/server/mocks.ts @@ -14,7 +14,7 @@ import { searchSourceCommonMock } from '@kbn/data-plugin/common/search/search_so import { rulesClientMock } from './rules_client.mock'; import { PluginSetupContract, PluginStartContract } from './plugin'; import { Alert, AlertFactoryDoneUtils } from './alert'; -import { AlertInstanceContext, AlertInstanceState } from './types'; +import { AlertInstanceContext, AlertInstanceState, PublicRuleMonitoringService } from './types'; export { rulesClientMock }; @@ -128,3 +128,17 @@ export const alertsMock = { createStart: createStartMock, createRuleExecutorServices: createRuleExecutorServicesMock, }; + +const createRuleMonitoringServiceMock = () => { + const mock = { + setLastRunMetricsTotalSearchDurationMs: jest.fn(), + setLastRunMetricsTotalIndexingDurationMs: jest.fn(), + setLastRunMetricsTotalAlertsDetected: jest.fn(), + setLastRunMetricsTotalAlertsCreated: jest.fn(), + setLastRunMetricsGapDurationS: jest.fn(), + } as unknown as jest.Mocked; + + return mock; +}; + +export const ruleMonitoringServiceMock = { create: createRuleMonitoringServiceMock }; diff --git a/x-pack/plugins/rule_registry/server/utils/rule_executor.test_helpers.ts b/x-pack/plugins/rule_registry/server/utils/rule_executor.test_helpers.ts index 70c827942ade3..330531b1fd27e 100644 --- a/x-pack/plugins/rule_registry/server/utils/rule_executor.test_helpers.ts +++ b/x-pack/plugins/rule_registry/server/utils/rule_executor.test_helpers.ts @@ -16,7 +16,7 @@ import { RuleTypeParams, RuleTypeState, } from '@kbn/alerting-plugin/server'; -import { alertsMock } from '@kbn/alerting-plugin/server/mocks'; +import { alertsMock, ruleMonitoringServiceMock } from '@kbn/alerting-plugin/server/mocks'; import { searchSourceCommonMock } from '@kbn/data-plugin/common/search/search_source/mocks'; import { Logger } from '@kbn/logging'; @@ -77,6 +77,7 @@ export const createDefaultAlertExecutorOptions = < shouldWriteAlerts: () => shouldWriteAlerts, shouldStopExecution: () => false, searchSourceClient: searchSourceCommonMock, + ruleMonitoringService: ruleMonitoringServiceMock.create(), }, state, previousStartedAt: null, diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/find.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/find.ts index 0bb9129230068..d10518aca575f 100644 --- a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/find.ts +++ b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/find.ts @@ -327,8 +327,8 @@ export default function createFindTests({ getService }: FtrProviderContext) { createdAt: match.createdAt, updatedAt: match.updatedAt, executionStatus: match.executionStatus, - ...(match.next_run ? { next_run: match.next_run } : {}), - ...(match.last_run ? { last_run: match.last_run } : {}), + ...(match.nextRun ? { nextRun: match.nextRun } : {}), + ...(match.lastRun ? { lastRun: match.lastRun } : {}), }); expect(Date.parse(match.createdAt)).to.be.greaterThan(0); expect(Date.parse(match.updatedAt)).to.be.greaterThan(0); diff --git a/x-pack/test/functional/es_archives/security_solution/legacy_actions/data.json b/x-pack/test/functional/es_archives/security_solution/legacy_actions/data.json index 25aa371e02144..f0c883c6b3756 100644 --- a/x-pack/test/functional/es_archives/security_solution/legacy_actions/data.json +++ b/x-pack/test/functional/es_archives/security_solution/legacy_actions/data.json @@ -776,7 +776,7 @@ "scheduledTaskId" : null, "legacyId" : "29ba2fa0-b076-11ec-bb3f-1f063f8e06cf", "monitoring" : { - "execution" : { + "run" : { "history" : [ { "duration" : 111, From 8da8345ce6b951cb9d3ca2c69f3596f88cbd0d57 Mon Sep 17 00:00:00 2001 From: Jiawei Wu Date: Tue, 8 Nov 2022 13:27:11 -0800 Subject: [PATCH 19/27] Fix SO migration snapshot, add more task runner tests --- .../migrations/check_registered_types.test.ts | 2 +- .../alerting/server/task_runner/fixtures.ts | 26 +++-- .../server/task_runner/task_runner.test.ts | 101 ++++++++++++++++++ .../utils/create_lifecycle_rule_type.test.ts | 1 + 4 files changed, 120 insertions(+), 10 deletions(-) diff --git a/src/core/server/integration_tests/saved_objects/migrations/check_registered_types.test.ts b/src/core/server/integration_tests/saved_objects/migrations/check_registered_types.test.ts index 0252a6be82215..5f97899d137bd 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/check_registered_types.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/check_registered_types.test.ts @@ -53,7 +53,7 @@ describe('checking migration metadata changes on all registered SO types', () => Object { "action": "7858e6d5a9f231bf23f6f2e57328eb0095b26735", "action_task_params": "bbd38cbfd74bf6713586fe078e3fa92db2234299", - "alert": "48461f3375d9ba22882ea23a318b62a5b0921a9b", + "alert": "eefada4a02ce05962387c0679d7b292771a931c4", "api_key_pending_invalidation": "9b4bc1235337da9a87ef05a1d1f4858b2a3b77c6", "apm-indices": "ceb0870f3a74e2ffc3a1cd3a3c73af76baca0999", "apm-server-schema": "2bfd2998d3873872e1366458ce553def85418f91", diff --git a/x-pack/plugins/alerting/server/task_runner/fixtures.ts b/x-pack/plugins/alerting/server/task_runner/fixtures.ts index 39b252ce940a5..b0e98263591a9 100644 --- a/x-pack/plugins/alerting/server/task_runner/fixtures.ts +++ b/x-pack/plugins/alerting/server/task_runner/fixtures.ts @@ -6,7 +6,7 @@ */ import { TaskStatus } from '@kbn/task-manager-plugin/server'; -import { Rule, RuleTypeParams, RecoveredActionGroup } from '../../common'; +import { Rule, RuleTypeParams, RecoveredActionGroup, RuleMonitoring } from '../../common'; import { getDefaultMonitoring } from '../lib/monitoring'; import { UntypedNormalizedRuleType } from '../rule_type_registry'; import { EVENT_LOG_ACTIONS } from '../plugin'; @@ -54,17 +54,30 @@ export const RULE_ACTIONS = [ }, ]; +const defaultHistory = [ + { + success: true, + timestamp: 0, + }, +]; + export const generateSavedObjectParams = ({ error = null, warning = null, status = 'ok', outcome = 'succeeded', + nextRun = '1970-01-01T00:00:10.000Z', + successRatio = 1, + history = defaultHistory, alertsCount, }: { error?: null | { reason: string; message: string }; warning?: null | { reason: string; message: string }; status?: string; outcome?: string; + nextRun?: string | null; + successRatio?: number; + history?: RuleMonitoring['run']['history']; alertsCount?: Record; }) => [ 'alert', @@ -73,14 +86,9 @@ export const generateSavedObjectParams = ({ monitoring: { run: { calculated_metrics: { - success_ratio: 1, + success_ratio: successRatio, }, - history: [ - { - success: true, - timestamp: 0, - }, - ], + history, last_run: { timestamp: '1970-01-01T00:00:00.000Z', metrics: { @@ -112,7 +120,7 @@ export const generateSavedObjectParams = ({ ...(alertsCount || {}), }, }, - nextRun: '1970-01-01T00:00:10.000Z', + nextRun, }, { refresh: false, namespace: undefined }, ]; diff --git a/x-pack/plugins/alerting/server/task_runner/task_runner.test.ts b/x-pack/plugins/alerting/server/task_runner/task_runner.test.ts index b9730b3e0195d..babac36771c54 100644 --- a/x-pack/plugins/alerting/server/task_runner/task_runner.test.ts +++ b/x-pack/plugins/alerting/server/task_runner/task_runner.test.ts @@ -258,6 +258,10 @@ describe('Task Runner', () => { 2, 'deprecated ruleRunStatus for test:1: {"lastExecutionDate":"1970-01-01T00:00:00.000Z","status":"ok"}' ); + expect(logger.debug).nthCalledWith( + 3, + 'ruleRunStatus for test:1: {"outcome":"succeeded","outcomeMsg":null,"warning":null,"alertsCount":{"active":0,"new":0,"recovered":0,"ignored":0}}' + ); expect(logger.debug).nthCalledWith( 4, 'ruleRunMetrics for test:1: {"numSearches":3,"totalSearchDurationMs":23423,"esSearchDurationMs":33,"numberOfTriggeredActions":0,"numberOfGeneratedActions":0,"numberOfActiveAlerts":0,"numberOfRecoveredAlerts":0,"numberOfNewAlerts":0,"hasReachedAlertLimit":false,"triggeredActionsStatus":"complete"}' @@ -337,6 +341,10 @@ describe('Task Runner', () => { 3, 'deprecated ruleRunStatus for test:1: {"lastExecutionDate":"1970-01-01T00:00:00.000Z","status":"active"}' ); + expect(logger.debug).nthCalledWith( + 4, + 'ruleRunStatus for test:1: {"outcome":"succeeded","outcomeMsg":null,"warning":null,"alertsCount":{"active":1,"new":1,"recovered":0,"ignored":0}}' + ); expect(logger.debug).nthCalledWith( 5, 'ruleRunMetrics for test:1: {"numSearches":3,"totalSearchDurationMs":23423,"esSearchDurationMs":33,"numberOfTriggeredActions":1,"numberOfGeneratedActions":1,"numberOfActiveAlerts":1,"numberOfRecoveredAlerts":0,"numberOfNewAlerts":1,"hasReachedAlertLimit":false,"triggeredActionsStatus":"complete"}' @@ -419,6 +427,10 @@ describe('Task Runner', () => { 4, 'deprecated ruleRunStatus for test:1: {"lastExecutionDate":"1970-01-01T00:00:00.000Z","status":"active"}' ); + expect(logger.debug).nthCalledWith( + 5, + 'ruleRunStatus for test:1: {"outcome":"succeeded","outcomeMsg":null,"warning":null,"alertsCount":{"active":1,"new":1,"recovered":0,"ignored":0}}' + ); expect(logger.debug).nthCalledWith( 6, 'ruleRunMetrics for test:1: {"numSearches":3,"totalSearchDurationMs":23423,"esSearchDurationMs":33,"numberOfTriggeredActions":0,"numberOfGeneratedActions":0,"numberOfActiveAlerts":1,"numberOfRecoveredAlerts":0,"numberOfNewAlerts":1,"hasReachedAlertLimit":false,"triggeredActionsStatus":"complete"}' @@ -593,6 +605,10 @@ describe('Task Runner', () => { 4, 'deprecated ruleRunStatus for test:1: {"lastExecutionDate":"1970-01-01T00:00:00.000Z","status":"active"}' ); + expect(logger.debug).nthCalledWith( + 5, + 'ruleRunStatus for test:1: {"outcome":"succeeded","outcomeMsg":null,"warning":null,"alertsCount":{"active":2,"new":2,"recovered":0,"ignored":0}}' + ); expect(logger.debug).nthCalledWith( 6, 'ruleRunMetrics for test:1: {"numSearches":3,"totalSearchDurationMs":23423,"esSearchDurationMs":33,"numberOfTriggeredActions":1,"numberOfGeneratedActions":1,"numberOfActiveAlerts":2,"numberOfRecoveredAlerts":0,"numberOfNewAlerts":2,"hasReachedAlertLimit":false,"triggeredActionsStatus":"complete"}' @@ -1024,6 +1040,10 @@ describe('Task Runner', () => { 4, 'deprecated ruleRunStatus for test:1: {"lastExecutionDate":"1970-01-01T00:00:00.000Z","status":"active"}' ); + expect(logger.debug).nthCalledWith( + 5, + 'ruleRunStatus for test:1: {"outcome":"succeeded","outcomeMsg":null,"warning":null,"alertsCount":{"active":1,"new":0,"recovered":1,"ignored":0}}' + ); expect(logger.debug).nthCalledWith( 6, 'ruleRunMetrics for test:1: {"numSearches":3,"totalSearchDurationMs":23423,"esSearchDurationMs":33,"numberOfTriggeredActions":2,"numberOfGeneratedActions":2,"numberOfActiveAlerts":1,"numberOfRecoveredAlerts":1,"numberOfNewAlerts":0,"hasReachedAlertLimit":false,"triggeredActionsStatus":"complete"}' @@ -1137,6 +1157,10 @@ describe('Task Runner', () => { 4, `deprecated ruleRunStatus for test:1: {"lastExecutionDate":"1970-01-01T00:00:00.000Z","status":"active"}` ); + expect(logger.debug).nthCalledWith( + 5, + 'ruleRunStatus for test:1: {"outcome":"succeeded","outcomeMsg":null,"warning":null,"alertsCount":{"active":1,"new":0,"recovered":1,"ignored":0}}' + ); expect(logger.debug).nthCalledWith( 6, `ruleRunMetrics for test:1: {"numSearches":3,"totalSearchDurationMs":23423,"esSearchDurationMs":33,"numberOfTriggeredActions":2,"numberOfGeneratedActions":2,"numberOfActiveAlerts":1,"numberOfRecoveredAlerts":1,"numberOfNewAlerts":0,"hasReachedAlertLimit":false,"triggeredActionsStatus":"complete"}` @@ -2083,6 +2107,10 @@ describe('Task Runner', () => { 2, 'deprecated ruleRunStatus for test:1: {"lastExecutionDate":"1970-01-01T00:00:00.000Z","status":"ok"}' ); + expect(logger.debug).nthCalledWith( + 3, + 'ruleRunStatus for test:1: {"outcome":"succeeded","outcomeMsg":null,"warning":null,"alertsCount":{"active":0,"new":0,"recovered":0,"ignored":0}}' + ); expect(logger.debug).nthCalledWith( 4, 'ruleRunMetrics for test:1: {"numSearches":3,"totalSearchDurationMs":23423,"esSearchDurationMs":33,"numberOfTriggeredActions":0,"numberOfGeneratedActions":0,"numberOfActiveAlerts":0,"numberOfRecoveredAlerts":0,"numberOfNewAlerts":0,"hasReachedAlertLimit":false,"triggeredActionsStatus":"complete"}' @@ -2176,6 +2204,79 @@ describe('Task Runner', () => { ); }); + test('successfully stores next run', async () => { + const taskRunner = new TaskRunner( + ruleType, + mockedTaskInstance, + taskRunnerFactoryInitializerParams, + inMemoryMetrics + ); + expect(AlertingEventLogger).toHaveBeenCalled(); + rulesClient.get.mockResolvedValue({ + ...mockedRuleTypeSavedObject, + schedule: { interval: '50s' }, + }); + encryptedSavedObjectsClient.getDecryptedAsInternalUser.mockResolvedValueOnce(SAVED_OBJECT); + + await taskRunner.run(); + expect( + taskRunnerFactoryInitializerParams.internalSavedObjectsRepository.update + ).toHaveBeenCalledWith( + ...generateSavedObjectParams({ + nextRun: '1970-01-01T00:00:50.000Z', + }) + ); + }); + + test('updates the rule saved object correctly when failed', async () => { + const taskRunner = new TaskRunner( + ruleType, + mockedTaskInstance, + taskRunnerFactoryInitializerParams, + inMemoryMetrics + ); + expect(AlertingEventLogger).toHaveBeenCalled(); + + rulesClient.get.mockResolvedValue(mockedRuleTypeSavedObject); + encryptedSavedObjectsClient.getDecryptedAsInternalUser.mockResolvedValue(SAVED_OBJECT); + + ruleType.executor.mockImplementation( + async ({ + services: executorServices, + }: RuleExecutorOptions< + RuleTypeParams, + RuleTypeState, + AlertInstanceState, + AlertInstanceContext, + string + >) => { + throw new Error(GENERIC_ERROR_MESSAGE); + } + ); + await taskRunner.run(); + ruleType.executor.mockClear(); + expect( + taskRunnerFactoryInitializerParams.internalSavedObjectsRepository.update + ).toHaveBeenCalledWith( + ...generateSavedObjectParams({ + error: { + message: GENERIC_ERROR_MESSAGE, + reason: 'execute', + }, + outcome: 'failed', + status: 'error', + successRatio: 0, + history: [ + { + success: false, + timestamp: 0, + }, + ], + nextRun: null, + }) + ); + }); + test('caps monitoring history at 200', async () => { const taskRunner = new TaskRunner( ruleType, diff --git a/x-pack/plugins/rule_registry/server/utils/create_lifecycle_rule_type.test.ts b/x-pack/plugins/rule_registry/server/utils/create_lifecycle_rule_type.test.ts index 74f79751dd57b..26f4c52aa2e3d 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_lifecycle_rule_type.test.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_lifecycle_rule_type.test.ts @@ -129,6 +129,7 @@ function createRule(shouldWriteAlerts: boolean = true) { shouldStopExecution: () => false, shouldWriteAlerts: () => shouldWriteAlerts, uiSettingsClient: {} as any, + ruleMonitoringService: {} as any, }, spaceId: 'spaceId', startedAt, From daac93e5eeb7e075f6fdf58ddb2376ad8fd814b4 Mon Sep 17 00:00:00 2001 From: Jiawei Wu Date: Tue, 8 Nov 2022 18:29:49 -0800 Subject: [PATCH 20/27] Fix type check --- .../mock/rule_details/alert_summary/index.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/triggers_actions_ui/public/application/mock/rule_details/alert_summary/index.ts b/x-pack/plugins/triggers_actions_ui/public/application/mock/rule_details/alert_summary/index.ts index 2d5af068ae55d..0c00abae8d855 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/mock/rule_details/alert_summary/index.ts +++ b/x-pack/plugins/triggers_actions_ui/public/application/mock/rule_details/alert_summary/index.ts @@ -33,7 +33,7 @@ export const mockRule = (): Rule => { lastExecutionDate: new Date('2020-08-20T19:23:38Z'), }, monitoring: { - execution: { + run: { history: [ { success: true, @@ -55,7 +55,13 @@ export const mockRule = (): Rule => { success_ratio: 0.66, p50: 200000, p95: 300000, - p99: 300000, + p99: 390000, + }, + last_run: { + timestamp: '2020-08-20T19:23:38Z', + metrics: { + duration: 500, + }, }, }, }, From 9a70231bbb300e56c9ea07abfc1470d70c058233 Mon Sep 17 00:00:00 2001 From: Jiawei Wu Date: Wed, 9 Nov 2022 09:55:31 -0800 Subject: [PATCH 21/27] Fix type check --- .../rule_details/components/rule_definition.test.tsx | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/rule_details/components/rule_definition.test.tsx b/x-pack/plugins/triggers_actions_ui/public/application/sections/rule_details/components/rule_definition.test.tsx index 5166642eaabba..6f215e139adf0 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/sections/rule_details/components/rule_definition.test.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/rule_details/components/rule_definition.test.tsx @@ -221,7 +221,7 @@ function mockRule(overwrite = {}): Rule { lastExecutionDate: new Date('2020-08-20T19:23:38Z'), }, monitoring: { - execution: { + run: { history: [ { success: true, @@ -245,6 +245,12 @@ function mockRule(overwrite = {}): Rule { p95: 300000, p99: 300000, }, + last_run: { + timestamp: '2020-08-20T19:23:38Z', + metrics: { + duration: 500, + }, + }, }, }, ...overwrite, From 897ec3f7e480fa3cc2112d4da57ad61c789714a4 Mon Sep 17 00:00:00 2001 From: Jiawei Wu Date: Wed, 9 Nov 2022 23:32:05 -0800 Subject: [PATCH 22/27] Fix more types --- x-pack/plugins/alerting/server/mocks.ts | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/x-pack/plugins/alerting/server/mocks.ts b/x-pack/plugins/alerting/server/mocks.ts index 67876537c3d3e..18f0a66bb1657 100644 --- a/x-pack/plugins/alerting/server/mocks.ts +++ b/x-pack/plugins/alerting/server/mocks.ts @@ -96,6 +96,18 @@ const createAbortableSearchServiceMock = () => { }; }; +const createRuleMonitoringServiceMock = () => { + const mock = { + setLastRunMetricsTotalSearchDurationMs: jest.fn(), + setLastRunMetricsTotalIndexingDurationMs: jest.fn(), + setLastRunMetricsTotalAlertsDetected: jest.fn(), + setLastRunMetricsTotalAlertsCreated: jest.fn(), + setLastRunMetricsGapDurationS: jest.fn(), + } as unknown as jest.Mocked; + + return mock; +}; + const createRuleExecutorServicesMock = < InstanceState extends AlertInstanceState = AlertInstanceState, InstanceContext extends AlertInstanceContext = AlertInstanceContext @@ -118,6 +130,7 @@ const createRuleExecutorServicesMock = < shouldStopExecution: () => true, search: createAbortableSearchServiceMock(), searchSourceClient: searchSourceCommonMock, + ruleMonitoringService: createRuleMonitoringServiceMock(), }; }; export type RuleExecutorServicesMock = ReturnType; @@ -129,16 +142,4 @@ export const alertsMock = { createRuleExecutorServices: createRuleExecutorServicesMock, }; -const createRuleMonitoringServiceMock = () => { - const mock = { - setLastRunMetricsTotalSearchDurationMs: jest.fn(), - setLastRunMetricsTotalIndexingDurationMs: jest.fn(), - setLastRunMetricsTotalAlertsDetected: jest.fn(), - setLastRunMetricsTotalAlertsCreated: jest.fn(), - setLastRunMetricsGapDurationS: jest.fn(), - } as unknown as jest.Mocked; - - return mock; -}; - export const ruleMonitoringServiceMock = { create: createRuleMonitoringServiceMock }; From 2ce6b6ed89b7982b36deb19b17646765f3d17dc3 Mon Sep 17 00:00:00 2001 From: Jiawei Wu Date: Sun, 13 Nov 2022 22:30:31 -0800 Subject: [PATCH 23/27] Address comments --- .../public/lib/common_transformations.test.ts | 148 ++++++++++++++++++ .../public/lib/common_transformations.ts | 2 + .../server/lib/last_run_status.test.ts | 85 ++++++++++ .../alerting/server/lib/monitoring.test.ts | 52 +++++- .../plugins/alerting/server/lib/monitoring.ts | 4 +- .../server/rules_client/rules_client.ts | 1 + .../server/rules_client/tests/disable.test.ts | 6 + .../server/task_runner/task_runner.test.ts | 19 ++- .../server/task_runner/task_runner.ts | 2 + x-pack/plugins/alerting/server/types.ts | 2 +- .../group1/tests/alerting/create.ts | 3 + .../group1/tests/alerting/get.ts | 3 + .../group2/tests/alerting/update.ts | 15 ++ .../spaces_only/tests/alerting/create.ts | 12 +- .../spaces_only/tests/alerting/get.ts | 10 +- .../spaces_only/tests/alerting/update.ts | 10 +- 16 files changed, 359 insertions(+), 15 deletions(-) create mode 100644 x-pack/plugins/alerting/server/lib/last_run_status.test.ts diff --git a/x-pack/plugins/alerting/public/lib/common_transformations.test.ts b/x-pack/plugins/alerting/public/lib/common_transformations.test.ts index 51d24538b449e..e804c9c1a2810 100644 --- a/x-pack/plugins/alerting/public/lib/common_transformations.test.ts +++ b/x-pack/plugins/alerting/public/lib/common_transformations.test.ts @@ -54,6 +54,43 @@ describe('common_transformations', () => { message: 'this is just a test', }, }, + monitoring: { + run: { + history: [ + { + timestamp: dateExecuted.getTime(), + duration: 42, + success: false, + outcome: 'failed', + }, + ], + calculated_metrics: { + success_ratio: 0, + p50: 0, + p95: 42, + p99: 42, + }, + last_run: { + timestamp: dateExecuted.toISOString(), + metrics: { + duration: 42, + total_search_duration_ms: 100, + }, + }, + }, + }, + last_run: { + outcome: 'failed', + outcome_msg: 'this is just a test', + warning: RuleExecutionStatusErrorReasons.Unknown, + alerts_count: { + new: 1, + active: 2, + recovered: 3, + ignored: 4, + }, + }, + next_run: dateUpdated.toISOString(), }; expect(transformRule(apiRule)).toMatchInlineSnapshot(` Object { @@ -89,12 +126,49 @@ describe('common_transformations', () => { "status": "error", }, "id": "some-id", + "lastRun": Object { + "alertsCount": Object { + "active": 2, + "ignored": 4, + "new": 1, + "recovered": 3, + }, + "outcome": "failed", + "outcomeMsg": "this is just a test", + "warning": "unknown", + }, + "monitoring": Object { + "run": Object { + "calculated_metrics": Object { + "p50": 0, + "p95": 42, + "p99": 42, + "success_ratio": 0, + }, + "history": Array [ + Object { + "duration": 42, + "outcome": "failed", + "success": false, + "timestamp": 1639571696789, + }, + ], + "last_run": Object { + "metrics": Object { + "duration": 42, + "total_search_duration_ms": 100, + }, + "timestamp": "2021-12-15T12:34:56.789Z", + }, + }, + }, "muteAll": false, "mutedInstanceIds": Array [ "bob", "jim", ], "name": "some-name", + "nextRun": 2021-12-15T12:34:55.789Z, "notifyWhen": "onActiveAlert", "params": Object { "bar": "foo", @@ -152,6 +226,43 @@ describe('common_transformations', () => { last_execution_date: dateExecuted.toISOString(), status: 'error', }, + monitoring: { + run: { + history: [ + { + timestamp: dateExecuted.getTime(), + duration: 42, + success: false, + outcome: 'failed', + }, + ], + calculated_metrics: { + success_ratio: 0, + p50: 0, + p95: 42, + p99: 42, + }, + last_run: { + timestamp: dateExecuted.toISOString(), + metrics: { + duration: 42, + total_search_duration_ms: 100, + }, + }, + }, + }, + last_run: { + outcome: 'failed', + outcome_msg: 'this is just a test', + warning: RuleExecutionStatusErrorReasons.Unknown, + alerts_count: { + new: 1, + active: 2, + recovered: 3, + ignored: 4, + }, + }, + next_run: dateUpdated.toISOString(), }; expect(transformRule(apiRule)).toMatchInlineSnapshot(` Object { @@ -176,12 +287,49 @@ describe('common_transformations', () => { "status": "error", }, "id": "some-id", + "lastRun": Object { + "alertsCount": Object { + "active": 2, + "ignored": 4, + "new": 1, + "recovered": 3, + }, + "outcome": "failed", + "outcomeMsg": "this is just a test", + "warning": "unknown", + }, + "monitoring": Object { + "run": Object { + "calculated_metrics": Object { + "p50": 0, + "p95": 42, + "p99": 42, + "success_ratio": 0, + }, + "history": Array [ + Object { + "duration": 42, + "outcome": "failed", + "success": false, + "timestamp": 1639571696789, + }, + ], + "last_run": Object { + "metrics": Object { + "duration": 42, + "total_search_duration_ms": 100, + }, + "timestamp": "2021-12-15T12:34:56.789Z", + }, + }, + }, "muteAll": false, "mutedInstanceIds": Array [ "bob", "jim", ], "name": "some-name", + "nextRun": 2021-12-15T12:34:55.789Z, "notifyWhen": "onActiveAlert", "params": Object {}, "schedule": Object { diff --git a/x-pack/plugins/alerting/public/lib/common_transformations.ts b/x-pack/plugins/alerting/public/lib/common_transformations.ts index d68bcaec29ecb..c48c1f882eaed 100644 --- a/x-pack/plugins/alerting/public/lib/common_transformations.ts +++ b/x-pack/plugins/alerting/public/lib/common_transformations.ts @@ -70,6 +70,7 @@ export type ApiRule = Omit< | 'alert_type_id' | 'muted_instance_ids' | 'last_run' + | 'next_run' > & { execution_status: ApiRuleExecutionStatus; actions: Array>; @@ -78,6 +79,7 @@ export type ApiRule = Omit< rule_type_id: string; muted_alert_ids: string[]; last_run?: AsApiContract; + next_run?: string; }; export function transformRule(input: ApiRule): Rule { diff --git a/x-pack/plugins/alerting/server/lib/last_run_status.test.ts b/x-pack/plugins/alerting/server/lib/last_run_status.test.ts new file mode 100644 index 0000000000000..da44325ba3cbd --- /dev/null +++ b/x-pack/plugins/alerting/server/lib/last_run_status.test.ts @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { lastRunFromState } from './last_run_status'; +import { ActionsCompletion } from '../../common'; +import { RuleRunMetrics } from './rule_run_metrics_store'; +const getMetrics = (): RuleRunMetrics => { + return { + triggeredActionsStatus: ActionsCompletion.COMPLETE, + esSearchDurationMs: 3, + numSearches: 1, + numberOfActiveAlerts: 10, + numberOfGeneratedActions: 15, + numberOfNewAlerts: 12, + numberOfRecoveredAlerts: 11, + numberOfTriggeredActions: 5, + totalSearchDurationMs: 2, + hasReachedAlertLimit: false, + }; +}; + +describe('lastRunFromState', () => { + it('successfuly outcome', () => { + const result = lastRunFromState({ metrics: getMetrics() }); + + expect(result.lastRun.outcome).toEqual('succeeded'); + expect(result.lastRun.outcomeMsg).toEqual(null); + expect(result.lastRun.warning).toEqual(null); + + expect(result.lastRun.alertsCount).toEqual({ + active: 10, + new: 12, + recovered: 11, + ignored: 0, + }); + }); + + it('limited reached outcome', () => { + const result = lastRunFromState({ + metrics: { + ...getMetrics(), + hasReachedAlertLimit: true, + }, + }); + + expect(result.lastRun.outcome).toEqual('warning'); + expect(result.lastRun.outcomeMsg).toEqual( + 'Rule reported more than the maximum number of alerts in a single run. Alerts may be missed and recovery notifications may be delayed' + ); + expect(result.lastRun.warning).toEqual('maxAlerts'); + + expect(result.lastRun.alertsCount).toEqual({ + active: 10, + new: 12, + recovered: 11, + ignored: 0, + }); + }); + + it('partial triggered actions status outcome', () => { + const result = lastRunFromState({ + metrics: { + ...getMetrics(), + triggeredActionsStatus: ActionsCompletion.PARTIAL, + }, + }); + + expect(result.lastRun.outcome).toEqual('warning'); + expect(result.lastRun.outcomeMsg).toEqual( + 'The maximum number of actions for this rule type was reached; excess actions were not triggered.' + ); + expect(result.lastRun.warning).toEqual('maxExecutableActions'); + + expect(result.lastRun.alertsCount).toEqual({ + active: 10, + new: 12, + recovered: 11, + ignored: 0, + }); + }); +}); diff --git a/x-pack/plugins/alerting/server/lib/monitoring.test.ts b/x-pack/plugins/alerting/server/lib/monitoring.test.ts index 270b270dc0de7..492e205a99508 100644 --- a/x-pack/plugins/alerting/server/lib/monitoring.test.ts +++ b/x-pack/plugins/alerting/server/lib/monitoring.test.ts @@ -5,7 +5,11 @@ * 2.0. */ -import { getExecutionDurationPercentiles } from './monitoring'; +import { + getExecutionDurationPercentiles, + updateMonitoring, + convertMonitoringFromRawAndVerify, +} from './monitoring'; import { RuleMonitoring } from '../types'; const mockHistory = [ @@ -47,6 +51,12 @@ const mockRuleMonitoring = { calculated_metrics: { success_ratio: 0, }, + last_run: { + timestamp: '2022-06-18T01:00:00.000Z', + metrics: { + duration: 123, + }, + }, }, } as RuleMonitoring; @@ -76,3 +86,43 @@ describe('getExecutionDurationPercentiles', () => { expect(Object.keys(percentiles).length).toEqual(0); }); }); + +describe('updateMonitoring', () => { + it('can update monitoring', () => { + const result = updateMonitoring({ + monitoring: mockRuleMonitoring, + timestamp: '2022-07-18T01:00:00.000Z', + duration: 1000, + }); + + expect(result.run.history).toEqual(mockRuleMonitoring.run.history); + expect(result.run.calculated_metrics).toEqual(mockRuleMonitoring.run.calculated_metrics); + expect(result.run.last_run.timestamp).toEqual('2022-07-18T01:00:00.000Z'); + expect(result.run.last_run.metrics.duration).toEqual(1000); + }); +}); + +describe('convertMonitoringFromRawAndVerify', () => { + it('can convert monitoring to raw and verify the duration', () => { + const monitoring = { + run: { + ...mockRuleMonitoring.run, + last_run: { + ...mockRuleMonitoring.run.last_run, + timestamp: 'invalid', + }, + }, + }; + + const mockLoggerDebug = jest.fn(); + const mockLogger = { + debug: mockLoggerDebug, + }; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const result = convertMonitoringFromRawAndVerify(mockLogger as any, '123', monitoring); + expect(mockLoggerDebug).toHaveBeenCalledWith( + 'invalid monitoring last_run.timestamp "invalid" in raw rule 123' + ); + expect(Date.parse(result!.run.last_run.timestamp)).toBeTruthy(); + }); +}); diff --git a/x-pack/plugins/alerting/server/lib/monitoring.ts b/x-pack/plugins/alerting/server/lib/monitoring.ts index aeb34e743f6db..93da6e2283152 100644 --- a/x-pack/plugins/alerting/server/lib/monitoring.ts +++ b/x-pack/plugins/alerting/server/lib/monitoring.ts @@ -59,7 +59,7 @@ export const updateMonitoring = ({ }: { monitoring: RuleMonitoring; timestamp: string; - duration: number; + duration?: number; }) => { const { run } = monitoring; const { last_run: lastRun, ...rest } = run; @@ -99,6 +99,6 @@ export const convertMonitoringFromRawAndVerify = ( return updateMonitoring({ monitoring, timestamp: new Date(parsedDateMillis).toISOString(), - duration: monitoring.run.last_run.metrics.duration || 0, + duration: monitoring.run.last_run.metrics.duration, }); }; diff --git a/x-pack/plugins/alerting/server/rules_client/rules_client.ts b/x-pack/plugins/alerting/server/rules_client/rules_client.ts index e8dd110a5c7de..67a347f5d45c1 100644 --- a/x-pack/plugins/alerting/server/rules_client/rules_client.ts +++ b/x-pack/plugins/alerting/server/rules_client/rules_client.ts @@ -2837,6 +2837,7 @@ export class RulesClient { scheduledTaskId: attributes.scheduledTaskId === id ? attributes.scheduledTaskId : null, updatedBy: await this.getUserName(), updatedAt: new Date().toISOString(), + nextRun: null, }), { version } ); diff --git a/x-pack/plugins/alerting/server/rules_client/tests/disable.test.ts b/x-pack/plugins/alerting/server/rules_client/tests/disable.test.ts index 558d33ecca87c..5af5ec3e60bd1 100644 --- a/x-pack/plugins/alerting/server/rules_client/tests/disable.test.ts +++ b/x-pack/plugins/alerting/server/rules_client/tests/disable.test.ts @@ -224,6 +224,7 @@ describe('disable()', () => { }, }, ], + nextRun: null, }, { version: '123', @@ -294,6 +295,7 @@ describe('disable()', () => { }, }, ], + nextRun: null, }, { version: '123', @@ -375,6 +377,7 @@ describe('disable()', () => { }, }, ], + nextRun: null, }, { version: '123', @@ -418,6 +421,7 @@ describe('disable()', () => { }, }, ], + nextRun: null, }, { version: '123', @@ -509,6 +513,7 @@ describe('disable()', () => { }, }, ], + nextRun: null, }, { version: '123', @@ -556,6 +561,7 @@ describe('disable()', () => { }, }, ], + nextRun: null, }, { version: '123', diff --git a/x-pack/plugins/alerting/server/task_runner/task_runner.test.ts b/x-pack/plugins/alerting/server/task_runner/task_runner.test.ts index babac36771c54..69b648f099e44 100644 --- a/x-pack/plugins/alerting/server/task_runner/task_runner.test.ts +++ b/x-pack/plugins/alerting/server/task_runner/task_runner.test.ts @@ -1568,10 +1568,15 @@ describe('Task Runner', () => { return taskRunner.run().catch((ex) => { expect(ex.toString()).toEqual(`Error: Saved object [alert/1] not found`); - const executeRuleDebugLogger = logger.debug.mock.calls[3][0]; - expect(executeRuleDebugLogger as string).toMatchInlineSnapshot( + const updateRuleDebugLogger = logger.debug.mock.calls[3][0]; + expect(updateRuleDebugLogger as string).toMatchInlineSnapshot( `"Updating rule task for test rule with id 1 - {\\"lastExecutionDate\\":\\"1970-01-01T00:00:00.000Z\\",\\"status\\":\\"error\\",\\"error\\":{\\"reason\\":\\"read\\",\\"message\\":\\"Saved object [alert/1] not found\\"}} - {\\"outcome\\":\\"failed\\",\\"warning\\":\\"read\\",\\"outcomeMsg\\":\\"Saved object [alert/1] not found\\",\\"alertsCount\\":{}}"` ); + const executeRuleDebugLogger = logger.debug.mock.calls[4][0]; + expect(executeRuleDebugLogger as string).toMatchInlineSnapshot( + `"Executing Rule foo:test:1 has resulted in Error: Saved object [alert/1] not found"` + ); + expect(logger.error).not.toHaveBeenCalled(); expect(logger.warn).toHaveBeenCalledTimes(1); expect(logger.warn).nthCalledWith( @@ -1650,10 +1655,15 @@ describe('Task Runner', () => { return taskRunner.run().catch((ex) => { expect(ex.toString()).toEqual(`Error: Saved object [alert/1] not found`); - const ruleExecuteDebugLog = logger.debug.mock.calls[3][0]; - expect(ruleExecuteDebugLog as string).toMatchInlineSnapshot( + const updateRuleDebugLogger = logger.debug.mock.calls[3][0]; + expect(updateRuleDebugLogger as string).toMatchInlineSnapshot( `"Updating rule task for test rule with id 1 - {\\"lastExecutionDate\\":\\"1970-01-01T00:00:00.000Z\\",\\"status\\":\\"error\\",\\"error\\":{\\"reason\\":\\"read\\",\\"message\\":\\"Saved object [alert/1] not found\\"}} - {\\"outcome\\":\\"failed\\",\\"warning\\":\\"read\\",\\"outcomeMsg\\":\\"Saved object [alert/1] not found\\",\\"alertsCount\\":{}}"` ); + const ruleExecuteDebugLog = logger.debug.mock.calls[4][0]; + expect(ruleExecuteDebugLog as string).toMatchInlineSnapshot( + `"Executing Rule test space:test:1 has resulted in Error: Saved object [alert/1] not found"` + ); + expect(logger.error).not.toHaveBeenCalled(); expect(logger.warn).toHaveBeenCalledTimes(1); expect(logger.warn).nthCalledWith( @@ -2272,7 +2282,6 @@ describe('Task Runner', () => { timestamp: 0, }, ], - nextRun: null, }) ); }); diff --git a/x-pack/plugins/alerting/server/task_runner/task_runner.ts b/x-pack/plugins/alerting/server/task_runner/task_runner.ts index 295b4fa0ae570..a06bd01cb9ce0 100644 --- a/x-pack/plugins/alerting/server/task_runner/task_runner.ts +++ b/x-pack/plugins/alerting/server/task_runner/task_runner.ts @@ -661,6 +661,8 @@ export class TaskRunner< let nextRun: string | null = null; if (isOk(schedule)) { nextRun = getNextRun({ startDate: startedAt, interval: schedule.value.interval }); + } else if (taskSchedule) { + nextRun = getNextRun({ startDate: startedAt, interval: taskSchedule.interval }); } const { executionStatus, executionMetrics } = await this.timer.runWithTimer( diff --git a/x-pack/plugins/alerting/server/types.ts b/x-pack/plugins/alerting/server/types.ts index 66cb65c095fd2..3eccd1d362127 100644 --- a/x-pack/plugins/alerting/server/types.ts +++ b/x-pack/plugins/alerting/server/types.ts @@ -82,7 +82,7 @@ export interface RuleExecutorServices< alertFactory: PublicAlertFactory; shouldWriteAlerts: () => boolean; shouldStopExecution: () => boolean; - ruleMonitoringService: PublicRuleMonitoringService; + ruleMonitoringService?: PublicRuleMonitoringService; } export interface RuleExecutorOptions< diff --git a/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/create.ts b/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/create.ts index 1fe600ee06d4a..cebfe68e279b0 100644 --- a/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/create.ts +++ b/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/create.ts @@ -130,6 +130,9 @@ export default function createAlertTests({ getService }: FtrProviderContext) { expect(typeof response.body.scheduled_task_id).to.be('string'); expect(Date.parse(response.body.created_at)).to.be.greaterThan(0); expect(Date.parse(response.body.updated_at)).to.be.greaterThan(0); + if (response.body.next_run) { + expect(Date.parse(response.body.next_run)).to.be.greaterThan(0); + } const taskRecord = await getScheduledTask(response.body.scheduled_task_id); expect(taskRecord.type).to.eql('task'); diff --git a/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/get.ts b/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/get.ts index 8da3e6d3c5def..b0900f74993cb 100644 --- a/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/get.ts +++ b/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/get.ts @@ -93,6 +93,9 @@ const getTestUtils = ( }); expect(Date.parse(response.body.created_at)).to.be.greaterThan(0); expect(Date.parse(response.body.updated_at)).to.be.greaterThan(0); + if (response.body.next_run) { + expect(Date.parse(response.body.next_run)).to.be.greaterThan(0); + } break; default: throw new Error(`Scenario untested: ${JSON.stringify(scenario)}`); diff --git a/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/alerting/update.ts b/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/alerting/update.ts index 6e9102b62c043..430d69274041a 100644 --- a/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/alerting/update.ts +++ b/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/alerting/update.ts @@ -140,6 +140,9 @@ export default function createUpdateTests({ getService }: FtrProviderContext) { expect(Date.parse(response.body.updated_at)).to.be.greaterThan( Date.parse(response.body.created_at) ); + if (response.body.next_run) { + expect(Date.parse(response.body.next_run)).to.be.greaterThan(0); + } // Ensure AAD isn't broken await checkAAD({ supertest, @@ -226,6 +229,9 @@ export default function createUpdateTests({ getService }: FtrProviderContext) { expect(Date.parse(response.body.updated_at)).to.be.greaterThan( Date.parse(response.body.created_at) ); + if (response.body.next_run) { + expect(Date.parse(response.body.next_run)).to.be.greaterThan(0); + } // Ensure AAD isn't broken await checkAAD({ supertest, @@ -323,6 +329,9 @@ export default function createUpdateTests({ getService }: FtrProviderContext) { expect(Date.parse(response.body.updated_at)).to.be.greaterThan( Date.parse(response.body.created_at) ); + if (response.body.next_run) { + expect(Date.parse(response.body.next_run)).to.be.greaterThan(0); + } // Ensure AAD isn't broken await checkAAD({ supertest, @@ -420,6 +429,9 @@ export default function createUpdateTests({ getService }: FtrProviderContext) { expect(Date.parse(response.body.updated_at)).to.be.greaterThan( Date.parse(response.body.created_at) ); + if (response.body.next_run) { + expect(Date.parse(response.body.next_run)).to.be.greaterThan(0); + } // Ensure AAD isn't broken await checkAAD({ supertest, @@ -515,6 +527,9 @@ export default function createUpdateTests({ getService }: FtrProviderContext) { expect(Date.parse(response.body.updated_at)).to.be.greaterThan( Date.parse(response.body.created_at) ); + if (response.body.next_run) { + expect(Date.parse(response.body.next_run)).to.be.greaterThan(0); + } // Ensure AAD isn't broken await checkAAD({ supertest, diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/create.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/create.ts index 8b00b773e0018..5cc7316f1c6e3 100644 --- a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/create.ts +++ b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/create.ts @@ -100,7 +100,9 @@ export default function createAlertTests({ getService }: FtrProviderContext) { expect(Date.parse(response.body.created_at)).to.be.greaterThan(0); expect(Date.parse(response.body.updated_at)).to.be.greaterThan(0); expect(Date.parse(response.body.updated_at)).to.eql(Date.parse(response.body.created_at)); - + if (response.body.next_run) { + expect(Date.parse(response.body.next_run)).to.be.greaterThan(0); + } expect(typeof response.body.scheduled_task_id).to.be('string'); const taskRecord = await getScheduledTask(response.body.scheduled_task_id); expect(taskRecord.type).to.eql('task'); @@ -196,6 +198,10 @@ export default function createAlertTests({ getService }: FtrProviderContext) { ...(response.body.last_run ? { last_run: response.body.last_run } : {}), }); + if (response.body.next_run) { + expect(Date.parse(response.body.next_run)).to.be.greaterThan(0); + } + const esResponse = await es.get>( { index: '.kibana', @@ -495,7 +501,9 @@ export default function createAlertTests({ getService }: FtrProviderContext) { expect(Date.parse(response.body.createdAt)).to.be.greaterThan(0); expect(Date.parse(response.body.updatedAt)).to.be.greaterThan(0); expect(Date.parse(response.body.updatedAt)).to.eql(Date.parse(response.body.createdAt)); - + if (response.body.next_run) { + expect(Date.parse(response.body.next_run)).to.be.greaterThan(0); + } expect(typeof response.body.scheduledTaskId).to.be('string'); const taskRecord = await getScheduledTask(response.body.scheduledTaskId); expect(taskRecord.type).to.eql('task'); diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/get.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/get.ts index 3ed3eeb067860..c91467f698dc1 100644 --- a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/get.ts +++ b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/get.ts @@ -66,6 +66,9 @@ const getTestUtils = ( }); expect(Date.parse(response.body.created_at)).to.be.greaterThan(0); expect(Date.parse(response.body.updated_at)).to.be.greaterThan(0); + if (response.body.next_run) { + expect(Date.parse(response.body.next_run)).to.be.greaterThan(0); + } }); it(`shouldn't find alert from another space`, async () => { @@ -151,11 +154,14 @@ export default function createGetTests({ getService }: FtrProviderContext) { createdAt: response.body.createdAt, updatedAt: response.body.updatedAt, executionStatus: response.body.executionStatus, - ...(response.body.next_run ? { next_run: response.body.next_run } : {}), - ...(response.body.last_run ? { last_run: response.body.last_run } : {}), + ...(response.body.nextRun ? { nextRun: response.body.nextRun } : {}), + ...(response.body.lastRun ? { lastRun: response.body.lastRun } : {}), }); expect(Date.parse(response.body.createdAt)).to.be.greaterThan(0); expect(Date.parse(response.body.updatedAt)).to.be.greaterThan(0); + if (response.body.nextRun) { + expect(Date.parse(response.body.nextRun)).to.be.greaterThan(0); + } }); }); }); diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/update.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/update.ts index 33897ac0df062..4c740b3be9b97 100644 --- a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/update.ts +++ b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/update.ts @@ -71,6 +71,9 @@ export default function createUpdateTests({ getService }: FtrProviderContext) { expect(Date.parse(response.body.updated_at)).to.be.greaterThan( Date.parse(response.body.created_at) ); + if (response.body.next_run) { + expect(Date.parse(response.body.next_run)).to.be.greaterThan(0); + } response = await supertest.get( `${getUrlPrefix( @@ -165,14 +168,17 @@ export default function createUpdateTests({ getService }: FtrProviderContext) { createdAt: response.body.createdAt, updatedAt: response.body.updatedAt, executionStatus: response.body.executionStatus, - ...(response.body.next_run ? { next_run: response.body.next_run } : {}), - ...(response.body.last_run ? { last_run: response.body.last_run } : {}), + ...(response.body.nextRun ? { nextRun: response.body.nextRun } : {}), + ...(response.body.lastRun ? { lastRun: response.body.lastRun } : {}), }); expect(Date.parse(response.body.createdAt)).to.be.greaterThan(0); expect(Date.parse(response.body.updatedAt)).to.be.greaterThan(0); expect(Date.parse(response.body.updatedAt)).to.be.greaterThan( Date.parse(response.body.createdAt) ); + if (response.body.nextRun) { + expect(Date.parse(response.body.nextRun)).to.be.greaterThan(0); + } // Ensure AAD isn't broken await checkAAD({ From d0b5298e431bc95ca00074baa46441d80622ae61 Mon Sep 17 00:00:00 2001 From: Jiawei Wu Date: Sun, 13 Nov 2022 22:34:51 -0800 Subject: [PATCH 24/27] Update rule monitoring mock --- .../server/utils/create_lifecycle_rule_type.test.ts | 1 - .../rule_registry/server/utils/rule_executor.test_helpers.ts | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/x-pack/plugins/rule_registry/server/utils/create_lifecycle_rule_type.test.ts b/x-pack/plugins/rule_registry/server/utils/create_lifecycle_rule_type.test.ts index 26f4c52aa2e3d..74f79751dd57b 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_lifecycle_rule_type.test.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_lifecycle_rule_type.test.ts @@ -129,7 +129,6 @@ function createRule(shouldWriteAlerts: boolean = true) { shouldStopExecution: () => false, shouldWriteAlerts: () => shouldWriteAlerts, uiSettingsClient: {} as any, - ruleMonitoringService: {} as any, }, spaceId: 'spaceId', startedAt, diff --git a/x-pack/plugins/rule_registry/server/utils/rule_executor.test_helpers.ts b/x-pack/plugins/rule_registry/server/utils/rule_executor.test_helpers.ts index 330531b1fd27e..70c827942ade3 100644 --- a/x-pack/plugins/rule_registry/server/utils/rule_executor.test_helpers.ts +++ b/x-pack/plugins/rule_registry/server/utils/rule_executor.test_helpers.ts @@ -16,7 +16,7 @@ import { RuleTypeParams, RuleTypeState, } from '@kbn/alerting-plugin/server'; -import { alertsMock, ruleMonitoringServiceMock } from '@kbn/alerting-plugin/server/mocks'; +import { alertsMock } from '@kbn/alerting-plugin/server/mocks'; import { searchSourceCommonMock } from '@kbn/data-plugin/common/search/search_source/mocks'; import { Logger } from '@kbn/logging'; @@ -77,7 +77,6 @@ export const createDefaultAlertExecutorOptions = < shouldWriteAlerts: () => shouldWriteAlerts, shouldStopExecution: () => false, searchSourceClient: searchSourceCommonMock, - ruleMonitoringService: ruleMonitoringServiceMock.create(), }, state, previousStartedAt: null, From d9a6b9d2b3999f18547c8db23ebd4337576955a4 Mon Sep 17 00:00:00 2001 From: Xavier Mouligneau Date: Mon, 14 Nov 2022 10:09:45 -0500 Subject: [PATCH 25/27] add more test --- .../public/lib/common_transformations.test.ts | 4 +- .../saved_objects/migrations/index.test.ts | 688 +++++++++--------- 2 files changed, 333 insertions(+), 359 deletions(-) diff --git a/x-pack/plugins/alerting/public/lib/common_transformations.test.ts b/x-pack/plugins/alerting/public/lib/common_transformations.test.ts index e804c9c1a2810..5815c9e56b91a 100644 --- a/x-pack/plugins/alerting/public/lib/common_transformations.test.ts +++ b/x-pack/plugins/alerting/public/lib/common_transformations.test.ts @@ -6,7 +6,7 @@ */ import { ApiRule, transformRule } from './common_transformations'; -import { RuleExecutionStatusErrorReasons } from '../../common'; +import { RuleExecutionStatusErrorReasons, RuleLastRunOutcomeValues } from '../../common'; beforeEach(() => jest.resetAllMocks()); @@ -90,7 +90,7 @@ describe('common_transformations', () => { ignored: 4, }, }, - next_run: dateUpdated.toISOString(), + next_run: dateUpdated, }; expect(transformRule(apiRule)).toMatchInlineSnapshot(` Object { diff --git a/x-pack/plugins/alerting/server/saved_objects/migrations/index.test.ts b/x-pack/plugins/alerting/server/saved_objects/migrations/index.test.ts index 147c4d1f9df82..f7f31d1d06786 100644 --- a/x-pack/plugins/alerting/server/saved_objects/migrations/index.test.ts +++ b/x-pack/plugins/alerting/server/saved_objects/migrations/index.test.ts @@ -2150,442 +2150,416 @@ describe('successful migrations', () => { expect(migratedAlert.attributes.params.outputIndex).toEqual(''); } ); + }); - describe('8.0.1', () => { - describe.each(Object.keys(ruleTypeMappings) as RuleType[])( - 'auto_disabled %p rule tags', - (ruleType) => { - const alert717Enabled = getMockData( - { - params: { outputIndex: 'output-index', type: ruleType }, - alertTypeId: 'siem.signals', - enabled: true, - scheduledTaskId: 'abcd', - }, - true - ); - const alert717Disabled = getMockData( - { - params: { outputIndex: 'output-index', type: ruleType }, - alertTypeId: 'siem.signals', - enabled: false, - }, - true - ); - const alert800 = getMockData( - { - params: { outputIndex: '', type: ruleType }, - alertTypeId: ruleTypeMappings[ruleType], - enabled: false, - scheduledTaskId: 'abcd', - }, - true - ); + describe('8.0.1', () => { + describe.each(Object.keys(ruleTypeMappings) as RuleType[])( + 'auto_disabled %p rule tags', + (ruleType) => { + const alert717Enabled = getMockData( + { + params: { outputIndex: 'output-index', type: ruleType }, + alertTypeId: 'siem.signals', + enabled: true, + scheduledTaskId: 'abcd', + }, + true + ); + const alert717Disabled = getMockData( + { + params: { outputIndex: 'output-index', type: ruleType }, + alertTypeId: 'siem.signals', + enabled: false, + }, + true + ); + const alert800 = getMockData( + { + params: { outputIndex: '', type: ruleType }, + alertTypeId: ruleTypeMappings[ruleType], + enabled: false, + scheduledTaskId: 'abcd', + }, + true + ); - test('Does not update rule tags if rule has already been enabled', () => { - const migrations = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured); - const migration800 = migrations['8.0.0']; - const migration801 = migrations['8.0.1']; + test('Does not update rule tags if rule has already been enabled', () => { + const migrations = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured); + const migration800 = migrations['8.0.0']; + const migration801 = migrations['8.0.1']; - // migrate to 8.0.0 - const migratedAlert800 = migration800(alert717Enabled, migrationContext); - expect(migratedAlert800.attributes.enabled).toEqual(false); + // migrate to 8.0.0 + const migratedAlert800 = migration800(alert717Enabled, migrationContext); + expect(migratedAlert800.attributes.enabled).toEqual(false); - // reenable rule - migratedAlert800.attributes.enabled = true; + // reenable rule + migratedAlert800.attributes.enabled = true; - // migrate to 8.0.1 - const migratedAlert801 = migration801(migratedAlert800, migrationContext); + // migrate to 8.0.1 + const migratedAlert801 = migration801(migratedAlert800, migrationContext); - expect(migratedAlert801.attributes.alertTypeId).toEqual(ruleTypeMappings[ruleType]); - expect(migratedAlert801.attributes.enabled).toEqual(true); - expect(migratedAlert801.attributes.params.outputIndex).toEqual(''); + expect(migratedAlert801.attributes.alertTypeId).toEqual(ruleTypeMappings[ruleType]); + expect(migratedAlert801.attributes.enabled).toEqual(true); + expect(migratedAlert801.attributes.params.outputIndex).toEqual(''); - // tags not updated - expect(migratedAlert801.attributes.tags).toEqual(['foo']); - }); + // tags not updated + expect(migratedAlert801.attributes.tags).toEqual(['foo']); + }); - test('Does not update rule tags if rule was already disabled before upgrading to 8.0', () => { - const migrations = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured); - const migration800 = migrations['8.0.0']; - const migration801 = migrations['8.0.1']; + test('Does not update rule tags if rule was already disabled before upgrading to 8.0', () => { + const migrations = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured); + const migration800 = migrations['8.0.0']; + const migration801 = migrations['8.0.1']; - // migrate to 8.0.0 - const migratedAlert800 = migration800(alert717Disabled, migrationContext); - expect(migratedAlert800.attributes.enabled).toEqual(false); + // migrate to 8.0.0 + const migratedAlert800 = migration800(alert717Disabled, migrationContext); + expect(migratedAlert800.attributes.enabled).toEqual(false); - // migrate to 8.0.1 - const migratedAlert801 = migration801(migratedAlert800, migrationContext); + // migrate to 8.0.1 + const migratedAlert801 = migration801(migratedAlert800, migrationContext); - expect(migratedAlert801.attributes.alertTypeId).toEqual(ruleTypeMappings[ruleType]); - expect(migratedAlert801.attributes.enabled).toEqual(false); - expect(migratedAlert801.attributes.params.outputIndex).toEqual(''); + expect(migratedAlert801.attributes.alertTypeId).toEqual(ruleTypeMappings[ruleType]); + expect(migratedAlert801.attributes.enabled).toEqual(false); + expect(migratedAlert801.attributes.params.outputIndex).toEqual(''); - // tags not updated - expect(migratedAlert801.attributes.tags).toEqual(['foo']); - }); + // tags not updated + expect(migratedAlert801.attributes.tags).toEqual(['foo']); + }); - test('Updates rule tags if rule was auto-disabled in 8.0 upgrade and not reenabled', () => { - const migrations = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured); - const migration800 = migrations['8.0.0']; - const migration801 = migrations['8.0.1']; + test('Updates rule tags if rule was auto-disabled in 8.0 upgrade and not reenabled', () => { + const migrations = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured); + const migration800 = migrations['8.0.0']; + const migration801 = migrations['8.0.1']; - // migrate to 8.0.0 - const migratedAlert800 = migration800(alert717Enabled, migrationContext); - expect(migratedAlert800.attributes.enabled).toEqual(false); + // migrate to 8.0.0 + const migratedAlert800 = migration800(alert717Enabled, migrationContext); + expect(migratedAlert800.attributes.enabled).toEqual(false); - // migrate to 8.0.1 - const migratedAlert801 = migration801(migratedAlert800, migrationContext); + // migrate to 8.0.1 + const migratedAlert801 = migration801(migratedAlert800, migrationContext); - expect(migratedAlert801.attributes.alertTypeId).toEqual(ruleTypeMappings[ruleType]); - expect(migratedAlert801.attributes.enabled).toEqual(false); - expect(migratedAlert801.attributes.params.outputIndex).toEqual(''); + expect(migratedAlert801.attributes.alertTypeId).toEqual(ruleTypeMappings[ruleType]); + expect(migratedAlert801.attributes.enabled).toEqual(false); + expect(migratedAlert801.attributes.params.outputIndex).toEqual(''); - // tags updated - expect(migratedAlert801.attributes.tags).toEqual(['foo', 'auto_disabled_8.0']); - }); + // tags updated + expect(migratedAlert801.attributes.tags).toEqual(['foo', 'auto_disabled_8.0']); + }); - test('Updates rule tags correctly if tags are undefined', () => { - const migrations = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured); - const migration801 = migrations['8.0.1']; + test('Updates rule tags correctly if tags are undefined', () => { + const migrations = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured); + const migration801 = migrations['8.0.1']; - const alert = { - ...alert800, - attributes: { - ...alert800.attributes, - tags: undefined, - }, - }; + const alert = { + ...alert800, + attributes: { + ...alert800.attributes, + tags: undefined, + }, + }; - // migrate to 8.0.1 - const migratedAlert801 = migration801(alert, migrationContext); + // migrate to 8.0.1 + const migratedAlert801 = migration801(alert, migrationContext); - expect(migratedAlert801.attributes.alertTypeId).toEqual(ruleTypeMappings[ruleType]); - expect(migratedAlert801.attributes.enabled).toEqual(false); - expect(migratedAlert801.attributes.params.outputIndex).toEqual(''); + expect(migratedAlert801.attributes.alertTypeId).toEqual(ruleTypeMappings[ruleType]); + expect(migratedAlert801.attributes.enabled).toEqual(false); + expect(migratedAlert801.attributes.params.outputIndex).toEqual(''); - // tags updated - expect(migratedAlert801.attributes.tags).toEqual(['auto_disabled_8.0']); - }); + // tags updated + expect(migratedAlert801.attributes.tags).toEqual(['auto_disabled_8.0']); + }); - test('Updates rule tags correctly if tags are null', () => { - const migrations = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured); - const migration801 = migrations['8.0.1']; + test('Updates rule tags correctly if tags are null', () => { + const migrations = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured); + const migration801 = migrations['8.0.1']; - const alert = { - ...alert800, - attributes: { - ...alert800.attributes, - tags: null, - }, - }; + const alert = { + ...alert800, + attributes: { + ...alert800.attributes, + tags: null, + }, + }; - // migrate to 8.0.1 - const migratedAlert801 = migration801(alert, migrationContext); + // migrate to 8.0.1 + const migratedAlert801 = migration801(alert, migrationContext); - expect(migratedAlert801.attributes.alertTypeId).toEqual(ruleTypeMappings[ruleType]); - expect(migratedAlert801.attributes.enabled).toEqual(false); - expect(migratedAlert801.attributes.params.outputIndex).toEqual(''); + expect(migratedAlert801.attributes.alertTypeId).toEqual(ruleTypeMappings[ruleType]); + expect(migratedAlert801.attributes.enabled).toEqual(false); + expect(migratedAlert801.attributes.params.outputIndex).toEqual(''); - // tags updated - expect(migratedAlert801.attributes.tags).toEqual(['auto_disabled_8.0']); - }); - } - ); - }); + // tags updated + expect(migratedAlert801.attributes.tags).toEqual(['auto_disabled_8.0']); + }); + } + ); + }); - describe('8.2.0', () => { - test('migrates params to mapped_params', () => { - const migration820 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)[ - '8.2.0' - ]; - const alert = getMockData( - { - params: { - risk_score: 60, - severity: 'high', - foo: 'bar', - }, - alertTypeId: 'siem.signals', + describe('8.2.0', () => { + test('migrates params to mapped_params', () => { + const migration820 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)['8.2.0']; + const alert = getMockData( + { + params: { + risk_score: 60, + severity: 'high', + foo: 'bar', }, - true - ); + alertTypeId: 'siem.signals', + }, + true + ); - const migratedAlert820 = migration820(alert, migrationContext); + const migratedAlert820 = migration820(alert, migrationContext); - expect(migratedAlert820.attributes.mapped_params).toEqual({ - risk_score: 60, - severity: '60-high', - }); + expect(migratedAlert820.attributes.mapped_params).toEqual({ + risk_score: 60, + severity: '60-high', }); }); + }); - describe('8.3.0', () => { - test('migrates snoozed rules to the new data model', () => { - const fakeTimer = sinon.useFakeTimers(); - const migration830 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)[ - '8.3.0' - ]; - const mutedAlert = getMockData( - { - snoozeEndTime: '1970-01-02T00:00:00.000Z', - }, - true - ); - const migratedMutedAlert830 = migration830(mutedAlert, migrationContext); + describe('8.3.0', () => { + test('migrates snoozed rules to the new data model', () => { + const fakeTimer = sinon.useFakeTimers(); + const migration830 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)['8.3.0']; + const mutedAlert = getMockData( + { + snoozeEndTime: '1970-01-02T00:00:00.000Z', + }, + true + ); + const migratedMutedAlert830 = migration830(mutedAlert, migrationContext); - expect(migratedMutedAlert830.attributes.snoozeSchedule.length).toEqual(1); - expect(migratedMutedAlert830.attributes.snoozeSchedule[0].rRule.dtstart).toEqual( - '1970-01-01T00:00:00.000Z' - ); - expect(migratedMutedAlert830.attributes.snoozeSchedule[0].duration).toEqual(86400000); - fakeTimer.restore(); - }); + expect(migratedMutedAlert830.attributes.snoozeSchedule.length).toEqual(1); + expect(migratedMutedAlert830.attributes.snoozeSchedule[0].rRule.dtstart).toEqual( + '1970-01-01T00:00:00.000Z' + ); + expect(migratedMutedAlert830.attributes.snoozeSchedule[0].duration).toEqual(86400000); + fakeTimer.restore(); + }); - test('migrates es_query alert params', () => { - const migration830 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)[ - '8.3.0' - ]; - const alert = getMockData( - { - params: { esQuery: '{ "query": "test-query" }' }, - alertTypeId: '.es-query', - }, - true - ); - const migratedAlert820 = migration830(alert, migrationContext); + test('migrates es_query alert params', () => { + const migration830 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)['8.3.0']; + const alert = getMockData( + { + params: { esQuery: '{ "query": "test-query" }' }, + alertTypeId: '.es-query', + }, + true + ); + const migratedAlert820 = migration830(alert, migrationContext); - expect(migratedAlert820.attributes.params).toEqual({ - esQuery: '{ "query": "test-query" }', - searchType: 'esQuery', - }); + expect(migratedAlert820.attributes.params).toEqual({ + esQuery: '{ "query": "test-query" }', + searchType: 'esQuery', }); + }); - test('removes internal tags', () => { - const migration830 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)[ - '8.3.0' - ]; - const alert = getMockData( - { - tags: [ - '__internal_immutable:false', - '__internal_rule_id:064e3fed-6328-416b-bb85-c08265088f41', - 'test-tag', - ], - alertTypeId: 'siem.queryRule', - }, - true - ); + test('removes internal tags', () => { + const migration830 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)['8.3.0']; + const alert = getMockData( + { + tags: [ + '__internal_immutable:false', + '__internal_rule_id:064e3fed-6328-416b-bb85-c08265088f41', + 'test-tag', + ], + alertTypeId: 'siem.queryRule', + }, + true + ); - const migratedAlert830 = migration830(alert, migrationContext); + const migratedAlert830 = migration830(alert, migrationContext); - expect(migratedAlert830.attributes.tags).toEqual(['test-tag']); - }); + expect(migratedAlert830.attributes.tags).toEqual(['test-tag']); + }); - test('do not remove internal tags if rule is not Security solution rule', () => { - const migration830 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)[ - '8.3.0' - ]; - const alert = getMockData( - { - tags: ['__internal_immutable:false', 'tag-1'], - }, - true - ); + test('do not remove internal tags if rule is not Security solution rule', () => { + const migration830 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)['8.3.0']; + const alert = getMockData( + { + tags: ['__internal_immutable:false', 'tag-1'], + }, + true + ); - const migratedAlert830 = migration830(alert, migrationContext); + const migratedAlert830 = migration830(alert, migrationContext); - expect(migratedAlert830.attributes.tags).toEqual(['__internal_immutable:false', 'tag-1']); - }); + expect(migratedAlert830.attributes.tags).toEqual(['__internal_immutable:false', 'tag-1']); }); + }); - describe('8.4.1', () => { - test('removes isSnoozedUntil', () => { - const migration841 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)[ - '8.4.1' - ]; - const mutedAlert = getMockData( - { - isSnoozedUntil: '1970-01-02T00:00:00.000Z', - }, - true - ); - expect(mutedAlert.attributes.isSnoozedUntil).toBeTruthy(); - const migratedAlert841 = migration841(mutedAlert, migrationContext); + describe('8.4.1', () => { + test('removes isSnoozedUntil', () => { + const migration841 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)['8.4.1']; + const mutedAlert = getMockData( + { + isSnoozedUntil: '1970-01-02T00:00:00.000Z', + }, + true + ); + expect(mutedAlert.attributes.isSnoozedUntil).toBeTruthy(); + const migratedAlert841 = migration841(mutedAlert, migrationContext); - expect(migratedAlert841.attributes.isSnoozedUntil).toBeFalsy(); - }); + expect(migratedAlert841.attributes.isSnoozedUntil).toBeFalsy(); + }); - test('works as expected if isSnoozedUntil is not populated', () => { - const migration841 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)[ - '8.4.1' - ]; - const mutedAlert = getMockData({}, true); - expect(mutedAlert.attributes.isSnoozedUntil).toBeFalsy(); - expect(() => migration841(mutedAlert, migrationContext)).not.toThrowError(); - }); + test('works as expected if isSnoozedUntil is not populated', () => { + const migration841 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)['8.4.1']; + const mutedAlert = getMockData({}, true); + expect(mutedAlert.attributes.isSnoozedUntil).toBeFalsy(); + expect(() => migration841(mutedAlert, migrationContext)).not.toThrowError(); }); + }); - describe('8.6.0', () => { - test('migrates executionStatus success', () => { - const migration860 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)[ - '8.6.0' - ]; + describe('8.6.0', () => { + test('migrates executionStatus success', () => { + const migration860 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)['8.6.0']; - let ruleWithExecutionStatus = getMockData({ - executionStatus: { - status: 'ok', - lastExecutionDate: '2022-01-02T00:00:00.000Z', - lastDuration: 60000, - }, - }); - - let migratedRule = migration860(ruleWithExecutionStatus, migrationContext); - expect(migratedRule.attributes.lastRun.outcome).toEqual('succeeded'); - expect(migratedRule.attributes.lastRun.warning).toEqual(null); - ruleWithExecutionStatus = getMockData({ - executionStatus: { - status: 'active', - lastExecutionDate: '2022-01-02T00:00:00.000Z', - lastDuration: 60000, - }, - }); + let ruleWithExecutionStatus = getMockData({ + executionStatus: { + status: 'ok', + lastExecutionDate: '2022-01-02T00:00:00.000Z', + lastDuration: 60000, + }, + }); - migratedRule = migration860(ruleWithExecutionStatus, migrationContext); - expect(migratedRule.attributes.lastRun.outcome).toEqual('succeeded'); - expect(migratedRule.attributes.lastRun.warning).toEqual(null); + let migratedRule = migration860(ruleWithExecutionStatus, migrationContext); + expect(migratedRule.attributes.lastRun.outcome).toEqual('succeeded'); + expect(migratedRule.attributes.lastRun.warning).toEqual(null); + ruleWithExecutionStatus = getMockData({ + executionStatus: { + status: 'active', + lastExecutionDate: '2022-01-02T00:00:00.000Z', + lastDuration: 60000, + }, }); - test('migrates executionStatus warning and error', () => { - const migration860 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)[ - '8.6.0' - ]; + migratedRule = migration860(ruleWithExecutionStatus, migrationContext); + expect(migratedRule.attributes.lastRun.outcome).toEqual('succeeded'); + expect(migratedRule.attributes.lastRun.warning).toEqual(null); + }); - let ruleWithExecutionStatus = getMockData({ - executionStatus: { - status: 'warning', - lastExecutionDate: '2022-01-02T00:00:00.000Z', - lastDuration: 60000, - warning: { - reason: 'warning reason', - message: 'warning message', - }, + test('migrates executionStatus warning and error', () => { + const migration860 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)['8.6.0']; + + let ruleWithExecutionStatus = getMockData({ + executionStatus: { + status: 'warning', + lastExecutionDate: '2022-01-02T00:00:00.000Z', + lastDuration: 60000, + warning: { + reason: 'warning reason', + message: 'warning message', }, - }); + }, + }); - let migratedRule = migration860(ruleWithExecutionStatus, migrationContext); - expect(migratedRule.attributes.lastRun.outcome).toEqual('warning'); - expect(migratedRule.attributes.lastRun.outcomeMsg).toEqual('warning message'); - expect(migratedRule.attributes.lastRun.warning).toEqual('warning reason'); + let migratedRule = migration860(ruleWithExecutionStatus, migrationContext); + expect(migratedRule.attributes.lastRun.outcome).toEqual('warning'); + expect(migratedRule.attributes.lastRun.outcomeMsg).toEqual('warning message'); + expect(migratedRule.attributes.lastRun.warning).toEqual('warning reason'); - ruleWithExecutionStatus = getMockData({ - executionStatus: { - status: 'error', - lastExecutionDate: '2022-01-02T00:00:00.000Z', - lastDuration: 60000, - error: { - reason: 'failed reason', - message: 'failed message', - }, + ruleWithExecutionStatus = getMockData({ + executionStatus: { + status: 'error', + lastExecutionDate: '2022-01-02T00:00:00.000Z', + lastDuration: 60000, + error: { + reason: 'failed reason', + message: 'failed message', }, - }); - - migratedRule = migration860(ruleWithExecutionStatus, migrationContext); - expect(migratedRule.attributes.lastRun.outcome).toEqual('failed'); - expect(migratedRule.attributes.lastRun.outcomeMsg).toEqual('failed message'); - expect(migratedRule.attributes.lastRun.warning).toEqual('failed reason'); + }, }); - test('migrates empty monitoring', () => { - const migration860 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)[ - '8.6.0' - ]; + migratedRule = migration860(ruleWithExecutionStatus, migrationContext); + expect(migratedRule.attributes.lastRun.outcome).toEqual('failed'); + expect(migratedRule.attributes.lastRun.outcomeMsg).toEqual('failed message'); + expect(migratedRule.attributes.lastRun.warning).toEqual('failed reason'); + }); - const ruleWithoutMonitoring = getMockData(); - const migratedRule = migration860(ruleWithoutMonitoring, migrationContext); + test('migrates empty monitoring', () => { + const migration860 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)['8.6.0']; - expect(migratedRule.attributes.monitoring).toBeUndefined(); - }); + const ruleWithoutMonitoring = getMockData(); + const migratedRule = migration860(ruleWithoutMonitoring, migrationContext); - test('migrates empty monitoring when executionStatus exists', () => { - const migration860 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)[ - '8.6.0' - ]; + expect(migratedRule.attributes.monitoring).toBeUndefined(); + }); - const ruleWithMonitoring = getMockData({ - executionStatus: { - status: 'ok', - lastExecutionDate: '2022-01-02T00:00:00.000Z', - lastDuration: 60000, - }, - }); - const migratedRule = migration860(ruleWithMonitoring, migrationContext); + test('migrates empty monitoring when executionStatus exists', () => { + const migration860 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)['8.6.0']; - expect(migratedRule.attributes.monitoring.run.history).toEqual([]); - expect(migratedRule.attributes.monitoring.run.last_run.timestamp).toEqual( - '2022-01-02T00:00:00.000Z' - ); - expect(migratedRule.attributes.monitoring.run.last_run.metrics.duration).toEqual(60000); + const ruleWithMonitoring = getMockData({ + executionStatus: { + status: 'ok', + lastExecutionDate: '2022-01-02T00:00:00.000Z', + lastDuration: 60000, + }, }); + const migratedRule = migration860(ruleWithMonitoring, migrationContext); + + expect(migratedRule.attributes.monitoring.run.history).toEqual([]); + expect(migratedRule.attributes.monitoring.run.last_run.timestamp).toEqual( + '2022-01-02T00:00:00.000Z' + ); + expect(migratedRule.attributes.monitoring.run.last_run.metrics.duration).toEqual(60000); }); + }); - describe('Metrics Inventory Threshold rule', () => { - test('Migrates incorrect action group spelling', () => { - const migration800 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)[ - '8.0.0' - ]; + describe('Metrics Inventory Threshold rule', () => { + test('Migrates incorrect action group spelling', () => { + const migration800 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)['8.0.0']; - const actions = [ - { - group: 'metrics.invenotry_threshold.fired', - params: { - level: 'info', - message: - '""{{alertName}} - {{context.group}} is in a state of {{context.alertState}} Reason: {{context.reason}}""', - }, - actionRef: 'action_0', - actionTypeId: '.server-log', + const actions = [ + { + group: 'metrics.invenotry_threshold.fired', + params: { + level: 'info', + message: + '""{{alertName}} - {{context.group}} is in a state of {{context.alertState}} Reason: {{context.reason}}""', }, - ]; + actionRef: 'action_0', + actionTypeId: '.server-log', + }, + ]; - const alert = getMockData({ alertTypeId: 'metrics.alert.inventory.threshold', actions }); + const alert = getMockData({ alertTypeId: 'metrics.alert.inventory.threshold', actions }); - expect(migration800(alert, migrationContext)).toMatchObject({ - ...alert, - attributes: { - ...alert.attributes, - actions: [{ ...actions[0], group: 'metrics.inventory_threshold.fired' }], - }, - }); + expect(migration800(alert, migrationContext)).toMatchObject({ + ...alert, + attributes: { + ...alert.attributes, + actions: [{ ...actions[0], group: 'metrics.inventory_threshold.fired' }], + }, }); + }); - test('Works with the correct action group spelling', () => { - const migration800 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)[ - '8.0.0' - ]; + test('Works with the correct action group spelling', () => { + const migration800 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)['8.0.0']; - const actions = [ - { - group: 'metrics.inventory_threshold.fired', - params: { - level: 'info', - message: - '""{{alertName}} - {{context.group}} is in a state of {{context.alertState}} Reason: {{context.reason}}""', - }, - actionRef: 'action_0', - actionTypeId: '.server-log', + const actions = [ + { + group: 'metrics.inventory_threshold.fired', + params: { + level: 'info', + message: + '""{{alertName}} - {{context.group}} is in a state of {{context.alertState}} Reason: {{context.reason}}""', }, - ]; + actionRef: 'action_0', + actionTypeId: '.server-log', + }, + ]; - const alert = getMockData({ alertTypeId: 'metrics.alert.inventory.threshold', actions }); + const alert = getMockData({ alertTypeId: 'metrics.alert.inventory.threshold', actions }); - expect(migration800(alert, migrationContext)).toMatchObject({ - ...alert, - attributes: { - ...alert.attributes, - actions: [{ ...actions[0], group: 'metrics.inventory_threshold.fired' }], - }, - }); + expect(migration800(alert, migrationContext)).toMatchObject({ + ...alert, + attributes: { + ...alert.attributes, + actions: [{ ...actions[0], group: 'metrics.inventory_threshold.fired' }], + }, }); }); }); From 1869e17e3973b071cb0ad3fdd599c1964a2ace1d Mon Sep 17 00:00:00 2001 From: Xavier Mouligneau Date: Mon, 14 Nov 2022 10:11:38 -0500 Subject: [PATCH 26/27] unit test --- .../alerting/public/lib/common_transformations.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/alerting/public/lib/common_transformations.test.ts b/x-pack/plugins/alerting/public/lib/common_transformations.test.ts index 5815c9e56b91a..9b98528586d3e 100644 --- a/x-pack/plugins/alerting/public/lib/common_transformations.test.ts +++ b/x-pack/plugins/alerting/public/lib/common_transformations.test.ts @@ -61,7 +61,7 @@ describe('common_transformations', () => { timestamp: dateExecuted.getTime(), duration: 42, success: false, - outcome: 'failed', + outcome: RuleLastRunOutcomeValues[2], }, ], calculated_metrics: { @@ -80,7 +80,7 @@ describe('common_transformations', () => { }, }, last_run: { - outcome: 'failed', + outcome: RuleLastRunOutcomeValues[2], outcome_msg: 'this is just a test', warning: RuleExecutionStatusErrorReasons.Unknown, alerts_count: { From f2e47355ca82fa05c36b908da9a2297a40098ca4 Mon Sep 17 00:00:00 2001 From: Xavier Mouligneau Date: Mon, 14 Nov 2022 12:30:15 -0500 Subject: [PATCH 27/27] fix type --- .../plugins/alerting/public/lib/common_transformations.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/alerting/public/lib/common_transformations.test.ts b/x-pack/plugins/alerting/public/lib/common_transformations.test.ts index 9b98528586d3e..6b7026f1ea593 100644 --- a/x-pack/plugins/alerting/public/lib/common_transformations.test.ts +++ b/x-pack/plugins/alerting/public/lib/common_transformations.test.ts @@ -90,7 +90,7 @@ describe('common_transformations', () => { ignored: 4, }, }, - next_run: dateUpdated, + next_run: dateUpdated.toISOString(), }; expect(transformRule(apiRule)).toMatchInlineSnapshot(` Object {