diff --git a/test/functional/apps/discover/_discover.js b/test/functional/apps/discover/_discover.js index e597cc14654bc..3c9996ca44ff8 100644 --- a/test/functional/apps/discover/_discover.js +++ b/test/functional/apps/discover/_discover.js @@ -32,7 +32,8 @@ export default function ({ getService, getPageObjects }) { defaultIndex: 'logstash-*', }; - describe('discover test', function describeIndexTests() { + // Failing: See https://github.com/elastic/kibana/issues/82915 + describe.skip('discover test', function describeIndexTests() { before(async function () { log.debug('load kibana index with default index pattern'); await esArchiver.load('discover'); diff --git a/x-pack/plugins/alerts/common/alert.ts b/x-pack/plugins/alerts/common/alert.ts index 79e6bb8f2cbba..97a9a58400e38 100644 --- a/x-pack/plugins/alerts/common/alert.ts +++ b/x-pack/plugins/alerts/common/alert.ts @@ -20,13 +20,12 @@ export interface IntervalSchedule extends SavedObjectAttributes { export const AlertExecutionStatusValues = ['ok', 'active', 'error', 'pending', 'unknown'] as const; export type AlertExecutionStatuses = typeof AlertExecutionStatusValues[number]; -export const AlertExecutionStatusErrorReasonValues = [ - 'read', - 'decrypt', - 'execute', - 'unknown', -] as const; -export type AlertExecutionStatusErrorReasons = typeof AlertExecutionStatusErrorReasonValues[number]; +export enum AlertExecutionStatusErrorReasons { + Read = 'read', + Decrypt = 'decrypt', + Execute = 'execute', + Unknown = 'unknown', +} export interface AlertExecutionStatus { status: AlertExecutionStatuses; @@ -74,3 +73,24 @@ export interface Alert { } export type SanitizedAlert = Omit; + +export enum HealthStatus { + OK = 'ok', + Warning = 'warn', + Error = 'error', +} + +export interface AlertsHealth { + decryptionHealth: { + status: HealthStatus; + timestamp: string; + }; + executionHealth: { + status: HealthStatus; + timestamp: string; + }; + readHealth: { + status: HealthStatus; + timestamp: string; + }; +} diff --git a/x-pack/plugins/alerts/common/index.ts b/x-pack/plugins/alerts/common/index.ts index ab71f77a049f6..65aeec840da7e 100644 --- a/x-pack/plugins/alerts/common/index.ts +++ b/x-pack/plugins/alerts/common/index.ts @@ -4,6 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ +import { AlertsHealth } from './alert'; + export * from './alert'; export * from './alert_type'; export * from './alert_instance'; @@ -19,6 +21,7 @@ export interface ActionGroup { export interface AlertingFrameworkHealth { isSufficientlySecure: boolean; hasPermanentEncryptionKey: boolean; + alertingFrameworkHeath: AlertsHealth; } export const BASE_ALERT_API_PATH = '/api/alerts'; diff --git a/x-pack/plugins/alerts/server/config.test.ts b/x-pack/plugins/alerts/server/config.test.ts new file mode 100644 index 0000000000000..93aa3c38a0460 --- /dev/null +++ b/x-pack/plugins/alerts/server/config.test.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { configSchema } from './config'; + +describe('config validation', () => { + test('alerts defaults', () => { + const config: Record = {}; + expect(configSchema.validate(config)).toMatchInlineSnapshot(` + Object { + "healthCheck": Object { + "interval": "60m", + }, + } + `); + }); +}); diff --git a/x-pack/plugins/alerts/server/config.ts b/x-pack/plugins/alerts/server/config.ts new file mode 100644 index 0000000000000..a6d2196a407b5 --- /dev/null +++ b/x-pack/plugins/alerts/server/config.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { schema, TypeOf } from '@kbn/config-schema'; +import { validateDurationSchema } from './lib'; + +export const configSchema = schema.object({ + healthCheck: schema.object({ + interval: schema.string({ validate: validateDurationSchema, defaultValue: '60m' }), + }), +}); + +export type AlertsConfig = TypeOf; diff --git a/x-pack/plugins/alerts/server/health/get_health.test.ts b/x-pack/plugins/alerts/server/health/get_health.test.ts new file mode 100644 index 0000000000000..34517a89f04d9 --- /dev/null +++ b/x-pack/plugins/alerts/server/health/get_health.test.ts @@ -0,0 +1,221 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { savedObjectsRepositoryMock } from '../../../../../src/core/server/mocks'; +import { AlertExecutionStatusErrorReasons, HealthStatus } from '../types'; +import { getHealth } from './get_health'; + +const savedObjectsRepository = savedObjectsRepositoryMock.create(); + +describe('getHealth()', () => { + test('return true if some of alerts has a decryption error', async () => { + const lastExecutionDateError = new Date().toISOString(); + const lastExecutionDate = new Date().toISOString(); + savedObjectsRepository.find.mockResolvedValueOnce({ + total: 1, + per_page: 1, + page: 1, + saved_objects: [ + { + id: '1', + type: 'alert', + attributes: { + alertTypeId: 'myType', + schedule: { interval: '10s' }, + params: { + bar: true, + }, + createdAt: new Date().toISOString(), + actions: [ + { + group: 'default', + actionRef: 'action_0', + params: { + foo: true, + }, + }, + ], + executionStatus: { + status: 'error', + lastExecutionDate: lastExecutionDateError, + error: { + reason: AlertExecutionStatusErrorReasons.Decrypt, + message: 'Failed decrypt', + }, + }, + }, + score: 1, + references: [ + { + name: 'action_0', + type: 'action', + id: '1', + }, + ], + }, + ], + }); + savedObjectsRepository.find.mockResolvedValueOnce({ + total: 0, + per_page: 10, + page: 1, + saved_objects: [], + }); + + savedObjectsRepository.find.mockResolvedValueOnce({ + total: 0, + per_page: 10, + page: 1, + saved_objects: [], + }); + + savedObjectsRepository.find.mockResolvedValueOnce({ + total: 1, + per_page: 1, + page: 1, + saved_objects: [ + { + id: '2', + type: 'alert', + attributes: { + alertTypeId: 'myType', + schedule: { interval: '1s' }, + params: { + bar: true, + }, + createdAt: new Date().toISOString(), + actions: [], + executionStatus: { + status: 'ok', + lastExecutionDate, + }, + }, + score: 1, + references: [], + }, + ], + }); + const result = await getHealth(savedObjectsRepository); + expect(result).toStrictEqual({ + executionHealth: { + status: HealthStatus.OK, + timestamp: lastExecutionDate, + }, + readHealth: { + status: HealthStatus.OK, + timestamp: lastExecutionDate, + }, + decryptionHealth: { + status: HealthStatus.Warning, + timestamp: lastExecutionDateError, + }, + }); + expect(savedObjectsRepository.find).toHaveBeenCalledTimes(4); + }); + + test('return false if no alerts with a decryption error', async () => { + const lastExecutionDateError = new Date().toISOString(); + const lastExecutionDate = new Date().toISOString(); + savedObjectsRepository.find.mockResolvedValueOnce({ + total: 0, + per_page: 10, + page: 1, + saved_objects: [], + }); + + savedObjectsRepository.find.mockResolvedValueOnce({ + total: 1, + per_page: 1, + page: 1, + saved_objects: [ + { + id: '1', + type: 'alert', + attributes: { + alertTypeId: 'myType', + schedule: { interval: '10s' }, + params: { + bar: true, + }, + createdAt: new Date().toISOString(), + actions: [ + { + group: 'default', + actionRef: 'action_0', + params: { + foo: true, + }, + }, + ], + executionStatus: { + status: 'error', + lastExecutionDate: lastExecutionDateError, + error: { + reason: AlertExecutionStatusErrorReasons.Execute, + message: 'Failed', + }, + }, + }, + score: 1, + references: [ + { + name: 'action_0', + type: 'action', + id: '1', + }, + ], + }, + ], + }); + savedObjectsRepository.find.mockResolvedValueOnce({ + total: 0, + per_page: 10, + page: 1, + saved_objects: [], + }); + + savedObjectsRepository.find.mockResolvedValueOnce({ + total: 1, + per_page: 1, + page: 1, + saved_objects: [ + { + id: '2', + type: 'alert', + attributes: { + alertTypeId: 'myType', + schedule: { interval: '1s' }, + params: { + bar: true, + }, + createdAt: new Date().toISOString(), + actions: [], + executionStatus: { + status: 'ok', + lastExecutionDate, + }, + }, + score: 1, + references: [], + }, + ], + }); + const result = await getHealth(savedObjectsRepository); + expect(result).toStrictEqual({ + executionHealth: { + status: HealthStatus.Warning, + timestamp: lastExecutionDateError, + }, + readHealth: { + status: HealthStatus.OK, + timestamp: lastExecutionDate, + }, + decryptionHealth: { + status: HealthStatus.OK, + timestamp: lastExecutionDate, + }, + }); + }); +}); diff --git a/x-pack/plugins/alerts/server/health/get_health.ts b/x-pack/plugins/alerts/server/health/get_health.ts new file mode 100644 index 0000000000000..b7b4582aa8d10 --- /dev/null +++ b/x-pack/plugins/alerts/server/health/get_health.ts @@ -0,0 +1,97 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { ISavedObjectsRepository } from 'src/core/server'; +import { AlertsHealth, HealthStatus, RawAlert, AlertExecutionStatusErrorReasons } from '../types'; + +export const getHealth = async ( + internalSavedObjectsRepository: ISavedObjectsRepository +): Promise => { + const healthStatuses = { + decryptionHealth: { + status: HealthStatus.OK, + timestamp: '', + }, + executionHealth: { + status: HealthStatus.OK, + timestamp: '', + }, + readHealth: { + status: HealthStatus.OK, + timestamp: '', + }, + }; + + const { saved_objects: decryptErrorData } = await internalSavedObjectsRepository.find({ + filter: `alert.attributes.executionStatus.status:error and alert.attributes.executionStatus.error.reason:${AlertExecutionStatusErrorReasons.Decrypt}`, + fields: ['executionStatus'], + type: 'alert', + sortField: 'executionStatus.lastExecutionDate', + sortOrder: 'desc', + page: 1, + perPage: 1, + }); + + if (decryptErrorData.length > 0) { + healthStatuses.decryptionHealth = { + status: HealthStatus.Warning, + timestamp: decryptErrorData[0].attributes.executionStatus.lastExecutionDate, + }; + } + + const { saved_objects: executeErrorData } = await internalSavedObjectsRepository.find({ + filter: `alert.attributes.executionStatus.status:error and alert.attributes.executionStatus.error.reason:${AlertExecutionStatusErrorReasons.Execute}`, + fields: ['executionStatus'], + type: 'alert', + sortField: 'executionStatus.lastExecutionDate', + sortOrder: 'desc', + page: 1, + perPage: 1, + }); + + if (executeErrorData.length > 0) { + healthStatuses.executionHealth = { + status: HealthStatus.Warning, + timestamp: executeErrorData[0].attributes.executionStatus.lastExecutionDate, + }; + } + + const { saved_objects: readErrorData } = await internalSavedObjectsRepository.find({ + filter: `alert.attributes.executionStatus.status:error and alert.attributes.executionStatus.error.reason:${AlertExecutionStatusErrorReasons.Read}`, + fields: ['executionStatus'], + type: 'alert', + sortField: 'executionStatus.lastExecutionDate', + sortOrder: 'desc', + page: 1, + perPage: 1, + }); + + if (readErrorData.length > 0) { + healthStatuses.readHealth = { + status: HealthStatus.Warning, + timestamp: readErrorData[0].attributes.executionStatus.lastExecutionDate, + }; + } + + const { saved_objects: noErrorData } = await internalSavedObjectsRepository.find({ + filter: 'not alert.attributes.executionStatus.status:error', + fields: ['executionStatus'], + type: 'alert', + sortField: 'executionStatus.lastExecutionDate', + sortOrder: 'desc', + }); + const lastExecutionDate = + noErrorData.length > 0 + ? noErrorData[0].attributes.executionStatus.lastExecutionDate + : new Date().toISOString(); + + for (const [, statusItem] of Object.entries(healthStatuses)) { + if (statusItem.status === HealthStatus.OK) { + statusItem.timestamp = lastExecutionDate; + } + } + + return healthStatuses; +}; diff --git a/x-pack/plugins/alerts/server/health/get_state.test.ts b/x-pack/plugins/alerts/server/health/get_state.test.ts new file mode 100644 index 0000000000000..86981c486da0f --- /dev/null +++ b/x-pack/plugins/alerts/server/health/get_state.test.ts @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { taskManagerMock } from '../../../task_manager/server/mocks'; +import { getHealthStatusStream } from '.'; +import { TaskStatus } from '../../../task_manager/server'; +import { HealthStatus } from '../types'; + +describe('getHealthStatusStream()', () => { + const mockTaskManager = taskManagerMock.createStart(); + + it('should return an object with the "unavailable" level and proper summary of "Alerting framework is unhealthy"', async () => { + mockTaskManager.get.mockReturnValue( + new Promise((_resolve, _reject) => { + return { + id: 'test', + attempts: 0, + status: TaskStatus.Running, + version: '123', + runAt: new Date(), + scheduledAt: new Date(), + startedAt: new Date(), + retryAt: new Date(Date.now() + 5 * 60 * 1000), + state: { + runs: 1, + health_status: HealthStatus.Warning, + }, + taskType: 'alerting:alerting_health_check', + params: { + alertId: '1', + }, + ownerId: null, + }; + }) + ); + getHealthStatusStream(mockTaskManager).subscribe( + (val: { level: Readonly; summary: string }) => { + expect(val.level).toBe(false); + } + ); + }); + + it('should return an object with the "available" level and proper summary of "Alerting framework is healthy"', async () => { + mockTaskManager.get.mockReturnValue( + new Promise((_resolve, _reject) => { + return { + id: 'test', + attempts: 0, + status: TaskStatus.Running, + version: '123', + runAt: new Date(), + scheduledAt: new Date(), + startedAt: new Date(), + retryAt: new Date(Date.now() + 5 * 60 * 1000), + state: { + runs: 1, + health_status: HealthStatus.OK, + }, + taskType: 'alerting:alerting_health_check', + params: { + alertId: '1', + }, + ownerId: null, + }; + }) + ); + getHealthStatusStream(mockTaskManager).subscribe( + (val: { level: Readonly; summary: string }) => { + expect(val.level).toBe(true); + } + ); + }); +}); diff --git a/x-pack/plugins/alerts/server/health/get_state.ts b/x-pack/plugins/alerts/server/health/get_state.ts new file mode 100644 index 0000000000000..476456ecad88a --- /dev/null +++ b/x-pack/plugins/alerts/server/health/get_state.ts @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { i18n } from '@kbn/i18n'; +import { interval, Observable } from 'rxjs'; +import { catchError, switchMap } from 'rxjs/operators'; +import { ServiceStatus, ServiceStatusLevels } from '../../../../../src/core/server'; +import { TaskManagerStartContract } from '../../../task_manager/server'; +import { HEALTH_TASK_ID } from './task'; +import { HealthStatus } from '../types'; + +async function getLatestTaskState(taskManager: TaskManagerStartContract) { + try { + const result = await taskManager.get(HEALTH_TASK_ID); + return result; + } catch (err) { + const errMessage = err && err.message ? err.message : err.toString(); + if (!errMessage.includes('NotInitialized')) { + throw err; + } + } + + return null; +} + +const LEVEL_SUMMARY = { + [ServiceStatusLevels.available.toString()]: i18n.translate( + 'xpack.alerts.server.healthStatus.available', + { + defaultMessage: 'Alerting framework is available', + } + ), + [ServiceStatusLevels.degraded.toString()]: i18n.translate( + 'xpack.alerts.server.healthStatus.degraded', + { + defaultMessage: 'Alerting framework is degraded', + } + ), + [ServiceStatusLevels.unavailable.toString()]: i18n.translate( + 'xpack.alerts.server.healthStatus.unavailable', + { + defaultMessage: 'Alerting framework is unavailable', + } + ), +}; + +export const getHealthStatusStream = ( + taskManager: TaskManagerStartContract +): Observable> => { + return interval(60000 * 5).pipe( + switchMap(async () => { + const doc = await getLatestTaskState(taskManager); + const level = + doc?.state?.health_status === HealthStatus.OK + ? ServiceStatusLevels.available + : doc?.state?.health_status === HealthStatus.Warning + ? ServiceStatusLevels.degraded + : ServiceStatusLevels.unavailable; + return { + level, + summary: LEVEL_SUMMARY[level.toString()], + }; + }), + catchError(async (error) => ({ + level: ServiceStatusLevels.unavailable, + summary: LEVEL_SUMMARY[ServiceStatusLevels.unavailable.toString()], + meta: { error }, + })) + ); +}; diff --git a/x-pack/plugins/alerts/server/health/index.ts b/x-pack/plugins/alerts/server/health/index.ts new file mode 100644 index 0000000000000..730c4596aa550 --- /dev/null +++ b/x-pack/plugins/alerts/server/health/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export { getHealthStatusStream } from './get_state'; +export { scheduleAlertingHealthCheck, initializeAlertingHealth } from './task'; diff --git a/x-pack/plugins/alerts/server/health/task.ts b/x-pack/plugins/alerts/server/health/task.ts new file mode 100644 index 0000000000000..6ea01a1083c13 --- /dev/null +++ b/x-pack/plugins/alerts/server/health/task.ts @@ -0,0 +1,94 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { CoreStart, Logger } from 'kibana/server'; +import { + RunContext, + TaskManagerSetupContract, + TaskManagerStartContract, +} from '../../../task_manager/server'; +import { AlertsConfig } from '../config'; +import { AlertingPluginsStart } from '../plugin'; +import { HealthStatus } from '../types'; +import { getHealth } from './get_health'; + +export const HEALTH_TASK_TYPE = 'alerting_health_check'; + +export const HEALTH_TASK_ID = `Alerting-${HEALTH_TASK_TYPE}`; + +export function initializeAlertingHealth( + logger: Logger, + taskManager: TaskManagerSetupContract, + coreStartServices: Promise<[CoreStart, AlertingPluginsStart, unknown]> +) { + registerAlertingHealthCheckTask(logger, taskManager, coreStartServices); +} + +export async function scheduleAlertingHealthCheck( + logger: Logger, + config: Promise, + taskManager: TaskManagerStartContract +) { + try { + const interval = (await config).healthCheck.interval; + await taskManager.ensureScheduled({ + id: HEALTH_TASK_ID, + taskType: HEALTH_TASK_TYPE, + schedule: { + interval, + }, + state: {}, + params: {}, + }); + } catch (e) { + logger.debug(`Error scheduling task, received ${e.message}`); + } +} + +function registerAlertingHealthCheckTask( + logger: Logger, + taskManager: TaskManagerSetupContract, + coreStartServices: Promise<[CoreStart, AlertingPluginsStart, unknown]> +) { + taskManager.registerTaskDefinitions({ + [HEALTH_TASK_TYPE]: { + title: 'Alerting framework health check task', + createTaskRunner: healthCheckTaskRunner(logger, coreStartServices), + }, + }); +} + +export function healthCheckTaskRunner( + logger: Logger, + coreStartServices: Promise<[CoreStart, AlertingPluginsStart, unknown]> +) { + return ({ taskInstance }: RunContext) => { + const { state } = taskInstance; + return { + async run() { + try { + const alertingHealthStatus = await getHealth( + (await coreStartServices)[0].savedObjects.createInternalRepository(['alert']) + ); + return { + state: { + runs: (state.runs || 0) + 1, + health_status: alertingHealthStatus.decryptionHealth.status, + }, + }; + } catch (errMsg) { + logger.warn(`Error executing alerting health check task: ${errMsg}`); + return { + state: { + runs: (state.runs || 0) + 1, + health_status: HealthStatus.Error, + }, + }; + } + }, + }; + }; +} diff --git a/x-pack/plugins/alerts/server/index.ts b/x-pack/plugins/alerts/server/index.ts index 1e442c5196cf2..64e585da5c654 100644 --- a/x-pack/plugins/alerts/server/index.ts +++ b/x-pack/plugins/alerts/server/index.ts @@ -5,8 +5,10 @@ */ import type { PublicMethodsOf } from '@kbn/utility-types'; import { AlertsClient as AlertsClientClass } from './alerts_client'; -import { PluginInitializerContext } from '../../../../src/core/server'; +import { PluginConfigDescriptor, PluginInitializerContext } from '../../../../src/core/server'; import { AlertingPlugin } from './plugin'; +import { configSchema } from './config'; +import { AlertsConfigType } from './types'; export type AlertsClient = PublicMethodsOf; @@ -30,3 +32,7 @@ export { AlertInstance } from './alert_instance'; export { parseDuration } from './lib'; export const plugin = (initContext: PluginInitializerContext) => new AlertingPlugin(initContext); + +export const config: PluginConfigDescriptor = { + schema: configSchema, +}; diff --git a/x-pack/plugins/alerts/server/lib/alert_execution_status.test.ts b/x-pack/plugins/alerts/server/lib/alert_execution_status.test.ts index 3372d19cd4090..bb24ab034d0dd 100644 --- a/x-pack/plugins/alerts/server/lib/alert_execution_status.test.ts +++ b/x-pack/plugins/alerts/server/lib/alert_execution_status.test.ts @@ -57,7 +57,9 @@ describe('AlertExecutionStatus', () => { }); test('error with a reason', () => { - const status = executionStatusFromError(new ErrorWithReason('execute', new Error('hoo!'))); + const status = executionStatusFromError( + new ErrorWithReason(AlertExecutionStatusErrorReasons.Execute, new Error('hoo!')) + ); expect(status.status).toBe('error'); expect(status.error).toMatchInlineSnapshot(` Object { @@ -71,7 +73,7 @@ describe('AlertExecutionStatus', () => { describe('alertExecutionStatusToRaw()', () => { const date = new Date('2020-09-03T16:26:58Z'); const status = 'ok'; - const reason: AlertExecutionStatusErrorReasons = 'decrypt'; + const reason = AlertExecutionStatusErrorReasons.Decrypt; const error = { reason, message: 'wops' }; test('status without an error', () => { @@ -102,7 +104,7 @@ describe('AlertExecutionStatus', () => { describe('alertExecutionStatusFromRaw()', () => { const date = new Date('2020-09-03T16:26:58Z').toISOString(); const status = 'active'; - const reason: AlertExecutionStatusErrorReasons = 'execute'; + const reason = AlertExecutionStatusErrorReasons.Execute; const error = { reason, message: 'wops' }; test('no input', () => { diff --git a/x-pack/plugins/alerts/server/lib/error_with_reason.test.ts b/x-pack/plugins/alerts/server/lib/error_with_reason.test.ts index f31f584400308..eff935966345f 100644 --- a/x-pack/plugins/alerts/server/lib/error_with_reason.test.ts +++ b/x-pack/plugins/alerts/server/lib/error_with_reason.test.ts @@ -5,20 +5,21 @@ */ import { ErrorWithReason, getReasonFromError, isErrorWithReason } from './error_with_reason'; +import { AlertExecutionStatusErrorReasons } from '../types'; describe('ErrorWithReason', () => { const plainError = new Error('well, actually'); - const errorWithReason = new ErrorWithReason('decrypt', plainError); + const errorWithReason = new ErrorWithReason(AlertExecutionStatusErrorReasons.Decrypt, plainError); test('ErrorWithReason class', () => { expect(errorWithReason.message).toBe(plainError.message); expect(errorWithReason.error).toBe(plainError); - expect(errorWithReason.reason).toBe('decrypt'); + expect(errorWithReason.reason).toBe(AlertExecutionStatusErrorReasons.Decrypt); }); test('getReasonFromError()', () => { expect(getReasonFromError(plainError)).toBe('unknown'); - expect(getReasonFromError(errorWithReason)).toBe('decrypt'); + expect(getReasonFromError(errorWithReason)).toBe(AlertExecutionStatusErrorReasons.Decrypt); }); test('isErrorWithReason()', () => { diff --git a/x-pack/plugins/alerts/server/lib/error_with_reason.ts b/x-pack/plugins/alerts/server/lib/error_with_reason.ts index 29eb666e64427..a732b44ef2238 100644 --- a/x-pack/plugins/alerts/server/lib/error_with_reason.ts +++ b/x-pack/plugins/alerts/server/lib/error_with_reason.ts @@ -21,7 +21,7 @@ export function getReasonFromError(error: Error): AlertExecutionStatusErrorReaso if (isErrorWithReason(error)) { return error.reason; } - return 'unknown'; + return AlertExecutionStatusErrorReasons.Unknown; } export function isErrorWithReason(error: Error | ErrorWithReason): error is ErrorWithReason { diff --git a/x-pack/plugins/alerts/server/lib/is_alert_not_found_error.test.ts b/x-pack/plugins/alerts/server/lib/is_alert_not_found_error.test.ts index b570957d82de4..ab21dc77fa251 100644 --- a/x-pack/plugins/alerts/server/lib/is_alert_not_found_error.test.ts +++ b/x-pack/plugins/alerts/server/lib/is_alert_not_found_error.test.ts @@ -8,6 +8,7 @@ import { isAlertSavedObjectNotFoundError } from './is_alert_not_found_error'; import { ErrorWithReason } from './error_with_reason'; import { SavedObjectsErrorHelpers } from '../../../../../src/core/server'; import uuid from 'uuid'; +import { AlertExecutionStatusErrorReasons } from '../types'; describe('isAlertSavedObjectNotFoundError', () => { const id = uuid.v4(); @@ -25,7 +26,7 @@ describe('isAlertSavedObjectNotFoundError', () => { }); test('identifies SavedObjects Not Found errors wrapped in an ErrorWithReason', () => { - const error = new ErrorWithReason('read', errorSONF); + const error = new ErrorWithReason(AlertExecutionStatusErrorReasons.Read, errorSONF); expect(isAlertSavedObjectNotFoundError(error, id)).toBe(true); }); }); diff --git a/x-pack/plugins/alerts/server/mocks.ts b/x-pack/plugins/alerts/server/mocks.ts index 05d64bdbb77f4..cfae4c650bd42 100644 --- a/x-pack/plugins/alerts/server/mocks.ts +++ b/x-pack/plugins/alerts/server/mocks.ts @@ -25,6 +25,7 @@ const createStartMock = () => { const mock: jest.Mocked = { listTypes: jest.fn(), getAlertsClientWithRequest: jest.fn().mockResolvedValue(alertsClientMock.create()), + getFrameworkHealth: jest.fn(), }; return mock; }; diff --git a/x-pack/plugins/alerts/server/plugin.test.ts b/x-pack/plugins/alerts/server/plugin.test.ts index b13a1c62f6602..715fbc6aeed45 100644 --- a/x-pack/plugins/alerts/server/plugin.test.ts +++ b/x-pack/plugins/alerts/server/plugin.test.ts @@ -5,7 +5,7 @@ */ import { AlertingPlugin, AlertingPluginsSetup, AlertingPluginsStart } from './plugin'; -import { coreMock } from '../../../../src/core/server/mocks'; +import { coreMock, statusServiceMock } from '../../../../src/core/server/mocks'; import { licensingMock } from '../../licensing/server/mocks'; import { encryptedSavedObjectsMock } from '../../encrypted_saved_objects/server/mocks'; import { taskManagerMock } from '../../task_manager/server/mocks'; @@ -13,15 +13,21 @@ import { eventLogServiceMock } from '../../event_log/server/event_log_service.mo import { KibanaRequest, CoreSetup } from 'kibana/server'; import { featuresPluginMock } from '../../features/server/mocks'; import { KibanaFeature } from '../../features/server'; +import { AlertsConfig } from './config'; describe('Alerting Plugin', () => { describe('setup()', () => { it('should log warning when Encrypted Saved Objects plugin is using an ephemeral encryption key', async () => { - const context = coreMock.createPluginInitializerContext(); + const context = coreMock.createPluginInitializerContext({ + healthCheck: { + interval: '5m', + }, + }); const plugin = new AlertingPlugin(context); const coreSetup = coreMock.createSetup(); const encryptedSavedObjectsSetup = encryptedSavedObjectsMock.createSetup(); + const statusMock = statusServiceMock.createSetupContract(); await plugin.setup( ({ ...coreSetup, @@ -29,6 +35,7 @@ describe('Alerting Plugin', () => { ...coreSetup.http, route: jest.fn(), }, + status: statusMock, } as unknown) as CoreSetup, ({ licensing: licensingMock.createSetup(), @@ -38,6 +45,7 @@ describe('Alerting Plugin', () => { } as unknown) as AlertingPluginsSetup ); + expect(statusMock.set).toHaveBeenCalledTimes(1); expect(encryptedSavedObjectsSetup.usingEphemeralEncryptionKey).toEqual(true); expect(context.logger.get().warn).toHaveBeenCalledWith( 'APIs are disabled due to the Encrypted Saved Objects plugin using an ephemeral encryption key. Please set xpack.encryptedSavedObjects.encryptionKey in kibana.yml.' @@ -55,7 +63,11 @@ describe('Alerting Plugin', () => { */ describe('getAlertsClientWithRequest()', () => { it('throws error when encryptedSavedObjects plugin has usingEphemeralEncryptionKey set to true', async () => { - const context = coreMock.createPluginInitializerContext(); + const context = coreMock.createPluginInitializerContext({ + healthCheck: { + interval: '5m', + }, + }); const plugin = new AlertingPlugin(context); const coreSetup = coreMock.createSetup(); @@ -98,7 +110,11 @@ describe('Alerting Plugin', () => { }); it(`doesn't throw error when encryptedSavedObjects plugin has usingEphemeralEncryptionKey set to false`, async () => { - const context = coreMock.createPluginInitializerContext(); + const context = coreMock.createPluginInitializerContext({ + healthCheck: { + interval: '5m', + }, + }); const plugin = new AlertingPlugin(context); const coreSetup = coreMock.createSetup(); diff --git a/x-pack/plugins/alerts/server/plugin.ts b/x-pack/plugins/alerts/server/plugin.ts index 75873a2845c15..1fa89606a76fc 100644 --- a/x-pack/plugins/alerts/server/plugin.ts +++ b/x-pack/plugins/alerts/server/plugin.ts @@ -6,6 +6,7 @@ import type { PublicMethodsOf } from '@kbn/utility-types'; import { first, map } from 'rxjs/operators'; import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; +import { combineLatest } from 'rxjs'; import { SecurityPluginSetup } from '../../security/server'; import { EncryptedSavedObjectsPluginSetup, @@ -30,6 +31,8 @@ import { SharedGlobalConfig, ElasticsearchServiceStart, ILegacyClusterClient, + StatusServiceSetup, + ServiceStatus, } from '../../../../src/core/server'; import { @@ -56,12 +59,19 @@ import { PluginSetupContract as ActionsPluginSetupContract, PluginStartContract as ActionsPluginStartContract, } from '../../actions/server'; -import { Services } from './types'; +import { AlertsHealth, Services } from './types'; import { registerAlertsUsageCollector } from './usage'; import { initializeAlertingTelemetry, scheduleAlertingTelemetry } from './usage/task'; import { IEventLogger, IEventLogService, IEventLogClientService } from '../../event_log/server'; import { PluginStartContract as FeaturesPluginStart } from '../../features/server'; import { setupSavedObjects } from './saved_objects'; +import { + getHealthStatusStream, + scheduleAlertingHealthCheck, + initializeAlertingHealth, +} from './health'; +import { AlertsConfig } from './config'; +import { getHealth } from './health/get_health'; export const EVENT_LOG_PROVIDER = 'alerting'; export const EVENT_LOG_ACTIONS = { @@ -78,6 +88,7 @@ export interface PluginSetupContract { export interface PluginStartContract { listTypes: AlertTypeRegistry['list']; getAlertsClientWithRequest(request: KibanaRequest): PublicMethodsOf; + getFrameworkHealth: () => Promise; } export interface AlertingPluginsSetup { @@ -89,6 +100,7 @@ export interface AlertingPluginsSetup { spaces?: SpacesPluginSetup; usageCollection?: UsageCollectionSetup; eventLog: IEventLogService; + statusService: StatusServiceSetup; } export interface AlertingPluginsStart { actions: ActionsPluginStartContract; @@ -99,6 +111,7 @@ export interface AlertingPluginsStart { } export class AlertingPlugin { + private readonly config: Promise; private readonly logger: Logger; private alertTypeRegistry?: AlertTypeRegistry; private readonly taskRunnerFactory: TaskRunnerFactory; @@ -115,6 +128,7 @@ export class AlertingPlugin { private eventLogger?: IEventLogger; constructor(initializerContext: PluginInitializerContext) { + this.config = initializerContext.config.create().pipe(first()).toPromise(); this.logger = initializerContext.logger.get('plugins', 'alerting'); this.taskRunnerFactory = new TaskRunnerFactory(); this.alertsClientFactory = new AlertsClientFactory(); @@ -186,6 +200,25 @@ export class AlertingPlugin { }); } + core.getStartServices().then(async ([, startPlugins]) => { + core.status.set( + combineLatest([ + core.status.derivedStatus$, + getHealthStatusStream(startPlugins.taskManager), + ]).pipe( + map(([derivedStatus, healthStatus]) => { + if (healthStatus.level > derivedStatus.level) { + return healthStatus as ServiceStatus; + } else { + return derivedStatus; + } + }) + ) + ); + }); + + initializeAlertingHealth(this.logger, plugins.taskManager, core.getStartServices()); + core.http.registerRouteHandlerContext('alerting', this.createRouteHandlerContext(core)); // Routes @@ -275,10 +308,13 @@ export class AlertingPlugin { }); scheduleAlertingTelemetry(this.telemetryLogger, plugins.taskManager); + scheduleAlertingHealthCheck(this.logger, this.config, plugins.taskManager); return { listTypes: alertTypeRegistry!.list.bind(this.alertTypeRegistry!), getAlertsClientWithRequest, + getFrameworkHealth: async () => + await getHealth(core.savedObjects.createInternalRepository(['alert'])), }; } @@ -293,6 +329,8 @@ export class AlertingPlugin { return alertsClientFactory!.create(request, savedObjects); }, listTypes: alertTypeRegistry!.list.bind(alertTypeRegistry!), + getFrameworkHealth: async () => + await getHealth(savedObjects.createInternalRepository(['alert'])), }; }; }; diff --git a/x-pack/plugins/alerts/server/routes/_mock_handler_arguments.ts b/x-pack/plugins/alerts/server/routes/_mock_handler_arguments.ts index 3d13fc65ab260..b3f407b20c142 100644 --- a/x-pack/plugins/alerts/server/routes/_mock_handler_arguments.ts +++ b/x-pack/plugins/alerts/server/routes/_mock_handler_arguments.ts @@ -14,7 +14,7 @@ import { identity } from 'lodash'; import type { MethodKeysOf } from '@kbn/utility-types'; import { httpServerMock } from '../../../../../src/core/server/mocks'; import { alertsClientMock, AlertsClientMock } from '../alerts_client.mock'; -import { AlertType } from '../../common'; +import { AlertsHealth, AlertType } from '../../common'; import { elasticsearchServiceMock } from '../../../../../src/core/server/mocks'; export function mockHandlerArguments( @@ -22,10 +22,13 @@ export function mockHandlerArguments( alertsClient = alertsClientMock.create(), listTypes: listTypesRes = [], esClient = elasticsearchServiceMock.createLegacyClusterClient(), + getFrameworkHealth, }: { alertsClient?: AlertsClientMock; listTypes?: AlertType[]; esClient?: jest.Mocked; + getFrameworkHealth?: jest.MockInstance, []> & + (() => Promise); }, req: unknown, res?: Array> @@ -39,6 +42,7 @@ export function mockHandlerArguments( getAlertsClient() { return alertsClient || alertsClientMock.create(); }, + getFrameworkHealth, }, } as unknown) as RequestHandlerContext, req as KibanaRequest, diff --git a/x-pack/plugins/alerts/server/routes/health.test.ts b/x-pack/plugins/alerts/server/routes/health.test.ts index ce782dbd631a5..d1967c6dd9bf8 100644 --- a/x-pack/plugins/alerts/server/routes/health.test.ts +++ b/x-pack/plugins/alerts/server/routes/health.test.ts @@ -11,13 +11,34 @@ import { elasticsearchServiceMock } from '../../../../../src/core/server/mocks'; import { verifyApiAccess } from '../lib/license_api_access'; import { mockLicenseState } from '../lib/license_state.mock'; import { encryptedSavedObjectsMock } from '../../../encrypted_saved_objects/server/mocks'; +import { alertsClientMock } from '../alerts_client.mock'; +import { HealthStatus } from '../types'; +import { alertsMock } from '../mocks'; +const alertsClient = alertsClientMock.create(); jest.mock('../lib/license_api_access.ts', () => ({ verifyApiAccess: jest.fn(), })); +const alerting = alertsMock.createStart(); + +const currentDate = new Date().toISOString(); beforeEach(() => { jest.resetAllMocks(); + alerting.getFrameworkHealth.mockResolvedValue({ + decryptionHealth: { + status: HealthStatus.OK, + timestamp: currentDate, + }, + executionHealth: { + status: HealthStatus.OK, + timestamp: currentDate, + }, + readHealth: { + status: HealthStatus.OK, + timestamp: currentDate, + }, + }); }); describe('healthRoute', () => { @@ -46,7 +67,7 @@ describe('healthRoute', () => { const esClient = elasticsearchServiceMock.createLegacyClusterClient(); esClient.callAsInternalUser.mockReturnValue(Promise.resolve({})); - const [context, req, res] = mockHandlerArguments({ esClient }, {}, ['ok']); + const [context, req, res] = mockHandlerArguments({ esClient, alertsClient }, {}, ['ok']); await handler(context, req, res); @@ -75,16 +96,32 @@ describe('healthRoute', () => { const esClient = elasticsearchServiceMock.createLegacyClusterClient(); esClient.callAsInternalUser.mockReturnValue(Promise.resolve({})); - const [context, req, res] = mockHandlerArguments({ esClient }, {}, ['ok']); + const [context, req, res] = mockHandlerArguments( + { esClient, alertsClient, getFrameworkHealth: alerting.getFrameworkHealth }, + {}, + ['ok'] + ); - expect(await handler(context, req, res)).toMatchInlineSnapshot(` - Object { - "body": Object { - "hasPermanentEncryptionKey": false, - "isSufficientlySecure": true, + expect(await handler(context, req, res)).toStrictEqual({ + body: { + alertingFrameworkHeath: { + decryptionHealth: { + status: HealthStatus.OK, + timestamp: currentDate, + }, + executionHealth: { + status: HealthStatus.OK, + timestamp: currentDate, + }, + readHealth: { + status: HealthStatus.OK, + timestamp: currentDate, + }, }, - } - `); + hasPermanentEncryptionKey: false, + isSufficientlySecure: true, + }, + }); }); it('evaluates missing security info from the usage api to mean that the security plugin is disbled', async () => { @@ -99,16 +136,32 @@ describe('healthRoute', () => { const esClient = elasticsearchServiceMock.createLegacyClusterClient(); esClient.callAsInternalUser.mockReturnValue(Promise.resolve({})); - const [context, req, res] = mockHandlerArguments({ esClient }, {}, ['ok']); + const [context, req, res] = mockHandlerArguments( + { esClient, alertsClient, getFrameworkHealth: alerting.getFrameworkHealth }, + {}, + ['ok'] + ); - expect(await handler(context, req, res)).toMatchInlineSnapshot(` - Object { - "body": Object { - "hasPermanentEncryptionKey": true, - "isSufficientlySecure": true, + expect(await handler(context, req, res)).toStrictEqual({ + body: { + alertingFrameworkHeath: { + decryptionHealth: { + status: HealthStatus.OK, + timestamp: currentDate, + }, + executionHealth: { + status: HealthStatus.OK, + timestamp: currentDate, + }, + readHealth: { + status: HealthStatus.OK, + timestamp: currentDate, + }, }, - } - `); + hasPermanentEncryptionKey: true, + isSufficientlySecure: true, + }, + }); }); it('evaluates missing security http info from the usage api to mean that the security plugin is disbled', async () => { @@ -123,16 +176,32 @@ describe('healthRoute', () => { const esClient = elasticsearchServiceMock.createLegacyClusterClient(); esClient.callAsInternalUser.mockReturnValue(Promise.resolve({ security: {} })); - const [context, req, res] = mockHandlerArguments({ esClient }, {}, ['ok']); + const [context, req, res] = mockHandlerArguments( + { esClient, alertsClient, getFrameworkHealth: alerting.getFrameworkHealth }, + {}, + ['ok'] + ); - expect(await handler(context, req, res)).toMatchInlineSnapshot(` - Object { - "body": Object { - "hasPermanentEncryptionKey": true, - "isSufficientlySecure": true, + expect(await handler(context, req, res)).toStrictEqual({ + body: { + alertingFrameworkHeath: { + decryptionHealth: { + status: HealthStatus.OK, + timestamp: currentDate, + }, + executionHealth: { + status: HealthStatus.OK, + timestamp: currentDate, + }, + readHealth: { + status: HealthStatus.OK, + timestamp: currentDate, + }, }, - } - `); + hasPermanentEncryptionKey: true, + isSufficientlySecure: true, + }, + }); }); it('evaluates security enabled, and missing ssl info from the usage api to mean that the user cannot generate keys', async () => { @@ -147,16 +216,32 @@ describe('healthRoute', () => { const esClient = elasticsearchServiceMock.createLegacyClusterClient(); esClient.callAsInternalUser.mockReturnValue(Promise.resolve({ security: { enabled: true } })); - const [context, req, res] = mockHandlerArguments({ esClient }, {}, ['ok']); + const [context, req, res] = mockHandlerArguments( + { esClient, alertsClient, getFrameworkHealth: alerting.getFrameworkHealth }, + {}, + ['ok'] + ); - expect(await handler(context, req, res)).toMatchInlineSnapshot(` - Object { - "body": Object { - "hasPermanentEncryptionKey": true, - "isSufficientlySecure": false, + expect(await handler(context, req, res)).toStrictEqual({ + body: { + alertingFrameworkHeath: { + decryptionHealth: { + status: HealthStatus.OK, + timestamp: currentDate, + }, + executionHealth: { + status: HealthStatus.OK, + timestamp: currentDate, + }, + readHealth: { + status: HealthStatus.OK, + timestamp: currentDate, + }, }, - } - `); + hasPermanentEncryptionKey: true, + isSufficientlySecure: false, + }, + }); }); it('evaluates security enabled, SSL info present but missing http info from the usage api to mean that the user cannot generate keys', async () => { @@ -173,16 +258,32 @@ describe('healthRoute', () => { Promise.resolve({ security: { enabled: true, ssl: {} } }) ); - const [context, req, res] = mockHandlerArguments({ esClient }, {}, ['ok']); + const [context, req, res] = mockHandlerArguments( + { esClient, alertsClient, getFrameworkHealth: alerting.getFrameworkHealth }, + {}, + ['ok'] + ); - expect(await handler(context, req, res)).toMatchInlineSnapshot(` - Object { - "body": Object { - "hasPermanentEncryptionKey": true, - "isSufficientlySecure": false, + expect(await handler(context, req, res)).toStrictEqual({ + body: { + alertingFrameworkHeath: { + decryptionHealth: { + status: HealthStatus.OK, + timestamp: currentDate, + }, + executionHealth: { + status: HealthStatus.OK, + timestamp: currentDate, + }, + readHealth: { + status: HealthStatus.OK, + timestamp: currentDate, + }, }, - } - `); + hasPermanentEncryptionKey: true, + isSufficientlySecure: false, + }, + }); }); it('evaluates security and tls enabled to mean that the user can generate keys', async () => { @@ -199,15 +300,31 @@ describe('healthRoute', () => { Promise.resolve({ security: { enabled: true, ssl: { http: { enabled: true } } } }) ); - const [context, req, res] = mockHandlerArguments({ esClient }, {}, ['ok']); + const [context, req, res] = mockHandlerArguments( + { esClient, alertsClient, getFrameworkHealth: alerting.getFrameworkHealth }, + {}, + ['ok'] + ); - expect(await handler(context, req, res)).toMatchInlineSnapshot(` - Object { - "body": Object { - "hasPermanentEncryptionKey": true, - "isSufficientlySecure": true, + expect(await handler(context, req, res)).toStrictEqual({ + body: { + alertingFrameworkHeath: { + decryptionHealth: { + status: HealthStatus.OK, + timestamp: currentDate, + }, + executionHealth: { + status: HealthStatus.OK, + timestamp: currentDate, + }, + readHealth: { + status: HealthStatus.OK, + timestamp: currentDate, + }, }, - } - `); + hasPermanentEncryptionKey: true, + isSufficientlySecure: true, + }, + }); }); }); diff --git a/x-pack/plugins/alerts/server/routes/health.ts b/x-pack/plugins/alerts/server/routes/health.ts index b66e28b24e8a7..bfd5b1e272287 100644 --- a/x-pack/plugins/alerts/server/routes/health.ts +++ b/x-pack/plugins/alerts/server/routes/health.ts @@ -43,6 +43,9 @@ export function healthRoute( res: KibanaResponseFactory ): Promise { verifyApiAccess(licenseState); + if (!context.alerting) { + return res.badRequest({ body: 'RouteHandlerContext is not registered for alerting' }); + } try { const { security: { @@ -57,9 +60,12 @@ export function healthRoute( path: '/_xpack/usage', }); + const alertingFrameworkHeath = await context.alerting.getFrameworkHealth(); + const frameworkHealth: AlertingFrameworkHealth = { isSufficientlySecure: !isSecurityEnabled || (isSecurityEnabled && isTLSEnabled), hasPermanentEncryptionKey: !encryptedSavedObjects.usingEphemeralEncryptionKey, + alertingFrameworkHeath, }; return res.ok({ diff --git a/x-pack/plugins/alerts/server/task_runner/task_runner.ts b/x-pack/plugins/alerts/server/task_runner/task_runner.ts index 6a49f67268d69..86bf7006e8d09 100644 --- a/x-pack/plugins/alerts/server/task_runner/task_runner.ts +++ b/x-pack/plugins/alerts/server/task_runner/task_runner.ts @@ -28,6 +28,7 @@ import { AlertExecutorOptions, SanitizedAlert, AlertExecutionStatus, + AlertExecutionStatusErrorReasons, } from '../types'; import { promiseResult, map, Resultable, asOk, asErr, resolveErr } from '../lib/result_type'; import { taskInstanceToAlertTaskInstance } from './alert_task_instance'; @@ -211,7 +212,7 @@ export class TaskRunner { event.event = event.event || {}; event.event.outcome = 'failure'; eventLogger.logEvent(event); - throw new ErrorWithReason('execute', err); + throw new ErrorWithReason(AlertExecutionStatusErrorReasons.Execute, err); } eventLogger.stopTiming(event); @@ -288,7 +289,7 @@ export class TaskRunner { try { apiKey = await this.getApiKeyForAlertPermissions(alertId, spaceId); } catch (err) { - throw new ErrorWithReason('decrypt', err); + throw new ErrorWithReason(AlertExecutionStatusErrorReasons.Decrypt, err); } const [services, alertsClient] = this.getServicesWithSpaceLevelPermissions(spaceId, apiKey); @@ -298,7 +299,7 @@ export class TaskRunner { try { alert = await alertsClient.get({ id: alertId }); } catch (err) { - throw new ErrorWithReason('read', err); + throw new ErrorWithReason(AlertExecutionStatusErrorReasons.Read, err); } return { diff --git a/x-pack/plugins/alerts/server/types.ts b/x-pack/plugins/alerts/server/types.ts index 42eef9bba10e5..9226461f6e30a 100644 --- a/x-pack/plugins/alerts/server/types.ts +++ b/x-pack/plugins/alerts/server/types.ts @@ -27,6 +27,7 @@ import { AlertInstanceState, AlertExecutionStatuses, AlertExecutionStatusErrorReasons, + AlertsHealth, } from '../common'; export type WithoutQueryAndParams = Pick>; @@ -39,6 +40,7 @@ declare module 'src/core/server' { alerting?: { getAlertsClient: () => AlertsClient; listTypes: AlertTypeRegistry['list']; + getFrameworkHealth: () => Promise; }; } } @@ -172,4 +174,10 @@ export interface AlertingPlugin { start: PluginStartContract; } +export interface AlertsConfigType { + healthCheck: { + interval: string; + }; +} + export type AlertTypeRegistry = PublicMethodsOf; diff --git a/x-pack/plugins/ingest_manager/common/services/package_to_package_policy.test.ts b/x-pack/plugins/ingest_manager/common/services/package_to_package_policy.test.ts index 91396bce359b0..e81207300a5f3 100644 --- a/x-pack/plugins/ingest_manager/common/services/package_to_package_policy.test.ts +++ b/x-pack/plugins/ingest_manager/common/services/package_to_package_policy.test.ts @@ -84,16 +84,14 @@ describe('Ingest Manager - packageToPackagePolicy', () => { { type: 'foo', enabled: true, - streams: [ - { id: 'foo-foo', enabled: true, data_stream: { dataset: 'foo', type: 'logs' } }, - ], + streams: [{ enabled: true, data_stream: { dataset: 'foo', type: 'logs' } }], }, { type: 'bar', enabled: true, streams: [ - { id: 'bar-bar', enabled: true, data_stream: { dataset: 'bar', type: 'logs' } }, - { id: 'bar-bar2', enabled: true, data_stream: { dataset: 'bar2', type: 'logs' } }, + { enabled: true, data_stream: { dataset: 'bar', type: 'logs' } }, + { enabled: true, data_stream: { dataset: 'bar2', type: 'logs' } }, ], }, ]); @@ -142,7 +140,6 @@ describe('Ingest Manager - packageToPackagePolicy', () => { enabled: true, streams: [ { - id: 'foo-foo', enabled: true, data_stream: { dataset: 'foo', type: 'logs' }, vars: { 'var-name': { value: 'foo-var-value' } }, @@ -154,13 +151,11 @@ describe('Ingest Manager - packageToPackagePolicy', () => { enabled: true, streams: [ { - id: 'bar-bar', enabled: true, data_stream: { dataset: 'bar', type: 'logs' }, vars: { 'var-name': { type: 'text', value: 'bar-var-value' } }, }, { - id: 'bar-bar2', enabled: true, data_stream: { dataset: 'bar2', type: 'logs' }, vars: { 'var-name': { type: 'yaml', value: 'bar2-var-value' } }, @@ -258,7 +253,6 @@ describe('Ingest Manager - packageToPackagePolicy', () => { }, streams: [ { - id: 'foo-foo', enabled: true, data_stream: { dataset: 'foo', type: 'logs' }, vars: { @@ -276,7 +270,6 @@ describe('Ingest Manager - packageToPackagePolicy', () => { }, streams: [ { - id: 'bar-bar', enabled: true, data_stream: { dataset: 'bar', type: 'logs' }, vars: { @@ -284,7 +277,6 @@ describe('Ingest Manager - packageToPackagePolicy', () => { }, }, { - id: 'bar-bar2', enabled: true, data_stream: { dataset: 'bar2', type: 'logs' }, vars: { @@ -298,7 +290,6 @@ describe('Ingest Manager - packageToPackagePolicy', () => { enabled: false, streams: [ { - id: 'with-disabled-streams-disabled', enabled: false, data_stream: { dataset: 'disabled', type: 'logs' }, vars: { @@ -306,7 +297,6 @@ describe('Ingest Manager - packageToPackagePolicy', () => { }, }, { - id: 'with-disabled-streams-disabled2', enabled: false, data_stream: { dataset: 'disabled2', type: 'logs' }, }, diff --git a/x-pack/plugins/ingest_manager/common/services/package_to_package_policy.ts b/x-pack/plugins/ingest_manager/common/services/package_to_package_policy.ts index 822747916ebc5..cbdfa25ed7f7e 100644 --- a/x-pack/plugins/ingest_manager/common/services/package_to_package_policy.ts +++ b/x-pack/plugins/ingest_manager/common/services/package_to_package_policy.ts @@ -8,11 +8,10 @@ import { RegistryPolicyTemplate, RegistryVarsEntry, RegistryStream, - PackagePolicy, PackagePolicyConfigRecord, PackagePolicyConfigRecordEntry, - PackagePolicyInput, - PackagePolicyInputStream, + NewPackagePolicyInput, + NewPackagePolicyInputStream, NewPackagePolicy, } from '../types'; @@ -42,8 +41,10 @@ const getStreamsForInputType = ( /* * This service creates a package policy inputs definition from defaults provided in package info */ -export const packageToPackagePolicyInputs = (packageInfo: PackageInfo): PackagePolicy['inputs'] => { - const inputs: PackagePolicy['inputs'] = []; +export const packageToPackagePolicyInputs = ( + packageInfo: PackageInfo +): NewPackagePolicy['inputs'] => { + const inputs: NewPackagePolicy['inputs'] = []; // Assume package will only ever ship one package policy template for now const packagePolicyTemplate: RegistryPolicyTemplate | null = @@ -71,12 +72,11 @@ export const packageToPackagePolicyInputs = (packageInfo: PackageInfo): PackageP }; // Map each package input stream into package policy input stream - const streams: PackagePolicyInputStream[] = getStreamsForInputType( + const streams: NewPackagePolicyInputStream[] = getStreamsForInputType( packageInput.type, packageInfo ).map((packageStream) => { - const stream: PackagePolicyInputStream = { - id: `${packageInput.type}-${packageStream.data_stream.dataset}`, + const stream: NewPackagePolicyInputStream = { enabled: packageStream.enabled === false ? false : true, data_stream: packageStream.data_stream, }; @@ -86,7 +86,7 @@ export const packageToPackagePolicyInputs = (packageInfo: PackageInfo): PackageP return stream; }); - const input: PackagePolicyInput = { + const input: NewPackagePolicyInput = { type: packageInput.type, enabled: streams.length ? !!streams.find((stream) => stream.enabled) : true, streams, diff --git a/x-pack/plugins/ingest_manager/common/types/models/package_policy.ts b/x-pack/plugins/ingest_manager/common/types/models/package_policy.ts index 724dbae5dac85..ae16899a4b6f9 100644 --- a/x-pack/plugins/ingest_manager/common/types/models/package_policy.ts +++ b/x-pack/plugins/ingest_manager/common/types/models/package_policy.ts @@ -18,7 +18,6 @@ export interface PackagePolicyConfigRecordEntry { export type PackagePolicyConfigRecord = Record; export interface NewPackagePolicyInputStream { - id: string; enabled: boolean; data_stream: { dataset: string; @@ -29,6 +28,7 @@ export interface NewPackagePolicyInputStream { } export interface PackagePolicyInputStream extends NewPackagePolicyInputStream { + id: string; compiled_stream?: any; } diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/create_package_policy_page/components/package_policy_input_config.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/create_package_policy_page/components/package_policy_input_config.tsx index 175bfb1469902..177354dad14dc 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/create_package_policy_page/components/package_policy_input_config.tsx +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/create_package_policy_page/components/package_policy_input_config.tsx @@ -14,7 +14,7 @@ import { EuiSpacer, EuiButtonEmpty, } from '@elastic/eui'; -import { PackagePolicyInput, RegistryVarsEntry } from '../../../../types'; +import { NewPackagePolicyInput, RegistryVarsEntry } from '../../../../types'; import { isAdvancedVar, PackagePolicyConfigValidationResults, @@ -28,8 +28,8 @@ const FlexItemWithMaxWidth = styled(EuiFlexItem)` export const PackagePolicyInputConfig: React.FunctionComponent<{ packageInputVars?: RegistryVarsEntry[]; - packagePolicyInput: PackagePolicyInput; - updatePackagePolicyInput: (updatedInput: Partial) => void; + packagePolicyInput: NewPackagePolicyInput; + updatePackagePolicyInput: (updatedInput: Partial) => void; inputVarsValidationResults: PackagePolicyConfigValidationResults; forceShowErrors?: boolean; }> = memo( diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/create_package_policy_page/components/package_policy_input_panel.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/create_package_policy_page/components/package_policy_input_panel.tsx index 1e43cc0d5938e..79ff0cc29850c 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/create_package_policy_page/components/package_policy_input_panel.tsx +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/create_package_policy_page/components/package_policy_input_panel.tsx @@ -17,7 +17,7 @@ import { EuiSpacer, } from '@elastic/eui'; import { - PackagePolicyInput, + NewPackagePolicyInput, PackagePolicyInputStream, RegistryInput, RegistryStream, @@ -40,7 +40,7 @@ const ShortenedHorizontalRule = styled(EuiHorizontalRule)` const shouldShowStreamsByDefault = ( packageInput: RegistryInput, packageInputStreams: Array, - packagePolicyInput: PackagePolicyInput + packagePolicyInput: NewPackagePolicyInput ): boolean => { return ( packagePolicyInput.enabled && @@ -63,8 +63,8 @@ const shouldShowStreamsByDefault = ( export const PackagePolicyInputPanel: React.FunctionComponent<{ packageInput: RegistryInput; packageInputStreams: Array; - packagePolicyInput: PackagePolicyInput; - updatePackagePolicyInput: (updatedInput: Partial) => void; + packagePolicyInput: NewPackagePolicyInput; + updatePackagePolicyInput: (updatedInput: Partial) => void; inputValidationResults: PackagePolicyInputValidationResults; forceShowErrors?: boolean; }> = memo( @@ -210,7 +210,7 @@ export const PackagePolicyInputPanel: React.FunctionComponent<{ ...updatedStream, }; - const updatedInput: Partial = { + const updatedInput: Partial = { streams: newStreams, }; @@ -227,7 +227,7 @@ export const PackagePolicyInputPanel: React.FunctionComponent<{ updatePackagePolicyInput(updatedInput); }} inputStreamValidationResults={ - inputValidationResults.streams![packagePolicyInputStream!.id] + inputValidationResults.streams![packagePolicyInputStream!.data_stream!.dataset] } forceShowErrors={forceShowErrors} /> diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/create_package_policy_page/components/package_policy_input_stream.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/create_package_policy_page/components/package_policy_input_stream.tsx index 3d33edd468151..963d0da50ce7f 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/create_package_policy_page/components/package_policy_input_stream.tsx +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/create_package_policy_page/components/package_policy_input_stream.tsx @@ -16,7 +16,7 @@ import { EuiSpacer, EuiButtonEmpty, } from '@elastic/eui'; -import { PackagePolicyInputStream, RegistryStream, RegistryVarsEntry } from '../../../../types'; +import { NewPackagePolicyInputStream, RegistryStream, RegistryVarsEntry } from '../../../../types'; import { isAdvancedVar, PackagePolicyConfigValidationResults, @@ -30,8 +30,8 @@ const FlexItemWithMaxWidth = styled(EuiFlexItem)` export const PackagePolicyInputStreamConfig: React.FunctionComponent<{ packageInputStream: RegistryStream; - packagePolicyInputStream: PackagePolicyInputStream; - updatePackagePolicyInputStream: (updatedStream: Partial) => void; + packagePolicyInputStream: NewPackagePolicyInputStream; + updatePackagePolicyInputStream: (updatedStream: Partial) => void; inputStreamValidationResults: PackagePolicyConfigValidationResults; forceShowErrors?: boolean; }> = memo( diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/create_package_policy_page/services/validate_package_policy.test..ts b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/create_package_policy_page/services/validate_package_policy.test.ts similarity index 91% rename from x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/create_package_policy_page/services/validate_package_policy.test..ts rename to x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/create_package_policy_page/services/validate_package_policy.test.ts index 9022e312ece79..8d46fed1ff14e 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/create_package_policy_page/services/validate_package_policy.test..ts +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/create_package_policy_page/services/validate_package_policy.test.ts @@ -154,7 +154,6 @@ describe('Ingest Manager - validatePackagePolicy()', () => { }, streams: [ { - id: 'foo-foo', data_stream: { dataset: 'foo', type: 'logs' }, enabled: true, vars: { 'var-name': { value: 'test_yaml: value', type: 'yaml' } }, @@ -170,13 +169,11 @@ describe('Ingest Manager - validatePackagePolicy()', () => { }, streams: [ { - id: 'bar-bar', data_stream: { dataset: 'bar', type: 'logs' }, enabled: true, vars: { 'var-name': { value: 'test_yaml: value', type: 'yaml' } }, }, { - id: 'bar-bar2', data_stream: { dataset: 'bar2', type: 'logs' }, enabled: true, vars: { 'var-name': { value: undefined, type: 'text' } }, @@ -193,13 +190,11 @@ describe('Ingest Manager - validatePackagePolicy()', () => { enabled: true, streams: [ { - id: 'with-disabled-streams-disabled', data_stream: { dataset: 'disabled', type: 'logs' }, enabled: false, vars: { 'var-name': { value: undefined, type: 'text' } }, }, { - id: 'with-disabled-streams-disabled-without-vars', data_stream: { dataset: 'disabled2', type: 'logs' }, enabled: false, }, @@ -213,8 +208,7 @@ describe('Ingest Manager - validatePackagePolicy()', () => { }, streams: [ { - id: 'with-no-stream-vars-bar', - data_stream: { dataset: 'bar', type: 'logs' }, + data_stream: { dataset: 'with-no-stream-vars-bar', type: 'logs' }, enabled: true, }, ], @@ -236,7 +230,6 @@ describe('Ingest Manager - validatePackagePolicy()', () => { }, streams: [ { - id: 'foo-foo', data_stream: { dataset: 'foo', type: 'logs' }, enabled: true, vars: { 'var-name': { value: 'invalidyaml: test\n foo bar:', type: 'yaml' } }, @@ -252,13 +245,11 @@ describe('Ingest Manager - validatePackagePolicy()', () => { }, streams: [ { - id: 'bar-bar', data_stream: { dataset: 'bar', type: 'logs' }, enabled: true, vars: { 'var-name': { value: ' \n\n', type: 'yaml' } }, }, { - id: 'bar-bar2', data_stream: { dataset: 'bar2', type: 'logs' }, enabled: true, vars: { 'var-name': { value: undefined, type: 'text' } }, @@ -275,7 +266,6 @@ describe('Ingest Manager - validatePackagePolicy()', () => { enabled: true, streams: [ { - id: 'with-disabled-streams-disabled', data_stream: { dataset: 'disabled', type: 'logs' }, enabled: false, vars: { @@ -286,7 +276,6 @@ describe('Ingest Manager - validatePackagePolicy()', () => { }, }, { - id: 'with-disabled-streams-disabled-without-vars', data_stream: { dataset: 'disabled2', type: 'logs' }, enabled: false, }, @@ -300,8 +289,7 @@ describe('Ingest Manager - validatePackagePolicy()', () => { }, streams: [ { - id: 'with-no-stream-vars-bar', - data_stream: { dataset: 'bar', type: 'logs' }, + data_stream: { dataset: 'with-no-stream-vars-bar', type: 'logs' }, enabled: true, }, ], @@ -320,21 +308,21 @@ describe('Ingest Manager - validatePackagePolicy()', () => { 'foo-input2-var-name': null, 'foo-input3-var-name': null, }, - streams: { 'foo-foo': { vars: { 'var-name': null } } }, + streams: { foo: { vars: { 'var-name': null } } }, }, bar: { vars: { 'bar-input-var-name': null, 'bar-input2-var-name': null }, streams: { - 'bar-bar': { vars: { 'var-name': null } }, - 'bar-bar2': { vars: { 'var-name': null } }, + bar: { vars: { 'var-name': null } }, + bar2: { vars: { 'var-name': null } }, }, }, 'with-disabled-streams': { streams: { - 'with-disabled-streams-disabled': { + disabled: { vars: { 'var-name': null }, }, - 'with-disabled-streams-disabled-without-vars': {}, + disabled2: {}, }, }, 'with-no-stream-vars': { @@ -364,7 +352,7 @@ describe('Ingest Manager - validatePackagePolicy()', () => { 'foo-input2-var-name': ['foo-input2-var-name is required'], 'foo-input3-var-name': ['foo-input3-var-name is required'], }, - streams: { 'foo-foo': { vars: { 'var-name': ['Invalid YAML format'] } } }, + streams: { foo: { vars: { 'var-name': ['Invalid YAML format'] } } }, }, bar: { vars: { @@ -372,14 +360,14 @@ describe('Ingest Manager - validatePackagePolicy()', () => { 'bar-input2-var-name': ['bar-input2-var-name is required'], }, streams: { - 'bar-bar': { vars: { 'var-name': ['var-name is required'] } }, - 'bar-bar2': { vars: { 'var-name': null } }, + bar: { vars: { 'var-name': ['var-name is required'] } }, + bar2: { vars: { 'var-name': null } }, }, }, 'with-disabled-streams': { streams: { - 'with-disabled-streams-disabled': { vars: { 'var-name': null } }, - 'with-disabled-streams-disabled-without-vars': {}, + disabled: { vars: { 'var-name': null } }, + disabled2: {}, }, }, 'with-no-stream-vars': { @@ -427,7 +415,7 @@ describe('Ingest Manager - validatePackagePolicy()', () => { 'foo-input2-var-name': ['foo-input2-var-name is required'], 'foo-input3-var-name': ['foo-input3-var-name is required'], }, - streams: { 'foo-foo': { vars: { 'var-name': null } } }, + streams: { foo: { vars: { 'var-name': null } } }, }, bar: { vars: { @@ -435,16 +423,16 @@ describe('Ingest Manager - validatePackagePolicy()', () => { 'bar-input2-var-name': ['bar-input2-var-name is required'], }, streams: { - 'bar-bar': { vars: { 'var-name': null } }, - 'bar-bar2': { vars: { 'var-name': null } }, + bar: { vars: { 'var-name': null } }, + bar2: { vars: { 'var-name': null } }, }, }, 'with-disabled-streams': { streams: { - 'with-disabled-streams-disabled': { + disabled: { vars: { 'var-name': null }, }, - 'with-disabled-streams-disabled-without-vars': {}, + disabled2: {}, }, }, 'with-no-stream-vars': { diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/create_package_policy_page/services/validate_package_policy.ts b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/create_package_policy_page/services/validate_package_policy.ts index 9ce73c0690ccb..1126cd7e58e18 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/create_package_policy_page/services/validate_package_policy.ts +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/create_package_policy_page/services/validate_package_policy.ts @@ -151,7 +151,7 @@ export const validatePackagePolicy = ( ); } - inputValidationResults.streams![stream.id] = streamValidationResults; + inputValidationResults.streams![stream.data_stream.dataset] = streamValidationResults; }); } else { delete inputValidationResults.streams; diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/create_package_policy_page/step_configure_package.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/create_package_policy_page/step_configure_package.tsx index d3d5e60c34e58..b335ff439684b 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/create_package_policy_page/step_configure_package.tsx +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/create_package_policy_page/step_configure_package.tsx @@ -5,7 +5,12 @@ */ import React from 'react'; import { EuiHorizontalRule, EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; -import { PackageInfo, RegistryStream, NewPackagePolicy, PackagePolicyInput } from '../../../types'; +import { + PackageInfo, + RegistryStream, + NewPackagePolicy, + NewPackagePolicyInput, +} from '../../../types'; import { Loading } from '../../../components'; import { PackagePolicyValidationResults } from './services'; import { PackagePolicyInputPanel, CustomPackagePolicy } from './components'; @@ -71,7 +76,7 @@ export const StepConfigurePackagePolicy: React.FunctionComponent<{ packageInput={packageInput} packageInputStreams={packageInputStreams} packagePolicyInput={packagePolicyInput} - updatePackagePolicyInput={(updatedInput: Partial) => { + updatePackagePolicyInput={(updatedInput: Partial) => { const indexOfUpdatedInput = packagePolicy.inputs.findIndex( (input) => input.type === packageInput.type ); diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/types/index.ts b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/types/index.ts index 386ffa5649cc2..1cf8077aeda40 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/types/index.ts +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/types/index.ts @@ -17,7 +17,9 @@ export { NewPackagePolicy, UpdatePackagePolicy, PackagePolicyInput, + NewPackagePolicyInput, PackagePolicyInputStream, + NewPackagePolicyInputStream, PackagePolicyConfigRecord, PackagePolicyConfigRecordEntry, Output, diff --git a/x-pack/plugins/ingest_manager/server/routes/package_policy/handlers.test.ts b/x-pack/plugins/ingest_manager/server/routes/package_policy/handlers.test.ts index 44c2ccda3bd2a..f47b8499a1b69 100644 --- a/x-pack/plugins/ingest_manager/server/routes/package_policy/handlers.test.ts +++ b/x-pack/plugins/ingest_manager/server/routes/package_policy/handlers.test.ts @@ -28,6 +28,13 @@ jest.mock('../../services/package_policy', (): { create: jest.fn((soClient, callCluster, newData) => Promise.resolve({ ...newData, + inputs: newData.inputs.map((input) => ({ + ...input, + streams: input.streams.map((stream) => ({ + id: stream.data_stream.dataset, + ...stream, + })), + })), id: '1', revision: 1, updated_at: new Date().toISOString(), diff --git a/x-pack/plugins/ingest_manager/server/routes/package_policy/handlers.ts b/x-pack/plugins/ingest_manager/server/routes/package_policy/handlers.ts index d9baeca4deb47..3a2b9ba7a744f 100644 --- a/x-pack/plugins/ingest_manager/server/routes/package_policy/handlers.ts +++ b/x-pack/plugins/ingest_manager/server/routes/package_policy/handlers.ts @@ -7,7 +7,6 @@ import { TypeOf } from '@kbn/config-schema'; import Boom from '@hapi/boom'; import { RequestHandler, SavedObjectsErrorHelpers } from '../../../../../../src/core/server'; import { appContextService, packagePolicyService } from '../../services'; -import { getPackageInfo } from '../../services/epm/packages'; import { GetPackagePoliciesRequestSchema, GetOnePackagePolicyRequestSchema, @@ -134,21 +133,11 @@ export const updatePackagePolicyHandler: RequestHandler< const newData = { ...request.body }; const pkg = newData.package || packagePolicy.package; const inputs = newData.inputs || packagePolicy.inputs; - if (pkg && (newData.inputs || newData.package)) { - const pkgInfo = await getPackageInfo({ - savedObjectsClient: soClient, - pkgName: pkg.name, - pkgVersion: pkg.version, - }); - newData.inputs = (await packagePolicyService.assignPackageStream(pkgInfo, inputs)) as TypeOf< - typeof CreatePackagePolicyRequestSchema.body - >['inputs']; - } const updatedPackagePolicy = await packagePolicyService.update( soClient, request.params.packagePolicyId, - newData, + { ...newData, package: pkg, inputs }, { user } ); return response.ok({ diff --git a/x-pack/plugins/ingest_manager/server/services/package_policy.test.ts b/x-pack/plugins/ingest_manager/server/services/package_policy.test.ts index 6064e5bae0634..6ae76c56436d5 100644 --- a/x-pack/plugins/ingest_manager/server/services/package_policy.test.ts +++ b/x-pack/plugins/ingest_manager/server/services/package_policy.test.ts @@ -34,6 +34,12 @@ jest.mock('./epm/packages/assets', () => { }; }); +jest.mock('./epm/packages', () => { + return { + getPackageInfo: () => ({}), + }; +}); + jest.mock('./epm/registry', () => { return { fetchInfo: () => ({}), diff --git a/x-pack/plugins/ingest_manager/server/services/package_policy.ts b/x-pack/plugins/ingest_manager/server/services/package_policy.ts index dc3a4495191c9..0f78c97a6f2bd 100644 --- a/x-pack/plugins/ingest_manager/server/services/package_policy.ts +++ b/x-pack/plugins/ingest_manager/server/services/package_policy.ts @@ -4,10 +4,12 @@ * you may not use this file except in compliance with the Elastic License. */ import { SavedObjectsClientContract } from 'src/core/server'; +import uuid from 'uuid'; import { AuthenticatedUser } from '../../../security/server'; import { DeletePackagePoliciesResponse, PackagePolicyInput, + NewPackagePolicyInput, PackagePolicyInputStream, PackageInfo, ListWithKuery, @@ -58,6 +60,11 @@ class PackagePolicyService { throw new Error('There is already a package with the same name on this agent policy'); } } + // Add ids to stream + const packagePolicyId = options?.id || uuid.v4(); + let inputs: PackagePolicyInput[] = packagePolicy.inputs.map((input) => + assignStreamIdToInput(packagePolicyId, input) + ); // Make sure the associated package is installed if (packagePolicy.package?.name) { @@ -85,7 +92,7 @@ class PackagePolicyService { } } - packagePolicy.inputs = await this.assignPackageStream(pkgInfo, packagePolicy.inputs); + inputs = await this.assignPackageStream(pkgInfo, inputs); } const isoDate = new Date().toISOString(); @@ -93,13 +100,15 @@ class PackagePolicyService { SAVED_OBJECT_TYPE, { ...packagePolicy, + inputs, revision: 1, created_at: isoDate, created_by: options?.user?.username ?? 'system', updated_at: isoDate, updated_by: options?.user?.username ?? 'system', }, - options + + { ...options, id: packagePolicyId } ); // Assign it to the given agent policy @@ -124,18 +133,28 @@ class PackagePolicyService { const isoDate = new Date().toISOString(); // eslint-disable-next-line @typescript-eslint/naming-convention const { saved_objects } = await soClient.bulkCreate( - packagePolicies.map((packagePolicy) => ({ - type: SAVED_OBJECT_TYPE, - attributes: { - ...packagePolicy, - policy_id: agentPolicyId, - revision: 1, - created_at: isoDate, - created_by: options?.user?.username ?? 'system', - updated_at: isoDate, - updated_by: options?.user?.username ?? 'system', - }, - })) + packagePolicies.map((packagePolicy) => { + const packagePolicyId = uuid.v4(); + + const inputs = packagePolicy.inputs.map((input) => + assignStreamIdToInput(packagePolicyId, input) + ); + + return { + type: SAVED_OBJECT_TYPE, + id: packagePolicyId, + attributes: { + ...packagePolicy, + inputs, + policy_id: agentPolicyId, + revision: 1, + created_at: isoDate, + created_by: options?.user?.username ?? 'system', + updated_at: isoDate, + updated_by: options?.user?.username ?? 'system', + }, + }; + }) ); // Filter out invalid SOs @@ -255,11 +274,26 @@ class PackagePolicyService { } } + let inputs = await restOfPackagePolicy.inputs.map((input) => + assignStreamIdToInput(oldPackagePolicy.id, input) + ); + + if (packagePolicy.package?.name) { + const pkgInfo = await getPackageInfo({ + savedObjectsClient: soClient, + pkgName: packagePolicy.package.name, + pkgVersion: packagePolicy.package.version, + }); + + inputs = await this.assignPackageStream(pkgInfo, inputs); + } + await soClient.update( SAVED_OBJECT_TYPE, id, { ...restOfPackagePolicy, + inputs, revision: oldPackagePolicy.revision + 1, updated_at: new Date().toISOString(), updated_by: options?.user?.username ?? 'system', @@ -353,6 +387,15 @@ class PackagePolicyService { } } +function assignStreamIdToInput(packagePolicyId: string, input: NewPackagePolicyInput) { + return { + ...input, + streams: input.streams.map((stream) => { + return { ...stream, id: `${input.type}-${stream.data_stream.dataset}-${packagePolicyId}` }; + }), + }; +} + async function _assignPackageStreamToInput( registryPkgInfo: RegistryPackage, pkgInfo: PackageInfo, diff --git a/x-pack/plugins/ingest_manager/server/types/models/package_policy.ts b/x-pack/plugins/ingest_manager/server/types/models/package_policy.ts index 6673c12d51511..20d29c0aa18c9 100644 --- a/x-pack/plugins/ingest_manager/server/types/models/package_policy.ts +++ b/x-pack/plugins/ingest_manager/server/types/models/package_policy.ts @@ -54,7 +54,7 @@ const PackagePolicyBaseSchema = { ), streams: schema.arrayOf( schema.object({ - id: schema.string(), + id: schema.maybe(schema.string()), // BWC < 7.11 enabled: schema.boolean(), data_stream: schema.object({ dataset: schema.string(), type: schema.string() }), vars: schema.maybe(ConfigRecordSchema), diff --git a/x-pack/plugins/maps/public/classes/layers/vector_tile_layer/vector_tile_layer.js b/x-pack/plugins/maps/public/classes/layers/vector_tile_layer/vector_tile_layer.js index 96dad0c01139e..dc3ace69e5a61 100644 --- a/x-pack/plugins/maps/public/classes/layers/vector_tile_layer/vector_tile_layer.js +++ b/x-pack/plugins/maps/public/classes/layers/vector_tile_layer/vector_tile_layer.js @@ -11,7 +11,7 @@ import { isRetina } from '../../../meta'; import { addSpriteSheetToMapFromImageData, loadSpriteSheetImageData, -} from '../../../connected_components/map/mb/utils'; //todo move this implementation +} from '../../../connected_components/mb_map/utils'; //todo move this implementation const MB_STYLE_TYPE_TO_OPACITY = { fill: ['fill-opacity'], diff --git a/x-pack/plugins/maps/public/connected_components/_index.scss b/x-pack/plugins/maps/public/connected_components/_index.scss index a952b3b545922..19c11d3fde662 100644 --- a/x-pack/plugins/maps/public/connected_components/_index.scss +++ b/x-pack/plugins/maps/public/connected_components/_index.scss @@ -2,4 +2,4 @@ @import 'layer_panel/index'; @import 'widget_overlay/index'; @import 'toolbar_overlay/index'; -@import 'map/features_tooltip/index'; +@import 'mb_map/features_tooltip/index'; diff --git a/x-pack/plugins/maps/public/connected_components/map_container/map_container.tsx b/x-pack/plugins/maps/public/connected_components/map_container/map_container.tsx index 352aed4a8cc93..169875e63a536 100644 --- a/x-pack/plugins/maps/public/connected_components/map_container/map_container.tsx +++ b/x-pack/plugins/maps/public/connected_components/map_container/map_container.tsx @@ -13,7 +13,7 @@ import uuid from 'uuid/v4'; import { Filter } from 'src/plugins/data/public'; import { ActionExecutionContext, Action } from 'src/plugins/ui_actions/public'; // @ts-expect-error -import { MBMap } from '../map/mb'; +import { MBMap } from '../mb_map'; // @ts-expect-error import { WidgetOverlay } from '../widget_overlay'; // @ts-expect-error diff --git a/x-pack/plugins/maps/public/connected_components/map/mb/draw_control/draw_circle.ts b/x-pack/plugins/maps/public/connected_components/mb_map/draw_control/draw_circle.ts similarity index 100% rename from x-pack/plugins/maps/public/connected_components/map/mb/draw_control/draw_circle.ts rename to x-pack/plugins/maps/public/connected_components/mb_map/draw_control/draw_circle.ts diff --git a/x-pack/plugins/maps/public/connected_components/map/mb/draw_control/draw_control.js b/x-pack/plugins/maps/public/connected_components/mb_map/draw_control/draw_control.js similarity index 97% rename from x-pack/plugins/maps/public/connected_components/map/mb/draw_control/draw_control.js rename to x-pack/plugins/maps/public/connected_components/mb_map/draw_control/draw_control.js index 0356a8267c18a..089d4be28dff7 100644 --- a/x-pack/plugins/maps/public/connected_components/map/mb/draw_control/draw_control.js +++ b/x-pack/plugins/maps/public/connected_components/mb_map/draw_control/draw_control.js @@ -6,7 +6,7 @@ import _ from 'lodash'; import React from 'react'; -import { DRAW_TYPE } from '../../../../../common/constants'; +import { DRAW_TYPE } from '../../../../common/constants'; import MapboxDraw from '@mapbox/mapbox-gl-draw/dist/mapbox-gl-draw-unminified'; import DrawRectangle from 'mapbox-gl-draw-rectangle-mode'; import { DrawCircle } from './draw_circle'; @@ -15,7 +15,7 @@ import { createSpatialFilterWithGeometry, getBoundingBoxGeometry, roundCoordinates, -} from '../../../../../common/elasticsearch_util'; +} from '../../../../common/elasticsearch_util'; import { DrawTooltip } from './draw_tooltip'; const DRAW_RECTANGLE = 'draw_rectangle'; diff --git a/x-pack/plugins/maps/public/connected_components/map/mb/draw_control/draw_tooltip.js b/x-pack/plugins/maps/public/connected_components/mb_map/draw_control/draw_tooltip.js similarity index 97% rename from x-pack/plugins/maps/public/connected_components/map/mb/draw_control/draw_tooltip.js rename to x-pack/plugins/maps/public/connected_components/mb_map/draw_control/draw_tooltip.js index c8bde29b94fb6..dd93b038ff8a1 100644 --- a/x-pack/plugins/maps/public/connected_components/map/mb/draw_control/draw_tooltip.js +++ b/x-pack/plugins/maps/public/connected_components/mb_map/draw_control/draw_tooltip.js @@ -8,7 +8,7 @@ import _ from 'lodash'; import React, { Component } from 'react'; import { EuiPopover, EuiText } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; -import { DRAW_TYPE } from '../../../../../common/constants'; +import { DRAW_TYPE } from '../../../../common/constants'; const noop = () => {}; diff --git a/x-pack/plugins/maps/public/connected_components/map/mb/draw_control/index.js b/x-pack/plugins/maps/public/connected_components/mb_map/draw_control/index.js similarity index 83% rename from x-pack/plugins/maps/public/connected_components/map/mb/draw_control/index.js rename to x-pack/plugins/maps/public/connected_components/mb_map/draw_control/index.js index bc026c41fcf0a..230ad5b3f39d5 100644 --- a/x-pack/plugins/maps/public/connected_components/map/mb/draw_control/index.js +++ b/x-pack/plugins/maps/public/connected_components/mb_map/draw_control/index.js @@ -6,8 +6,8 @@ import { connect } from 'react-redux'; import { DrawControl } from './draw_control'; -import { updateDrawState } from '../../../../actions'; -import { getDrawState, isDrawingFilter } from '../../../../selectors/map_selectors'; +import { updateDrawState } from '../../../actions'; +import { getDrawState, isDrawingFilter } from '../../../selectors/map_selectors'; function mapStateToProps(state = {}) { return { diff --git a/x-pack/plugins/maps/public/connected_components/map/features_tooltip/__snapshots__/feature_properties.test.js.snap b/x-pack/plugins/maps/public/connected_components/mb_map/features_tooltip/__snapshots__/feature_properties.test.js.snap similarity index 100% rename from x-pack/plugins/maps/public/connected_components/map/features_tooltip/__snapshots__/feature_properties.test.js.snap rename to x-pack/plugins/maps/public/connected_components/mb_map/features_tooltip/__snapshots__/feature_properties.test.js.snap diff --git a/x-pack/plugins/maps/public/connected_components/map/features_tooltip/__snapshots__/tooltip_header.test.js.snap b/x-pack/plugins/maps/public/connected_components/mb_map/features_tooltip/__snapshots__/tooltip_header.test.js.snap similarity index 100% rename from x-pack/plugins/maps/public/connected_components/map/features_tooltip/__snapshots__/tooltip_header.test.js.snap rename to x-pack/plugins/maps/public/connected_components/mb_map/features_tooltip/__snapshots__/tooltip_header.test.js.snap diff --git a/x-pack/plugins/maps/public/connected_components/map/features_tooltip/_index.scss b/x-pack/plugins/maps/public/connected_components/mb_map/features_tooltip/_index.scss similarity index 100% rename from x-pack/plugins/maps/public/connected_components/map/features_tooltip/_index.scss rename to x-pack/plugins/maps/public/connected_components/mb_map/features_tooltip/_index.scss diff --git a/x-pack/plugins/maps/public/connected_components/map/features_tooltip/feature_geometry_filter_form.js b/x-pack/plugins/maps/public/connected_components/mb_map/features_tooltip/feature_geometry_filter_form.js similarity index 100% rename from x-pack/plugins/maps/public/connected_components/map/features_tooltip/feature_geometry_filter_form.js rename to x-pack/plugins/maps/public/connected_components/mb_map/features_tooltip/feature_geometry_filter_form.js diff --git a/x-pack/plugins/maps/public/connected_components/map/features_tooltip/feature_properties.js b/x-pack/plugins/maps/public/connected_components/mb_map/features_tooltip/feature_properties.js similarity index 100% rename from x-pack/plugins/maps/public/connected_components/map/features_tooltip/feature_properties.js rename to x-pack/plugins/maps/public/connected_components/mb_map/features_tooltip/feature_properties.js diff --git a/x-pack/plugins/maps/public/connected_components/map/features_tooltip/feature_properties.test.js b/x-pack/plugins/maps/public/connected_components/mb_map/features_tooltip/feature_properties.test.js similarity index 100% rename from x-pack/plugins/maps/public/connected_components/map/features_tooltip/feature_properties.test.js rename to x-pack/plugins/maps/public/connected_components/mb_map/features_tooltip/feature_properties.test.js diff --git a/x-pack/plugins/maps/public/connected_components/map/features_tooltip/features_tooltip.js b/x-pack/plugins/maps/public/connected_components/mb_map/features_tooltip/features_tooltip.js similarity index 100% rename from x-pack/plugins/maps/public/connected_components/map/features_tooltip/features_tooltip.js rename to x-pack/plugins/maps/public/connected_components/mb_map/features_tooltip/features_tooltip.js diff --git a/x-pack/plugins/maps/public/connected_components/map/features_tooltip/tooltip_header.js b/x-pack/plugins/maps/public/connected_components/mb_map/features_tooltip/tooltip_header.js similarity index 100% rename from x-pack/plugins/maps/public/connected_components/map/features_tooltip/tooltip_header.js rename to x-pack/plugins/maps/public/connected_components/mb_map/features_tooltip/tooltip_header.js diff --git a/x-pack/plugins/maps/public/connected_components/map/features_tooltip/tooltip_header.test.js b/x-pack/plugins/maps/public/connected_components/mb_map/features_tooltip/tooltip_header.test.js similarity index 100% rename from x-pack/plugins/maps/public/connected_components/map/features_tooltip/tooltip_header.test.js rename to x-pack/plugins/maps/public/connected_components/mb_map/features_tooltip/tooltip_header.test.js diff --git a/x-pack/plugins/maps/public/connected_components/map/mb/get_initial_view.ts b/x-pack/plugins/maps/public/connected_components/mb_map/get_initial_view.ts similarity index 87% rename from x-pack/plugins/maps/public/connected_components/map/mb/get_initial_view.ts rename to x-pack/plugins/maps/public/connected_components/mb_map/get_initial_view.ts index 20fb8186f9870..853819eb289a3 100644 --- a/x-pack/plugins/maps/public/connected_components/map/mb/get_initial_view.ts +++ b/x-pack/plugins/maps/public/connected_components/mb_map/get_initial_view.ts @@ -4,9 +4,9 @@ * you may not use this file except in compliance with the Elastic License. */ -import { INITIAL_LOCATION } from '../../../../common/constants'; -import { Goto, MapCenterAndZoom } from '../../../../common/descriptor_types'; -import { MapSettings } from '../../../reducers/map'; +import { INITIAL_LOCATION } from '../../../common/constants'; +import { Goto, MapCenterAndZoom } from '../../../common/descriptor_types'; +import { MapSettings } from '../../reducers/map'; export async function getInitialView( goto: Goto | null, diff --git a/x-pack/plugins/maps/public/connected_components/map/mb/image_utils.js b/x-pack/plugins/maps/public/connected_components/mb_map/image_utils.js similarity index 100% rename from x-pack/plugins/maps/public/connected_components/map/mb/image_utils.js rename to x-pack/plugins/maps/public/connected_components/mb_map/image_utils.js diff --git a/x-pack/plugins/maps/public/connected_components/map/mb/index.js b/x-pack/plugins/maps/public/connected_components/mb_map/index.js similarity index 84% rename from x-pack/plugins/maps/public/connected_components/map/mb/index.js rename to x-pack/plugins/maps/public/connected_components/mb_map/index.js index 4b8df07bd1f39..cccd5e571d3e8 100644 --- a/x-pack/plugins/maps/public/connected_components/map/mb/index.js +++ b/x-pack/plugins/maps/public/connected_components/mb_map/index.js @@ -5,7 +5,7 @@ */ import { connect } from 'react-redux'; -import { MBMap } from './view'; +import { MBMap } from './mb_map'; import { mapExtentChanged, mapReady, @@ -14,7 +14,7 @@ import { clearMouseCoordinates, clearGoto, setMapInitError, -} from '../../../actions'; +} from '../../actions'; import { getLayerList, getMapReady, @@ -25,9 +25,9 @@ import { isViewControlHidden, getSpatialFiltersLayer, getMapSettings, -} from '../../../selectors/map_selectors'; +} from '../../selectors/map_selectors'; -import { getInspectorAdapters } from '../../../reducers/non_serializable_instances'; +import { getInspectorAdapters } from '../../reducers/non_serializable_instances'; function mapStateToProps(state = {}) { return { @@ -72,7 +72,5 @@ function mapDispatchToProps(dispatch) { }; } -const connectedMBMap = connect(mapStateToProps, mapDispatchToProps, null, { - forwardRef: true, -})(MBMap); -export { connectedMBMap as MBMap }; +const connected = connect(mapStateToProps, mapDispatchToProps)(MBMap); +export { connected as MBMap }; diff --git a/x-pack/plugins/maps/public/connected_components/map/mb/mb.utils.test.js b/x-pack/plugins/maps/public/connected_components/mb_map/mb.utils.test.js similarity index 98% rename from x-pack/plugins/maps/public/connected_components/map/mb/mb.utils.test.js rename to x-pack/plugins/maps/public/connected_components/mb_map/mb.utils.test.js index e2050724ef684..a28cc75f6d89d 100644 --- a/x-pack/plugins/maps/public/connected_components/map/mb/mb.utils.test.js +++ b/x-pack/plugins/maps/public/connected_components/mb_map/mb.utils.test.js @@ -5,7 +5,7 @@ */ import { removeOrphanedSourcesAndLayers } from './utils'; -import { SPATIAL_FILTERS_LAYER_ID } from '../../../../common/constants'; +import { SPATIAL_FILTERS_LAYER_ID } from '../../../common/constants'; import _ from 'lodash'; class MockMbMap { diff --git a/x-pack/plugins/maps/public/connected_components/map/mb/view.js b/x-pack/plugins/maps/public/connected_components/mb_map/mb_map.js similarity index 97% rename from x-pack/plugins/maps/public/connected_components/map/mb/view.js rename to x-pack/plugins/maps/public/connected_components/mb_map/mb_map.js index ddc48cfc9c329..04c376a093623 100644 --- a/x-pack/plugins/maps/public/connected_components/map/mb/view.js +++ b/x-pack/plugins/maps/public/connected_components/mb_map/mb_map.js @@ -6,15 +6,15 @@ import _ from 'lodash'; import React from 'react'; -import { ResizeChecker } from '../../../../../../../src/plugins/kibana_utils/public'; +import { ResizeChecker } from '../../../../../../src/plugins/kibana_utils/public'; import { removeOrphanedSourcesAndLayers, addSpritesheetToMap } from './utils'; import { syncLayerOrder } from './sort_layers'; -import { getGlyphUrl, isRetina } from '../../../meta'; +import { getGlyphUrl, isRetina } from '../../meta'; import { DECIMAL_DEGREES_PRECISION, KBN_TOO_MANY_FEATURES_IMAGE_ID, ZOOM_PRECISION, -} from '../../../../common/constants'; +} from '../../../common/constants'; import mapboxgl from 'mapbox-gl/dist/mapbox-gl-csp'; import mbWorkerUrl from '!!file-loader!mapbox-gl/dist/mapbox-gl-csp-worker'; import mbRtlPlugin from '!!file-loader!@mapbox/mapbox-gl-rtl-text/mapbox-gl-rtl-text.min.js'; @@ -23,9 +23,9 @@ import sprites1 from '@elastic/maki/dist/sprite@1.png'; import sprites2 from '@elastic/maki/dist/sprite@2.png'; import { DrawControl } from './draw_control'; import { TooltipControl } from './tooltip_control'; -import { clampToLatBounds, clampToLonBounds } from '../../../../common/elasticsearch_util'; +import { clampToLatBounds, clampToLonBounds } from '../../../common/elasticsearch_util'; import { getInitialView } from './get_initial_view'; -import { getPreserveDrawingBuffer } from '../../../kibana_services'; +import { getPreserveDrawingBuffer } from '../../kibana_services'; mapboxgl.workerUrl = mbWorkerUrl; mapboxgl.setRTLTextPlugin(mbRtlPlugin); diff --git a/x-pack/plugins/maps/public/connected_components/map/mb/sort_layers.test.ts b/x-pack/plugins/maps/public/connected_components/mb_map/sort_layers.test.ts similarity index 98% rename from x-pack/plugins/maps/public/connected_components/map/mb/sort_layers.test.ts rename to x-pack/plugins/maps/public/connected_components/mb_map/sort_layers.test.ts index e26a1e43509c8..9e85c7b04b266 100644 --- a/x-pack/plugins/maps/public/connected_components/map/mb/sort_layers.test.ts +++ b/x-pack/plugins/maps/public/connected_components/mb_map/sort_layers.test.ts @@ -8,8 +8,8 @@ import _ from 'lodash'; import { Map as MbMap, Layer as MbLayer, Style as MbStyle } from 'mapbox-gl'; import { getIsTextLayer, syncLayerOrder } from './sort_layers'; -import { SPATIAL_FILTERS_LAYER_ID } from '../../../../common/constants'; -import { ILayer } from '../../../classes/layers/layer'; +import { SPATIAL_FILTERS_LAYER_ID } from '../../../common/constants'; +import { ILayer } from '../../classes/layers/layer'; let moveCounter = 0; diff --git a/x-pack/plugins/maps/public/connected_components/map/mb/sort_layers.ts b/x-pack/plugins/maps/public/connected_components/mb_map/sort_layers.ts similarity index 98% rename from x-pack/plugins/maps/public/connected_components/map/mb/sort_layers.ts rename to x-pack/plugins/maps/public/connected_components/mb_map/sort_layers.ts index 0c970fe663557..dda43269e32d8 100644 --- a/x-pack/plugins/maps/public/connected_components/map/mb/sort_layers.ts +++ b/x-pack/plugins/maps/public/connected_components/mb_map/sort_layers.ts @@ -5,7 +5,7 @@ */ import { Map as MbMap, Layer as MbLayer } from 'mapbox-gl'; -import { ILayer } from '../../../classes/layers/layer'; +import { ILayer } from '../../classes/layers/layer'; // "Layer" is overloaded and can mean the following // 1) Map layer (ILayer): A single map layer consists of one to many mapbox layers. diff --git a/x-pack/plugins/maps/public/connected_components/map/mb/tooltip_control/__snapshots__/tooltip_control.test.js.snap b/x-pack/plugins/maps/public/connected_components/mb_map/tooltip_control/__snapshots__/tooltip_control.test.js.snap similarity index 100% rename from x-pack/plugins/maps/public/connected_components/map/mb/tooltip_control/__snapshots__/tooltip_control.test.js.snap rename to x-pack/plugins/maps/public/connected_components/mb_map/tooltip_control/__snapshots__/tooltip_control.test.js.snap diff --git a/x-pack/plugins/maps/public/connected_components/map/mb/tooltip_control/__snapshots__/tooltip_popover.test.js.snap b/x-pack/plugins/maps/public/connected_components/mb_map/tooltip_control/__snapshots__/tooltip_popover.test.js.snap similarity index 100% rename from x-pack/plugins/maps/public/connected_components/map/mb/tooltip_control/__snapshots__/tooltip_popover.test.js.snap rename to x-pack/plugins/maps/public/connected_components/mb_map/tooltip_control/__snapshots__/tooltip_popover.test.js.snap diff --git a/x-pack/plugins/maps/public/connected_components/map/mb/tooltip_control/index.js b/x-pack/plugins/maps/public/connected_components/mb_map/tooltip_control/index.js similarity index 94% rename from x-pack/plugins/maps/public/connected_components/map/mb/tooltip_control/index.js rename to x-pack/plugins/maps/public/connected_components/mb_map/tooltip_control/index.js index 407dcf1997aeb..7d2f2b05d6f11 100644 --- a/x-pack/plugins/maps/public/connected_components/map/mb/tooltip_control/index.js +++ b/x-pack/plugins/maps/public/connected_components/mb_map/tooltip_control/index.js @@ -11,13 +11,13 @@ import { openOnClickTooltip, closeOnHoverTooltip, openOnHoverTooltip, -} from '../../../../actions'; +} from '../../../actions'; import { getLayerList, getOpenTooltips, getHasLockedTooltips, isDrawingFilter, -} from '../../../../selectors/map_selectors'; +} from '../../../selectors/map_selectors'; function mapStateToProps(state = {}) { return { diff --git a/x-pack/plugins/maps/public/connected_components/map/mb/tooltip_control/tooltip_control.js b/x-pack/plugins/maps/public/connected_components/mb_map/tooltip_control/tooltip_control.js similarity index 98% rename from x-pack/plugins/maps/public/connected_components/map/mb/tooltip_control/tooltip_control.js rename to x-pack/plugins/maps/public/connected_components/mb_map/tooltip_control/tooltip_control.js index edfeb3c76b104..b178eef6fa5d3 100644 --- a/x-pack/plugins/maps/public/connected_components/map/mb/tooltip_control/tooltip_control.js +++ b/x-pack/plugins/maps/public/connected_components/mb_map/tooltip_control/tooltip_control.js @@ -6,9 +6,9 @@ import _ from 'lodash'; import React from 'react'; -import { FEATURE_ID_PROPERTY_NAME, LON_INDEX } from '../../../../../common/constants'; +import { FEATURE_ID_PROPERTY_NAME, LON_INDEX } from '../../../../common/constants'; import { TooltipPopover } from './tooltip_popover'; -import { EXCLUDE_TOO_MANY_FEATURES_BOX } from '../../../../classes/util/mb_filter_expressions'; +import { EXCLUDE_TOO_MANY_FEATURES_BOX } from '../../../classes/util/mb_filter_expressions'; function justifyAnchorLocation(mbLngLat, targetFeature) { let popupAnchorLocation = [mbLngLat.lng, mbLngLat.lat]; // default popup location to mouse location diff --git a/x-pack/plugins/maps/public/connected_components/map/mb/tooltip_control/tooltip_control.test.js b/x-pack/plugins/maps/public/connected_components/mb_map/tooltip_control/tooltip_control.test.js similarity index 100% rename from x-pack/plugins/maps/public/connected_components/map/mb/tooltip_control/tooltip_control.test.js rename to x-pack/plugins/maps/public/connected_components/mb_map/tooltip_control/tooltip_control.test.js diff --git a/x-pack/plugins/maps/public/connected_components/map/mb/tooltip_control/tooltip_popover.js b/x-pack/plugins/maps/public/connected_components/mb_map/tooltip_control/tooltip_popover.js similarity index 97% rename from x-pack/plugins/maps/public/connected_components/map/mb/tooltip_control/tooltip_popover.js rename to x-pack/plugins/maps/public/connected_components/mb_map/tooltip_control/tooltip_popover.js index 4cfddf0034039..ca4864f79940e 100644 --- a/x-pack/plugins/maps/public/connected_components/map/mb/tooltip_control/tooltip_popover.js +++ b/x-pack/plugins/maps/public/connected_components/mb_map/tooltip_control/tooltip_popover.js @@ -5,8 +5,8 @@ */ import React, { Component } from 'react'; -import { LAT_INDEX, LON_INDEX } from '../../../../../common/constants'; -import { FeaturesTooltip } from '../../features_tooltip/features_tooltip'; +import { LAT_INDEX, LON_INDEX } from '../../../../common/constants'; +import { FeaturesTooltip } from '../features_tooltip/features_tooltip'; import { EuiPopover, EuiText } from '@elastic/eui'; const noop = () => {}; diff --git a/x-pack/plugins/maps/public/connected_components/map/mb/tooltip_control/tooltip_popover.test.js b/x-pack/plugins/maps/public/connected_components/mb_map/tooltip_control/tooltip_popover.test.js similarity index 98% rename from x-pack/plugins/maps/public/connected_components/map/mb/tooltip_control/tooltip_popover.test.js rename to x-pack/plugins/maps/public/connected_components/mb_map/tooltip_control/tooltip_popover.test.js index 205ca7337277d..b15c3fce6c0b7 100644 --- a/x-pack/plugins/maps/public/connected_components/map/mb/tooltip_control/tooltip_popover.test.js +++ b/x-pack/plugins/maps/public/connected_components/mb_map/tooltip_control/tooltip_popover.test.js @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -jest.mock('../../features_tooltip/features_tooltip', () => ({ +jest.mock('../features_tooltip/features_tooltip', () => ({ FeaturesTooltip: () => { return
mockFeaturesTooltip
; }, diff --git a/x-pack/plugins/maps/public/connected_components/map/mb/utils.js b/x-pack/plugins/maps/public/connected_components/mb_map/utils.js similarity index 100% rename from x-pack/plugins/maps/public/connected_components/map/mb/utils.js rename to x-pack/plugins/maps/public/connected_components/mb_map/utils.js diff --git a/x-pack/plugins/observability/common/annotations.ts b/x-pack/plugins/observability/common/annotations.ts index 6aea4d3d92f9b..f7ab243cf73f3 100644 --- a/x-pack/plugins/observability/common/annotations.ts +++ b/x-pack/plugins/observability/common/annotations.ts @@ -5,7 +5,24 @@ */ import * as t from 'io-ts'; -import { dateAsStringRt } from '../../apm/common/runtime_types/date_as_string_rt'; +import { either } from 'fp-ts/lib/Either'; + +/** + * Checks whether a string is a valid ISO timestamp, + * but doesn't convert it into a Date object when decoding. + * + * Copied from x-pack/plugins/apm/common/runtime_types/date_as_string_rt.ts. + */ +const dateAsStringRt = new t.Type( + 'DateAsString', + t.string.is, + (input, context) => + either.chain(t.string.validate(input, context), (str) => { + const date = new Date(str); + return isNaN(date.getTime()) ? t.failure(input, context) : t.success(str); + }), + t.identity +); export const createAnnotationRt = t.intersection([ t.type({ diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.ts index 6ffbf4e4c8d4c..1b0417cf59bc2 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.ts @@ -48,6 +48,8 @@ import { } from '../common/schemas'; import { threat_index, + concurrent_searches, + items_per_search, threat_query, threat_filters, threat_mapping, @@ -130,6 +132,8 @@ export const addPrepackagedRulesSchema = t.intersection([ threat_query, // defaults to "undefined" if not set during decode threat_index, // defaults to "undefined" if not set during decode threat_language, // defaults "undefined" if not set during decode + concurrent_searches, // defaults to "undefined" if not set during decode + items_per_search, // defaults to "undefined" if not set during decode }) ), ]); diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.test.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.test.ts index a4f002b589ef5..1b6a8d6f27762 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.test.ts @@ -1702,5 +1702,23 @@ describe('create rules schema', () => { expect(getPaths(left(message.errors))).toEqual([]); expect(message.schema).toEqual(expected); }); + + test('You can set a threat query, index, mapping, filters, concurrent_searches, items_per_search with a when creating a rule', () => { + const payload: CreateRulesSchema = { + ...getCreateThreatMatchRulesSchemaMock(), + concurrent_searches: 10, + items_per_search: 10, + }; + const decoded = createRulesSchema.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + const expected: CreateRulesSchemaDecoded = { + ...getCreateThreatMatchRulesSchemaDecodedMock(), + concurrent_searches: 10, + items_per_search: 10, + }; + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(expected); + }); }); }); diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.ts index d8e7614fcb840..2fe52bbe470a5 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.ts @@ -49,6 +49,8 @@ import { } from '../common/schemas'; import { threat_index, + concurrent_searches, + items_per_search, threat_query, threat_filters, threat_mapping, @@ -126,6 +128,8 @@ export const createRulesSchema = t.intersection([ threat_filters, // defaults to "undefined" if not set during decode threat_index, // defaults to "undefined" if not set during decode threat_language, // defaults "undefined" if not set during decode + concurrent_searches, // defaults "undefined" if not set during decode + items_per_search, // defaults "undefined" if not set during decode }) ), ]); diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_type_dependents.test.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_type_dependents.test.ts index 75ad92578318c..a78b41cd0da18 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_type_dependents.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_type_dependents.test.ts @@ -125,4 +125,36 @@ describe('create_rules_type_dependents', () => { const errors = createRuleValidateTypeDependents(schema); expect(errors).toEqual([]); }); + + test('validates that both "items_per_search" and "concurrent_searches" works when together', () => { + const schema: CreateRulesSchema = { + ...getCreateThreatMatchRulesSchemaMock(), + concurrent_searches: 10, + items_per_search: 10, + }; + const errors = createRuleValidateTypeDependents(schema); + expect(errors).toEqual([]); + }); + + test('does NOT validate when only "items_per_search" is present', () => { + const schema: CreateRulesSchema = { + ...getCreateThreatMatchRulesSchemaMock(), + items_per_search: 10, + }; + const errors = createRuleValidateTypeDependents(schema); + expect(errors).toEqual([ + 'when "items_per_search" exists, "concurrent_searches" must also exist', + ]); + }); + + test('does NOT validate when only "concurrent_searches" is present', () => { + const schema: CreateRulesSchema = { + ...getCreateThreatMatchRulesSchemaMock(), + concurrent_searches: 10, + }; + const errors = createRuleValidateTypeDependents(schema); + expect(errors).toEqual([ + 'when "concurrent_searches" exists, "items_per_search" must also exist', + ]); + }); }); diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_type_dependents.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_type_dependents.ts index c2a41005ebf4d..c93b0f0b14f6a 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_type_dependents.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_type_dependents.ts @@ -110,17 +110,23 @@ export const validateThreshold = (rule: CreateRulesSchema): string[] => { export const validateThreatMapping = (rule: CreateRulesSchema): string[] => { let errors: string[] = []; if (isThreatMatchRule(rule.type)) { - if (!rule.threat_mapping) { + if (rule.threat_mapping == null) { errors = ['when "type" is "threat_match", "threat_mapping" is required', ...errors]; } else if (rule.threat_mapping.length === 0) { errors = ['threat_mapping" must have at least one element', ...errors]; } - if (!rule.threat_query) { + if (rule.threat_query == null) { errors = ['when "type" is "threat_match", "threat_query" is required', ...errors]; } - if (!rule.threat_index) { + if (rule.threat_index == null) { errors = ['when "type" is "threat_match", "threat_index" is required', ...errors]; } + if (rule.concurrent_searches == null && rule.items_per_search != null) { + errors = ['when "items_per_search" exists, "concurrent_searches" must also exist', ...errors]; + } + if (rule.concurrent_searches != null && rule.items_per_search == null) { + errors = ['when "concurrent_searches" exists, "items_per_search" must also exist', ...errors]; + } } return errors; }; diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.ts index 852394b74767b..4f28c46923865 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.ts @@ -55,6 +55,8 @@ import { } from '../common/schemas'; import { threat_index, + items_per_search, + concurrent_searches, threat_query, threat_filters, threat_mapping, @@ -149,6 +151,8 @@ export const importRulesSchema = t.intersection([ threat_query, // defaults to "undefined" if not set during decode threat_index, // defaults to "undefined" if not set during decode threat_language, // defaults "undefined" if not set during decode + concurrent_searches, // defaults to "undefined" if not set during decode + items_per_search, // defaults to "undefined" if not set during decode }) ), ]); diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_schema.ts index f4dce5c7ac05f..45fcfbaa3c76a 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_schema.ts @@ -50,6 +50,8 @@ import { } from '../common/schemas'; import { threat_index, + concurrent_searches, + items_per_search, threat_query, threat_filters, threat_mapping, @@ -109,6 +111,8 @@ export const patchRulesSchema = t.exact( threat_filters, threat_mapping, threat_language, + concurrent_searches, + items_per_search, }) ); diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_schema.ts index b0cd8b1c53688..5d759fc12cd52 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_schema.ts @@ -51,6 +51,8 @@ import { } from '../common/schemas'; import { threat_index, + concurrent_searches, + items_per_search, threat_query, threat_filters, threat_mapping, @@ -134,6 +136,8 @@ export const updateRulesSchema = t.intersection([ threat_filters, // defaults to "undefined" if not set during decode threat_index, // defaults to "undefined" if not set during decode threat_language, // defaults "undefined" if not set during decode + concurrent_searches, // defaults to "undefined" if not set during decode + items_per_search, // defaults to "undefined" if not set during decode }) ), ]); diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.test.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.test.ts index 82675768a11b7..3508526e182d7 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.test.ts @@ -762,9 +762,9 @@ describe('rules_schema', () => { expect(fields).toEqual(expected); }); - test('should return 5 fields for a rule of type "threat_match"', () => { + test('should return 8 fields for a rule of type "threat_match"', () => { const fields = addThreatMatchFields({ type: 'threat_match' }); - expect(fields.length).toEqual(6); + expect(fields.length).toEqual(8); }); }); diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts index e85beddf0e51e..0f7d04763a36f 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts @@ -63,6 +63,8 @@ import { } from '../common/schemas'; import { threat_index, + concurrent_searches, + items_per_search, threat_query, threat_filters, threat_mapping, @@ -144,6 +146,8 @@ export const dependentRulesSchema = t.partial({ threat_filters, threat_index, threat_query, + concurrent_searches, + items_per_search, threat_mapping, threat_language, }); @@ -282,6 +286,12 @@ export const addThreatMatchFields = (typeAndTimelineOnly: TypeAndTimelineOnly): t.exact(t.partial({ threat_language: dependentRulesSchema.props.threat_language })), t.exact(t.partial({ threat_filters: dependentRulesSchema.props.threat_filters })), t.exact(t.partial({ saved_id: dependentRulesSchema.props.saved_id })), + t.exact(t.partial({ concurrent_searches: dependentRulesSchema.props.concurrent_searches })), + t.exact( + t.partial({ + items_per_search: dependentRulesSchema.props.items_per_search, + }) + ), ]; } else { return []; diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/types/threat_mapping.test.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/types/threat_mapping.test.ts index 63d593ea84e67..d8f61e4309b17 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/types/threat_mapping.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/types/threat_mapping.test.ts @@ -5,6 +5,8 @@ */ import { + concurrent_searches, + items_per_search, ThreatMapping, threatMappingEntries, ThreatMappingEntries, @@ -33,7 +35,7 @@ describe('threat_mapping', () => { expect(message.schema).toEqual(payload); }); - test('it should NOT validate an extra entry item', () => { + test('it should fail validation with an extra entry item', () => { const payload: ThreatMappingEntries & Array<{ extra: string }> = [ { field: 'field.one', @@ -50,7 +52,7 @@ describe('threat_mapping', () => { expect(message.schema).toEqual({}); }); - test('it should NOT validate a non string', () => { + test('it should fail validation with a non string', () => { const payload = ([ { field: 5, @@ -66,7 +68,7 @@ describe('threat_mapping', () => { expect(message.schema).toEqual({}); }); - test('it should NOT validate a wrong type', () => { + test('it should fail validation with a wrong type', () => { const payload = ([ { field: 'field.one', @@ -107,7 +109,7 @@ describe('threat_mapping', () => { }); }); - test('it should NOT validate an extra key', () => { + test('it should fail validate with an extra key', () => { const payload: ThreatMapping & Array<{ extra: string }> = [ { entries: [ @@ -129,7 +131,7 @@ describe('threat_mapping', () => { expect(message.schema).toEqual({}); }); - test('it should NOT validate an extra inner entry', () => { + test('it should fail validate with an extra inner entry', () => { const payload: ThreatMapping & Array<{ entries: Array<{ extra: string }> }> = [ { entries: [ @@ -151,7 +153,7 @@ describe('threat_mapping', () => { expect(message.schema).toEqual({}); }); - test('it should NOT validate an extra inner entry with the wrong data type', () => { + test('it should fail validate with an extra inner entry with the wrong data type', () => { const payload = ([ { entries: [ @@ -173,4 +175,48 @@ describe('threat_mapping', () => { ]); expect(message.schema).toEqual({}); }); + + test('it should fail validation when concurrent_searches is < 0', () => { + const payload = -1; + const decoded = concurrent_searches.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual([ + 'Invalid value "-1" supplied to "PositiveIntegerGreaterThanZero"', + ]); + expect(message.schema).toEqual({}); + }); + + test('it should fail validation when concurrent_searches is 0', () => { + const payload = 0; + const decoded = concurrent_searches.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual([ + 'Invalid value "0" supplied to "PositiveIntegerGreaterThanZero"', + ]); + expect(message.schema).toEqual({}); + }); + + test('it should fail validation when items_per_search is 0', () => { + const payload = 0; + const decoded = items_per_search.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual([ + 'Invalid value "0" supplied to "PositiveIntegerGreaterThanZero"', + ]); + expect(message.schema).toEqual({}); + }); + + test('it should fail validation when items_per_search is < 0', () => { + const payload = -1; + const decoded = items_per_search.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual([ + 'Invalid value "-1" supplied to "PositiveIntegerGreaterThanZero"', + ]); + expect(message.schema).toEqual({}); + }); }); diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/types/threat_mapping.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/types/threat_mapping.ts index a1be6485f596b..dec8ddd000132 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/types/threat_mapping.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/types/threat_mapping.ts @@ -9,6 +9,7 @@ import * as t from 'io-ts'; import { language } from '../common/schemas'; import { NonEmptyString } from './non_empty_string'; +import { PositiveIntegerGreaterThanZero } from './positive_integer_greater_than_zero'; export const threat_query = t.string; export type ThreatQuery = t.TypeOf; @@ -55,3 +56,13 @@ export const threat_language = t.union([language, t.undefined]); export type ThreatLanguage = t.TypeOf; export const threatLanguageOrUndefined = t.union([threat_language, t.undefined]); export type ThreatLanguageOrUndefined = t.TypeOf; + +export const concurrent_searches = PositiveIntegerGreaterThanZero; +export type ConcurrentSearches = t.TypeOf; +export const concurrentSearchesOrUndefined = t.union([concurrent_searches, t.undefined]); +export type ConcurrentSearchesOrUndefined = t.TypeOf; + +export const items_per_search = PositiveIntegerGreaterThanZero; +export type ItemsPerSearch = t.TypeOf; +export const itemsPerSearchOrUndefined = t.union([items_per_search, t.undefined]); +export type ItemsPerSearchOrUndefined = t.TypeOf; diff --git a/x-pack/plugins/security_solution/cypress/integration/events_viewer.spec.ts b/x-pack/plugins/security_solution/cypress/integration/events_viewer.spec.ts index 13fa9592469e4..9eb49c19c23f6 100644 --- a/x-pack/plugins/security_solution/cypress/integration/events_viewer.spec.ts +++ b/x-pack/plugins/security_solution/cypress/integration/events_viewer.spec.ts @@ -142,10 +142,11 @@ describe('Events Viewer', () => { }); }); - context.skip('Events columns', () => { + context('Events columns', () => { before(() => { loginAndWaitForPage(HOSTS_URL); openEvents(); + cy.scrollTo('bottom'); waitsForEventsToBeLoaded(); }); @@ -160,9 +161,8 @@ describe('Events Viewer', () => { const expectedOrderAfterDragAndDrop = 'message@timestamphost.nameevent.moduleevent.datasetevent.actionuser.namesource.ipdestination.ip'; - cy.scrollTo('bottom'); cy.get(HEADERS_GROUP).invoke('text').should('equal', originalColumnOrder); - dragAndDropColumn({ column: 0, newPosition: 1 }); + dragAndDropColumn({ column: 0, newPosition: 0 }); cy.get(HEADERS_GROUP).invoke('text').should('equal', expectedOrderAfterDragAndDrop); }); }); diff --git a/x-pack/plugins/security_solution/cypress/integration/timeline_local_storage.spec.ts b/x-pack/plugins/security_solution/cypress/integration/timeline_local_storage.spec.ts index 383ebe2220585..d518f9e42f21f 100644 --- a/x-pack/plugins/security_solution/cypress/integration/timeline_local_storage.spec.ts +++ b/x-pack/plugins/security_solution/cypress/integration/timeline_local_storage.spec.ts @@ -13,7 +13,8 @@ import { TABLE_COLUMN_EVENTS_MESSAGE } from '../screens/hosts/external_events'; import { waitsForEventsToBeLoaded, openEventsViewerFieldsBrowser } from '../tasks/hosts/events'; import { removeColumn, resetFields } from '../tasks/timeline'; -describe('persistent timeline', () => { +// Failing: See https://github.com/elastic/kibana/issues/75794 +describe.skip('persistent timeline', () => { before(() => { loginAndWaitForPage(HOSTS_URL); openEvents(); diff --git a/x-pack/plugins/security_solution/cypress/screens/hosts/events.ts b/x-pack/plugins/security_solution/cypress/screens/hosts/events.ts index 0434de7bff88e..cf507924a753f 100644 --- a/x-pack/plugins/security_solution/cypress/screens/hosts/events.ts +++ b/x-pack/plugins/security_solution/cypress/screens/hosts/events.ts @@ -34,3 +34,6 @@ export const LOAD_MORE = '[data-test-subj="events-viewer-panel"] [data-test-subj="TimelineMoreButton"'; export const SERVER_SIDE_EVENT_COUNT = '[data-test-subj="server-side-event-count"]'; + +export const EVENTS_VIEWER_PAGINATION = + '[data-test-subj="events-viewer-panel"] [data-test-subj="timeline-pagination"]'; diff --git a/x-pack/plugins/security_solution/cypress/tasks/common.ts b/x-pack/plugins/security_solution/cypress/tasks/common.ts index e16db54599981..bb009f34b02d6 100644 --- a/x-pack/plugins/security_solution/cypress/tasks/common.ts +++ b/x-pack/plugins/security_solution/cypress/tasks/common.ts @@ -23,14 +23,14 @@ export const drag = (subject: JQuery) => { clientY: subjectLocation.top, force: true, }) - .wait(3000) + .wait(300) .trigger('mousemove', { button: primaryButton, clientX: subjectLocation.left + dndSloppyClickDetectionThreshold, clientY: subjectLocation.top, force: true, }) - .wait(3000); + .wait(300); }; /** Drags the subject being dragged on the specified drop target, but does not drop it */ @@ -42,11 +42,17 @@ export const dragWithoutDrop = (dropTarget: JQuery) => { /** "Drops" the subject being dragged on the specified drop target */ export const drop = (dropTarget: JQuery) => { + const targetLocation = dropTarget[0].getBoundingClientRect(); cy.wrap(dropTarget) - .trigger('mousemove', { button: primaryButton, force: true }) - .wait(3000) + .trigger('mousemove', { + button: primaryButton, + clientX: targetLocation.left, + clientY: targetLocation.top, + force: true, + }) + .wait(300) .trigger('mouseup', { force: true }) - .wait(3000); + .wait(300); }; export const reload = (afterReload: () => void) => { diff --git a/x-pack/plugins/security_solution/cypress/tasks/hosts/events.ts b/x-pack/plugins/security_solution/cypress/tasks/hosts/events.ts index 226178cd92f18..401a78767ac57 100644 --- a/x-pack/plugins/security_solution/cypress/tasks/hosts/events.ts +++ b/x-pack/plugins/security_solution/cypress/tasks/hosts/events.ts @@ -8,6 +8,7 @@ import { drag, drop } from '../common'; import { CLOSE_MODAL, EVENTS_VIEWER_FIELDS_BUTTON, + EVENTS_VIEWER_PAGINATION, FIELDS_BROWSER_CONTAINER, HOST_GEO_CITY_NAME_CHECKBOX, HOST_GEO_COUNTRY_NAME_CHECKBOX, @@ -16,6 +17,7 @@ import { SERVER_SIDE_EVENT_COUNT, } from '../../screens/hosts/events'; import { DRAGGABLE_HEADER } from '../../screens/timeline'; +import { REFRESH_BUTTON } from '../../screens/security_header'; export const addsHostGeoCityNameToHeader = () => { cy.get(HOST_GEO_CITY_NAME_CHECKBOX).check({ @@ -53,7 +55,9 @@ export const opensInspectQueryModal = () => { }; export const waitsForEventsToBeLoaded = () => { - cy.get(SERVER_SIDE_EVENT_COUNT).should('exist').invoke('text').should('not.equal', '0'); + cy.get(SERVER_SIDE_EVENT_COUNT).should('not.have.text', '0'); + cy.get(REFRESH_BUTTON).should('not.have.text', 'Updating'); + cy.get(EVENTS_VIEWER_PAGINATION).should('exist'); }; export const dragAndDropColumn = ({ diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/translations.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/translations.tsx index 164b1df8463e6..221963767caad 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/translations.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/translations.tsx @@ -95,7 +95,7 @@ export const THREAT_MATCH_INDEX_HELPER_TEXT = i18n.translate( export const THREAT_MATCH_REQUIRED = i18n.translate( 'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.customThreatQueryFieldRequiredError', { - defaultMessage: 'At least one threat match is required.', + defaultMessage: 'At least one indicator match is required.', } ); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts index 94b820344b37c..773e84d9c88fc 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts @@ -407,6 +407,8 @@ export const getResult = (): RuleAlertType => ({ note: '# Investigative notes', version: 1, exceptionsList: getListArrayMock(), + concurrentSearches: undefined, + itemsPerSearch: undefined, }, createdAt: new Date('2019-12-13T16:40:33.400Z'), updatedAt: new Date('2019-12-13T16:40:33.400Z'), diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_bulk_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_bulk_route.ts index 8c7a19869ce18..aa409580df965 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_bulk_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_bulk_route.ts @@ -102,6 +102,8 @@ export const createRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) => threat_mapping: threatMapping, threat_query: threatQuery, threat_language: threatLanguage, + concurrent_searches: concurrentSearches, + items_per_search: itemsPerSearch, threshold, throttle, timestamp_override: timestampOverride, @@ -193,6 +195,8 @@ export const createRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) => threatQuery, threatIndex, threatLanguage, + concurrentSearches, + itemsPerSearch, threshold, timestampOverride, references, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts index 6ba7bc78fbded..97c05b4626ddc 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts @@ -85,6 +85,8 @@ export const createRulesRoute = (router: IRouter, ml: SetupPlugins['ml']): void threat_query: threatQuery, threat_mapping: threatMapping, threat_language: threatLanguage, + concurrent_searches: concurrentSearches, + items_per_search: itemsPerSearch, throttle, timestamp_override: timestampOverride, to, @@ -182,6 +184,8 @@ export const createRulesRoute = (router: IRouter, ml: SetupPlugins['ml']): void threatQuery, threatMapping, threatLanguage, + concurrentSearches, + itemsPerSearch, timestampOverride, references, note, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rules_status_route.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rules_status_route.test.ts index 4d992c6c7029d..4b75127af1bc7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rules_status_route.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rules_status_route.test.ts @@ -9,6 +9,7 @@ import { getFindResultStatus, ruleStatusRequest, getResult } from '../__mocks__/ import { serverMock, requestContextMock, requestMock } from '../__mocks__'; import { findRulesStatusesRoute } from './find_rules_status_route'; import { RuleStatusResponse } from '../../rules/types'; +import { AlertExecutionStatusErrorReasons } from '../../../../../../alerts/common'; jest.mock('../../signals/rule_status_service'); @@ -57,7 +58,7 @@ describe('find_statuses', () => { status: 'error', lastExecutionDate: failingExecutionRule.executionStatus.lastExecutionDate, error: { - reason: 'read', + reason: AlertExecutionStatusErrorReasons.Read, message: 'oops', }, }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/import_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/import_rules_route.ts index 7cbcf25590921..688036c59c8ff 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/import_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/import_rules_route.ts @@ -169,6 +169,8 @@ export const importRulesRoute = (router: IRouter, config: ConfigType, ml: SetupP threat_query: threatQuery, threat_mapping: threatMapping, threat_language: threatLanguage, + concurrent_searches: concurrentSearches, + items_per_search: itemsPerSearch, threshold, timestamp_override: timestampOverride, to, @@ -235,6 +237,8 @@ export const importRulesRoute = (router: IRouter, config: ConfigType, ml: SetupP threatQuery, threatMapping, threatLanguage, + concurrentSearches, + itemsPerSearch, timestampOverride, references, note, @@ -284,6 +288,8 @@ export const importRulesRoute = (router: IRouter, config: ConfigType, ml: SetupP threatQuery, threatMapping, threatLanguage, + concurrentSearches, + itemsPerSearch, references, note, version, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_bulk_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_bulk_route.ts index 4c310774ec72b..7dfb4daa1a0a2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_bulk_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_bulk_route.ts @@ -97,6 +97,8 @@ export const patchRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) => threat_query: threatQuery, threat_mapping: threatMapping, threat_language: threatLanguage, + concurrent_searches: concurrentSearches, + items_per_search: itemsPerSearch, timestamp_override: timestampOverride, throttle, references, @@ -162,6 +164,8 @@ export const patchRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) => threatQuery, threatMapping, threatLanguage, + concurrentSearches, + itemsPerSearch, timestampOverride, references, note, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_route.ts index dbdcd9844c0a7..aadb13ef54e72 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_route.ts @@ -83,6 +83,8 @@ export const patchRulesRoute = (router: IRouter, ml: SetupPlugins['ml']) => { threat_query: threatQuery, threat_mapping: threatMapping, threat_language: threatLanguage, + concurrent_searches: concurrentSearches, + items_per_search: itemsPerSearch, timestamp_override: timestampOverride, throttle, references, @@ -161,6 +163,8 @@ export const patchRulesRoute = (router: IRouter, ml: SetupPlugins['ml']) => { threatQuery, threatMapping, threatLanguage, + concurrentSearches, + itemsPerSearch, timestampOverride, references, note, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_bulk_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_bulk_route.ts index b93b3f319193f..f4a31c2bb456d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_bulk_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_bulk_route.ts @@ -102,6 +102,8 @@ export const updateRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) => threat_query: threatQuery, threat_mapping: threatMapping, threat_language: threatLanguage, + concurrent_searches: concurrentSearches, + items_per_search: itemsPerSearch, throttle, timestamp_override: timestampOverride, references, @@ -174,6 +176,8 @@ export const updateRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) => threatQuery, threatMapping, threatLanguage, + concurrentSearches, + itemsPerSearch, timestampOverride, references, note, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_route.ts index ea19fed5d6668..7ad525b67f7aa 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_route.ts @@ -86,6 +86,8 @@ export const updateRulesRoute = (router: IRouter, ml: SetupPlugins['ml']) => { threat_query: threatQuery, threat_mapping: threatMapping, threat_language: threatLanguage, + concurrent_searches: concurrentSearches, + items_per_search: itemsPerSearch, throttle, timestamp_override: timestampOverride, references, @@ -163,6 +165,8 @@ export const updateRulesRoute = (router: IRouter, ml: SetupPlugins['ml']) => { threatQuery, threatMapping, threatLanguage, + concurrentSearches, + itemsPerSearch, timestampOverride, references, note, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.ts index fb4ba855f6536..7360dc77aac22 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.ts @@ -151,6 +151,8 @@ export const transformAlertToRule = ( threat_query: alert.params.threatQuery, threat_mapping: alert.params.threatMapping, threat_language: alert.params.threatLanguage, + concurrent_searches: alert.params.concurrentSearches, + items_per_search: alert.params.itemsPerSearch, throttle: ruleActions?.ruleThrottle || 'no_actions', timestamp_override: alert.params.timestampOverride, note: alert.params.note, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/utils.test.ts index 25e47b38e8a56..b613061ac85f2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/utils.test.ts @@ -27,6 +27,7 @@ import { import { responseMock } from './__mocks__'; import { exampleRuleStatus, exampleFindRuleStatusResponse } from '../signals/__mocks__/es_results'; import { getResult } from './__mocks__/request_responses'; +import { AlertExecutionStatusErrorReasons } from '../../../../../alerts/common'; let alertsClient: ReturnType; @@ -464,7 +465,7 @@ describe('utils', () => { status: 'error', lastExecutionDate: foundRule.executionStatus.lastExecutionDate, error: { - reason: 'read', + reason: AlertExecutionStatusErrorReasons.Read, message: 'oops', }, }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.mock.ts index 271b1043ea568..68199c531a2fe 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.mock.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.mock.ts @@ -43,6 +43,8 @@ export const getCreateRulesOptionsMock = (): CreateRulesOptions => ({ threatFilters: undefined, threatMapping: undefined, threatLanguage: undefined, + concurrentSearches: undefined, + itemsPerSearch: undefined, threatQuery: undefined, threatIndex: undefined, threshold: undefined, @@ -94,6 +96,8 @@ export const getCreateMlRulesOptionsMock = (): CreateRulesOptions => ({ threatMapping: undefined, threatQuery: undefined, threatLanguage: undefined, + concurrentSearches: undefined, + itemsPerSearch: undefined, threshold: undefined, timestampOverride: undefined, to: 'now', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.ts index 776882d0f8494..3c814ce7e6606 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.ts @@ -46,6 +46,8 @@ export const createRules = async ({ threatFilters, threatIndex, threatLanguage, + concurrentSearches, + itemsPerSearch, threatQuery, threatMapping, threshold, @@ -96,6 +98,8 @@ export const createRules = async ({ threatFilters, threatIndex, threatQuery, + concurrentSearches, + itemsPerSearch, threatMapping, threatLanguage, timestampOverride, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/install_prepacked_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/install_prepacked_rules.ts index 0a43c652234d0..4c01318f02cde 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/install_prepacked_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/install_prepacked_rules.ts @@ -51,6 +51,8 @@ export const installPrepackagedRules = ( threat_filters: threatFilters, threat_mapping: threatMapping, threat_language: threatLanguage, + concurrent_searches: concurrentSearches, + items_per_search: itemsPerSearch, threat_query: threatQuery, threat_index: threatIndex, threshold, @@ -103,6 +105,8 @@ export const installPrepackagedRules = ( threatFilters, threatMapping, threatLanguage, + concurrentSearches, + itemsPerSearch, threatQuery, threatIndex, threshold, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.mock.ts index ef7cd35f28f1b..60f1d599470e3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.mock.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.mock.ts @@ -154,6 +154,8 @@ export const getPatchRulesOptionsMock = (): PatchRulesOptions => ({ threatQuery: undefined, threatMapping: undefined, threatLanguage: undefined, + concurrentSearches: undefined, + itemsPerSearch: undefined, timestampOverride: undefined, to: 'now', type: 'query', @@ -203,6 +205,8 @@ export const getPatchMlRulesOptionsMock = (): PatchRulesOptions => ({ threatQuery: undefined, threatMapping: undefined, threatLanguage: undefined, + concurrentSearches: undefined, + itemsPerSearch: undefined, timestampOverride: undefined, to: 'now', type: 'machine_learning', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.ts index 1982dcf9dd9b6..22b2593283696 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.ts @@ -49,6 +49,8 @@ export const patchRules = async ({ threatQuery, threatMapping, threatLanguage, + concurrentSearches, + itemsPerSearch, timestampOverride, to, type, @@ -97,6 +99,8 @@ export const patchRules = async ({ threatQuery, threatMapping, threatLanguage, + concurrentSearches, + itemsPerSearch, timestampOverride, to, type, @@ -141,6 +145,8 @@ export const patchRules = async ({ threatQuery, threatMapping, threatLanguage, + concurrentSearches, + itemsPerSearch, timestampOverride, to, type, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts index fb4763a982f43..f6ab3fb0c3ed2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts @@ -92,6 +92,8 @@ import { ThreatMappingOrUndefined, ThreatFiltersOrUndefined, ThreatLanguageOrUndefined, + ConcurrentSearchesOrUndefined, + ItemsPerSearchOrUndefined, } from '../../../../common/detection_engine/schemas/types/threat_mapping'; import { AlertsClient, PartialAlert } from '../../../../../alerts/server'; @@ -234,6 +236,8 @@ export interface CreateRulesOptions { threatIndex: ThreatIndexOrUndefined; threatQuery: ThreatQueryOrUndefined; threatMapping: ThreatMappingOrUndefined; + concurrentSearches: ConcurrentSearchesOrUndefined; + itemsPerSearch: ItemsPerSearchOrUndefined; threatLanguage: ThreatLanguageOrUndefined; timestampOverride: TimestampOverrideOrUndefined; to: To; @@ -284,6 +288,8 @@ export interface UpdateRulesOptions { threatIndex: ThreatIndexOrUndefined; threatQuery: ThreatQueryOrUndefined; threatMapping: ThreatMappingOrUndefined; + itemsPerSearch: ItemsPerSearchOrUndefined; + concurrentSearches: ConcurrentSearchesOrUndefined; threatLanguage: ThreatLanguageOrUndefined; timestampOverride: TimestampOverrideOrUndefined; to: To; @@ -327,6 +333,8 @@ export interface PatchRulesOptions { severityMapping: SeverityMappingOrUndefined; tags: TagsOrUndefined; threat: ThreatOrUndefined; + itemsPerSearch: ItemsPerSearchOrUndefined; + concurrentSearches: ConcurrentSearchesOrUndefined; threshold: ThresholdOrUndefined; threatFilters: ThreatFiltersOrUndefined; threatIndex: ThreatIndexOrUndefined; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_prepacked_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_prepacked_rules.ts index c685c4198c119..3d4b27b74c0af 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_prepacked_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_prepacked_rules.ts @@ -52,6 +52,8 @@ export const updatePrepackagedRules = async ( threat_query: threatQuery, threat_mapping: threatMapping, threat_language: threatLanguage, + concurrent_searches: concurrentSearches, + items_per_search: itemsPerSearch, timestamp_override: timestampOverride, references, version, @@ -107,6 +109,8 @@ export const updatePrepackagedRules = async ( threatQuery, threatMapping, threatLanguage, + concurrentSearches, + itemsPerSearch, references, version, note, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.mock.ts index a33651580ef22..34be0f6ad843d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.mock.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.mock.ts @@ -49,6 +49,8 @@ export const getUpdateRulesOptionsMock = (): UpdateRulesOptions => ({ threatMapping: undefined, threatLanguage: undefined, timestampOverride: undefined, + concurrentSearches: undefined, + itemsPerSearch: undefined, to: 'now', type: 'query', references: ['http://www.example.com'], @@ -99,6 +101,8 @@ export const getUpdateMlRulesOptionsMock = (): UpdateRulesOptions => ({ threatMapping: undefined, threatLanguage: undefined, timestampOverride: undefined, + concurrentSearches: undefined, + itemsPerSearch: undefined, to: 'now', type: 'machine_learning', references: ['http://www.example.com'], diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.ts index 3da921ed47f26..5168affca5c62 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.ts @@ -50,6 +50,8 @@ export const updateRules = async ({ threatQuery, threatMapping, threatLanguage, + concurrentSearches, + itemsPerSearch, timestampOverride, to, type, @@ -99,6 +101,8 @@ export const updateRules = async ({ threatQuery, threatMapping, threatLanguage, + concurrentSearches, + itemsPerSearch, timestampOverride, to, type, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.test.ts index 654383ff97c7a..8555af424ecd7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.test.ts @@ -60,6 +60,8 @@ describe('utils', () => { threatQuery: undefined, threatMapping: undefined, threatLanguage: undefined, + concurrentSearches: undefined, + itemsPerSearch: undefined, to: undefined, timestampOverride: undefined, type: undefined, @@ -108,6 +110,8 @@ describe('utils', () => { threatQuery: undefined, threatMapping: undefined, threatLanguage: undefined, + concurrentSearches: undefined, + itemsPerSearch: undefined, to: undefined, timestampOverride: undefined, type: undefined, @@ -158,6 +162,8 @@ describe('utils', () => { threatLanguage: undefined, to: undefined, timestampOverride: undefined, + concurrentSearches: undefined, + itemsPerSearch: undefined, type: undefined, references: undefined, version: undefined, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.ts index a9a100543b528..83d9e3fd3e59f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.ts @@ -43,6 +43,8 @@ import { } from '../../../../common/detection_engine/schemas/common/schemas'; import { PartialFilter } from '../types'; import { + ConcurrentSearchesOrUndefined, + ItemsPerSearchOrUndefined, ListArrayOrUndefined, ThreatFiltersOrUndefined, ThreatIndexOrUndefined, @@ -98,6 +100,8 @@ export interface UpdateProperties { threatQuery: ThreatQueryOrUndefined; threatMapping: ThreatMappingOrUndefined; threatLanguage: ThreatLanguageOrUndefined; + concurrentSearches: ConcurrentSearchesOrUndefined; + itemsPerSearch: ItemsPerSearchOrUndefined; timestampOverride: TimestampOverrideOrUndefined; to: ToOrUndefined; type: TypeOrUndefined; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/create_threat_data.sh b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/create_threat_data.sh index 23c1914387c44..4807afd71e8d2 100755 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/create_threat_data.sh +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/create_threat_data.sh @@ -12,7 +12,7 @@ set -e # Adds port mock data to a threat list for testing. # Example: ./create_threat_data.sh -# Example: ./create_threat_data.sh 1000 2000 +# Example: ./create_threat_data.sh 1 500 START=${1:-1} END=${2:-1000} @@ -22,7 +22,7 @@ do { curl -s -k \ -H "Content-Type: application/json" \ -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ - -X PUT ${ELASTICSEARCH_URL}/mock-threat-list/_doc/$i \ + -X PUT ${ELASTICSEARCH_URL}/mock-threat-list-1/_doc/$i \ --data " { \"@timestamp\": \"$(date -u +"%Y-%m-%dT%H:%M:%SZ")\", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/query_with_threat_mapping_perf.json b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/query_with_threat_mapping_perf.json new file mode 100644 index 0000000000000..c573db7fbca35 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/query_with_threat_mapping_perf.json @@ -0,0 +1,32 @@ +{ + "concurrent_searches": 10, + "items_per_search": 10, + "index": ["auditbeat-*", "endgame-*", "filebeat-*", "logs-*", "packetbeat-*", "winlogbeat-*"], + "name": "Indicator Match Concurrent Searches", + "description": "Does 100 Concurrent searches with 10 items per search", + "rule_id": "indicator_concurrent_search", + "risk_score": 1, + "severity": "high", + "type": "threat_match", + "query": "*:*", + "tags": ["concurrent_searches_test", "from_script"], + "threat_index": ["mock-threat-list-1"], + "threat_language": "kuery", + "threat_query": "*:*", + "threat_mapping": [ + { + "entries": [ + { + "field": "source.port", + "type": "mapping", + "value": "source.port" + }, + { + "field": "source.ip", + "type": "mapping", + "value": "source.ip" + } + ] + } + ] +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts index 4559a658c9583..92e6b9562d970 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts @@ -68,6 +68,8 @@ export const sampleRuleAlertParams = ( threat: undefined, version: 1, exceptionsList: getListArrayMock(), + concurrentSearches: undefined, + itemsPerSearch: undefined, }); export const sampleRuleSO = (): SavedObject => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts index cfe71f66395b0..50e740e81830f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts @@ -54,6 +54,8 @@ const signalSchema = schema.object({ threatQuery: schema.maybe(schema.string()), threatMapping: schema.maybe(schema.arrayOf(schema.object({}, { unknowns: 'allow' }))), threatLanguage: schema.maybe(schema.string()), + concurrentSearches: schema.maybe(schema.number()), + itemsPerSearch: schema.maybe(schema.number()), }); /** diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts index 415abc9d995fb..dc68e3949eb36 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts @@ -504,7 +504,7 @@ describe('rules_notification_alert_type', () => { await alert.executor(payload); expect(logger.error).toHaveBeenCalled(); expect(logger.error.mock.calls[0][0]).toContain( - 'An error occurred during rule execution: message: "Threat Match rule is missing threatQuery and/or threatIndex and/or threatMapping: threatQuery: "undefined" threatIndex: "undefined" threatMapping: "undefined"" name: "Detect Root/Admin Users" id: "04128c15-0d1b-4716-a4c5-46997ac7f3bd" rule id: "rule-1" signals index: ".siem-signals"' + 'An error occurred during rule execution: message: "Indicator match is missing threatQuery and/or threatIndex and/or threatMapping: threatQuery: "undefined" threatIndex: "undefined" threatMapping: "undefined"" name: "Detect Root/Admin Users" id: "04128c15-0d1b-4716-a4c5-46997ac7f3bd" rule id: "rule-1" signals index: ".siem-signals"' ); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index a0d5c833b208c..1d2b1c23f868f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -119,6 +119,8 @@ export const signalRulesAlertType = ({ timestampOverride, type, exceptionsList, + concurrentSearches, + itemsPerSearch, } = params; const searchAfterSize = Math.min(maxSignals, DEFAULT_SEARCH_AFTER_PAGE_SIZE); @@ -360,7 +362,7 @@ export const signalRulesAlertType = ({ ) { throw new Error( [ - 'Threat Match rule is missing threatQuery and/or threatIndex and/or threatMapping:', + 'Indicator match is missing threatQuery and/or threatIndex and/or threatMapping:', `threatQuery: "${threatQuery}"`, `threatIndex: "${threatIndex}"`, `threatMapping: "${threatMapping}"`, @@ -403,6 +405,8 @@ export const signalRulesAlertType = ({ threatLanguage, buildRuleMessage, threatIndex, + concurrentSearches: concurrentSearches ?? 1, + itemsPerSearch: itemsPerSearch ?? 9000, }); } else if (type === 'query' || type === 'saved_query') { const inputIndex = await getInputIndex(services, version, index); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.test.ts index 85d172b3631a9..8eed838fc9680 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.test.ts @@ -19,28 +19,32 @@ import { } from './build_threat_mapping_filter'; import { getThreatMappingMock, - getThreatListSearchResponseMock, getThreatListItemMock, getThreatMappingFilterMock, getFilterThreatMapping, getThreatMappingFiltersShouldMock, getThreatMappingFilterShouldMock, + getThreatListSearchResponseMock, } from './build_threat_mapping_filter.mock'; -import { BooleanFilter } from './types'; +import { BooleanFilter, ThreatListItem } from './types'; describe('build_threat_mapping_filter', () => { describe('buildThreatMappingFilter', () => { test('it should throw if given a chunk over 1024 in size', () => { const threatMapping = getThreatMappingMock(); - const threatList = getThreatListSearchResponseMock(); + const threatList = getThreatListSearchResponseMock().hits.hits; expect(() => - buildThreatMappingFilter({ threatMapping, threatList, chunkSize: 1025 }) + buildThreatMappingFilter({ + threatMapping, + threatList, + chunkSize: 1025, + }) ).toThrow('chunk sizes cannot exceed 1024 in size'); }); test('it should NOT throw if given a chunk under 1024 in size', () => { const threatMapping = getThreatMappingMock(); - const threatList = getThreatListSearchResponseMock(); + const threatList = getThreatListSearchResponseMock().hits.hits; expect(() => buildThreatMappingFilter({ threatMapping, threatList, chunkSize: 1023 }) ).not.toThrow(); @@ -48,30 +52,30 @@ describe('build_threat_mapping_filter', () => { test('it should create the correct entries when using the default mocks', () => { const threatMapping = getThreatMappingMock(); - const threatList = getThreatListSearchResponseMock(); + const threatList = getThreatListSearchResponseMock().hits.hits; const filter = buildThreatMappingFilter({ threatMapping, threatList }); expect(filter).toEqual(getThreatMappingFilterMock()); }); test('it should not mutate the original threatMapping', () => { const threatMapping = getThreatMappingMock(); - const threatList = getThreatListSearchResponseMock(); + const threatList = getThreatListSearchResponseMock().hits.hits; buildThreatMappingFilter({ threatMapping, threatList }); expect(threatMapping).toEqual(getThreatMappingMock()); }); test('it should not mutate the original threatListItem', () => { const threatMapping = getThreatMappingMock(); - const threatList = getThreatListSearchResponseMock(); + const threatList = getThreatListSearchResponseMock().hits.hits; buildThreatMappingFilter({ threatMapping, threatList }); - expect(threatList).toEqual(getThreatListSearchResponseMock()); + expect(threatList).toEqual(getThreatListSearchResponseMock().hits.hits); }); }); describe('filterThreatMapping', () => { test('it should not remove any entries when using the default mocks', () => { const threatMapping = getThreatMappingMock(); - const threatListItem = getThreatListItemMock(); + const threatListItem = getThreatListSearchResponseMock().hits.hits[0]; const item = filterThreatMapping({ threatMapping, threatListItem }); const expected = getFilterThreatMapping(); @@ -80,7 +84,7 @@ describe('build_threat_mapping_filter', () => { test('it should only give one filtered element if only 1 element is defined', () => { const [firstElement] = getThreatMappingMock(); // get only the first element - const threatListItem = getThreatListItemMock(); + const threatListItem = getThreatListSearchResponseMock().hits.hits[0]; const item = filterThreatMapping({ threatMapping: [firstElement], threatListItem }); const [firstElementFilter] = getFilterThreatMapping(); // get only the first element to compare @@ -89,7 +93,7 @@ describe('build_threat_mapping_filter', () => { test('it should not mutate the original threatMapping', () => { const threatMapping = getThreatMappingMock(); - const threatListItem = getThreatListItemMock(); + const threatListItem = getThreatListSearchResponseMock().hits.hits[0]; filterThreatMapping({ threatMapping, @@ -100,13 +104,13 @@ describe('build_threat_mapping_filter', () => { test('it should not mutate the original threatListItem', () => { const threatMapping = getThreatMappingMock(); - const threatListItem = getThreatListItemMock(); + const threatListItem = getThreatListSearchResponseMock().hits.hits[0]; filterThreatMapping({ threatMapping, threatListItem, }); - expect(threatListItem).toEqual(getThreatListItemMock()); + expect(threatListItem).toEqual(getThreatListSearchResponseMock().hits.hits[0]); }); test('it should remove the entire "AND" clause if one of the pieces of data is missing from the list', () => { @@ -166,9 +170,11 @@ describe('build_threat_mapping_filter', () => { }, ], threatListItem: { - '@timestamp': '2020-09-09T21:59:13Z', - host: { - name: 'host-1', + _source: { + '@timestamp': '2020-09-09T21:59:13Z', + host: { + name: 'host-1', + }, }, }, }); @@ -189,7 +195,7 @@ describe('build_threat_mapping_filter', () => { describe('createInnerAndClauses', () => { test('it should return two clauses given a single entry', () => { const [{ entries: threatMappingEntries }] = getThreatMappingMock(); // get the first element - const threatListItem = getThreatListItemMock(); + const threatListItem = getThreatListSearchResponseMock().hits.hits[0]; const innerClause = createInnerAndClauses({ threatMappingEntries, threatListItem }); const { bool: { @@ -219,7 +225,7 @@ describe('build_threat_mapping_filter', () => { type: 'mapping', }, ]; - const threatListItem = getThreatListItemMock(); + const threatListItem = getThreatListSearchResponseMock().hits.hits[0]; const innerClause = createInnerAndClauses({ threatMappingEntries, threatListItem }); const { bool: { @@ -248,7 +254,7 @@ describe('build_threat_mapping_filter', () => { type: 'mapping', }, ]; - const threatListItem = getThreatListItemMock(); + const threatListItem = getThreatListSearchResponseMock().hits.hits[0]; const innerClause = createInnerAndClauses({ threatMappingEntries, threatListItem }); const { bool: { @@ -275,7 +281,7 @@ describe('build_threat_mapping_filter', () => { type: 'mapping', }, ]; - const threatListItem = getThreatListItemMock(); + const threatListItem = getThreatListSearchResponseMock().hits.hits[0]; const innerClause = createInnerAndClauses({ threatMappingEntries, threatListItem }); expect(innerClause).toEqual([]); }); @@ -284,27 +290,31 @@ describe('build_threat_mapping_filter', () => { describe('createAndOrClauses', () => { test('it should return all clauses given the entries', () => { const threatMapping = getThreatMappingMock(); - const threatListItem = getThreatListItemMock(); + const threatListItem = getThreatListSearchResponseMock().hits.hits[0]; const innerClause = createAndOrClauses({ threatMapping, threatListItem }); expect(innerClause).toEqual(getThreatMappingFilterShouldMock()); }); test('it should filter out data from entries that do not have mappings', () => { const threatMapping = getThreatMappingMock(); - const threatListItem = { ...getThreatListItemMock(), foo: 'bar' }; + const threatListItem = getThreatListSearchResponseMock().hits.hits[0]; + threatListItem._source = { + ...getThreatListSearchResponseMock().hits.hits[0]._source, + foo: 'bar', + }; const innerClause = createAndOrClauses({ threatMapping, threatListItem }); expect(innerClause).toEqual(getThreatMappingFilterShouldMock()); }); test('it should return an empty boolean given an empty array', () => { - const threatListItem = getThreatListItemMock(); + const threatListItem = getThreatListSearchResponseMock().hits.hits[0]; const innerClause = createAndOrClauses({ threatMapping: [], threatListItem }); expect(innerClause).toEqual({ bool: { minimum_should_match: 1, should: [] } }); }); test('it should return an empty boolean clause given an empty object for a threat list item', () => { const threatMapping = getThreatMappingMock(); - const innerClause = createAndOrClauses({ threatMapping, threatListItem: {} }); + const innerClause = createAndOrClauses({ threatMapping, threatListItem: { _source: {} } }); expect(innerClause).toEqual({ bool: { minimum_should_match: 1, should: [] } }); }); }); @@ -312,7 +322,7 @@ describe('build_threat_mapping_filter', () => { describe('buildEntriesMappingFilter', () => { test('it should return all clauses given the entries', () => { const threatMapping = getThreatMappingMock(); - const threatList = getThreatListSearchResponseMock(); + const threatList = getThreatListSearchResponseMock().hits.hits; const mapping = buildEntriesMappingFilter({ threatMapping, threatList, @@ -326,8 +336,7 @@ describe('build_threat_mapping_filter', () => { test('it should return empty "should" given an empty threat list', () => { const threatMapping = getThreatMappingMock(); - const threatList = getThreatListSearchResponseMock(); - threatList.hits.hits = []; + const threatList: ThreatListItem[] = []; const mapping = buildEntriesMappingFilter({ threatMapping, threatList, @@ -340,7 +349,7 @@ describe('build_threat_mapping_filter', () => { }); test('it should return empty "should" given an empty threat mapping', () => { - const threatList = getThreatListSearchResponseMock(); + const threatList = getThreatListSearchResponseMock().hits.hits; const mapping = buildEntriesMappingFilter({ threatMapping: [], threatList, @@ -374,7 +383,7 @@ describe('build_threat_mapping_filter', () => { }, ], ]; - const threatList = getThreatListSearchResponseMock(); + const threatList = getThreatListSearchResponseMock().hits.hits; const mapping = buildEntriesMappingFilter({ threatMapping, threatList, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.ts index 346f156a9ec33..294d97e0bf2f1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.ts @@ -53,9 +53,9 @@ export const filterThreatMapping = ({ }: FilterThreatMappingOptions): ThreatMapping => threatMapping .map((threatMap) => { - const atLeastOneItemMissingInThreatList = threatMap.entries.some( - (entry) => get(entry.value, threatListItem) == null - ); + const atLeastOneItemMissingInThreatList = threatMap.entries.some((entry) => { + return get(entry.value, threatListItem._source) == null; + }); if (atLeastOneItemMissingInThreatList) { return { ...threatMap, entries: [] }; } else { @@ -69,7 +69,7 @@ export const createInnerAndClauses = ({ threatListItem, }: CreateInnerAndClausesOptions): BooleanFilter[] => { return threatMappingEntries.reduce((accum, threatMappingEntry) => { - const value = get(threatMappingEntry.value, threatListItem); + const value = get(threatMappingEntry.value, threatListItem._source); if (value != null) { // These values could be potentially 10k+ large so mutating the array intentionally accum.push({ @@ -114,24 +114,21 @@ export const buildEntriesMappingFilter = ({ threatList, chunkSize, }: BuildEntriesMappingFilterOptions): BooleanFilter => { - const combinedShould = threatList.hits.hits.reduce( - (accum, threatListSearchItem) => { - const filteredEntries = filterThreatMapping({ - threatMapping, - threatListItem: threatListSearchItem._source, - }); - const queryWithAndOrClause = createAndOrClauses({ - threatMapping: filteredEntries, - threatListItem: threatListSearchItem._source, - }); - if (queryWithAndOrClause.bool.should.length !== 0) { - // These values can be 10k+ large, so using a push here for performance - accum.push(queryWithAndOrClause); - } - return accum; - }, - [] - ); + const combinedShould = threatList.reduce((accum, threatListSearchItem) => { + const filteredEntries = filterThreatMapping({ + threatMapping, + threatListItem: threatListSearchItem, + }); + const queryWithAndOrClause = createAndOrClauses({ + threatMapping: filteredEntries, + threatListItem: threatListSearchItem, + }); + if (queryWithAndOrClause.bool.should.length !== 0) { + // These values can be 10k+ large, so using a push here for performance + accum.push(queryWithAndOrClause); + } + return accum; + }, []); const should = splitShouldClauses({ should: combinedShould, chunkSize }); return { bool: { should, minimum_should_match: 1 } }; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts index 037f91240edfa..43fb759d07620 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts @@ -4,13 +4,12 @@ * you may not use this file except in compliance with the Elastic License. */ -import { getThreatList } from './get_threat_list'; import { buildThreatMappingFilter } from './build_threat_mapping_filter'; import { getFilter } from '../get_filter'; import { searchAfterAndBulkCreate } from '../search_after_bulk_create'; -import { CreateThreatSignalOptions, ThreatSignalResults } from './types'; -import { combineResults } from './utils'; +import { CreateThreatSignalOptions } from './types'; +import { SearchAfterAndBulkCreateReturnType } from '../types'; export const createThreatSignal = async ({ threatMapping, @@ -41,28 +40,11 @@ export const createThreatSignal = async ({ refresh, tags, throttle, - threatFilters, - threatQuery, - threatLanguage, buildRuleMessage, - threatIndex, name, currentThreatList, currentResult, -}: CreateThreatSignalOptions): Promise => { - const threatList = await getThreatList({ - callCluster: services.callCluster, - exceptionItems, - query: threatQuery, - language: threatLanguage, - threatFilters, - index: threatIndex, - searchAfter: currentThreatList.hits.hits[currentThreatList.hits.hits.length - 1].sort, - sortField: undefined, - sortOrder: undefined, - listClient, - }); - +}: CreateThreatSignalOptions): Promise => { const threatFilter = buildThreatMappingFilter({ threatMapping, threatList: currentThreatList, @@ -71,7 +53,12 @@ export const createThreatSignal = async ({ if (threatFilter.query.bool.should.length === 0) { // empty threat list and we do not want to return everything as being // a hit so opt to return the existing result. - return { threatList, results: currentResult }; + logger.debug( + buildRuleMessage( + 'Indicator items are empty after filtering for missing data, returning without attempting a match' + ) + ); + return currentResult; } else { const esFilter = await getFilter({ type, @@ -83,7 +70,13 @@ export const createThreatSignal = async ({ index: inputIndex, lists: exceptionItems, }); - const newResult = await searchAfterAndBulkCreate({ + + logger.debug( + buildRuleMessage( + `${threatFilter.query.bool.should.length} indicator items are being checked for existence of matches` + ) + ); + const result = await searchAfterAndBulkCreate({ gap, previousStartedAt, listClient, @@ -110,7 +103,15 @@ export const createThreatSignal = async ({ throttle, buildRuleMessage, }); - const results = combineResults(currentResult, newResult); - return { threatList, results }; + logger.debug( + buildRuleMessage( + `${ + threatFilter.query.bool.should.length + } items have completed match checks and the total times to search were ${ + result.searchAfterTimes.length !== 0 ? result.searchAfterTimes : '(unknown) ' + }ms` + ) + ); + return result; } }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts index 8be76dc8caf0f..e90c45d40de95 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts @@ -4,11 +4,13 @@ * you may not use this file except in compliance with the Elastic License. */ -import { getThreatList } from './get_threat_list'; +import chunk from 'lodash/fp/chunk'; +import { getThreatList, getThreatListCount } from './get_threat_list'; import { CreateThreatSignalsOptions } from './types'; import { createThreatSignal } from './create_threat_signal'; import { SearchAfterAndBulkCreateReturnType } from '../types'; +import { combineConcurrentResults } from './utils'; export const createThreatSignals = async ({ threatMapping, @@ -45,7 +47,12 @@ export const createThreatSignals = async ({ buildRuleMessage, threatIndex, name, + concurrentSearches, + itemsPerSearch, }: CreateThreatSignalsOptions): Promise => { + logger.debug(buildRuleMessage('Indicator matching rule starting')); + const perPage = concurrentSearches * itemsPerSearch; + let results: SearchAfterAndBulkCreateReturnType = { success: true, bulkCreateTimes: [], @@ -55,6 +62,16 @@ export const createThreatSignals = async ({ errors: [], }; + let threatListCount = await getThreatListCount({ + callCluster: services.callCluster, + exceptionItems, + threatFilters, + query: threatQuery, + language: threatLanguage, + index: threatIndex, + }); + logger.debug(buildRuleMessage(`Total indicator items: ${threatListCount}`)); + let threatList = await getThreatList({ callCluster: services.callCluster, exceptionItems, @@ -66,47 +83,89 @@ export const createThreatSignals = async ({ searchAfter: undefined, sortField: undefined, sortOrder: undefined, + logger, + buildRuleMessage, + perPage, }); - while (threatList.hits.hits.length !== 0 && results.createdSignalsCount <= params.maxSignals) { - ({ threatList, results } = await createThreatSignal({ - threatMapping, - query, - inputIndex, - type, - filters, - language, - savedId, - services, + while (threatList.hits.hits.length !== 0) { + const chunks = chunk(itemsPerSearch, threatList.hits.hits); + logger.debug(buildRuleMessage(`${chunks.length} concurrent indicator searches are starting.`)); + const concurrentSearchesPerformed = chunks.map>( + (slicedChunk) => + createThreatSignal({ + threatMapping, + query, + inputIndex, + type, + filters, + language, + savedId, + services, + exceptionItems, + gap, + previousStartedAt, + listClient, + logger, + eventsTelemetry, + alertId, + outputIndex, + params, + searchAfterSize, + actions, + createdBy, + createdAt, + updatedBy, + updatedAt, + interval, + enabled, + tags, + refresh, + throttle, + buildRuleMessage, + name, + currentThreatList: slicedChunk, + currentResult: results, + }) + ); + const searchesPerformed = await Promise.all(concurrentSearchesPerformed); + results = combineConcurrentResults(results, searchesPerformed); + threatListCount -= threatList.hits.hits.length; + logger.debug( + buildRuleMessage( + `Concurrent indicator match searches completed with ${results.createdSignalsCount} signals found`, + `search times of ${results.searchAfterTimes}ms,`, + `bulk create times ${results.bulkCreateTimes}ms,`, + `all successes are ${results.success}` + ) + ); + if (results.createdSignalsCount >= params.maxSignals) { + logger.debug( + buildRuleMessage( + `Indicator match has reached its max signals count ${params.maxSignals}. Additional indicator items not checked are ${threatListCount}` + ) + ); + break; + } + logger.debug(buildRuleMessage(`Indicator items left to check are ${threatListCount}`)); + + threatList = await getThreatList({ + callCluster: services.callCluster, exceptionItems, - gap, - previousStartedAt, - listClient, - logger, - eventsTelemetry, - alertId, - outputIndex, - params, - searchAfterSize, - actions, - createdBy, - createdAt, - updatedBy, - updatedAt, - interval, - enabled, - tags, - refresh, - throttle, + query: threatQuery, + language: threatLanguage, threatFilters, - threatQuery, + index: threatIndex, + searchAfter: threatList.hits.hits[threatList.hits.hits.length - 1].sort, + sortField: undefined, + sortOrder: undefined, + listClient, buildRuleMessage, - threatIndex, - threatLanguage, - name, - currentThreatList: threatList, - currentResult: results, - })); + logger, + perPage, + }); } + + logger.debug(buildRuleMessage('Indicator matching rule has completed')); return results; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_threat_list.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_threat_list.ts index 3147eb1705168..aba3f6f69d706 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_threat_list.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_threat_list.ts @@ -10,6 +10,7 @@ import { GetSortWithTieBreakerOptions, GetThreatListOptions, SortWithTieBreaker, + ThreatListCountOptions, ThreatListItem, } from './types'; @@ -30,6 +31,8 @@ export const getThreatList = async ({ exceptionItems, threatFilters, listClient, + buildRuleMessage, + logger, }: GetThreatListOptions): Promise> => { const calculatedPerPage = perPage ?? MAX_PER_PAGE; if (calculatedPerPage > 10000) { @@ -43,6 +46,11 @@ export const getThreatList = async ({ exceptionItems ); + logger.debug( + buildRuleMessage( + `Querying the indicator items from the index: "${index}" with searchAfter: "${searchAfter}" for up to ${calculatedPerPage} indicator items` + ) + ); const response: SearchResponse = await callCluster('search', { body: { query: queryFilter, @@ -58,6 +66,8 @@ export const getThreatList = async ({ index, size: calculatedPerPage, }); + + logger.debug(buildRuleMessage(`Retrieved indicator items of size: ${response.hits.hits.length}`)); return response; }; @@ -89,3 +99,30 @@ export const getSortWithTieBreaker = ({ } } }; + +export const getThreatListCount = async ({ + callCluster, + query, + language, + threatFilters, + index, + exceptionItems, +}: ThreatListCountOptions): Promise => { + const queryFilter = getQueryFilter( + query, + language ?? 'kuery', + threatFilters, + index, + exceptionItems + ); + const response: { + count: number; + } = await callCluster('count', { + body: { + query: queryFilter, + }, + ignoreUnavailable: true, + index, + }); + return response.count; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts index 0078cf1b3c64f..2e32a4e682403 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts @@ -5,7 +5,6 @@ */ import { Duration } from 'moment'; -import { SearchResponse } from 'elasticsearch'; import { ListClient } from '../../../../../../lists/server'; import { Type, @@ -17,6 +16,8 @@ import { ThreatMappingEntries, ThreatIndex, ThreatLanguageOrUndefined, + ConcurrentSearches, + ItemsPerSearch, } from '../../../../../common/detection_engine/schemas/types/threat_mapping'; import { PartialFilter, RuleTypeParams } from '../../types'; import { AlertServices } from '../../../../../../alerts/server'; @@ -62,6 +63,8 @@ export interface CreateThreatSignalsOptions { threatIndex: ThreatIndex; threatLanguage: ThreatLanguageOrUndefined; name: string; + concurrentSearches: ConcurrentSearches; + itemsPerSearch: ItemsPerSearch; } export interface CreateThreatSignalOptions { @@ -93,24 +96,15 @@ export interface CreateThreatSignalOptions { tags: string[]; refresh: false | 'wait_for'; throttle: string; - threatFilters: PartialFilter[]; - threatQuery: ThreatQuery; buildRuleMessage: BuildRuleMessage; - threatIndex: ThreatIndex; - threatLanguage: ThreatLanguageOrUndefined; name: string; - currentThreatList: SearchResponse; + currentThreatList: ThreatListItem[]; currentResult: SearchAfterAndBulkCreateReturnType; } -export interface ThreatSignalResults { - threatList: SearchResponse; - results: SearchAfterAndBulkCreateReturnType; -} - export interface BuildThreatMappingFilterOptions { threatMapping: ThreatMapping; - threatList: SearchResponse; + threatList: ThreatListItem[]; chunkSize?: number; } @@ -131,7 +125,7 @@ export interface CreateAndOrClausesOptions { export interface BuildEntriesMappingFilterOptions { threatMapping: ThreatMapping; - threatList: SearchResponse; + threatList: ThreatListItem[]; chunkSize: number; } @@ -156,6 +150,17 @@ export interface GetThreatListOptions { threatFilters: PartialFilter[]; exceptionItems: ExceptionListItemSchema[]; listClient: ListClient; + buildRuleMessage: BuildRuleMessage; + logger: Logger; +} + +export interface ThreatListCountOptions { + callCluster: ILegacyScopedClusterClient['callAsCurrentUser']; + query: string; + language: ThreatLanguageOrUndefined; + threatFilters: PartialFilter[]; + index: string[]; + exceptionItems: ExceptionListItemSchema[]; } export interface GetSortWithTieBreakerOptions { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts index 27593b40b0c8f..840d64381c793 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts @@ -6,7 +6,13 @@ import { SearchAfterAndBulkCreateReturnType } from '../types'; -import { calculateAdditiveMax, combineResults } from './utils'; +import { + calculateAdditiveMax, + calculateMax, + calculateMaxLookBack, + combineConcurrentResults, + combineResults, +} from './utils'; describe('utils', () => { describe('calculateAdditiveMax', () => { @@ -156,4 +162,383 @@ describe('utils', () => { ); }); }); + + describe('calculateMax', () => { + test('it should return 0 for two empty arrays', () => { + const max = calculateMax([], []); + expect(max).toEqual('0'); + }); + + test('it should return 5 for two arrays with the numbers 5', () => { + const max = calculateMax(['5'], ['5']); + expect(max).toEqual('5'); + }); + + test('it should return 5 for two arrays with second array having just 5', () => { + const max = calculateMax([], ['5']); + expect(max).toEqual('5'); + }); + + test('it should return 5 for two arrays with first array having just 5', () => { + const max = calculateMax(['5'], []); + expect(max).toEqual('5'); + }); + + test('it should return 10 for the max of the two arrays when the max of each array is 10', () => { + const max = calculateMax(['3', '5', '1'], ['3', '5', '10']); + expect(max).toEqual('10'); + }); + + test('it should return 10 for the max of the two arrays when the max of the first is 10', () => { + const max = calculateMax(['3', '5', '10'], ['3', '5', '1']); + expect(max).toEqual('10'); + }); + }); + + describe('calculateMaxLookBack', () => { + test('it should return null if both are null', () => { + const max = calculateMaxLookBack(null, null); + expect(max).toEqual(null); + }); + + test('it should return undefined if both are undefined', () => { + const max = calculateMaxLookBack(undefined, undefined); + expect(max).toEqual(undefined); + }); + + test('it should return null if both one is null and other other is undefined', () => { + const max = calculateMaxLookBack(undefined, null); + expect(max).toEqual(null); + }); + + test('it should return null if both one is null and other other is undefined with flipped arguments', () => { + const max = calculateMaxLookBack(null, undefined); + expect(max).toEqual(null); + }); + + test('it should return a date time if one argument is null', () => { + const max = calculateMaxLookBack(null, new Date('2020-09-16T03:34:32.390Z')); + expect(max).toEqual(new Date('2020-09-16T03:34:32.390Z')); + }); + + test('it should return a date time if one argument is null with flipped arguments', () => { + const max = calculateMaxLookBack(new Date('2020-09-16T03:34:32.390Z'), null); + expect(max).toEqual(new Date('2020-09-16T03:34:32.390Z')); + }); + + test('it should return a date time if one argument is undefined', () => { + const max = calculateMaxLookBack(new Date('2020-09-16T03:34:32.390Z'), undefined); + expect(max).toEqual(new Date('2020-09-16T03:34:32.390Z')); + }); + + test('it should return a date time if one argument is undefined with flipped arguments', () => { + const max = calculateMaxLookBack(undefined, new Date('2020-09-16T03:34:32.390Z')); + expect(max).toEqual(new Date('2020-09-16T03:34:32.390Z')); + }); + + test('it should return a date time that is larger than the other', () => { + const max = calculateMaxLookBack( + new Date('2020-10-16T03:34:32.390Z'), + new Date('2020-09-16T03:34:32.390Z') + ); + expect(max).toEqual(new Date('2020-10-16T03:34:32.390Z')); + }); + + test('it should return a date time that is larger than the other with arguments flipped', () => { + const max = calculateMaxLookBack( + new Date('2020-09-16T03:34:32.390Z'), + new Date('2020-10-16T03:34:32.390Z') + ); + expect(max).toEqual(new Date('2020-10-16T03:34:32.390Z')); + }); + }); + + describe('combineConcurrentResults', () => { + test('it should use the maximum found if given an empty array for newResults', () => { + const existingResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: [], + }; + const expectedResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['30'], // max value from existingResult.searchAfterTimes + bulkCreateTimes: ['25'], // max value from existingResult.bulkCreateTimes + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: [], + }; + const combinedResults = combineConcurrentResults(existingResult, []); + expect(combinedResults).toEqual(expectedResult); + }); + + test('it should work with empty arrays for searchAfterTimes and bulkCreateTimes', () => { + const existingResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: [], + }; + const newResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: [], + bulkCreateTimes: [], + lastLookBackDate: undefined, + createdSignalsCount: 0, + errors: [], + }; + const expectedResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['30'], // max value from existingResult.searchAfterTimes + bulkCreateTimes: ['25'], // max value from existingResult.bulkCreateTimes + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: [], + }; + + const combinedResults = combineConcurrentResults(existingResult, [newResult]); + expect(combinedResults).toEqual(expectedResult); + }); + + test('it should get the max of two new results and then combine the result with an existingResult correctly', () => { + const existingResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], // max is 30 + bulkCreateTimes: ['5', '15', '25'], // max is 25 + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: [], + }; + const newResult1: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: new Date('2020-09-16T03:34:32.390Z'), + createdSignalsCount: 5, + errors: [], + }; + const newResult2: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['40', '5', '15'], + bulkCreateTimes: ['50', '5', '15'], + lastLookBackDate: new Date('2020-09-16T04:34:32.390Z'), + createdSignalsCount: 8, + errors: [], + }; + + const expectedResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['70'], // max value between newResult1 and newResult2 + max array value of existingResult (40 + 30 = 70) + bulkCreateTimes: ['75'], // max value between newResult1 and newResult2 + max array value of existingResult (50 + 25 = 75) + lastLookBackDate: new Date('2020-09-16T04:34:32.390Z'), // max lastLookBackDate + createdSignalsCount: 16, // all the signals counted together (8 + 5 + 3) + errors: [], + }; + + const combinedResults = combineConcurrentResults(existingResult, [newResult1, newResult2]); + expect(combinedResults).toEqual(expectedResult); + }); + + test('it should get the max of two new results and then combine the result with an existingResult correctly when the results are flipped around', () => { + const existingResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], // max is 30 + bulkCreateTimes: ['5', '15', '25'], // max is 25 + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: [], + }; + const newResult1: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: new Date('2020-09-16T03:34:32.390Z'), + createdSignalsCount: 5, + errors: [], + }; + const newResult2: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['40', '5', '15'], + bulkCreateTimes: ['50', '5', '15'], + lastLookBackDate: new Date('2020-09-16T04:34:32.390Z'), + createdSignalsCount: 8, + errors: [], + }; + + const expectedResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['70'], // max value between newResult1 and newResult2 + max array value of existingResult (40 + 30 = 70) + bulkCreateTimes: ['75'], // max value between newResult1 and newResult2 + max array value of existingResult (50 + 25 = 75) + lastLookBackDate: new Date('2020-09-16T04:34:32.390Z'), // max lastLookBackDate + createdSignalsCount: 16, // all the signals counted together (8 + 5 + 3) + errors: [], + }; + + const combinedResults = combineConcurrentResults(existingResult, [newResult2, newResult1]); // two array elements are flipped + expect(combinedResults).toEqual(expectedResult); + }); + + test('it should return the max date correctly if one date contains a null', () => { + const existingResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], // max is 30 + bulkCreateTimes: ['5', '15', '25'], // max is 25 + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: [], + }; + const newResult1: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: new Date('2020-09-16T03:34:32.390Z'), + createdSignalsCount: 5, + errors: [], + }; + const newResult2: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['40', '5', '15'], + bulkCreateTimes: ['50', '5', '15'], + lastLookBackDate: null, + createdSignalsCount: 8, + errors: [], + }; + + const expectedResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['70'], // max value between newResult1 and newResult2 + max array value of existingResult (40 + 30 = 70) + bulkCreateTimes: ['75'], // max value between newResult1 and newResult2 + max array value of existingResult (50 + 25 = 75) + lastLookBackDate: new Date('2020-09-16T03:34:32.390Z'), // max lastLookBackDate + createdSignalsCount: 16, // all the signals counted together (8 + 5 + 3) + errors: [], + }; + + const combinedResults = combineConcurrentResults(existingResult, [newResult1, newResult2]); + expect(combinedResults).toEqual(expectedResult); + }); + + test('it should combine two results with success set to "true" if both are "true"', () => { + const existingResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: [], + }; + + const newResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: [], + }; + const combinedResults = combineConcurrentResults(existingResult, [newResult]); + expect(combinedResults.success).toEqual(true); + }); + + test('it should combine two results with success set to "false" if one of them is "false"', () => { + const existingResult: SearchAfterAndBulkCreateReturnType = { + success: false, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: [], + }; + + const newResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: [], + }; + const combinedResults = combineConcurrentResults(existingResult, [newResult]); + expect(combinedResults.success).toEqual(false); + }); + + test('it should use the latest date if it is set in the new result', () => { + const existingResult: SearchAfterAndBulkCreateReturnType = { + success: false, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: [], + }; + + const newResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: new Date('2020-09-16T03:34:32.390Z'), + createdSignalsCount: 3, + errors: [], + }; + const combinedResults = combineConcurrentResults(existingResult, [newResult]); + expect(combinedResults.lastLookBackDate?.toISOString()).toEqual('2020-09-16T03:34:32.390Z'); + }); + + test('it should combine the searchAfterTimes and the bulkCreateTimes', () => { + const existingResult: SearchAfterAndBulkCreateReturnType = { + success: false, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: [], + }; + + const newResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: new Date('2020-09-16T03:34:32.390Z'), + createdSignalsCount: 3, + errors: [], + }; + const combinedResults = combineConcurrentResults(existingResult, [newResult]); + expect(combinedResults).toEqual( + expect.objectContaining({ + searchAfterTimes: ['60'], + bulkCreateTimes: ['50'], + }) + ); + }); + + test('it should combine errors together without duplicates', () => { + const existingResult: SearchAfterAndBulkCreateReturnType = { + success: false, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: ['error 1', 'error 2', 'error 3'], + }; + + const newResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: new Date('2020-09-16T03:34:32.390Z'), + createdSignalsCount: 3, + errors: ['error 4', 'error 1', 'error 3', 'error 5'], + }; + const combinedResults = combineConcurrentResults(existingResult, [newResult]); + expect(combinedResults).toEqual( + expect.objectContaining({ + errors: ['error 1', 'error 2', 'error 3', 'error 4', 'error 5'], + }) + ); + }); + }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.ts index 401a4a1acb065..d6c91fad6d9cb 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.ts @@ -19,6 +19,41 @@ export const calculateAdditiveMax = (existingTimers: string[], newTimers: string return [String(numericNewTimerMax + numericExistingTimerMax)]; }; +/** + * Given two timers this will take the max of each and then get the max from each. + * Max(Max(timer_array_1), Max(timer_array_2)) + * @param existingTimers String array of existing timers + * @param newTimers String array of new timers. + * @returns String array of the new maximum between the two timers + */ +export const calculateMax = (existingTimers: string[], newTimers: string[]): string => { + const numericNewTimerMax = Math.max(0, ...newTimers.map((time) => +time)); + const numericExistingTimerMax = Math.max(0, ...existingTimers.map((time) => +time)); + return String(Math.max(numericNewTimerMax, numericExistingTimerMax)); +}; + +/** + * Given two dates this will return the larger of the two unless one of them is null + * or undefined. If both one or the other is null/undefined it will return the newDate. + * If there is a mix of "undefined" and "null", this will prefer to set it to "null" as having + * a higher value than "undefined" + * @param existingDate The existing date which can be undefined or null or a date + * @param newDate The new date which can be undefined or null or a date + */ +export const calculateMaxLookBack = ( + existingDate: Date | null | undefined, + newDate: Date | null | undefined +): Date | null | undefined => { + const newDateValue = newDate === null ? 1 : newDate === undefined ? 0 : newDate.valueOf(); + const existingDateValue = + existingDate === null ? 1 : existingDate === undefined ? 0 : existingDate.valueOf(); + if (newDateValue >= existingDateValue) { + return newDate; + } else { + return existingDate; + } +}; + /** * Combines two results together and returns the results combined * @param currentResult The current result to combine with a newResult @@ -38,3 +73,39 @@ export const combineResults = ( createdSignalsCount: currentResult.createdSignalsCount + newResult.createdSignalsCount, errors: [...new Set([...currentResult.errors, ...newResult.errors])], }); + +/** + * Combines two results together and returns the results combined + * @param currentResult The current result to combine with a newResult + * @param newResult The new result to combine + */ +export const combineConcurrentResults = ( + currentResult: SearchAfterAndBulkCreateReturnType, + newResult: SearchAfterAndBulkCreateReturnType[] +): SearchAfterAndBulkCreateReturnType => { + const maxedNewResult = newResult.reduce( + (accum, item) => { + const maxSearchAfterTime = calculateMax(accum.searchAfterTimes, item.searchAfterTimes); + const maxBulkCreateTimes = calculateMax(accum.bulkCreateTimes, item.bulkCreateTimes); + const lastLookBackDate = calculateMaxLookBack(accum.lastLookBackDate, item.lastLookBackDate); + return { + success: accum.success && item.success, + searchAfterTimes: [maxSearchAfterTime], + bulkCreateTimes: [maxBulkCreateTimes], + lastLookBackDate, + createdSignalsCount: accum.createdSignalsCount + item.createdSignalsCount, + errors: [...new Set([...accum.errors, ...item.errors])], + }; + }, + { + success: true, + searchAfterTimes: [], + bulkCreateTimes: [], + lastLookBackDate: undefined, + createdSignalsCount: 0, + errors: [], + } + ); + + return combineResults(currentResult, maxedNewResult); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/types.ts index cf4d989c1f4c8..5cac76e2b0c01 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/types.ts @@ -44,6 +44,8 @@ import { ThreatQueryOrUndefined, ThreatMappingOrUndefined, ThreatLanguageOrUndefined, + ConcurrentSearchesOrUndefined, + ItemsPerSearchOrUndefined, } from '../../../common/detection_engine/schemas/types/threat_mapping'; import { LegacyCallAPIOptions } from '../../../../../../src/core/server'; @@ -93,6 +95,8 @@ export interface RuleTypeParams { references: References; version: Version; exceptionsList: ListArrayOrUndefined; + concurrentSearches: ConcurrentSearchesOrUndefined; + itemsPerSearch: ItemsPerSearchOrUndefined; } // eslint-disable-next-line @typescript-eslint/no-explicit-any diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/alert_details/components/alert_details.test.tsx b/x-pack/plugins/triggers_actions_ui/public/application/sections/alert_details/components/alert_details.test.tsx index 662db81101eee..70b6fb0b750dd 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/sections/alert_details/components/alert_details.test.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/alert_details/components/alert_details.test.tsx @@ -11,7 +11,10 @@ import { Alert, ActionType, ValidationResult } from '../../../../types'; import { EuiTitle, EuiBadge, EuiFlexItem, EuiSwitch, EuiButtonEmpty, EuiText } from '@elastic/eui'; import { ViewInApp } from './view_in_app'; import { coreMock } from 'src/core/public/mocks'; -import { ALERTS_FEATURE_ID } from '../../../../../../alerts/common'; +import { + AlertExecutionStatusErrorReasons, + ALERTS_FEATURE_ID, +} from '../../../../../../alerts/common'; const mockes = coreMock.createSetup(); @@ -125,7 +128,7 @@ describe('alert_details', () => { status: 'error', lastExecutionDate: new Date('2020-08-20T19:23:38Z'), error: { - reason: 'unknown', + reason: AlertExecutionStatusErrorReasons.Unknown, message: 'test', }, }, diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_list/components/alerts_list.test.tsx b/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_list/components/alerts_list.test.tsx index 18cc7b540296e..c434ca9d21402 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_list/components/alerts_list.test.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_list/components/alerts_list.test.tsx @@ -17,7 +17,10 @@ import { AppContextProvider } from '../../../app_context'; import { chartPluginMock } from '../../../../../../../../src/plugins/charts/public/mocks'; import { dataPluginMock } from '../../../../../../../../src/plugins/data/public/mocks'; import { alertingPluginMock } from '../../../../../../alerts/public/mocks'; -import { ALERTS_FEATURE_ID } from '../../../../../../alerts/common'; +import { + AlertExecutionStatusErrorReasons, + ALERTS_FEATURE_ID, +} from '../../../../../../alerts/common'; import { featuresPluginMock } from '../../../../../../features/public/mocks'; jest.mock('../../../lib/action_connector_api', () => ({ @@ -245,7 +248,7 @@ describe('alerts_list component with items', () => { status: 'error', lastExecutionDate: new Date('2020-08-20T19:23:38Z'), error: { - reason: 'unknown', + reason: AlertExecutionStatusErrorReasons.Unknown, message: 'test', }, }, diff --git a/x-pack/test/alerting_api_integration/security_and_spaces/tests/alerting/execution_status.ts b/x-pack/test/alerting_api_integration/security_and_spaces/tests/alerting/execution_status.ts index 8fb89042e4a90..4058b71356280 100644 --- a/x-pack/test/alerting_api_integration/security_and_spaces/tests/alerting/execution_status.ts +++ b/x-pack/test/alerting_api_integration/security_and_spaces/tests/alerting/execution_status.ts @@ -5,6 +5,7 @@ */ import expect from '@kbn/expect'; +import { AlertExecutionStatusErrorReasons } from '../../../../../plugins/alerts/common'; import { Spaces } from '../../scenarios'; import { getUrlPrefix, getTestAlertData, ObjectRemover } from '../../../common/lib'; import { FtrProviderContext } from '../../../common/ftr_provider_context'; @@ -49,7 +50,7 @@ export default function executionStatusAlertTests({ getService }: FtrProviderCon executionStatus = await waitForStatus(alertId, new Set(['error'])); expect(executionStatus.error).to.be.ok(); - expect(executionStatus.error.reason).to.be('decrypt'); + expect(executionStatus.error.reason).to.be(AlertExecutionStatusErrorReasons.Decrypt); expect(executionStatus.error.message).to.be('Unable to decrypt attribute "apiKey"'); }); });