diff --git a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts index 8234c3a9a599d..70fe2b6187aa6 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts @@ -22,7 +22,7 @@ import { Query, Language, Index, TimestampOverrideOrUndefined } from './schemas/ export const getQueryFilter = ( query: Query, language: Language, - filters: Array>, + filters: unknown, index: Index, lists: Array, excludeExceptions: boolean = true @@ -48,7 +48,7 @@ export const getQueryFilter = ( chunkSize: 1024, }); const initialQuery = { query, language }; - const allFilters = getAllFilters((filters as unknown) as Filter[], exceptionFilter); + const allFilters = getAllFilters(filters as Filter[], exceptionFilter); return buildEsQuery(indexPattern, initialQuery, allFilters, config); }; diff --git a/x-pack/plugins/security_solution/common/validate.ts b/x-pack/plugins/security_solution/common/validate.ts index 22e4179ae7050..79a0351b824e8 100644 --- a/x-pack/plugins/security_solution/common/validate.ts +++ b/x-pack/plugins/security_solution/common/validate.ts @@ -26,6 +26,19 @@ export const validate = ( return pipe(checked, fold(left, right)); }; +export const validateNonExact = ( + obj: object, + schema: T +): [t.TypeOf | null, string | null] => { + const decoded = schema.decode(obj); + const left = (errors: t.Errors): [T | null, string | null] => [ + null, + formatErrors(errors).join(','), + ]; + const right = (output: T): [T | null, string | null] => [output, null]; + return pipe(decoded, fold(left, right)); +}; + export const validateEither = ( schema: T, obj: A diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/schemas/rule_schemas.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/schemas/rule_schemas.mock.ts new file mode 100644 index 0000000000000..a855bcb7cb6d0 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/schemas/rule_schemas.mock.ts @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getListArrayMock } from '../../../../common/detection_engine/schemas/types/lists.mock'; +import { + BaseRuleParams, + EqlRuleParams, + MachineLearningRuleParams, + ThresholdRuleParams, +} from './rule_schemas'; + +const getBaseRuleParams = (): BaseRuleParams => { + return { + author: ['Elastic'], + buildingBlockType: 'default', + ruleId: 'rule-1', + description: 'Detecting root and admin users', + falsePositives: [], + immutable: false, + from: 'now-6m', + to: 'now', + severity: 'high', + severityMapping: [], + license: 'Elastic License', + outputIndex: '.siem-signals', + references: ['http://google.com'], + riskScore: 50, + riskScoreMapping: [], + ruleNameOverride: undefined, + maxSignals: 10000, + note: '', + timelineId: undefined, + timelineTitle: undefined, + timestampOverride: undefined, + meta: undefined, + threat: [], + version: 1, + exceptionsList: getListArrayMock(), + }; +}; + +export const getThresholdRuleParams = (): ThresholdRuleParams => { + return { + ...getBaseRuleParams(), + type: 'threshold', + language: 'kuery', + index: ['some-index'], + query: 'host.name: *', + filters: undefined, + savedId: undefined, + threshold: { + field: 'host.id', + value: 5, + }, + }; +}; + +export const getEqlRuleParams = (): EqlRuleParams => { + return { + ...getBaseRuleParams(), + type: 'eql', + language: 'eql', + index: ['some-index'], + query: 'any where true', + filters: undefined, + eventCategoryOverride: undefined, + }; +}; + +export const getMlRuleParams = (): MachineLearningRuleParams => { + return { + ...getBaseRuleParams(), + type: 'machine_learning', + anomalyThreshold: 42, + machineLearningJobId: 'my-job', + }; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/schemas/rule_schemas.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/schemas/rule_schemas.ts index abbcfcaa79107..144b751491b2c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/schemas/rule_schemas.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/schemas/rule_schemas.ts @@ -104,6 +104,8 @@ const eqlSpecificRuleParams = t.type({ filters: filtersOrUndefined, eventCategoryOverride: eventCategoryOverrideOrUndefined, }); +export const eqlRuleParams = t.intersection([baseRuleParams, eqlSpecificRuleParams]); +export type EqlRuleParams = t.TypeOf; const threatSpecificRuleParams = t.type({ type: t.literal('threat_match'), @@ -121,6 +123,8 @@ const threatSpecificRuleParams = t.type({ concurrentSearches: concurrentSearchesOrUndefined, itemsPerSearch: itemsPerSearchOrUndefined, }); +export const threatRuleParams = t.intersection([baseRuleParams, threatSpecificRuleParams]); +export type ThreatRuleParams = t.TypeOf; const querySpecificRuleParams = t.exact( t.type({ @@ -132,6 +136,8 @@ const querySpecificRuleParams = t.exact( savedId: savedIdOrUndefined, }) ); +export const queryRuleParams = t.intersection([baseRuleParams, querySpecificRuleParams]); +export type QueryRuleParams = t.TypeOf; const savedQuerySpecificRuleParams = t.type({ type: t.literal('saved_query'), @@ -143,6 +149,8 @@ const savedQuerySpecificRuleParams = t.type({ filters: filtersOrUndefined, savedId: saved_id, }); +export const savedQueryRuleParams = t.intersection([baseRuleParams, savedQuerySpecificRuleParams]); +export type SavedQueryRuleParams = t.TypeOf; const thresholdSpecificRuleParams = t.type({ type: t.literal('threshold'), @@ -153,12 +161,19 @@ const thresholdSpecificRuleParams = t.type({ savedId: savedIdOrUndefined, threshold, }); +export const thresholdRuleParams = t.intersection([baseRuleParams, thresholdSpecificRuleParams]); +export type ThresholdRuleParams = t.TypeOf; const machineLearningSpecificRuleParams = t.type({ type: t.literal('machine_learning'), anomalyThreshold: anomaly_threshold, machineLearningJobId: machine_learning_job_id, }); +export const machineLearningRuleParams = t.intersection([ + baseRuleParams, + machineLearningSpecificRuleParams, +]); +export type MachineLearningRuleParams = t.TypeOf; export const typeSpecificRuleParams = t.union([ eqlSpecificRuleParams, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.test.ts new file mode 100644 index 0000000000000..f8f77bd2bf6e6 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.test.ts @@ -0,0 +1,94 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { loggingSystemMock } from 'src/core/server/mocks'; +import { alertsMock, AlertServicesMock } from '../../../../../../alerting/server/mocks'; +import { RuleStatusService } from '../rule_status_service'; +import { eqlExecutor } from './eql'; +import { getExceptionListItemSchemaMock } from '../../../../../../lists/common/schemas/response/exception_list_item_schema.mock'; +import { getEntryListMock } from '../../../../../../lists/common/schemas/types/entry_list.mock'; +import { getEqlRuleParams } from '../../schemas/rule_schemas.mock'; +import { getIndexVersion } from '../../routes/index/get_index_version'; +import { SIGNALS_TEMPLATE_VERSION } from '../../routes/index/get_signals_template'; +// eslint-disable-next-line @kbn/eslint/no-restricted-paths +import { elasticsearchClientMock } from 'src/core/server/elasticsearch/client/mocks'; + +jest.mock('../../routes/index/get_index_version'); + +describe('eql_executor', () => { + const version = '8.0.0'; + let logger: ReturnType; + let alertServices: AlertServicesMock; + let ruleStatusService: Record; + (getIndexVersion as jest.Mock).mockReturnValue(SIGNALS_TEMPLATE_VERSION); + const eqlSO = { + id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + type: 'alert', + version: '1', + updated_at: '2020-03-27T22:55:59.577Z', + attributes: { + actions: [], + enabled: true, + name: 'rule-name', + tags: ['some fake tag 1', 'some fake tag 2'], + createdBy: 'sample user', + createdAt: '2020-03-27T22:55:59.577Z', + updatedBy: 'sample user', + schedule: { + interval: '5m', + }, + throttle: 'no_actions', + params: getEqlRuleParams(), + }, + references: [], + }; + const searchAfterSize = 7; + + beforeEach(() => { + alertServices = alertsMock.createAlertServices(); + logger = loggingSystemMock.createLogger(); + ruleStatusService = { + success: jest.fn(), + find: jest.fn(), + goingToRun: jest.fn(), + error: jest.fn(), + partialFailure: jest.fn(), + }; + alertServices.scopedClusterClient.asCurrentUser.transport.request.mockResolvedValue( + elasticsearchClientMock.createSuccessTransportRequestPromise({ + hits: { + total: { value: 10 }, + }, + }) + ); + }); + + describe('eqlExecutor', () => { + it('should set a warning when exception list for eql rule contains value list exceptions', async () => { + const exceptionItems = [getExceptionListItemSchemaMock({ entries: [getEntryListMock()] })]; + try { + await eqlExecutor({ + rule: eqlSO, + exceptionItems, + ruleStatusService: (ruleStatusService as unknown) as RuleStatusService, + services: alertServices, + version, + logger, + refresh: false, + searchAfterSize, + }); + } catch (err) { + // eqlExecutor will throw until we have an EQL response mock that conforms to the + // expected EQL response format, so just catch the error and check the status service + } + expect(ruleStatusService.partialFailure).toHaveBeenCalled(); + expect(ruleStatusService.partialFailure.mock.calls[0][0]).toContain( + 'Exceptions that use "is in list" or "is not in list" operators are not applied to EQL rules' + ); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.ts new file mode 100644 index 0000000000000..a4763f67004f6 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.ts @@ -0,0 +1,129 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ApiResponse } from '@elastic/elasticsearch'; +import { performance } from 'perf_hooks'; +import { Logger } from 'src/core/server'; +import { SavedObject } from 'src/core/types'; +import { + AlertInstanceContext, + AlertInstanceState, + AlertServices, +} from '../../../../../../alerting/server'; +import { buildEqlSearchRequest } from '../../../../../common/detection_engine/get_query_filter'; +import { hasLargeValueItem } from '../../../../../common/detection_engine/utils'; +import { ExceptionListItemSchema } from '../../../../../common/shared_imports'; +import { isOutdated } from '../../migrations/helpers'; +import { getIndexVersion } from '../../routes/index/get_index_version'; +import { MIN_EQL_RULE_INDEX_VERSION } from '../../routes/index/get_signals_template'; +import { RefreshTypes } from '../../types'; +import { buildSignalFromEvent, buildSignalGroupFromSequence } from '../build_bulk_body'; +import { getInputIndex } from '../get_input_output_index'; +import { RuleStatusService } from '../rule_status_service'; +import { bulkInsertSignals, filterDuplicateSignals } from '../single_bulk_create'; +import { + EqlRuleAttributes, + EqlSignalSearchResponse, + SearchAfterAndBulkCreateReturnType, + WrappedSignalHit, +} from '../types'; +import { createSearchAfterReturnType, makeFloatString, wrapSignal } from '../utils'; + +export const eqlExecutor = async ({ + rule, + exceptionItems, + ruleStatusService, + services, + version, + searchAfterSize, + logger, + refresh, +}: { + rule: SavedObject; + exceptionItems: ExceptionListItemSchema[]; + ruleStatusService: RuleStatusService; + services: AlertServices; + version: string; + searchAfterSize: number; + logger: Logger; + refresh: RefreshTypes; +}): Promise => { + const result = createSearchAfterReturnType(); + const ruleParams = rule.attributes.params; + if (hasLargeValueItem(exceptionItems)) { + await ruleStatusService.partialFailure( + 'Exceptions that use "is in list" or "is not in list" operators are not applied to EQL rules' + ); + result.warning = true; + } + try { + const signalIndexVersion = await getIndexVersion( + services.scopedClusterClient.asCurrentUser, + ruleParams.outputIndex + ); + if (isOutdated({ current: signalIndexVersion, target: MIN_EQL_RULE_INDEX_VERSION })) { + throw new Error( + `EQL based rules require an update to version ${MIN_EQL_RULE_INDEX_VERSION} of the detection alerts index mapping` + ); + } + } catch (err) { + if (err.statusCode === 403) { + throw new Error( + `EQL based rules require the user that created it to have the view_index_metadata, read, and write permissions for index: ${ruleParams.outputIndex}` + ); + } else { + throw err; + } + } + const inputIndex = await getInputIndex(services, version, ruleParams.index); + const request = buildEqlSearchRequest( + ruleParams.query, + inputIndex, + ruleParams.from, + ruleParams.to, + searchAfterSize, + ruleParams.timestampOverride, + exceptionItems, + ruleParams.eventCategoryOverride + ); + const eqlSignalSearchStart = performance.now(); + // TODO: fix this later + const { body: response } = (await services.scopedClusterClient.asCurrentUser.transport.request( + request + )) as ApiResponse; + const eqlSignalSearchEnd = performance.now(); + const eqlSearchDuration = makeFloatString(eqlSignalSearchEnd - eqlSignalSearchStart); + result.searchAfterTimes = [eqlSearchDuration]; + let newSignals: WrappedSignalHit[] | undefined; + if (response.hits.sequences !== undefined) { + newSignals = response.hits.sequences.reduce( + (acc: WrappedSignalHit[], sequence) => + acc.concat(buildSignalGroupFromSequence(sequence, rule, ruleParams.outputIndex)), + [] + ); + } else if (response.hits.events !== undefined) { + newSignals = filterDuplicateSignals( + rule.id, + response.hits.events.map((event) => + wrapSignal(buildSignalFromEvent(event, rule, true), ruleParams.outputIndex) + ) + ); + } else { + throw new Error( + 'eql query response should have either `sequences` or `events` but had neither' + ); + } + + if (newSignals.length > 0) { + const insertResult = await bulkInsertSignals(newSignals, logger, services, refresh); + result.bulkCreateTimes.push(insertResult.bulkCreateDuration); + result.createdSignalsCount += insertResult.createdItemsCount; + result.createdSignals = insertResult.createdItems; + } + result.success = true; + return result; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.test.ts new file mode 100644 index 0000000000000..a3db2e5cbfd99 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.test.ts @@ -0,0 +1,159 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { loggingSystemMock } from 'src/core/server/mocks'; +import { alertsMock, AlertServicesMock } from '../../../../../../alerting/server/mocks'; +import { RuleStatusService } from '../rule_status_service'; +import { mlExecutor } from './ml'; +import { getExceptionListItemSchemaMock } from '../../../../../../lists/common/schemas/response/exception_list_item_schema.mock'; +import { getMlRuleParams } from '../../schemas/rule_schemas.mock'; +import { buildRuleMessageFactory } from '../rule_messages'; +import { getListClientMock } from '../../../../../../lists/server/services/lists/list_client.mock'; +import { findMlSignals } from '../find_ml_signals'; +import { bulkCreateMlSignals } from '../bulk_create_ml_signals'; + +jest.mock('../find_ml_signals'); +jest.mock('../bulk_create_ml_signals'); + +describe('ml_executor', () => { + const jobsSummaryMock = jest.fn(); + const mlMock = { + mlClient: { + callAsInternalUser: jest.fn(), + close: jest.fn(), + asScoped: jest.fn(), + }, + jobServiceProvider: jest.fn().mockReturnValue({ + jobsSummary: jobsSummaryMock, + }), + anomalyDetectorsProvider: jest.fn(), + mlSystemProvider: jest.fn(), + modulesProvider: jest.fn(), + resultsServiceProvider: jest.fn(), + alertingServiceProvider: jest.fn(), + }; + const exceptionItems = [getExceptionListItemSchemaMock()]; + let logger: ReturnType; + let alertServices: AlertServicesMock; + let ruleStatusService: Record; + const mlSO = { + id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + type: 'alert', + version: '1', + updated_at: '2020-03-27T22:55:59.577Z', + attributes: { + actions: [], + enabled: true, + name: 'rule-name', + tags: ['some fake tag 1', 'some fake tag 2'], + createdBy: 'sample user', + createdAt: '2020-03-27T22:55:59.577Z', + updatedBy: 'sample user', + schedule: { + interval: '5m', + }, + throttle: 'no_actions', + params: getMlRuleParams(), + }, + references: [], + }; + const buildRuleMessage = buildRuleMessageFactory({ + id: mlSO.id, + ruleId: mlSO.attributes.params.ruleId, + name: mlSO.attributes.name, + index: mlSO.attributes.params.outputIndex, + }); + + beforeEach(() => { + alertServices = alertsMock.createAlertServices(); + logger = loggingSystemMock.createLogger(); + ruleStatusService = { + success: jest.fn(), + find: jest.fn(), + goingToRun: jest.fn(), + error: jest.fn(), + partialFailure: jest.fn(), + }; + (findMlSignals as jest.Mock).mockResolvedValue({ + _shards: {}, + hits: { + hits: [], + }, + }); + (bulkCreateMlSignals as jest.Mock).mockResolvedValue({ + success: true, + bulkCreateDuration: 0, + createdItemsCount: 0, + errors: [], + }); + }); + + it('should throw an error if ML plugin was not available', async () => { + await expect( + mlExecutor({ + rule: mlSO, + ml: undefined, + exceptionItems, + ruleStatusService: (ruleStatusService as unknown) as RuleStatusService, + services: alertServices, + logger, + refresh: false, + buildRuleMessage, + listClient: getListClientMock(), + }) + ).rejects.toThrow('ML plugin unavailable during rule execution'); + }); + + it('should throw an error if Machine learning job summary was null', async () => { + jobsSummaryMock.mockResolvedValue([]); + await mlExecutor({ + rule: mlSO, + ml: mlMock, + exceptionItems, + ruleStatusService: (ruleStatusService as unknown) as RuleStatusService, + services: alertServices, + logger, + refresh: false, + buildRuleMessage, + listClient: getListClientMock(), + }); + expect(logger.warn).toHaveBeenCalled(); + expect(logger.warn.mock.calls[0][0]).toContain('Machine learning job is not started'); + expect(ruleStatusService.error).toHaveBeenCalled(); + expect(ruleStatusService.error.mock.calls[0][0]).toContain( + 'Machine learning job is not started' + ); + }); + + it('should log an error if Machine learning job was not started', async () => { + jobsSummaryMock.mockResolvedValue([ + { + id: 'some_job_id', + jobState: 'starting', + datafeedState: 'started', + }, + ]); + + await mlExecutor({ + rule: mlSO, + ml: mlMock, + exceptionItems, + ruleStatusService: (ruleStatusService as unknown) as RuleStatusService, + services: alertServices, + logger, + refresh: false, + buildRuleMessage, + listClient: getListClientMock(), + }); + expect(logger.warn).toHaveBeenCalled(); + expect(logger.warn.mock.calls[0][0]).toContain('Machine learning job is not started'); + expect(ruleStatusService.error).toHaveBeenCalled(); + expect(ruleStatusService.error.mock.calls[0][0]).toContain( + 'Machine learning job is not started' + ); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.ts new file mode 100644 index 0000000000000..12ebca1aa3e7c --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.ts @@ -0,0 +1,145 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { KibanaRequest, Logger } from 'src/core/server'; +import { SavedObject } from 'src/core/types'; +import { + AlertInstanceContext, + AlertInstanceState, + AlertServices, +} from '../../../../../../alerting/server'; +import { ListClient } from '../../../../../../lists/server'; +import { isJobStarted } from '../../../../../common/machine_learning/helpers'; +import { ExceptionListItemSchema } from '../../../../../common/shared_imports'; +import { SetupPlugins } from '../../../../plugin'; +import { RefreshTypes } from '../../types'; +import { bulkCreateMlSignals } from '../bulk_create_ml_signals'; +import { filterEventsAgainstList } from '../filters/filter_events_against_list'; +import { findMlSignals } from '../find_ml_signals'; +import { BuildRuleMessage } from '../rule_messages'; +import { RuleStatusService } from '../rule_status_service'; +import { MachineLearningRuleAttributes } from '../types'; +import { createErrorsFromShard, createSearchAfterReturnType, mergeReturns } from '../utils'; + +export const mlExecutor = async ({ + rule, + ml, + listClient, + exceptionItems, + ruleStatusService, + services, + logger, + refresh, + buildRuleMessage, +}: { + rule: SavedObject; + ml: SetupPlugins['ml']; + listClient: ListClient; + exceptionItems: ExceptionListItemSchema[]; + ruleStatusService: RuleStatusService; + services: AlertServices; + logger: Logger; + refresh: RefreshTypes; + buildRuleMessage: BuildRuleMessage; +}) => { + const result = createSearchAfterReturnType(); + const ruleParams = rule.attributes.params; + if (ml == null) { + throw new Error('ML plugin unavailable during rule execution'); + } + + // Using fake KibanaRequest as it is needed to satisfy the ML Services API, but can be empty as it is + // currently unused by the jobsSummary function. + const fakeRequest = {} as KibanaRequest; + const summaryJobs = await ml + .jobServiceProvider(fakeRequest, services.savedObjectsClient) + .jobsSummary([ruleParams.machineLearningJobId]); + const jobSummary = summaryJobs.find((job) => job.id === ruleParams.machineLearningJobId); + + if (jobSummary == null || !isJobStarted(jobSummary.jobState, jobSummary.datafeedState)) { + const errorMessage = buildRuleMessage( + 'Machine learning job is not started:', + `job id: "${ruleParams.machineLearningJobId}"`, + `job status: "${jobSummary?.jobState}"`, + `datafeed status: "${jobSummary?.datafeedState}"` + ); + logger.warn(errorMessage); + result.warning = true; + // TODO: change this to partialFailure since we don't immediately exit rule function and still do actions at the end? + await ruleStatusService.error(errorMessage); + } + + const anomalyResults = await findMlSignals({ + ml, + // Using fake KibanaRequest as it is needed to satisfy the ML Services API, but can be empty as it is + // currently unused by the mlAnomalySearch function. + request: ({} as unknown) as KibanaRequest, + savedObjectsClient: services.savedObjectsClient, + jobId: ruleParams.machineLearningJobId, + anomalyThreshold: ruleParams.anomalyThreshold, + from: ruleParams.from, + to: ruleParams.to, + exceptionItems, + }); + + const filteredAnomalyResults = await filterEventsAgainstList({ + listClient, + exceptionsList: exceptionItems, + logger, + eventSearchResult: anomalyResults, + buildRuleMessage, + }); + + const anomalyCount = filteredAnomalyResults.hits.hits.length; + if (anomalyCount) { + logger.info(buildRuleMessage(`Found ${anomalyCount} signals from ML anomalies.`)); + } + const { + success, + errors, + bulkCreateDuration, + createdItemsCount, + createdItems, + } = await bulkCreateMlSignals({ + actions: rule.attributes.actions, + throttle: rule.attributes.throttle, + someResult: filteredAnomalyResults, + ruleParams, + services, + logger, + id: rule.id, + signalsIndex: ruleParams.outputIndex, + name: rule.attributes.name, + createdBy: rule.attributes.createdBy, + createdAt: rule.attributes.createdAt, + updatedBy: rule.attributes.updatedBy, + updatedAt: rule.updated_at ?? '', + interval: rule.attributes.schedule.interval, + enabled: rule.attributes.enabled, + refresh, + tags: rule.attributes.tags, + buildRuleMessage, + }); + // The legacy ES client does not define failures when it can be present on the structure, hence why I have the & { failures: [] } + const shardFailures = + (filteredAnomalyResults._shards as typeof filteredAnomalyResults._shards & { + failures: []; + }).failures ?? []; + const searchErrors = createErrorsFromShard({ + errors: shardFailures, + }); + return mergeReturns([ + result, + createSearchAfterReturnType({ + success: success && filteredAnomalyResults._shards.failed === 0, + errors: [...errors, ...searchErrors], + createdSignalsCount: createdItemsCount, + createdSignals: createdItems, + bulkCreateTimes: bulkCreateDuration ? [bulkCreateDuration] : [], + }), + ]); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/query.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/query.ts new file mode 100644 index 0000000000000..9914eb04c6ca6 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/query.ts @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SavedObject } from 'src/core/types'; +import { Logger } from 'src/core/server'; +import { + AlertInstanceContext, + AlertInstanceState, + AlertServices, +} from '../../../../../../alerting/server'; +import { ListClient } from '../../../../../../lists/server'; +import { ExceptionListItemSchema } from '../../../../../common/shared_imports'; +import { RefreshTypes } from '../../types'; +import { getFilter } from '../get_filter'; +import { getInputIndex } from '../get_input_output_index'; +import { searchAfterAndBulkCreate } from '../search_after_bulk_create'; +import { QueryRuleAttributes, RuleRangeTuple } from '../types'; +import { TelemetryEventsSender } from '../../../telemetry/sender'; +import { BuildRuleMessage } from '../rule_messages'; + +export const queryExecutor = async ({ + rule, + tuples, + listClient, + exceptionItems, + services, + version, + searchAfterSize, + logger, + refresh, + eventsTelemetry, + buildRuleMessage, +}: { + rule: SavedObject; + tuples: RuleRangeTuple[]; + listClient: ListClient; + exceptionItems: ExceptionListItemSchema[]; + services: AlertServices; + version: string; + searchAfterSize: number; + logger: Logger; + refresh: RefreshTypes; + eventsTelemetry: TelemetryEventsSender | undefined; + buildRuleMessage: BuildRuleMessage; +}) => { + const ruleParams = rule.attributes.params; + const inputIndex = await getInputIndex(services, version, ruleParams.index); + const esFilter = await getFilter({ + type: ruleParams.type, + filters: ruleParams.filters, + language: ruleParams.language, + query: ruleParams.query, + savedId: ruleParams.savedId, + services, + index: inputIndex, + lists: exceptionItems, + }); + + return searchAfterAndBulkCreate({ + tuples, + listClient, + exceptionsList: exceptionItems, + ruleParams, + services, + logger, + eventsTelemetry, + id: rule.id, + inputIndexPattern: inputIndex, + signalsIndex: ruleParams.outputIndex, + filter: esFilter, + actions: rule.attributes.actions, + name: rule.attributes.name, + createdBy: rule.attributes.createdBy, + createdAt: rule.attributes.createdAt, + updatedBy: rule.attributes.updatedBy, + updatedAt: rule.updated_at ?? '', + interval: rule.attributes.schedule.interval, + enabled: rule.attributes.enabled, + pageSize: searchAfterSize, + refresh, + tags: rule.attributes.tags, + throttle: rule.attributes.throttle, + buildRuleMessage, + }); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threat_match.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threat_match.ts new file mode 100644 index 0000000000000..5a8e945c3b06e --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threat_match.ts @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SavedObject } from 'src/core/types'; +import { Logger } from 'src/core/server'; +import { + AlertInstanceContext, + AlertInstanceState, + AlertServices, +} from '../../../../../../alerting/server'; +import { ListClient } from '../../../../../../lists/server'; +import { ExceptionListItemSchema } from '../../../../../common/shared_imports'; +import { RefreshTypes } from '../../types'; +import { getInputIndex } from '../get_input_output_index'; +import { RuleRangeTuple, ThreatRuleAttributes } from '../types'; +import { TelemetryEventsSender } from '../../../telemetry/sender'; +import { BuildRuleMessage } from '../rule_messages'; +import { createThreatSignals } from '../threat_mapping/create_threat_signals'; + +export const threatMatchExecutor = async ({ + rule, + tuples, + listClient, + exceptionItems, + services, + version, + searchAfterSize, + logger, + refresh, + eventsTelemetry, + buildRuleMessage, +}: { + rule: SavedObject; + tuples: RuleRangeTuple[]; + listClient: ListClient; + exceptionItems: ExceptionListItemSchema[]; + services: AlertServices; + version: string; + searchAfterSize: number; + logger: Logger; + refresh: RefreshTypes; + eventsTelemetry: TelemetryEventsSender | undefined; + buildRuleMessage: BuildRuleMessage; +}) => { + const ruleParams = rule.attributes.params; + const inputIndex = await getInputIndex(services, version, ruleParams.index); + return createThreatSignals({ + tuples, + threatMapping: ruleParams.threatMapping, + query: ruleParams.query, + inputIndex, + type: ruleParams.type, + filters: ruleParams.filters ?? [], + language: ruleParams.language, + name: rule.attributes.name, + savedId: ruleParams.savedId, + services, + exceptionItems, + listClient, + logger, + eventsTelemetry, + alertId: rule.id, + outputIndex: ruleParams.outputIndex, + params: ruleParams, + searchAfterSize, + actions: rule.attributes.actions, + createdBy: rule.attributes.createdBy, + createdAt: rule.attributes.createdAt, + updatedBy: rule.attributes.updatedBy, + interval: rule.attributes.schedule.interval, + updatedAt: rule.updated_at ?? '', + enabled: rule.attributes.enabled, + refresh, + tags: rule.attributes.tags, + throttle: rule.attributes.throttle, + threatFilters: ruleParams.threatFilters ?? [], + threatQuery: ruleParams.threatQuery, + threatLanguage: ruleParams.threatLanguage, + buildRuleMessage, + threatIndex: ruleParams.threatIndex, + threatIndicatorPath: ruleParams.threatIndicatorPath, + concurrentSearches: ruleParams.concurrentSearches ?? 1, + itemsPerSearch: ruleParams.itemsPerSearch ?? 9000, + }); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.test.ts new file mode 100644 index 0000000000000..5d62b28b73ae8 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.test.ts @@ -0,0 +1,83 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { loggingSystemMock } from 'src/core/server/mocks'; +import { alertsMock, AlertServicesMock } from '../../../../../../alerting/server/mocks'; +import { RuleStatusService } from '../rule_status_service'; +import { thresholdExecutor } from './threshold'; +import { getExceptionListItemSchemaMock } from '../../../../../../lists/common/schemas/response/exception_list_item_schema.mock'; +import { getEntryListMock } from '../../../../../../lists/common/schemas/types/entry_list.mock'; +import { getThresholdRuleParams } from '../../schemas/rule_schemas.mock'; +import { buildRuleMessageFactory } from '../rule_messages'; + +describe('threshold_executor', () => { + const version = '8.0.0'; + let logger: ReturnType; + let alertServices: AlertServicesMock; + let ruleStatusService: Record; + const thresholdSO = { + id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + type: 'alert', + version: '1', + updated_at: '2020-03-27T22:55:59.577Z', + attributes: { + actions: [], + enabled: true, + name: 'rule-name', + tags: ['some fake tag 1', 'some fake tag 2'], + createdBy: 'sample user', + createdAt: '2020-03-27T22:55:59.577Z', + updatedBy: 'sample user', + schedule: { + interval: '5m', + }, + throttle: 'no_actions', + params: getThresholdRuleParams(), + }, + references: [], + }; + const buildRuleMessage = buildRuleMessageFactory({ + id: thresholdSO.id, + ruleId: thresholdSO.attributes.params.ruleId, + name: thresholdSO.attributes.name, + index: thresholdSO.attributes.params.outputIndex, + }); + + beforeEach(() => { + alertServices = alertsMock.createAlertServices(); + logger = loggingSystemMock.createLogger(); + ruleStatusService = { + success: jest.fn(), + find: jest.fn(), + goingToRun: jest.fn(), + error: jest.fn(), + partialFailure: jest.fn(), + }; + }); + + describe('thresholdExecutor', () => { + it('should set a warning when exception list for threshold rule contains value list exceptions', async () => { + const exceptionItems = [getExceptionListItemSchemaMock({ entries: [getEntryListMock()] })]; + await thresholdExecutor({ + rule: thresholdSO, + tuples: [], + exceptionItems, + ruleStatusService: (ruleStatusService as unknown) as RuleStatusService, + services: alertServices, + version, + logger, + refresh: false, + buildRuleMessage, + startedAt: new Date(), + }); + expect(ruleStatusService.partialFailure).toHaveBeenCalled(); + expect(ruleStatusService.partialFailure.mock.calls[0][0]).toContain( + 'Exceptions that use "is in list" or "is not in list" operators are not applied to Threshold rules' + ); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.ts new file mode 100644 index 0000000000000..c8f70449251f6 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.ts @@ -0,0 +1,173 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Logger } from 'src/core/server'; +import { SavedObject } from 'src/core/types'; +import { + AlertInstanceContext, + AlertInstanceState, + AlertServices, +} from '../../../../../../alerting/server'; +import { + hasLargeValueItem, + normalizeThresholdField, +} from '../../../../../common/detection_engine/utils'; +import { ExceptionListItemSchema } from '../../../../../common/shared_imports'; +import { RefreshTypes } from '../../types'; +import { getFilter } from '../get_filter'; +import { getInputIndex } from '../get_input_output_index'; +import { BuildRuleMessage } from '../rule_messages'; +import { RuleStatusService } from '../rule_status_service'; +import { + bulkCreateThresholdSignals, + findThresholdSignals, + getThresholdBucketFilters, + getThresholdSignalHistory, +} from '../threshold'; +import { + RuleRangeTuple, + SearchAfterAndBulkCreateReturnType, + ThresholdRuleAttributes, +} from '../types'; +import { + createSearchAfterReturnType, + createSearchAfterReturnTypeFromResponse, + mergeReturns, +} from '../utils'; + +export const thresholdExecutor = async ({ + rule, + tuples, + exceptionItems, + ruleStatusService, + services, + version, + logger, + refresh, + buildRuleMessage, + startedAt, +}: { + rule: SavedObject; + tuples: RuleRangeTuple[]; + exceptionItems: ExceptionListItemSchema[]; + ruleStatusService: RuleStatusService; + services: AlertServices; + version: string; + logger: Logger; + refresh: RefreshTypes; + buildRuleMessage: BuildRuleMessage; + startedAt: Date; +}): Promise => { + let result = createSearchAfterReturnType(); + const ruleParams = rule.attributes.params; + if (hasLargeValueItem(exceptionItems)) { + await ruleStatusService.partialFailure( + 'Exceptions that use "is in list" or "is not in list" operators are not applied to Threshold rules' + ); + result.warning = true; + } + const inputIndex = await getInputIndex(services, version, ruleParams.index); + + for (const tuple of tuples) { + const { + thresholdSignalHistory, + searchErrors: previousSearchErrors, + } = await getThresholdSignalHistory({ + indexPattern: [ruleParams.outputIndex], + from: tuple.from.toISOString(), + to: tuple.to.toISOString(), + services, + logger, + ruleId: ruleParams.ruleId, + bucketByFields: normalizeThresholdField(ruleParams.threshold.field), + timestampOverride: ruleParams.timestampOverride, + buildRuleMessage, + }); + + const bucketFilters = await getThresholdBucketFilters({ + thresholdSignalHistory, + timestampOverride: ruleParams.timestampOverride, + }); + + const esFilter = await getFilter({ + type: ruleParams.type, + filters: ruleParams.filters ? ruleParams.filters.concat(bucketFilters) : bucketFilters, + language: ruleParams.language, + query: ruleParams.query, + savedId: ruleParams.savedId, + services, + index: inputIndex, + lists: exceptionItems, + }); + + const { + searchResult: thresholdResults, + searchErrors, + searchDuration: thresholdSearchDuration, + } = await findThresholdSignals({ + inputIndexPattern: inputIndex, + from: tuple.from.toISOString(), + to: tuple.to.toISOString(), + services, + logger, + filter: esFilter, + threshold: ruleParams.threshold, + timestampOverride: ruleParams.timestampOverride, + buildRuleMessage, + }); + + const { + success, + bulkCreateDuration, + createdItemsCount, + createdItems, + errors, + } = await bulkCreateThresholdSignals({ + actions: rule.attributes.actions, + throttle: rule.attributes.throttle, + someResult: thresholdResults, + ruleParams, + filter: esFilter, + services, + logger, + id: rule.id, + inputIndexPattern: inputIndex, + signalsIndex: ruleParams.outputIndex, + timestampOverride: ruleParams.timestampOverride, + startedAt, + from: tuple.from.toDate(), + name: rule.attributes.name, + createdBy: rule.attributes.createdBy, + createdAt: rule.attributes.createdAt, + updatedBy: rule.attributes.updatedBy, + updatedAt: rule.updated_at ?? '', + interval: rule.attributes.schedule.interval, + enabled: rule.attributes.enabled, + refresh, + tags: rule.attributes.tags, + thresholdSignalHistory, + buildRuleMessage, + }); + + result = mergeReturns([ + result, + createSearchAfterReturnTypeFromResponse({ + searchResult: thresholdResults, + timestampOverride: ruleParams.timestampOverride, + }), + createSearchAfterReturnType({ + success, + errors: [...errors, ...previousSearchErrors, ...searchErrors], + createdSignalsCount: createdItemsCount, + createdSignals: createdItems, + bulkCreateTimes: bulkCreateDuration ? [bulkCreateDuration] : [], + searchAfterTimes: [thresholdSearchDuration], + }), + ]); + } + return result; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.ts index d8ee4619fb4da..86940e9b77084 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.ts @@ -27,7 +27,7 @@ import { QueryFilter } from './types'; interface GetFilterArgs { type: Type; - filters: PartialFilter[] | undefined; + filters: unknown | undefined; language: LanguageOrUndefined; query: QueryOrUndefined; savedId: SavedIdOrUndefined; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts index 930cafe57fb0a..ba7776af9d36a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts @@ -8,36 +8,32 @@ import moment from 'moment'; import type { estypes } from '@elastic/elasticsearch'; import { loggingSystemMock } from 'src/core/server/mocks'; -import { - getResult, - getMlResult, - getThresholdResult, - getEqlResult, -} from '../routes/__mocks__/request_responses'; +import { getResult, getMlResult } from '../routes/__mocks__/request_responses'; import { signalRulesAlertType } from './signal_rule_alert_type'; import { alertsMock, AlertServicesMock } from '../../../../../alerting/server/mocks'; import { ruleStatusServiceFactory } from './rule_status_service'; -import { getListsClient, getExceptions, sortExceptionItems, checkPrivileges } from './utils'; -import { parseScheduleDates } from '../../../../common/detection_engine/parse_schedule_dates'; +import { + getListsClient, + getExceptions, + checkPrivileges, + createSearchAfterReturnType, +} from './utils'; +import * as parseScheduleDates from '../../../../common/detection_engine/parse_schedule_dates'; import { RuleExecutorOptions, SearchAfterAndBulkCreateReturnType } from './types'; -import { searchAfterAndBulkCreate } from './search_after_bulk_create'; import { scheduleNotificationActions } from '../notifications/schedule_notification_actions'; import { RuleAlertType } from '../rules/types'; -import { findMlSignals } from './find_ml_signals'; -import { bulkCreateMlSignals } from './bulk_create_ml_signals'; import { listMock } from '../../../../../lists/server/mocks'; import { getListClientMock } from '../../../../../lists/server/services/lists/list_client.mock'; import { getExceptionListClientMock } from '../../../../../lists/server/services/exception_lists/exception_list_client.mock'; import { getExceptionListItemSchemaMock } from '../../../../../lists/common/schemas/response/exception_list_item_schema.mock'; import { ApiResponse } from '@elastic/elasticsearch/lib/Transport'; -import { getEntryListMock } from '../../../../../lists/common/schemas/types/entry_list.mock'; // eslint-disable-next-line @kbn/eslint/no-restricted-paths import { elasticsearchClientMock } from 'src/core/server/elasticsearch/client/mocks'; +import { queryExecutor } from './executors/query'; +import { mlExecutor } from './executors/ml'; jest.mock('./rule_status_saved_objects_client'); jest.mock('./rule_status_service'); -jest.mock('./search_after_bulk_create'); -jest.mock('./get_filter'); jest.mock('./utils', () => { const original = jest.requireActual('./utils'); return { @@ -49,9 +45,8 @@ jest.mock('./utils', () => { }; }); jest.mock('../notifications/schedule_notification_actions'); -jest.mock('./find_ml_signals'); -jest.mock('./bulk_create_ml_signals'); -jest.mock('../../../../common/detection_engine/parse_schedule_dates'); +jest.mock('./executors/query'); +jest.mock('./executors/ml'); const getPayload = ( ruleAlert: RuleAlertType, @@ -78,7 +73,7 @@ const getPayload = ( updatedBy: 'elastic', }); -describe('rules_notification_alert_type', () => { +describe('signal_rule_alert_type', () => { const version = '8.0.0'; const jobsSummaryMock = jest.fn(); const mlMock = { @@ -118,16 +113,6 @@ describe('rules_notification_alert_type', () => { exceptionsClient: getExceptionListClientMock(), }); (getExceptions as jest.Mock).mockReturnValue([getExceptionListItemSchemaMock()]); - (sortExceptionItems as jest.Mock).mockReturnValue({ - exceptionsWithoutValueLists: [getExceptionListItemSchemaMock()], - exceptionsWithValueLists: [], - }); - (searchAfterAndBulkCreate as jest.Mock).mockClear(); - (searchAfterAndBulkCreate as jest.Mock).mockResolvedValue({ - success: true, - searchAfterTimes: [], - createdSignalsCount: 10, - }); (checkPrivileges as jest.Mock).mockImplementation(async (_, indices) => { return { index: indices.reduce( @@ -143,13 +128,13 @@ describe('rules_notification_alert_type', () => { ), }; }); - alertServices.scopedClusterClient.asCurrentUser.transport.request.mockResolvedValue( - elasticsearchClientMock.createSuccessTransportRequestPromise({ - hits: { - total: { value: 10 }, - }, - }) - ); + const executorReturnValue = createSearchAfterReturnType({ + createdSignalsCount: 10, + }); + (queryExecutor as jest.Mock).mockClear(); + (queryExecutor as jest.Mock).mockResolvedValue(executorReturnValue); + (mlExecutor as jest.Mock).mockClear(); + (mlExecutor as jest.Mock).mockResolvedValue(executorReturnValue); const value: Partial> = { statusCode: 200, body: { @@ -189,6 +174,12 @@ describe('rules_notification_alert_type', () => { }); describe('executor', () => { + it('should call ruleStatusService.success if signals were created', async () => { + payload.previousStartedAt = null; + await alert.executor(payload); + expect(ruleStatusService.success).toHaveBeenCalled(); + }); + it('should warn about the gap between runs if gap is very large', async () => { payload.previousStartedAt = moment().subtract(100, 'm').toDate(); await alert.executor(payload); @@ -225,30 +216,6 @@ describe('rules_notification_alert_type', () => { ); }); - it('should set a warning when exception list for threshold rule contains value list exceptions', async () => { - (getExceptions as jest.Mock).mockReturnValue([ - getExceptionListItemSchemaMock({ entries: [getEntryListMock()] }), - ]); - payload = getPayload(getThresholdResult(), alertServices); - await alert.executor(payload); - expect(ruleStatusService.partialFailure).toHaveBeenCalled(); - expect(ruleStatusService.partialFailure.mock.calls[0][0]).toContain( - 'Exceptions that use "is in list" or "is not in list" operators are not applied to Threshold rules' - ); - }); - - it('should set a warning when exception list for EQL rule contains value list exceptions', async () => { - (getExceptions as jest.Mock).mockReturnValue([ - getExceptionListItemSchemaMock({ entries: [getEntryListMock()] }), - ]); - payload = getPayload(getEqlResult(), alertServices); - await alert.executor(payload); - expect(ruleStatusService.partialFailure).toHaveBeenCalled(); - expect(ruleStatusService.partialFailure.mock.calls[0][0]).toContain( - 'Exceptions that use "is in list" or "is not in list" operators are not applied to EQL rules' - ); - }); - it('should set a failure status for when rules cannot read ANY provided indices', async () => { (checkPrivileges as jest.Mock).mockResolvedValueOnce({ username: 'elastic', @@ -300,14 +267,12 @@ describe('rules_notification_alert_type', () => { attributes: ruleAlert, }); await alert.executor(payload); - expect((searchAfterAndBulkCreate as jest.Mock).mock.calls[0][0].refresh).toEqual('wait_for'); - (searchAfterAndBulkCreate as jest.Mock).mockClear(); + expect((queryExecutor as jest.Mock).mock.calls[0][0].refresh).toEqual('wait_for'); }); it('should set refresh to false when actions are not present', async () => { await alert.executor(payload); - expect((searchAfterAndBulkCreate as jest.Mock).mock.calls[0][0].refresh).toEqual(false); - (searchAfterAndBulkCreate as jest.Mock).mockClear(); + expect((queryExecutor as jest.Mock).mock.calls[0][0].refresh).toEqual(false); }); it('should call scheduleActions if signalsCount was greater than 0 and rule has actions defined', async () => { @@ -361,9 +326,13 @@ describe('rules_notification_alert_type', () => { references: [], attributes: ruleAlert, }); - (parseScheduleDates as jest.Mock).mockReturnValue(moment(100)); payload.params.meta = {}; + + const parseScheduleDatesSpy = jest + .spyOn(parseScheduleDates, 'parseScheduleDates') + .mockReturnValue(moment(100)); await alert.executor(payload); + parseScheduleDatesSpy.mockRestore(); expect(scheduleNotificationActions).toHaveBeenCalledWith( expect.objectContaining({ @@ -394,9 +363,13 @@ describe('rules_notification_alert_type', () => { references: [], attributes: ruleAlert, }); - (parseScheduleDates as jest.Mock).mockReturnValue(moment(100)); delete payload.params.meta; + + const parseScheduleDatesSpy = jest + .spyOn(parseScheduleDates, 'parseScheduleDates') + .mockReturnValue(moment(100)); await alert.executor(payload); + parseScheduleDatesSpy.mockRestore(); expect(scheduleNotificationActions).toHaveBeenCalledWith( expect.objectContaining({ @@ -427,9 +400,13 @@ describe('rules_notification_alert_type', () => { references: [], attributes: ruleAlert, }); - (parseScheduleDates as jest.Mock).mockReturnValue(moment(100)); payload.params.meta = { kibana_siem_app_url: 'http://localhost' }; + + const parseScheduleDatesSpy = jest + .spyOn(parseScheduleDates, 'parseScheduleDates') + .mockReturnValue(moment(100)); await alert.executor(payload); + parseScheduleDatesSpy.mockRestore(); expect(scheduleNotificationActions).toHaveBeenCalledWith( expect.objectContaining({ @@ -440,206 +417,21 @@ describe('rules_notification_alert_type', () => { }); describe('ML rule', () => { - it('should throw an error if ML plugin was not available', async () => { - const ruleAlert = getMlResult(); - payload = getPayload(ruleAlert, alertServices) as jest.Mocked; - alert = signalRulesAlertType({ - logger, - eventsTelemetry: undefined, - version, - ml: undefined, - lists: undefined, - }); - await alert.executor(payload); - expect(logger.error).toHaveBeenCalled(); - expect(logger.error.mock.calls[0][0]).toContain( - 'ML plugin unavailable during rule execution' - ); - }); - - it('should throw an error if machineLearningJobId or anomalyThreshold was not null', async () => { - const ruleAlert = getMlResult(); - ruleAlert.params.anomalyThreshold = undefined; - payload = getPayload(ruleAlert, alertServices) as jest.Mocked; - payload.previousStartedAt = null; - await alert.executor(payload); - expect(logger.error).toHaveBeenCalled(); - expect(logger.error.mock.calls[0][0]).toContain( - 'Machine learning rule is missing job id and/or anomaly threshold' - ); - }); - - it('should throw an error if Machine learning job summary was null', async () => { - const ruleAlert = getMlResult(); - payload = getPayload(ruleAlert, alertServices) as jest.Mocked; - payload.previousStartedAt = null; - jobsSummaryMock.mockResolvedValue([]); - await alert.executor(payload); - expect(logger.warn).toHaveBeenCalled(); - expect(logger.warn.mock.calls[0][0]).toContain('Machine learning job is not started'); - expect(ruleStatusService.error).toHaveBeenCalled(); - expect(ruleStatusService.error.mock.calls[0][0]).toContain( - 'Machine learning job is not started' - ); - }); - - it('should log an error if Machine learning job was not started', async () => { - const ruleAlert = getMlResult(); - payload = getPayload(ruleAlert, alertServices) as jest.Mocked; - payload.previousStartedAt = null; - jobsSummaryMock.mockResolvedValue([ - { - id: 'some_job_id', - jobState: 'starting', - datafeedState: 'started', - }, - ]); - (findMlSignals as jest.Mock).mockResolvedValue({ - _shards: {}, - hits: { - hits: [], - }, - }); - await alert.executor(payload); - expect(logger.warn).toHaveBeenCalled(); - expect(logger.warn.mock.calls[0][0]).toContain('Machine learning job is not started'); - expect(ruleStatusService.error).toHaveBeenCalled(); - expect(ruleStatusService.error.mock.calls[0][0]).toContain( - 'Machine learning job is not started' - ); - }); - - it('should not call ruleStatusService.success if no anomalies were found', async () => { - const ruleAlert = getMlResult(); - payload = getPayload(ruleAlert, alertServices) as jest.Mocked; - jobsSummaryMock.mockResolvedValue([]); - (findMlSignals as jest.Mock).mockResolvedValue({ - _shards: {}, - hits: { - hits: [], - }, - }); - (bulkCreateMlSignals as jest.Mock).mockResolvedValue({ - success: true, - bulkCreateDuration: 0, - createdItemsCount: 0, - errors: [], - }); - await alert.executor(payload); - expect(ruleStatusService.success).not.toHaveBeenCalled(); - }); - - it('should call ruleStatusService.success if signals were created', async () => { - const ruleAlert = getMlResult(); - payload = getPayload(ruleAlert, alertServices) as jest.Mocked; - payload.previousStartedAt = null; - jobsSummaryMock.mockResolvedValue([ - { - id: 'some_job_id', - jobState: 'started', - datafeedState: 'started', - }, - ]); - (findMlSignals as jest.Mock).mockResolvedValue({ - _shards: { failed: 0 }, - hits: { - hits: [{}], - }, - }); - (bulkCreateMlSignals as jest.Mock).mockResolvedValue({ - success: true, - bulkCreateDuration: 1, - createdItemsCount: 1, - errors: [], - }); - await alert.executor(payload); - expect(ruleStatusService.success).toHaveBeenCalled(); - }); - it('should not call checkPrivileges if ML rule', async () => { const ruleAlert = getMlResult(); - payload = getPayload(ruleAlert, alertServices) as jest.Mocked; - payload.previousStartedAt = null; - jobsSummaryMock.mockResolvedValue([ - { - id: 'some_job_id', - jobState: 'started', - datafeedState: 'started', - }, - ]); - (findMlSignals as jest.Mock).mockResolvedValue({ - _shards: { failed: 0 }, - hits: { - hits: [{}], - }, - }); - (bulkCreateMlSignals as jest.Mock).mockResolvedValue({ - success: true, - bulkCreateDuration: 1, - createdItemsCount: 1, - errors: [], - }); - (checkPrivileges as jest.Mock).mockClear(); - - await alert.executor(payload); - expect(checkPrivileges).toHaveBeenCalledTimes(0); - expect(ruleStatusService.success).toHaveBeenCalled(); - }); - - it('should call scheduleActions if signalsCount was greater than 0 and rule has actions defined', async () => { - const ruleAlert = getMlResult(); - ruleAlert.actions = [ - { - actionTypeId: '.slack', - params: { - message: - 'Rule generated {{state.signals_count}} signals\n\n{{context.rule.name}}\n{{{context.results_link}}}', - }, - group: 'default', - id: '99403909-ca9b-49ba-9d7a-7e5320e68d05', - }, - ]; - payload = getPayload(ruleAlert, alertServices) as jest.Mocked; alertServices.savedObjectsClient.get.mockResolvedValue({ id: 'id', type: 'type', references: [], attributes: ruleAlert, }); - jobsSummaryMock.mockResolvedValue([]); - (findMlSignals as jest.Mock).mockResolvedValue({ - _shards: { failed: 0 }, - hits: { - hits: [{}], - }, - }); - (bulkCreateMlSignals as jest.Mock).mockResolvedValue({ - success: true, - bulkCreateDuration: 1, - createdItemsCount: 1, - errors: [], - }); - - await alert.executor(payload); - - expect(scheduleNotificationActions).toHaveBeenCalledWith( - expect.objectContaining({ - signalsCount: 1, - }) - ); - }); - }); + payload = getPayload(ruleAlert, alertServices) as jest.Mocked; + payload.previousStartedAt = null; + (checkPrivileges as jest.Mock).mockClear(); - describe('threat match', () => { - it('should throw an error if threatQuery or threatIndex or threatMapping was not null', async () => { - const result = getResult(); - result.params.type = 'threat_match'; - payload = getPayload(result, alertServices) as jest.Mocked; await alert.executor(payload); - expect(logger.error).toHaveBeenCalled(); - expect(logger.error.mock.calls[0][0]).toContain( - 'An error occurred during rule execution: message: "Indicator match is missing threatQuery and/or threatIndex and/or threatMapping: threatQuery: "undefined" threatIndex: "undefined" threatMapping: "undefined"" name: "Detect Root/Admin Users" id: "04128c15-0d1b-4716-a4c5-46997ac7f3bd" rule id: "rule-1" signals index: ".siem-signals"' - ); + expect(checkPrivileges).toHaveBeenCalledTimes(0); + expect(ruleStatusService.success).toHaveBeenCalled(); }); }); }); @@ -648,6 +440,7 @@ describe('rules_notification_alert_type', () => { it('when bulk indexing failed', async () => { const result: SearchAfterAndBulkCreateReturnType = { success: false, + warning: false, searchAfterTimes: [], bulkCreateTimes: [], lastLookBackDate: null, @@ -655,7 +448,7 @@ describe('rules_notification_alert_type', () => { createdSignals: [], errors: ['Error that bubbled up.'], }; - (searchAfterAndBulkCreate as jest.Mock).mockResolvedValue(result); + (queryExecutor as jest.Mock).mockResolvedValue(result); await alert.executor(payload); expect(logger.error).toHaveBeenCalled(); expect(logger.error.mock.calls[0][0]).toContain( @@ -665,7 +458,7 @@ describe('rules_notification_alert_type', () => { }); it('when error was thrown', async () => { - (searchAfterAndBulkCreate as jest.Mock).mockRejectedValue({}); + (queryExecutor as jest.Mock).mockRejectedValue({}); await alert.executor(payload); expect(logger.error).toHaveBeenCalled(); expect(logger.error.mock.calls[0][0]).toContain('An error occurred during rule execution'); @@ -673,7 +466,7 @@ describe('rules_notification_alert_type', () => { }); it('and call ruleStatusService with the default message', async () => { - (searchAfterAndBulkCreate as jest.Mock).mockRejectedValue( + (queryExecutor as jest.Mock).mockRejectedValue( elasticsearchClientMock.createErrorTransportRequestPromise({}) ); await alert.executor(payload); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index cd77cab01bb01..52ceafbdb69b3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -4,16 +4,16 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - /* eslint-disable complexity */ -import { Logger, KibanaRequest } from 'src/core/server'; +import { Logger, SavedObject } from 'src/core/server'; import isEmpty from 'lodash/isEmpty'; import { chain, tryCatch } from 'fp-ts/lib/TaskEither'; import { flow } from 'fp-ts/lib/function'; -import { ApiResponse } from '@elastic/elasticsearch'; -import { performance } from 'perf_hooks'; +import * as t from 'io-ts'; +import { pickBy } from 'lodash/fp'; +import { validateNonExact } from '../../../../common/validate'; import { toError, toPromise } from '../../../../common/fp_utils'; import { @@ -21,49 +21,28 @@ import { DEFAULT_SEARCH_AFTER_PAGE_SIZE, SERVER_APP_ID, } from '../../../../common/constants'; -import { isJobStarted, isMlRule } from '../../../../common/machine_learning/helpers'; +import { isMlRule } from '../../../../common/machine_learning/helpers'; import { isThresholdRule, isEqlRule, isThreatMatchRule, - hasLargeValueItem, - normalizeThresholdField, + isQueryRule, } from '../../../../common/detection_engine/utils'; import { parseScheduleDates } from '../../../../common/detection_engine/parse_schedule_dates'; import { SetupPlugins } from '../../../plugin'; import { getInputIndex } from './get_input_output_index'; -import { searchAfterAndBulkCreate } from './search_after_bulk_create'; -import { getFilter } from './get_filter'; -import { - SignalRuleAlertTypeDefinition, - RuleAlertAttributes, - EqlSignalSearchResponse, - WrappedSignalHit, -} from './types'; +import { SignalRuleAlertTypeDefinition, RuleAlertAttributes } from './types'; import { getListsClient, getExceptions, - wrapSignal, - createErrorsFromShard, createSearchAfterReturnType, - mergeReturns, - createSearchAfterReturnTypeFromResponse, checkPrivileges, hasTimestampFields, hasReadIndexPrivileges, getRuleRangeTuples, - makeFloatString, } from './utils'; import { signalParamsSchema } from './signal_params_schema'; import { siemRuleActionGroups } from './siem_rule_action_groups'; -import { findMlSignals } from './find_ml_signals'; -import { - bulkCreateThresholdSignals, - getThresholdBucketFilters, - getThresholdSignalHistory, - findThresholdSignals, -} from './threshold'; -import { bulkCreateMlSignals } from './bulk_create_ml_signals'; import { scheduleNotificationActions, NotificationRuleTypeParams, @@ -73,15 +52,19 @@ import { buildRuleMessageFactory } from './rule_messages'; import { ruleStatusSavedObjectsClientFactory } from './rule_status_saved_objects_client'; import { getNotificationResultsLink } from '../notifications/utils'; import { TelemetryEventsSender } from '../../telemetry/sender'; -import { buildEqlSearchRequest } from '../../../../common/detection_engine/get_query_filter'; -import { bulkInsertSignals, filterDuplicateSignals } from './single_bulk_create'; -import { buildSignalFromEvent, buildSignalGroupFromSequence } from './build_bulk_body'; -import { createThreatSignals } from './threat_mapping/create_threat_signals'; -import { getIndexVersion } from '../routes/index/get_index_version'; -import { MIN_EQL_RULE_INDEX_VERSION } from '../routes/index/get_signals_template'; -import { filterEventsAgainstList } from './filters/filter_events_against_list'; -import { isOutdated } from '../migrations/helpers'; import { RuleTypeParams } from '../types'; +import { eqlExecutor } from './executors/eql'; +import { queryExecutor } from './executors/query'; +import { threatMatchExecutor } from './executors/threat_match'; +import { thresholdExecutor } from './executors/threshold'; +import { mlExecutor } from './executors/ml'; +import { + eqlRuleParams, + machineLearningRuleParams, + queryRuleParams, + threatRuleParams, + thresholdRuleParams, +} from '../schemas/rule_schemas'; export const signalRulesAlertType = ({ logger, @@ -124,34 +107,7 @@ export const signalRulesAlertType = ({ spaceId, updatedBy: updatedByUser, }) { - const { - anomalyThreshold, - from, - ruleId, - index, - eventCategoryOverride, - filters, - language, - maxSignals, - meta, - machineLearningJobId, - outputIndex, - savedId, - query, - to, - threshold, - threatFilters, - threatQuery, - threatIndex, - threatIndicatorPath, - threatMapping, - threatLanguage, - timestampOverride, - type, - exceptionsList, - concurrentSearches, - itemsPerSearch, - } = params; + const { ruleId, index, maxSignals, meta, outputIndex, timestampOverride, type } = params; const searchAfterSize = Math.min(maxSignals, DEFAULT_SEARCH_AFTER_PAGE_SIZE); let hasError: boolean = false; @@ -168,15 +124,8 @@ export const signalRulesAlertType = ({ const { actions, name, - tags, - createdAt, - createdBy, - updatedBy, - enabled, schedule: { interval }, - throttle, } = savedObject.attributes; - const updatedAt = savedObject.updated_at ?? ''; const refresh = actions.length ? 'wait_for' : false; const buildRuleMessage = buildRuleMessageFactory({ id: alertId, @@ -240,8 +189,8 @@ export const signalRulesAlertType = ({ const { tuples, remainingGap } = getRuleRangeTuples({ logger, previousStartedAt, - from, - to, + from: params.from, + to: params.to, interval, maxSignals, buildRuleMessage, @@ -266,392 +215,80 @@ export const signalRulesAlertType = ({ }); const exceptionItems = await getExceptions({ client: exceptionsClient, - lists: exceptionsList ?? [], + lists: params.exceptionsList ?? [], }); - if (isMlRule(type)) { - if (ml == null) { - throw new Error('ML plugin unavailable during rule execution'); - } - if (machineLearningJobId == null || anomalyThreshold == null) { - throw new Error( - [ - 'Machine learning rule is missing job id and/or anomaly threshold:', - `job id: "${machineLearningJobId}"`, - `anomaly threshold: "${anomalyThreshold}"`, - ].join(' ') - ); - } - - // Using fake KibanaRequest as it is needed to satisfy the ML Services API, but can be empty as it is - // currently unused by the jobsSummary function. - const fakeRequest = {} as KibanaRequest; - const summaryJobs = await ml - .jobServiceProvider(fakeRequest, services.savedObjectsClient) - .jobsSummary([machineLearningJobId]); - const jobSummary = summaryJobs.find((job) => job.id === machineLearningJobId); - - if (jobSummary == null || !isJobStarted(jobSummary.jobState, jobSummary.datafeedState)) { - const errorMessage = buildRuleMessage( - 'Machine learning job is not started:', - `job id: "${machineLearningJobId}"`, - `job status: "${jobSummary?.jobState}"`, - `datafeed status: "${jobSummary?.datafeedState}"` - ); - logger.warn(errorMessage); - hasError = true; - await ruleStatusService.error(errorMessage); - } - - const anomalyResults = await findMlSignals({ + const mlRuleSO = asTypeSpecificSO(savedObject, machineLearningRuleParams); + result = await mlExecutor({ + rule: mlRuleSO, ml, - // Using fake KibanaRequest as it is needed to satisfy the ML Services API, but can be empty as it is - // currently unused by the mlAnomalySearch function. - request: ({} as unknown) as KibanaRequest, - savedObjectsClient: services.savedObjectsClient, - jobId: machineLearningJobId, - anomalyThreshold, - from, - to, - exceptionItems: exceptionItems ?? [], - }); - - const filteredAnomalyResults = await filterEventsAgainstList({ listClient, - exceptionsList: exceptionItems ?? [], + exceptionItems, + ruleStatusService, + services, logger, - eventSearchResult: anomalyResults, + refresh, buildRuleMessage, }); - - const anomalyCount = filteredAnomalyResults.hits.hits.length; - if (anomalyCount) { - logger.info(buildRuleMessage(`Found ${anomalyCount} signals from ML anomalies.`)); - } - - const { - success, - errors, - bulkCreateDuration, - createdItemsCount, - createdItems, - } = await bulkCreateMlSignals({ - actions, - throttle, - someResult: filteredAnomalyResults, - ruleParams: params, + } else if (isThresholdRule(type)) { + const thresholdRuleSO = asTypeSpecificSO(savedObject, thresholdRuleParams); + result = await thresholdExecutor({ + rule: thresholdRuleSO, + tuples, + exceptionItems, + ruleStatusService, services, + version, logger, - id: alertId, - signalsIndex: outputIndex, - name, - createdBy, - createdAt, - updatedBy, - updatedAt, - interval, - enabled, refresh, - tags, buildRuleMessage, + startedAt, }); - // The legacy ES client does not define failures when it can be present on the structure, hence why I have the & { failures: [] } - const shardFailures = - (filteredAnomalyResults._shards as typeof filteredAnomalyResults._shards & { - failures: []; - }).failures ?? []; - const searchErrors = createErrorsFromShard({ - errors: shardFailures, - }); - result = mergeReturns([ - result, - createSearchAfterReturnType({ - success: success && filteredAnomalyResults._shards.failed === 0, - errors: [...errors, ...searchErrors], - createdSignalsCount: createdItemsCount, - createdSignals: createdItems, - bulkCreateTimes: bulkCreateDuration ? [bulkCreateDuration] : [], - }), - ]); - } else if (isThresholdRule(type) && threshold) { - if (hasLargeValueItem(exceptionItems ?? [])) { - await ruleStatusService.partialFailure( - 'Exceptions that use "is in list" or "is not in list" operators are not applied to Threshold rules' - ); - wroteWarningStatus = true; - } - const inputIndex = await getInputIndex(services, version, index); - - for (const tuple of tuples) { - const { - thresholdSignalHistory, - searchErrors: previousSearchErrors, - } = await getThresholdSignalHistory({ - indexPattern: [outputIndex], - from: tuple.from.toISOString(), - to: tuple.to.toISOString(), - services, - logger, - ruleId, - bucketByFields: normalizeThresholdField(threshold.field), - timestampOverride, - buildRuleMessage, - }); - - const bucketFilters = await getThresholdBucketFilters({ - thresholdSignalHistory, - timestampOverride, - }); - - const esFilter = await getFilter({ - type, - filters: filters ? filters.concat(bucketFilters) : bucketFilters, - language, - query, - savedId, - services, - index: inputIndex, - lists: exceptionItems ?? [], - }); - - const { - searchResult: thresholdResults, - searchErrors, - searchDuration: thresholdSearchDuration, - } = await findThresholdSignals({ - inputIndexPattern: inputIndex, - from: tuple.from.toISOString(), - to: tuple.to.toISOString(), - services, - logger, - filter: esFilter, - threshold, - timestampOverride, - buildRuleMessage, - }); - - const { - success, - bulkCreateDuration, - createdItemsCount, - createdItems, - errors, - } = await bulkCreateThresholdSignals({ - actions, - throttle, - someResult: thresholdResults, - ruleParams: params, - filter: esFilter, - services, - logger, - id: alertId, - inputIndexPattern: inputIndex, - signalsIndex: outputIndex, - timestampOverride, - startedAt, - from: tuple.from.toDate(), - name, - createdBy, - createdAt, - updatedBy, - updatedAt, - interval, - enabled, - refresh, - tags, - thresholdSignalHistory, - buildRuleMessage, - }); - - result = mergeReturns([ - result, - createSearchAfterReturnTypeFromResponse({ - searchResult: thresholdResults, - timestampOverride, - }), - createSearchAfterReturnType({ - success, - errors: [...errors, ...previousSearchErrors, ...searchErrors], - createdSignalsCount: createdItemsCount, - createdSignals: createdItems, - bulkCreateTimes: bulkCreateDuration ? [bulkCreateDuration] : [], - searchAfterTimes: [thresholdSearchDuration], - }), - ]); - } } else if (isThreatMatchRule(type)) { - if ( - threatQuery == null || - threatIndex == null || - threatMapping == null || - query == null - ) { - throw new Error( - [ - 'Indicator match is missing threatQuery and/or threatIndex and/or threatMapping:', - `threatQuery: "${threatQuery}"`, - `threatIndex: "${threatIndex}"`, - `threatMapping: "${threatMapping}"`, - ].join(' ') - ); - } - const inputIndex = await getInputIndex(services, version, index); - result = await createThreatSignals({ + const threatRuleSO = asTypeSpecificSO(savedObject, threatRuleParams); + result = await threatMatchExecutor({ + rule: threatRuleSO, tuples, - threatMapping, - query, - inputIndex, - type, - filters: filters ?? [], - language, - name, - savedId, - services, - exceptionItems: exceptionItems ?? [], listClient, - logger, - eventsTelemetry, - alertId, - outputIndex, - params, + exceptionItems, + services, + version, searchAfterSize, - actions, - createdBy, - createdAt, - updatedBy, - interval, - updatedAt, - enabled, + logger, refresh, - tags, - throttle, - threatFilters: threatFilters ?? [], - threatQuery, - threatLanguage, + eventsTelemetry, buildRuleMessage, - threatIndex, - threatIndicatorPath, - concurrentSearches: concurrentSearches ?? 1, - itemsPerSearch: itemsPerSearch ?? 9000, - }); - } else if (type === 'query' || type === 'saved_query') { - const inputIndex = await getInputIndex(services, version, index); - const esFilter = await getFilter({ - type, - filters, - language, - query, - savedId, - services, - index: inputIndex, - lists: exceptionItems ?? [], }); - - result = await searchAfterAndBulkCreate({ + } else if (isQueryRule(type)) { + const queryRuleSO = asTypeSpecificSO(savedObject, queryRuleParams); + result = await queryExecutor({ + rule: queryRuleSO, tuples, listClient, - exceptionsList: exceptionItems ?? [], - ruleParams: params, + exceptionItems, services, + version, + searchAfterSize, logger, - eventsTelemetry, - id: alertId, - inputIndexPattern: inputIndex, - signalsIndex: outputIndex, - filter: esFilter, - actions, - name, - createdBy, - createdAt, - updatedBy, - updatedAt, - interval, - enabled, - pageSize: searchAfterSize, refresh, - tags, - throttle, + eventsTelemetry, buildRuleMessage, }); } else if (isEqlRule(type)) { - if (query === undefined) { - throw new Error('EQL query rule must have a query defined'); - } - if (hasLargeValueItem(exceptionItems ?? [])) { - await ruleStatusService.partialFailure( - 'Exceptions that use "is in list" or "is not in list" operators are not applied to EQL rules' - ); - wroteWarningStatus = true; - } - try { - const signalIndexVersion = await getIndexVersion( - services.scopedClusterClient.asCurrentUser, - outputIndex - ); - if (isOutdated({ current: signalIndexVersion, target: MIN_EQL_RULE_INDEX_VERSION })) { - throw new Error( - `EQL based rules require an update to version ${MIN_EQL_RULE_INDEX_VERSION} of the detection alerts index mapping` - ); - } - } catch (err) { - if (err.statusCode === 403) { - throw new Error( - `EQL based rules require the user that created it to have the view_index_metadata, read, and write permissions for index: ${outputIndex}` - ); - } else { - throw err; - } - } - const inputIndex = await getInputIndex(services, version, index); - const request = buildEqlSearchRequest( - query, - inputIndex, - from, - to, + const eqlRuleSO = asTypeSpecificSO(savedObject, eqlRuleParams); + result = await eqlExecutor({ + rule: eqlRuleSO, + exceptionItems, + ruleStatusService, + services, + version, searchAfterSize, - timestampOverride, - exceptionItems ?? [], - eventCategoryOverride - ); - const eqlSignalSearchStart = performance.now(); - const { - body: response, - } = (await services.scopedClusterClient.asCurrentUser.transport.request( - request - )) as ApiResponse; - const eqlSignalSearchEnd = performance.now(); - const eqlSearchDuration = makeFloatString(eqlSignalSearchEnd - eqlSignalSearchStart); - result.searchAfterTimes = [eqlSearchDuration]; - let newSignals: WrappedSignalHit[] | undefined; - if (response.hits.sequences !== undefined) { - newSignals = response.hits.sequences.reduce( - (acc: WrappedSignalHit[], sequence) => - acc.concat(buildSignalGroupFromSequence(sequence, savedObject, outputIndex)), - [] - ); - } else if (response.hits.events !== undefined) { - newSignals = filterDuplicateSignals( - savedObject.id, - response.hits.events.map((event) => - wrapSignal(buildSignalFromEvent(event, savedObject, true), outputIndex) - ) - ); - } else { - throw new Error( - 'eql query response should have either `sequences` or `events` but had neither' - ); - } - if (newSignals.length > 0) { - const insertResult = await bulkInsertSignals(newSignals, logger, services, refresh); - result.bulkCreateTimes.push(insertResult.bulkCreateDuration); - result.createdSignalsCount += insertResult.createdItemsCount; - result.createdSignals = insertResult.createdItems; - } - result.success = true; + logger, + refresh, + }); } else { throw new Error(`unknown rule type ${type}`); } - if (result.success) { if (actions.length) { const notificationRuleParams: NotificationRuleTypeParams = { @@ -743,3 +380,31 @@ export const signalRulesAlertType = ({ }, }; }; + +/** + * This function takes a generic rule SavedObject and a type-specific schema for the rule params + * and validates the SavedObject params against the schema. If they validate, it returns a SavedObject + * where the params have been replaced with the validated params. This eliminates the need for logic that + * checks if the required type specific fields actually exist on the SO and prevents rule executors from + * accessing fields that only exist on other rule types. + * + * @param ruleSO SavedObject typed as an object with all fields from all different rule types + * @param schema io-ts schema for the specific rule type the SavedObject claims to be + */ +export const asTypeSpecificSO = ( + ruleSO: SavedObject, + schema: T +) => { + const nonNullParams = pickBy((value: unknown) => value !== null, ruleSO.attributes.params); + const [validated, errors] = validateNonExact(nonNullParams, schema); + if (validated == null || errors != null) { + throw new Error(`Rule attempted to execute with invalid params: ${errors}`); + } + return { + ...ruleSO, + attributes: { + ...ruleSO.attributes, + params: validated, + }, + }; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts index 0146572941331..8e42e60768bf0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts @@ -57,6 +57,7 @@ export const createThreatSignals = async ({ let results: SearchAfterAndBulkCreateReturnType = { success: true, + warning: false, bulkCreateTimes: [], searchAfterTimes: [], lastLookBackDate: null, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts index 65b59d4df0791..aeed8da7ac3d9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts @@ -20,7 +20,7 @@ import { ItemsPerSearch, ThreatIndicatorPathOrUndefined, } from '../../../../../common/detection_engine/schemas/types/threat_mapping'; -import { PartialFilter, RuleTypeParams } from '../../types'; +import { RuleTypeParams } from '../../types'; import { AlertInstanceContext, AlertInstanceState, @@ -41,7 +41,7 @@ export interface CreateThreatSignalsOptions { query: string; inputIndex: string[]; type: Type; - filters: PartialFilter[]; + filters: unknown[]; language: LanguageOrUndefined; savedId: string | undefined; services: AlertServices; @@ -63,7 +63,7 @@ export interface CreateThreatSignalsOptions { tags: string[]; refresh: false | 'wait_for'; throttle: string; - threatFilters: PartialFilter[]; + threatFilters: unknown[]; threatQuery: ThreatQuery; buildRuleMessage: BuildRuleMessage; threatIndex: ThreatIndex; @@ -81,7 +81,7 @@ export interface CreateThreatSignalOptions { query: string; inputIndex: string[]; type: Type; - filters: PartialFilter[]; + filters: unknown[]; language: LanguageOrUndefined; savedId: string | undefined; services: AlertServices; @@ -154,7 +154,7 @@ export interface GetThreatListOptions { searchAfter: string[] | undefined; sortField: string | undefined; sortOrder: SortOrderOrUndefined; - threatFilters: PartialFilter[]; + threatFilters: unknown[]; exceptionItems: ExceptionListItemSchema[]; listClient: ListClient; buildRuleMessage: BuildRuleMessage; @@ -165,7 +165,7 @@ export interface ThreatListCountOptions { esClient: ElasticsearchClient; query: string; language: ThreatLanguageOrUndefined; - threatFilters: PartialFilter[]; + threatFilters: unknown[]; index: string[]; exceptionItems: ExceptionListItemSchema[]; } @@ -210,7 +210,7 @@ export interface BuildThreatEnrichmentOptions { listClient: ListClient; logger: Logger; services: AlertServices; - threatFilters: PartialFilter[]; + threatFilters: unknown[]; threatIndex: ThreatIndex; threatIndicatorPath: ThreatIndicatorPathOrUndefined; threatLanguage: ThreatLanguageOrUndefined; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts index 6219da93036ee..6c6447bad0975 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts @@ -51,6 +51,7 @@ describe('utils', () => { test('it should combine two results with success set to "true" if both are "true"', () => { const existingResult: SearchAfterAndBulkCreateReturnType = { success: true, + warning: false, searchAfterTimes: ['10', '20', '30'], bulkCreateTimes: ['5', '15', '25'], lastLookBackDate: undefined, @@ -61,6 +62,7 @@ describe('utils', () => { const newResult: SearchAfterAndBulkCreateReturnType = { success: true, + warning: false, searchAfterTimes: ['10', '20', '30'], bulkCreateTimes: ['5', '15', '25'], lastLookBackDate: undefined, @@ -75,6 +77,7 @@ describe('utils', () => { test('it should combine two results with success set to "false" if one of them is "false"', () => { const existingResult: SearchAfterAndBulkCreateReturnType = { success: false, + warning: false, searchAfterTimes: ['10', '20', '30'], bulkCreateTimes: ['5', '15', '25'], lastLookBackDate: undefined, @@ -85,6 +88,7 @@ describe('utils', () => { const newResult: SearchAfterAndBulkCreateReturnType = { success: true, + warning: false, searchAfterTimes: ['10', '20', '30'], bulkCreateTimes: ['5', '15', '25'], lastLookBackDate: undefined, @@ -99,6 +103,7 @@ describe('utils', () => { test('it should use the latest date if it is set in the new result', () => { const existingResult: SearchAfterAndBulkCreateReturnType = { success: false, + warning: false, searchAfterTimes: ['10', '20', '30'], bulkCreateTimes: ['5', '15', '25'], lastLookBackDate: undefined, @@ -109,6 +114,7 @@ describe('utils', () => { const newResult: SearchAfterAndBulkCreateReturnType = { success: true, + warning: false, searchAfterTimes: ['10', '20', '30'], bulkCreateTimes: ['5', '15', '25'], lastLookBackDate: new Date('2020-09-16T03:34:32.390Z'), @@ -123,6 +129,7 @@ describe('utils', () => { test('it should combine the searchAfterTimes and the bulkCreateTimes', () => { const existingResult: SearchAfterAndBulkCreateReturnType = { success: false, + warning: false, searchAfterTimes: ['10', '20', '30'], bulkCreateTimes: ['5', '15', '25'], lastLookBackDate: undefined, @@ -133,6 +140,7 @@ describe('utils', () => { const newResult: SearchAfterAndBulkCreateReturnType = { success: true, + warning: false, searchAfterTimes: ['10', '20', '30'], bulkCreateTimes: ['5', '15', '25'], lastLookBackDate: new Date('2020-09-16T03:34:32.390Z'), @@ -152,6 +160,7 @@ describe('utils', () => { test('it should combine errors together without duplicates', () => { const existingResult: SearchAfterAndBulkCreateReturnType = { success: false, + warning: false, searchAfterTimes: ['10', '20', '30'], bulkCreateTimes: ['5', '15', '25'], lastLookBackDate: undefined, @@ -162,6 +171,7 @@ describe('utils', () => { const newResult: SearchAfterAndBulkCreateReturnType = { success: true, + warning: false, searchAfterTimes: ['10', '20', '30'], bulkCreateTimes: ['5', '15', '25'], lastLookBackDate: new Date('2020-09-16T03:34:32.390Z'), @@ -272,6 +282,7 @@ describe('utils', () => { test('it should use the maximum found if given an empty array for newResults', () => { const existingResult: SearchAfterAndBulkCreateReturnType = { success: true, + warning: false, searchAfterTimes: ['10', '20', '30'], bulkCreateTimes: ['5', '15', '25'], lastLookBackDate: undefined, @@ -281,6 +292,7 @@ describe('utils', () => { }; const expectedResult: SearchAfterAndBulkCreateReturnType = { success: true, + warning: false, searchAfterTimes: ['30'], // max value from existingResult.searchAfterTimes bulkCreateTimes: ['25'], // max value from existingResult.bulkCreateTimes lastLookBackDate: undefined, @@ -295,6 +307,7 @@ describe('utils', () => { test('it should work with empty arrays for searchAfterTimes and bulkCreateTimes and createdSignals', () => { const existingResult: SearchAfterAndBulkCreateReturnType = { success: true, + warning: false, searchAfterTimes: ['10', '20', '30'], bulkCreateTimes: ['5', '15', '25'], lastLookBackDate: undefined, @@ -304,6 +317,7 @@ describe('utils', () => { }; const newResult: SearchAfterAndBulkCreateReturnType = { success: true, + warning: false, searchAfterTimes: [], bulkCreateTimes: [], lastLookBackDate: undefined, @@ -313,6 +327,7 @@ describe('utils', () => { }; const expectedResult: SearchAfterAndBulkCreateReturnType = { success: true, + warning: false, searchAfterTimes: ['30'], // max value from existingResult.searchAfterTimes bulkCreateTimes: ['25'], // max value from existingResult.bulkCreateTimes lastLookBackDate: undefined, @@ -328,6 +343,7 @@ describe('utils', () => { test('it should get the max of two new results and then combine the result with an existingResult correctly', () => { const existingResult: SearchAfterAndBulkCreateReturnType = { success: true, + warning: false, searchAfterTimes: ['10', '20', '30'], // max is 30 bulkCreateTimes: ['5', '15', '25'], // max is 25 lastLookBackDate: undefined, @@ -337,6 +353,7 @@ describe('utils', () => { }; const newResult1: SearchAfterAndBulkCreateReturnType = { success: true, + warning: false, searchAfterTimes: ['10', '20', '30'], bulkCreateTimes: ['5', '15', '25'], lastLookBackDate: new Date('2020-09-16T03:34:32.390Z'), @@ -346,6 +363,7 @@ describe('utils', () => { }; const newResult2: SearchAfterAndBulkCreateReturnType = { success: true, + warning: false, searchAfterTimes: ['40', '5', '15'], bulkCreateTimes: ['50', '5', '15'], lastLookBackDate: new Date('2020-09-16T04:34:32.390Z'), @@ -356,6 +374,7 @@ describe('utils', () => { const expectedResult: SearchAfterAndBulkCreateReturnType = { success: true, + warning: false, searchAfterTimes: ['70'], // max value between newResult1 and newResult2 + max array value of existingResult (40 + 30 = 70) bulkCreateTimes: ['75'], // max value between newResult1 and newResult2 + max array value of existingResult (50 + 25 = 75) lastLookBackDate: new Date('2020-09-16T04:34:32.390Z'), // max lastLookBackDate @@ -371,6 +390,7 @@ describe('utils', () => { test('it should get the max of two new results and then combine the result with an existingResult correctly when the results are flipped around', () => { const existingResult: SearchAfterAndBulkCreateReturnType = { success: true, + warning: false, searchAfterTimes: ['10', '20', '30'], // max is 30 bulkCreateTimes: ['5', '15', '25'], // max is 25 lastLookBackDate: undefined, @@ -380,6 +400,7 @@ describe('utils', () => { }; const newResult1: SearchAfterAndBulkCreateReturnType = { success: true, + warning: false, searchAfterTimes: ['10', '20', '30'], bulkCreateTimes: ['5', '15', '25'], lastLookBackDate: new Date('2020-09-16T03:34:32.390Z'), @@ -389,6 +410,7 @@ describe('utils', () => { }; const newResult2: SearchAfterAndBulkCreateReturnType = { success: true, + warning: false, searchAfterTimes: ['40', '5', '15'], bulkCreateTimes: ['50', '5', '15'], lastLookBackDate: new Date('2020-09-16T04:34:32.390Z'), @@ -399,6 +421,7 @@ describe('utils', () => { const expectedResult: SearchAfterAndBulkCreateReturnType = { success: true, + warning: false, searchAfterTimes: ['70'], // max value between newResult1 and newResult2 + max array value of existingResult (40 + 30 = 70) bulkCreateTimes: ['75'], // max value between newResult1 and newResult2 + max array value of existingResult (50 + 25 = 75) lastLookBackDate: new Date('2020-09-16T04:34:32.390Z'), // max lastLookBackDate @@ -414,6 +437,7 @@ describe('utils', () => { test('it should return the max date correctly if one date contains a null', () => { const existingResult: SearchAfterAndBulkCreateReturnType = { success: true, + warning: false, searchAfterTimes: ['10', '20', '30'], // max is 30 bulkCreateTimes: ['5', '15', '25'], // max is 25 lastLookBackDate: undefined, @@ -423,6 +447,7 @@ describe('utils', () => { }; const newResult1: SearchAfterAndBulkCreateReturnType = { success: true, + warning: false, searchAfterTimes: ['10', '20', '30'], bulkCreateTimes: ['5', '15', '25'], lastLookBackDate: new Date('2020-09-16T03:34:32.390Z'), @@ -432,6 +457,7 @@ describe('utils', () => { }; const newResult2: SearchAfterAndBulkCreateReturnType = { success: true, + warning: false, searchAfterTimes: ['40', '5', '15'], bulkCreateTimes: ['50', '5', '15'], lastLookBackDate: null, @@ -442,6 +468,7 @@ describe('utils', () => { const expectedResult: SearchAfterAndBulkCreateReturnType = { success: true, + warning: false, searchAfterTimes: ['70'], // max value between newResult1 and newResult2 + max array value of existingResult (40 + 30 = 70) bulkCreateTimes: ['75'], // max value between newResult1 and newResult2 + max array value of existingResult (50 + 25 = 75) lastLookBackDate: new Date('2020-09-16T03:34:32.390Z'), // max lastLookBackDate @@ -457,6 +484,7 @@ describe('utils', () => { test('it should combine two results with success set to "true" if both are "true"', () => { const existingResult: SearchAfterAndBulkCreateReturnType = { success: true, + warning: false, searchAfterTimes: ['10', '20', '30'], bulkCreateTimes: ['5', '15', '25'], lastLookBackDate: undefined, @@ -467,6 +495,7 @@ describe('utils', () => { const newResult: SearchAfterAndBulkCreateReturnType = { success: true, + warning: false, searchAfterTimes: ['10', '20', '30'], bulkCreateTimes: ['5', '15', '25'], lastLookBackDate: undefined, @@ -481,6 +510,7 @@ describe('utils', () => { test('it should combine two results with success set to "false" if one of them is "false"', () => { const existingResult: SearchAfterAndBulkCreateReturnType = { success: false, + warning: false, searchAfterTimes: ['10', '20', '30'], bulkCreateTimes: ['5', '15', '25'], lastLookBackDate: undefined, @@ -491,6 +521,7 @@ describe('utils', () => { const newResult: SearchAfterAndBulkCreateReturnType = { success: true, + warning: false, searchAfterTimes: ['10', '20', '30'], bulkCreateTimes: ['5', '15', '25'], lastLookBackDate: undefined, @@ -505,6 +536,7 @@ describe('utils', () => { test('it should use the latest date if it is set in the new result', () => { const existingResult: SearchAfterAndBulkCreateReturnType = { success: false, + warning: false, searchAfterTimes: ['10', '20', '30'], bulkCreateTimes: ['5', '15', '25'], lastLookBackDate: undefined, @@ -515,6 +547,7 @@ describe('utils', () => { const newResult: SearchAfterAndBulkCreateReturnType = { success: true, + warning: false, searchAfterTimes: ['10', '20', '30'], bulkCreateTimes: ['5', '15', '25'], lastLookBackDate: new Date('2020-09-16T03:34:32.390Z'), @@ -529,6 +562,7 @@ describe('utils', () => { test('it should combine the searchAfterTimes and the bulkCreateTimes', () => { const existingResult: SearchAfterAndBulkCreateReturnType = { success: false, + warning: false, searchAfterTimes: ['10', '20', '30'], bulkCreateTimes: ['5', '15', '25'], lastLookBackDate: undefined, @@ -539,6 +573,7 @@ describe('utils', () => { const newResult: SearchAfterAndBulkCreateReturnType = { success: true, + warning: false, searchAfterTimes: ['10', '20', '30'], bulkCreateTimes: ['5', '15', '25'], lastLookBackDate: new Date('2020-09-16T03:34:32.390Z'), @@ -558,6 +593,7 @@ describe('utils', () => { test('it should combine errors together without duplicates', () => { const existingResult: SearchAfterAndBulkCreateReturnType = { success: false, + warning: false, searchAfterTimes: ['10', '20', '30'], bulkCreateTimes: ['5', '15', '25'], lastLookBackDate: undefined, @@ -568,6 +604,7 @@ describe('utils', () => { const newResult: SearchAfterAndBulkCreateReturnType = { success: true, + warning: false, searchAfterTimes: ['10', '20', '30'], bulkCreateTimes: ['5', '15', '25'], lastLookBackDate: new Date('2020-09-16T03:34:32.390Z'), diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.ts index 805aca563701c..47a32915dd83f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.ts @@ -66,6 +66,7 @@ export const combineResults = ( newResult: SearchAfterAndBulkCreateReturnType ): SearchAfterAndBulkCreateReturnType => ({ success: currentResult.success === false ? false : newResult.success, + warning: currentResult.warning || newResult.warning, bulkCreateTimes: calculateAdditiveMax(currentResult.bulkCreateTimes, newResult.bulkCreateTimes), searchAfterTimes: calculateAdditiveMax( currentResult.searchAfterTimes, @@ -93,6 +94,7 @@ export const combineConcurrentResults = ( const lastLookBackDate = calculateMaxLookBack(accum.lastLookBackDate, item.lastLookBackDate); return { success: accum.success && item.success, + warning: accum.warning || item.warning, searchAfterTimes: [maxSearchAfterTime], bulkCreateTimes: [maxBulkCreateTimes], lastLookBackDate, @@ -103,6 +105,7 @@ export const combineConcurrentResults = ( }, { success: true, + warning: false, searchAfterTimes: [], bulkCreateTimes: [], lastLookBackDate: undefined, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts index 559c5875c90d0..615b91d60bb1b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts @@ -31,6 +31,13 @@ import { Logger } from '../../../../../../../src/core/server'; import { ExceptionListItemSchema } from '../../../../../lists/common/schemas'; import { BuildRuleMessage } from './rule_messages'; import { TelemetryEventsSender } from '../../telemetry/sender'; +import { + EqlRuleParams, + MachineLearningRuleParams, + QueryRuleParams, + ThreatRuleParams, + ThresholdRuleParams, +} from '../schemas/rule_schemas'; // used for gap detection code // eslint-disable-next-line @typescript-eslint/naming-convention @@ -241,6 +248,26 @@ export interface RuleAlertAttributes extends AlertAttributes { params: RuleTypeParams; } +export interface MachineLearningRuleAttributes extends AlertAttributes { + params: MachineLearningRuleParams; +} + +export interface ThresholdRuleAttributes extends AlertAttributes { + params: ThresholdRuleParams; +} + +export interface ThreatRuleAttributes extends AlertAttributes { + params: ThreatRuleParams; +} + +export interface QueryRuleAttributes extends AlertAttributes { + params: QueryRuleParams; +} + +export interface EqlRuleAttributes extends AlertAttributes { + params: EqlRuleParams; +} + export type BulkResponseErrorAggregation = Record; /** @@ -291,6 +318,7 @@ export interface SearchAfterAndBulkCreateParams { export interface SearchAfterAndBulkCreateReturnType { success: boolean; + warning: boolean; searchAfterTimes: string[]; bulkCreateTimes: string[]; lastLookBackDate: Date | null | undefined; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts index 249305ebcd9a1..37959a5ee877b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts @@ -1108,6 +1108,7 @@ describe('utils', () => { lastLookBackDate: null, searchAfterTimes: [], success: true, + warning: false, }; expect(newSearchResult).toEqual(expected); }); @@ -1126,6 +1127,7 @@ describe('utils', () => { lastLookBackDate: new Date('2020-04-20T21:27:45.000Z'), searchAfterTimes: [], success: true, + warning: false, }; expect(newSearchResult).toEqual(expected); }); @@ -1331,6 +1333,7 @@ describe('utils', () => { lastLookBackDate: null, searchAfterTimes: [], success: true, + warning: false, }; expect(searchAfterReturnType).toEqual(expected); }); @@ -1344,6 +1347,7 @@ describe('utils', () => { lastLookBackDate: new Date('2020-09-21T18:51:25.193Z'), searchAfterTimes: ['123'], success: false, + warning: true, }); const expected: SearchAfterAndBulkCreateReturnType = { bulkCreateTimes: ['123'], @@ -1353,6 +1357,7 @@ describe('utils', () => { lastLookBackDate: new Date('2020-09-21T18:51:25.193Z'), searchAfterTimes: ['123'], success: false, + warning: true, }; expect(searchAfterReturnType).toEqual(expected); }); @@ -1371,6 +1376,7 @@ describe('utils', () => { lastLookBackDate: null, searchAfterTimes: [], success: true, + warning: false, }; expect(searchAfterReturnType).toEqual(expected); }); @@ -1387,6 +1393,7 @@ describe('utils', () => { lastLookBackDate: null, searchAfterTimes: [], success: true, + warning: false, }; expect(merged).toEqual(expected); }); @@ -1460,6 +1467,7 @@ describe('utils', () => { lastLookBackDate: new Date('2020-09-21T18:51:25.193Z'), // takes the next lastLookBackDate searchAfterTimes: ['123', '567'], // concatenates the searchAfterTimes together success: true, // Defaults to success true is all of it was successful + warning: false, }; expect(merged).toEqual(expected); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts index 28edd97de0a0e..fb0166fd4dbee 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts @@ -226,7 +226,7 @@ export const getExceptions = async ({ }: { client: ExceptionListClient; lists: ListArray; -}): Promise => { +}): Promise => { if (lists.length > 0) { try { const listIds = lists.map(({ list_id: listId }) => listId); @@ -622,6 +622,7 @@ export const createSearchAfterReturnTypeFromResponse = ({ export const createSearchAfterReturnType = ({ success, + warning, searchAfterTimes, bulkCreateTimes, lastLookBackDate, @@ -630,6 +631,7 @@ export const createSearchAfterReturnType = ({ errors, }: { success?: boolean | undefined; + warning?: boolean; searchAfterTimes?: string[] | undefined; bulkCreateTimes?: string[] | undefined; lastLookBackDate?: Date | undefined; @@ -639,6 +641,7 @@ export const createSearchAfterReturnType = ({ } = {}): SearchAfterAndBulkCreateReturnType => { return { success: success ?? true, + warning: warning ?? false, searchAfterTimes: searchAfterTimes ?? [], bulkCreateTimes: bulkCreateTimes ?? [], lastLookBackDate: lastLookBackDate ?? null, @@ -673,6 +676,7 @@ export const mergeReturns = ( return searchAfters.reduce((prev, next) => { const { success: existingSuccess, + warning: existingWarning, searchAfterTimes: existingSearchAfterTimes, bulkCreateTimes: existingBulkCreateTimes, lastLookBackDate: existingLastLookBackDate, @@ -683,6 +687,7 @@ export const mergeReturns = ( const { success: newSuccess, + warning: newWarning, searchAfterTimes: newSearchAfterTimes, bulkCreateTimes: newBulkCreateTimes, lastLookBackDate: newLastLookBackDate, @@ -693,6 +698,7 @@ export const mergeReturns = ( return { success: existingSuccess && newSuccess, + warning: existingWarning || newWarning, searchAfterTimes: [...existingSearchAfterTimes, ...newSearchAfterTimes], bulkCreateTimes: [...existingBulkCreateTimes, ...newBulkCreateTimes], lastLookBackDate: newLastLookBackDate ?? existingLastLookBackDate, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/types.ts index 5bac992fd8da4..85c8483a0b988 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/types.ts @@ -58,27 +58,27 @@ import { AlertTypeParams } from '../../../../alerting/common'; export type PartialFilter = Partial; export interface RuleTypeParams extends AlertTypeParams { - anomalyThreshold: AnomalyThresholdOrUndefined; + anomalyThreshold?: AnomalyThresholdOrUndefined; author: AuthorOrUndefined; buildingBlockType: BuildingBlockTypeOrUndefined; description: Description; note: NoteOrUndefined; - eventCategoryOverride: EventCategoryOverrideOrUndefined; + eventCategoryOverride?: EventCategoryOverrideOrUndefined; falsePositives: FalsePositives; from: From; ruleId: RuleId; immutable: Immutable; - index: IndexOrUndefined; - language: LanguageOrUndefined; + index?: IndexOrUndefined; + language?: LanguageOrUndefined; license: LicenseOrUndefined; outputIndex: OutputIndex; - savedId: SavedIdOrUndefined; + savedId?: SavedIdOrUndefined; timelineId: TimelineIdOrUndefined; timelineTitle: TimelineTitleOrUndefined; meta: MetaOrUndefined; - machineLearningJobId: MachineLearningJobIdOrUndefined; - query: QueryOrUndefined; - filters: PartialFilter[] | undefined; + machineLearningJobId?: MachineLearningJobIdOrUndefined; + query?: QueryOrUndefined; + filters?: unknown[]; maxSignals: MaxSignals; riskScore: RiskScore; riskScoreMapping: RiskScoreMappingOrUndefined; @@ -86,21 +86,21 @@ export interface RuleTypeParams extends AlertTypeParams { severity: Severity; severityMapping: SeverityMappingOrUndefined; threat: ThreatsOrUndefined; - threshold: ThresholdOrUndefined; - threatFilters: PartialFilter[] | undefined; - threatIndex: ThreatIndexOrUndefined; - threatIndicatorPath: ThreatIndicatorPathOrUndefined; - threatQuery: ThreatQueryOrUndefined; - threatMapping: ThreatMappingOrUndefined; - threatLanguage: ThreatLanguageOrUndefined; + threshold?: ThresholdOrUndefined; + threatFilters?: unknown[] | undefined; + threatIndex?: ThreatIndexOrUndefined; + threatIndicatorPath?: ThreatIndicatorPathOrUndefined; + threatQuery?: ThreatQueryOrUndefined; + threatMapping?: ThreatMappingOrUndefined; + threatLanguage?: ThreatLanguageOrUndefined; timestampOverride: TimestampOverrideOrUndefined; to: To; type: Type; references: References; version: Version; exceptionsList: ListArrayOrUndefined; - concurrentSearches: ConcurrentSearchesOrUndefined; - itemsPerSearch: ItemsPerSearchOrUndefined; + concurrentSearches?: ConcurrentSearchesOrUndefined; + itemsPerSearch?: ItemsPerSearchOrUndefined; } // eslint-disable-next-line @typescript-eslint/no-explicit-any