Skip to content

Commit

Permalink
[Security Solution][Detections] Pull gap detection logic out in prepa…
Browse files Browse the repository at this point in the history 
…ration for sharing between rule types (#91966)

* Pull gap detection logic out in preparation for sharing between rule types

* Remove comments and unused import

* Remove unncessary function, cleanup, comment

* Update comment

* Address PR comments

* remove unneeded mocks

* Undo change to parseInterval

* Remove another unneeded mock

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
  • Loading branch information
@marshallmain @kibanamachine
marshallmain and kibanamachine authored Feb 25, 2021
1 parent 45bea2b commit f0838e6
Show file tree
Hide file tree
Showing 10 changed files with 279 additions and 467 deletions.
75 changes: 30 additions & 45 deletions ...ns/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
* 2.0.
*/

import moment from 'moment';
import {
sampleRuleAlertParams,
sampleEmptyDocSearchResults,
Expand All @@ -23,9 +22,10 @@ import { alertsMock, AlertServicesMock } from '../../../../../alerts/server/mock
import uuid from 'uuid';
import { listMock } from '../../../../../lists/server/mocks';
import { getExceptionListItemSchemaMock } from '../../../../../lists/common/schemas/response/exception_list_item_schema.mock';
import { BulkResponse } from './types';
import { BulkResponse, RuleRangeTuple } from './types';
import { SearchListItemArraySchema } from '../../../../../lists/common/schemas';
import { getSearchListItemResponseMock } from '../../../../../lists/common/schemas/response/search_list_item_schema.mock';
import { getRuleRangeTuples } from './utils';

const buildRuleMessage = buildRuleMessageFactory({
id: 'fake id',
Expand All @@ -39,16 +39,26 @@ describe('searchAfterAndBulkCreate', () => {
let inputIndexPattern: string[] = [];
let listClient = listMock.getListClient();
const someGuids = Array.from({ length: 13 }).map(() => uuid.v4());
const sampleParams = sampleRuleAlertParams(30);
let tuples: RuleRangeTuple[];
beforeEach(() => {
jest.clearAllMocks();
listClient = listMock.getListClient();
listClient.searchListItemByValues = jest.fn().mockResolvedValue([]);
inputIndexPattern = ['auditbeat-*'];
mockService = alertsMock.createAlertServices();
({ tuples } = getRuleRangeTuples({
logger: mockLogger,
previousStartedAt: new Date(),
from: sampleParams.from,
to: sampleParams.to,
interval: '5m',
maxSignals: sampleParams.maxSignals,
buildRuleMessage,
}));
});

test('should return success with number of searches less than max signals', async () => {
const sampleParams = sampleRuleAlertParams(30);
mockService.callCluster
.mockResolvedValueOnce(repeatedSearchResultsWithSortId(4, 1, someGuids.slice(0, 3)))
.mockResolvedValueOnce({
Expand Down Expand Up @@ -112,11 +122,9 @@ describe('searchAfterAndBulkCreate', () => {
},
},
];

const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({
ruleParams: sampleParams,
gap: null,
previousStartedAt: new Date(),
tuples,
listClient,
exceptionsList: [exceptionItem],
services: mockService,
Expand Down Expand Up @@ -147,7 +155,6 @@ describe('searchAfterAndBulkCreate', () => {
});

test('should return success with number of searches less than max signals with gap', async () => {
const sampleParams = sampleRuleAlertParams(30);
mockService.callCluster
.mockResolvedValueOnce(repeatedSearchResultsWithSortId(4, 1, someGuids.slice(0, 3)))
.mockResolvedValueOnce({
Expand Down Expand Up @@ -201,8 +208,7 @@ describe('searchAfterAndBulkCreate', () => {
];
const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({
ruleParams: sampleParams,
gap: moment.duration(2, 'm'),
previousStartedAt: moment().subtract(10, 'm').toDate(),
tuples,
listClient,
exceptionsList: [exceptionItem],
services: mockService,
Expand Down Expand Up @@ -233,7 +239,6 @@ describe('searchAfterAndBulkCreate', () => {
});

test('should return success when no search results are in the allowlist', async () => {
const sampleParams = sampleRuleAlertParams(30);
mockService.callCluster
.mockResolvedValueOnce(repeatedSearchResultsWithSortId(4, 4, someGuids.slice(0, 3)))
.mockResolvedValueOnce({
Expand Down Expand Up @@ -278,8 +283,7 @@ describe('searchAfterAndBulkCreate', () => {
];
const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({
ruleParams: sampleParams,
gap: null,
previousStartedAt: new Date(),
tuples,
listClient,
exceptionsList: [exceptionItem],
services: mockService,
Expand All @@ -305,7 +309,7 @@ describe('searchAfterAndBulkCreate', () => {
});
expect(success).toEqual(true);
expect(mockService.callCluster).toHaveBeenCalledTimes(3);
expect(createdSignalsCount).toEqual(4); // should not create any signals because all events were in the allowlist
expect(createdSignalsCount).toEqual(4);
expect(lastLookBackDate).toEqual(new Date('2020-04-20T21:27:45+0000'));
});

Expand All @@ -316,7 +320,6 @@ describe('searchAfterAndBulkCreate', () => {
{ ...getSearchListItemResponseMock(), value: ['3.3.3.3'] },
];
listClient.searchListItemByValues = jest.fn().mockResolvedValue(searchListItems);
const sampleParams = sampleRuleAlertParams(30);
mockService.callCluster
.mockResolvedValueOnce(
repeatedSearchResultsWithSortId(4, 4, someGuids.slice(0, 3), [
Expand All @@ -342,8 +345,7 @@ describe('searchAfterAndBulkCreate', () => {
];
const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({
ruleParams: sampleParams,
gap: null,
previousStartedAt: new Date(),
tuples,
listClient,
exceptionsList: [exceptionItem],
services: mockService,
Expand Down Expand Up @@ -382,7 +384,6 @@ describe('searchAfterAndBulkCreate', () => {
];

listClient.searchListItemByValues = jest.fn().mockResolvedValue(searchListItems);
const sampleParams = sampleRuleAlertParams(30);
mockService.callCluster.mockResolvedValueOnce(
repeatedSearchResultsWithNoSortId(4, 4, someGuids.slice(0, 3), [
'1.1.1.1',
Expand All @@ -406,8 +407,7 @@ describe('searchAfterAndBulkCreate', () => {
];
const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({
ruleParams: sampleParams,
gap: null,
previousStartedAt: new Date(),
tuples,
listClient,
exceptionsList: [exceptionItem],
services: mockService,
Expand Down Expand Up @@ -437,13 +437,12 @@ describe('searchAfterAndBulkCreate', () => {
expect(lastLookBackDate).toEqual(new Date('2020-04-20T21:27:45+0000'));
// I don't like testing log statements since logs change but this is the best
// way I can think of to ensure this section is getting hit with this test case.
expect(((mockLogger.debug as unknown) as jest.Mock).mock.calls[8][0]).toContain(
expect(((mockLogger.debug as unknown) as jest.Mock).mock.calls[7][0]).toContain(
'ran out of sort ids to sort on name: "fake name" id: "fake id" rule id: "fake rule id" signals index: "fakeindex"'
);
});

test('should return success when no sortId present but search results are in the allowlist', async () => {
const sampleParams = sampleRuleAlertParams(30);
mockService.callCluster
.mockResolvedValueOnce(repeatedSearchResultsWithNoSortId(4, 4, someGuids.slice(0, 3)))
.mockResolvedValueOnce({
Expand Down Expand Up @@ -487,8 +486,7 @@ describe('searchAfterAndBulkCreate', () => {
];
const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({
ruleParams: sampleParams,
gap: null,
previousStartedAt: new Date(),
tuples,
listClient,
exceptionsList: [exceptionItem],
services: mockService,
Expand All @@ -514,17 +512,16 @@ describe('searchAfterAndBulkCreate', () => {
});
expect(success).toEqual(true);
expect(mockService.callCluster).toHaveBeenCalledTimes(2);
expect(createdSignalsCount).toEqual(4); // should not create any signals because all events were in the allowlist
expect(createdSignalsCount).toEqual(4);
expect(lastLookBackDate).toEqual(new Date('2020-04-20T21:27:45+0000'));
// I don't like testing log statements since logs change but this is the best
// way I can think of to ensure this section is getting hit with this test case.
expect(((mockLogger.debug as unknown) as jest.Mock).mock.calls[15][0]).toContain(
expect(((mockLogger.debug as unknown) as jest.Mock).mock.calls[14][0]).toContain(
'ran out of sort ids to sort on name: "fake name" id: "fake id" rule id: "fake rule id" signals index: "fakeindex"'
);
});

test('should return success when no exceptions list provided', async () => {
const sampleParams = sampleRuleAlertParams(30);
mockService.callCluster
.mockResolvedValueOnce(repeatedSearchResultsWithSortId(4, 4, someGuids.slice(0, 3)))
.mockResolvedValueOnce({
Expand Down Expand Up @@ -565,8 +562,7 @@ describe('searchAfterAndBulkCreate', () => {
);
const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({
ruleParams: sampleParams,
gap: null,
previousStartedAt: new Date(),
tuples,
listClient,
exceptionsList: [],
services: mockService,
Expand All @@ -592,7 +588,7 @@ describe('searchAfterAndBulkCreate', () => {
});
expect(success).toEqual(true);
expect(mockService.callCluster).toHaveBeenCalledTimes(3);
expect(createdSignalsCount).toEqual(4); // should not create any signals because all events were in the allowlist
expect(createdSignalsCount).toEqual(4);
expect(lastLookBackDate).toEqual(new Date('2020-04-20T21:27:45+0000'));
});

Expand All @@ -609,15 +605,13 @@ describe('searchAfterAndBulkCreate', () => {
},
},
];
const sampleParams = sampleRuleAlertParams(10);
mockService.callCluster
.mockResolvedValueOnce(repeatedSearchResultsWithSortId(4, 1, someGuids.slice(0, 3)))
.mockRejectedValue(new Error('bulk failed')); // Added this recently
const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({
listClient,
exceptionsList: [exceptionItem],
gap: null,
previousStartedAt: new Date(),
tuples,
ruleParams: sampleParams,
services: mockService,
logger: mockLogger,
Expand Down Expand Up @@ -659,7 +653,6 @@ describe('searchAfterAndBulkCreate', () => {
},
},
];
const sampleParams = sampleRuleAlertParams(30);
mockService.callCluster.mockResolvedValueOnce(sampleEmptyDocSearchResults());
listClient.searchListItemByValues = jest.fn(({ value }) =>
Promise.resolve(
Expand All @@ -672,8 +665,7 @@ describe('searchAfterAndBulkCreate', () => {
const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({
listClient,
exceptionsList: [exceptionItem],
gap: null,
previousStartedAt: new Date(),
tuples,
ruleParams: sampleParams,
services: mockService,
logger: mockLogger,
Expand Down Expand Up @@ -702,7 +694,6 @@ describe('searchAfterAndBulkCreate', () => {
});

test('if returns false when singleSearchAfter throws an exception', async () => {
const sampleParams = sampleRuleAlertParams(10);
mockService.callCluster
.mockResolvedValueOnce({
took: 100,
Expand Down Expand Up @@ -741,8 +732,7 @@ describe('searchAfterAndBulkCreate', () => {
const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({
listClient,
exceptionsList: [exceptionItem],
gap: null,
previousStartedAt: new Date(),
tuples,
ruleParams: sampleParams,
services: mockService,
logger: mockLogger,
Expand Down Expand Up @@ -771,7 +761,6 @@ describe('searchAfterAndBulkCreate', () => {
});

test('it returns error array when singleSearchAfter returns errors', async () => {
const sampleParams = sampleRuleAlertParams(30);
const bulkItem: BulkResponse = {
took: 100,
errors: true,
Expand Down Expand Up @@ -832,16 +821,14 @@ describe('searchAfterAndBulkCreate', () => {
],
})
.mockResolvedValueOnce(sampleDocSearchResultsNoSortIdNoHits());

const {
success,
createdSignalsCount,
lastLookBackDate,
errors,
} = await searchAfterAndBulkCreate({
ruleParams: sampleParams,
gap: null,
previousStartedAt: new Date(),
tuples,
listClient,
exceptionsList: [],
services: mockService,
Expand Down Expand Up @@ -873,7 +860,6 @@ describe('searchAfterAndBulkCreate', () => {
});

it('invokes the enrichment callback with signal search results', async () => {
const sampleParams = sampleRuleAlertParams(30);
mockService.callCluster
.mockResolvedValueOnce(repeatedSearchResultsWithSortId(4, 1, someGuids.slice(0, 3)))
.mockResolvedValueOnce({
Expand Down Expand Up @@ -917,8 +903,7 @@ describe('searchAfterAndBulkCreate', () => {
const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({
enrichment: mockEnrichment,
ruleParams: sampleParams,
gap: moment.duration(2, 'm'),
previousStartedAt: moment().subtract(10, 'm').toDate(),
tuples,
listClient,
exceptionsList: [],
services: mockService,
Expand Down
14 changes: 1 addition & 13 deletions ...plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,16 +17,14 @@ import {
createSearchResultReturnType,
createSearchAfterReturnTypeFromResponse,
createTotalHitsFromSearchResult,
getSignalTimeTuples,
mergeReturns,
mergeSearchResults,
} from './utils';
import { SearchAfterAndBulkCreateParams, SearchAfterAndBulkCreateReturnType } from './types';

// search_after through documents and re-index using bulk endpoint.
export const searchAfterAndBulkCreate = async ({
gap,
previousStartedAt,
tuples: totalToFromTuples,
ruleParams,
exceptionsList,
services,
Expand Down Expand Up @@ -64,16 +62,6 @@ export const searchAfterAndBulkCreate = async ({
// to ensure we don't exceed maxSignals
let signalsCreatedCount = 0;

const totalToFromTuples = getSignalTimeTuples({
logger,
ruleParamsFrom: ruleParams.from,
ruleParamsTo: ruleParams.to,
ruleParamsMaxSignals: ruleParams.maxSignals,
gap,
previousStartedAt,
interval,
buildRuleMessage,
});
const tuplesToBeLogged = [...totalToFromTuples];
logger.debug(buildRuleMessage(`totalToFromTuples: ${totalToFromTuples.length}`));

Expand Down
Loading

0 comments on commit f0838e6

Please sign in to comment.