Skip to content

Commit

Permalink
[Security Solution][Alerts] adds data generator to functional tests (#…
Browse files Browse the repository at this point in the history 
…151192)

## Summary

- partially addresses #145406 by
adding dataGenerator
- once dataGenerator API agreed, we can proceed with tests refactoring
if needed
- added README how to use data generating tools -
https://github.com/vitaliidm/kibana/blob/alerts/functional-tests-data-generator/x-pack/test/detection_engine_api_integration/utils/data_generator/README.md
- partially migrated a few `new terms` tests, to give an example how to
use the created tool
  • Loading branch information
@vitaliidm
vitaliidm authored Mar 6, 2023
1 parent 61f0a39 commit 2858052
Show file tree
Hide file tree
Showing 13 changed files with 999 additions and 69 deletions.
110 changes: 104 additions & 6 deletions ...st/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
*/

import expect from '@kbn/expect';
import { v4 as uuidv4 } from 'uuid';

import { NewTermsRuleCreateProps } from '@kbn/security-solution-plugin/common/detection_engine/rule_schema';
import { orderBy } from 'lodash';
Expand All @@ -26,16 +27,54 @@ import {
import { FtrProviderContext } from '../../common/ftr_provider_context';
import { previewRuleWithExceptionEntries } from '../../utils/preview_rule_with_exception_entries';
import { deleteAllExceptions } from '../../../lists_api_integration/utils';
import { dataGeneratorFactory } from '../../utils/data_generator';

import { largeArraysBuckets } from './mocks/new_terms';
import { removeRandomValuedProperties } from './utils';

const historicalWindowStart = '2022-10-13T05:00:04.000Z';
const ruleExecutionStart = '2022-10-19T05:00:04.000Z';

// eslint-disable-next-line import/no-default-export
export default ({ getService }: FtrProviderContext) => {
const supertest = getService('supertest');
const esArchiver = getService('esArchiver');
const es = getService('es');
const log = getService('log');
const { indexEnhancedDocuments } = dataGeneratorFactory({
es,
index: 'new_terms',
log,
});

/**
* indexes 2 sets of documents:
* - documents in historical window
* - documents in rule execution window
* @returns id of documents
*/
const newTermsTestExecutionSetup = async ({
historicalDocuments,
ruleExecutionDocuments,
}: {
historicalDocuments: Array<Record<string, unknown>>;
ruleExecutionDocuments: Array<Record<string, unknown>>;
}) => {
const testId = uuidv4();

await indexEnhancedDocuments({
interval: [historicalWindowStart, ruleExecutionStart],
id: testId,
documents: historicalDocuments,
});

await indexEnhancedDocuments({
id: testId,
documents: ruleExecutionDocuments,
});

return testId;
};

describe('New terms type rules', () => {
before(async () => {
Expand Down Expand Up @@ -246,13 +285,36 @@ export default ({ getService }: FtrProviderContext) => {
});

it('should generate 1 alert for unique combination of existing terms', async () => {
// historical window documents
const historicalDocuments = [
{
host: { name: 'host-0', ip: '127.0.0.1' },
},
{
host: { name: 'host-1', ip: '127.0.0.2' },
},
];

// rule execution documents
const ruleExecutionDocuments = [
{
host: { name: 'host-0', ip: '127.0.0.2' },
},
];

const testId = await newTermsTestExecutionSetup({
historicalDocuments,
ruleExecutionDocuments,
});

// ensure there are no alerts for single new terms fields, it means values are not new
const rule: NewTermsRuleCreateProps = {
...getCreateNewTermsRulesSchemaMock('rule-1', true),
index: ['new_terms'],
new_terms_fields: ['host.name', 'host.ip'],
from: '2020-10-19T05:00:04.000Z',
history_window_start: '2020-10-13T05:00:04.000Z',
from: ruleExecutionStart,
history_window_start: historicalWindowStart,
query: `id: "${testId}"`,
};
// shouldn't be terms for 'host.ip'
const hostIpPreview = await previewRule({
Expand Down Expand Up @@ -285,12 +347,36 @@ export default ({ getService }: FtrProviderContext) => {
});

it('should generate 5 alerts, 1 for each new unique combination in 2 fields', async () => {
const historicalDocuments = [
{
'source.ip': ['192.168.1.1'],
tags: ['tag-1', 'tag-2'],
},
{
'source.ip': ['192.168.1.1'],
tags: ['tag-1'],
},
];

const ruleExecutionDocuments = [
{
'source.ip': ['192.168.1.1', '192.168.1.2'],
tags: ['tag-new-1', 'tag-2', 'tag-new-3'],
},
];

const testId = await newTermsTestExecutionSetup({
historicalDocuments,
ruleExecutionDocuments,
});

const rule: NewTermsRuleCreateProps = {
...getCreateNewTermsRulesSchemaMock('rule-1', true),
index: ['new_terms'],
new_terms_fields: ['source.ip', 'tags'],
from: '2020-10-19T05:00:04.000Z',
history_window_start: '2020-10-13T05:00:04.000Z',
from: ruleExecutionStart,
history_window_start: historicalWindowStart,
query: `id: "${testId}"`,
};

const { previewId } = await previewRule({ supertest, rule });
Expand All @@ -313,12 +399,24 @@ export default ({ getService }: FtrProviderContext) => {
});

it('should generate 1 alert for unique combination of terms, one of which is a number', async () => {
const historicalDocuments = [
{ user: { name: 'user-0', id: 0 } },
{ user: { name: 'user-1', id: 1 } },
];
const ruleExecutionDocuments = [{ user: { name: 'user-0', id: 1 } }];

const testId = await newTermsTestExecutionSetup({
historicalDocuments,
ruleExecutionDocuments,
});

const rule: NewTermsRuleCreateProps = {
...getCreateNewTermsRulesSchemaMock('rule-1', true),
index: ['new_terms'],
new_terms_fields: ['user.name', 'user.id'],
from: '2020-10-19T05:00:04.000Z',
history_window_start: '2020-10-13T05:00:04.000Z',
from: ruleExecutionStart,
history_window_start: historicalWindowStart,
query: `id: "${testId}"`,
};

const { previewId } = await previewRule({ supertest, rule });
Expand Down
19 changes: 5 additions & 14 deletions ...tection_engine_api_integration/security_and_spaces/rule_execution_logic/non_ecs_fields.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,37 +6,29 @@
*/

import expect from 'expect';
import { v4 as uuidv4 } from 'uuid';

import {
deleteAllRules,
deleteSignalsIndex,
getPreviewAlerts,
getRuleForSignalTesting,
previewRule,
} from '../../utils';
import { indexDocumentsFactory } from '../../utils/data_generator';
import { dataGeneratorFactory, enhanceDocument } from '../../utils/data_generator';
import { FtrProviderContext } from '../../common/ftr_provider_context';

const getQueryRule = (docIdToQuery: string) => ({
...getRuleForSignalTesting(['ecs_non_compliant']),
query: `id: "${docIdToQuery}"`,
});

const getDocument = (id: string, doc: Record<string, unknown>) => ({
id,
'@timestamp': new Date().toISOString(),
...doc,
});

// eslint-disable-next-line import/no-default-export
export default ({ getService }: FtrProviderContext) => {
const supertest = getService('supertest');
const esArchiver = getService('esArchiver');
const es = getService('es');
const log = getService('log');

const indexDocuments = indexDocumentsFactory({
const { indexListOfDocuments } = dataGeneratorFactory({
es,
index: 'ecs_non_compliant',
log,
Expand All @@ -49,13 +41,12 @@ export default ({ getService }: FtrProviderContext) => {
* 3. return created preview alert and errors logs
*/
const indexAndCreatePreviewAlert = async (document: Record<string, unknown>) => {
const documentId = uuidv4();

await indexDocuments([getDocument(documentId, document)]);
const enhancedDocument = enhanceDocument({ document });
await indexListOfDocuments([enhancedDocument]);

const { previewId, logs } = await previewRule({
supertest,
rule: getQueryRule(documentId),
rule: getQueryRule(enhancedDocument.id),
});
const previewAlerts = await getPreviewAlerts({ es, previewId });

Expand Down
50 changes: 23 additions & 27 deletions ...k/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/query.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ import {
setSignalStatus,
} from '../../utils';
import { FtrProviderContext } from '../../common/ftr_provider_context';
import { indexDocumentsFactory } from '../../utils/data_generator';
import { dataGeneratorFactory } from '../../utils/data_generator';
import { patchRule } from '../../utils/patch_rule';

/**
Expand Down Expand Up @@ -730,7 +730,7 @@ export default ({ getService }: FtrProviderContext) => {
});

describe('with a suppression time window', async () => {
const indexDocuments = indexDocumentsFactory({
const { indexListOfDocuments, indexGeneratedDocuments } = dataGeneratorFactory({
es,
index: 'ecs_compliant',
log,
Expand Down Expand Up @@ -758,7 +758,7 @@ export default ({ getService }: FtrProviderContext) => {
name: 'agent-1',
},
};
await indexDocuments([firstDocument, firstDocument]);
await indexListOfDocuments([firstDocument, firstDocument]);

const rule: QueryRuleCreateProps = {
...getRuleForSignalTesting(['ecs_compliant']),
Expand Down Expand Up @@ -799,7 +799,7 @@ export default ({ getService }: FtrProviderContext) => {
};
// Add a new document, then disable and re-enable to trigger another rule run. The second doc should
// trigger an update to the existing alert without changing the timestamp
await indexDocuments([secondDocument, secondDocument]);
await indexListOfDocuments([secondDocument, secondDocument]);
await patchRule(supertest, log, { id: createdRule.id, enabled: false });
await patchRule(supertest, log, { id: createdRule.id, enabled: true });
const afterTimestamp = new Date();
Expand Down Expand Up @@ -839,7 +839,7 @@ export default ({ getService }: FtrProviderContext) => {
name: 'agent-1',
},
};
await indexDocuments([firstDocument, firstDocument]);
await indexListOfDocuments([firstDocument, firstDocument]);

const rule: QueryRuleCreateProps = {
...getRuleForSignalTesting(['ecs_compliant']),
Expand Down Expand Up @@ -875,7 +875,7 @@ export default ({ getService }: FtrProviderContext) => {
};
// Add new documents, then disable and re-enable to trigger another rule run. The second doc should
// trigger a new alert since the first one is now closed.
await indexDocuments([secondDocument, secondDocument]);
await indexListOfDocuments([secondDocument, secondDocument]);
await patchRule(supertest, log, { id: createdRule.id, enabled: false });
await patchRule(supertest, log, { id: createdRule.id, enabled: true });
const afterTimestamp = new Date();
Expand Down Expand Up @@ -1160,7 +1160,7 @@ export default ({ getService }: FtrProviderContext) => {
ingested: '2020-10-28T06:10:00.000Z',
},
};
await indexDocuments([docWithoutOverride, docWithOverride]);
await indexListOfDocuments([docWithoutOverride, docWithOverride]);

const rule: QueryRuleCreateProps = {
...getRuleForSignalTesting(['ecs_compliant']),
Expand Down Expand Up @@ -1209,26 +1209,22 @@ export default ({ getService }: FtrProviderContext) => {
it('should generate and update up to max_signals alerts', async () => {
const id = uuidv4();
const timestamp = '2020-10-28T06:00:00.000Z';
const docs = Array(150)
.fill({})
.map((_, i) => ({
id,
'@timestamp': timestamp,
agent: {
name: `agent-${i}`,
},
}));
const laterTimestamp = '2020-10-28T07:00:00.000Z';
const laterDocs = Array(150)
.fill({})
.map((_, i) => ({
id,
'@timestamp': laterTimestamp,
agent: {
name: `agent-${i}`,
},
}));
await indexDocuments([...docs, ...laterDocs]);

await Promise.all(
[timestamp, laterTimestamp].map((t) =>
indexGeneratedDocuments({
docsCount: 150,
seed: (index) => ({
id,
'@timestamp': t,
agent: {
name: `agent-${index}`,
},
}),
})
)
);

const rule: QueryRuleCreateProps = {
...getRuleForSignalTesting(['ecs_compliant']),
Expand Down Expand Up @@ -1315,7 +1311,7 @@ export default ({ getService }: FtrProviderContext) => {
name: 'agent-2',
},
};
await indexDocuments([firstDoc, secondDoc, thirdDoc]);
await indexListOfDocuments([firstDoc, secondDoc, thirdDoc]);

const rule: QueryRuleCreateProps = {
...getRuleForSignalTesting(['ecs_compliant']),
Expand Down
Loading

0 comments on commit 2858052

Please sign in to comment.