From 60f6d4a49bdf3425d595beb33238bcdbc5511370 Mon Sep 17 00:00:00 2001 From: FrankHassanabad Date: Thu, 3 Jun 2021 10:04:59 -0600 Subject: [PATCH 1/5] wip --- .../security_and_spaces/tests/index.ts | 1 + .../security_and_spaces/tests/timestamps.ts | 70 +++++++++++++++++++ .../es_archives/security_solution/README.md | 11 +++ .../timestamp_in_seconds/data.json | 10 +++ .../timestamp_in_seconds/mappings.json | 22 ++++++ 5 files changed, 114 insertions(+) create mode 100644 x-pack/test/detection_engine_api_integration/security_and_spaces/tests/timestamps.ts create mode 100644 x-pack/test/functional/es_archives/security_solution/README.md create mode 100644 x-pack/test/functional/es_archives/security_solution/timestamp_in_seconds/data.json create mode 100644 x-pack/test/functional/es_archives/security_solution/timestamp_in_seconds/mappings.json diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts index 57b24f6de2a48..477f1f37f50d2 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts @@ -42,6 +42,7 @@ export default ({ loadTestFile }: FtrProviderContext): void => { loadTestFile(require.resolve('./create_signals_migrations')); loadTestFile(require.resolve('./finalize_signals_migrations')); loadTestFile(require.resolve('./delete_signals_migrations')); + loadTestFile(require.resolve('./timestamps')); }); // That split here enable us on using a different ciGroup to run the tests diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/timestamps.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/timestamps.ts new file mode 100644 index 0000000000000..29a2ed4c9f666 --- /dev/null +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/timestamps.ts @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { CreateRulesSchema } from '../../../../plugins/security_solution/common/detection_engine/schemas/request'; + +import { + DETECTION_ENGINE_RULES_URL, + DETECTION_ENGINE_RULES_STATUS_URL, +} from '../../../../plugins/security_solution/common/constants'; +import { FtrProviderContext } from '../../common/ftr_provider_context'; +import { + createSignalsIndex, + deleteAllAlerts, + deleteSignalsIndex, + createRule, + getSimpleRule, + getSimpleRuleOutput, + getSimpleRuleOutputWithoutRuleId, + getSimpleRuleWithoutRuleId, + removeServerGeneratedProperties, + removeServerGeneratedPropertiesIncludingRuleId, + getSimpleMlRule, + getSimpleMlRuleOutput, + waitForRuleSuccessOrStatus, + waitForSignalsToBePresent, + waitForAlertToComplete, + getRuleForSignalTesting, + getSignalsByIds, + getRuleForSignalTestingWithTimestampOverride, +} from '../../utils'; +import { ROLES } from '../../../../plugins/security_solution/common/test'; +import { createUserAndRole, deleteUserAndRole } from '../roles_users_utils'; +import { RuleStatusResponse } from '../../../../plugins/security_solution/server/lib/detection_engine/rules/types'; + +// eslint-disable-next-line import/no-default-export +export default ({ getService }: FtrProviderContext) => { + const supertest = getService('supertest'); + const supertestWithoutAuth = getService('supertestWithoutAuth'); + const esArchiver = getService('esArchiver'); + + describe('timestamps', () => { + describe('source index with timestamp in milliseconds', () => { + beforeEach(async () => { + await createSignalsIndex(supertest); + await esArchiver.load('security_solution/timestamp_in_seconds'); + }); + + afterEach(async () => { + await deleteSignalsIndex(supertest); + await deleteAllAlerts(supertest); + await esArchiver.unload('security_solution/timestamp_in_seconds'); + }); + + it('should convert a timestamp in epoch_seconds to the correct ISO format', async () => { + const rule = getRuleForSignalTesting(['timestamp_in_seconds']); + const { id } = await createRule(supertest, rule); + await waitForRuleSuccessOrStatus(supertest, id); + await waitForSignalsToBePresent(supertest, 1, [id]); + const signalsOpen = await getSignalsByIds(supertest, [id]); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.signal.original_time).sort(); + expect(hits).to.eql([]); + }); + }); + }); +}; diff --git a/x-pack/test/functional/es_archives/security_solution/README.md b/x-pack/test/functional/es_archives/security_solution/README.md new file mode 100644 index 0000000000000..c832e0835bbbc --- /dev/null +++ b/x-pack/test/functional/es_archives/security_solution/README.md @@ -0,0 +1,11 @@ +Collection of data sets for use within various tests. Most of the tests to these live in either: + +``` +x-pack/test/detection_engine_api_integrations/security_and_spaces/tests +``` + +or + +``` +x-pack/test/api_integration/apis/security_solution +``` diff --git a/x-pack/test/functional/es_archives/security_solution/timestamp_in_seconds/data.json b/x-pack/test/functional/es_archives/security_solution/timestamp_in_seconds/data.json new file mode 100644 index 0000000000000..46b30b239bbc7 --- /dev/null +++ b/x-pack/test/functional/es_archives/security_solution/timestamp_in_seconds/data.json @@ -0,0 +1,10 @@ +{ + "type": "doc", + "value": { + "index": "timestamp_in_seconds", + "source": { + "@timestamp": 1622676795 + }, + "type": "_doc" + } +} diff --git a/x-pack/test/functional/es_archives/security_solution/timestamp_in_seconds/mappings.json b/x-pack/test/functional/es_archives/security_solution/timestamp_in_seconds/mappings.json new file mode 100644 index 0000000000000..fd8880fe0bc49 --- /dev/null +++ b/x-pack/test/functional/es_archives/security_solution/timestamp_in_seconds/mappings.json @@ -0,0 +1,22 @@ +{ + "type": "index", + "value": { + "index": "timestamp_in_seconds", + "mappings": { + "dynamic": "strict", + "properties": { + "@timestamp": { + "type": "date", + "format": "epoch_second" + } + } + }, + "settings": { + "index": { + "refresh_interval": "1s", + "number_of_replicas": "1", + "number_of_shards": "1" + } + } + } +} From b8c57f5febf4230e0a2d9d773694c2858df6acc7 Mon Sep 17 00:00:00 2001 From: FrankHassanabad Date: Thu, 3 Jun 2021 17:35:44 -0600 Subject: [PATCH 2/5] Fixed query to work with different source timestamps --- .../signals/__mocks__/es_results.ts | 8 +- .../signals/build_bulk_body.test.ts | 61 +++---- .../signals/build_event_type_signal.test.ts | 3 - .../signals/build_events_query.test.ts | 63 +++---- .../signals/build_events_query.ts | 4 +- .../signals/build_rule.test.ts | 1 - .../signals/build_signal.test.ts | 12 +- .../detection_engine/signals/build_signal.ts | 9 +- .../build_rule_name_from_mapping.test.ts | 1 - .../lib/detection_engine/signals/utils.ts | 52 ++++-- .../tests/generating_signals.ts | 113 ------------ .../security_and_spaces/tests/timestamps.ts | 169 +++++++++++++++--- 12 files changed, 248 insertions(+), 248 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts index 1590a4f0fbb04..33a6878fcfb9c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts @@ -146,7 +146,7 @@ export const sampleDocWithSortId = ( export const sampleDocNoSortId = ( someUuid: string = sampleIdGuid, ip?: string -): SignalSourceHit => ({ +): SignalSourceHit & { _source: Required['_source'] } => ({ _index: 'myFakeSignalIndex', _type: 'doc', _score: 100, @@ -225,12 +225,12 @@ export const sampleWrappedSignalHit = (): WrappedSignalHit => { }; }; -export const sampleDocWithAncestors = (): SignalSearchResponse => { +export const sampleDocWithAncestors = (): SignalSearchResponse & { + hits: { hits: Array> }; +} => { const sampleDoc = sampleDocNoSortId(); delete sampleDoc.sort; - // @ts-expect-error @elastic/elasticsearch _source is optional delete sampleDoc._source.source; - // @ts-expect-error @elastic/elasticsearch _source is optional sampleDoc._source.signal = { parent: { id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts index 743d9580218a3..6cb00e08e3f86 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts @@ -24,6 +24,10 @@ import { SignalHit, SignalSourceHit } from './types'; import { SIGNALS_TEMPLATE_VERSION } from '../routes/index/get_signals_template'; import { getQueryRuleParams, getThresholdRuleParams } from '../schemas/rule_schemas.mock'; +type SignalHitOptionalTimestamp = Omit & { + '@timestamp'?: SignalHit['@timestamp']; +}; + describe('buildBulkBody', () => { beforeEach(() => { jest.clearAllMocks(); @@ -32,11 +36,9 @@ describe('buildBulkBody', () => { test('bulk body builds well-defined body', () => { const ruleSO = sampleRuleSO(getQueryRuleParams()); const doc = sampleDocNoSortId(); - // @ts-expect-error @elastic/elasticsearch _source is optional delete doc._source.source; - const fakeSignalSourceHit = buildBulkBody(ruleSO, doc); + const fakeSignalSourceHit: SignalHitOptionalTimestamp = buildBulkBody(ruleSO, doc); // Timestamp will potentially always be different so remove it for the test - // @ts-expect-error delete fakeSignalSourceHit['@timestamp']; const expected: Omit & { someKey: 'someValue' } = { someKey: 'someValue', @@ -69,7 +71,7 @@ describe('buildBulkBody', () => { depth: 0, }, ], - original_time: '2020-04-20T21:27:45+0000', + original_time: '2020-04-20T21:27:45.000Z', status: 'open', rule: expectedRule(), depth: 1, @@ -81,9 +83,8 @@ describe('buildBulkBody', () => { test('bulk body builds well-defined body with threshold results', () => { const ruleSO = sampleRuleSO(getThresholdRuleParams()); const baseDoc = sampleDocNoSortId(); - const doc: SignalSourceHit = { + const doc: SignalSourceHit & { _source: Required['_source'] } = { ...baseDoc, - // @ts-expect-error @elastic/elasticsearch _source is optional _source: { ...baseDoc._source, threshold_result: { @@ -96,11 +97,9 @@ describe('buildBulkBody', () => { }, }, }; - // @ts-expect-error @elastic/elasticsearch _source is optional delete doc._source.source; - const fakeSignalSourceHit = buildBulkBody(ruleSO, doc); + const fakeSignalSourceHit: SignalHitOptionalTimestamp = buildBulkBody(ruleSO, doc); // Timestamp will potentially always be different so remove it for the test - // @ts-expect-error delete fakeSignalSourceHit['@timestamp']; const expected: Omit & { someKey: 'someValue' } = { someKey: 'someValue', @@ -133,7 +132,7 @@ describe('buildBulkBody', () => { depth: 0, }, ], - original_time: '2020-04-20T21:27:45+0000', + original_time: '2020-04-20T21:27:45.000Z', status: 'open', rule: { ...expectedRule(), @@ -167,18 +166,15 @@ describe('buildBulkBody', () => { test('bulk body builds original_event if it exists on the event to begin with', () => { const ruleSO = sampleRuleSO(getQueryRuleParams()); const doc = sampleDocNoSortId(); - // @ts-expect-error @elastic/elasticsearch _source is optional delete doc._source.source; - // @ts-expect-error @elastic/elasticsearch _source is optional doc._source.event = { action: 'socket_opened', module: 'system', dataset: 'socket', kind: 'event', }; - const fakeSignalSourceHit = buildBulkBody(ruleSO, doc); + const fakeSignalSourceHit: SignalHitOptionalTimestamp = buildBulkBody(ruleSO, doc); // Timestamp will potentially always be different so remove it for the test - // @ts-expect-error delete fakeSignalSourceHit['@timestamp']; const expected: Omit & { someKey: 'someValue' } = { someKey: 'someValue', @@ -220,7 +216,7 @@ describe('buildBulkBody', () => { depth: 0, }, ], - original_time: '2020-04-20T21:27:45+0000', + original_time: '2020-04-20T21:27:45.000Z', status: 'open', rule: expectedRule(), depth: 1, @@ -232,17 +228,14 @@ describe('buildBulkBody', () => { test('bulk body builds original_event if it exists on the event to begin with but no kind information', () => { const ruleSO = sampleRuleSO(getQueryRuleParams()); const doc = sampleDocNoSortId(); - // @ts-expect-error @elastic/elasticsearch _source is optional delete doc._source.source; - // @ts-expect-error @elastic/elasticsearch _source is optional doc._source.event = { action: 'socket_opened', module: 'system', dataset: 'socket', }; - const fakeSignalSourceHit = buildBulkBody(ruleSO, doc); + const fakeSignalSourceHit: SignalHitOptionalTimestamp = buildBulkBody(ruleSO, doc); // Timestamp will potentially always be different so remove it for the test - // @ts-expect-error delete fakeSignalSourceHit['@timestamp']; const expected: Omit & { someKey: 'someValue' } = { someKey: 'someValue', @@ -283,7 +276,7 @@ describe('buildBulkBody', () => { depth: 0, }, ], - original_time: '2020-04-20T21:27:45+0000', + original_time: '2020-04-20T21:27:45.000Z', status: 'open', rule: expectedRule(), depth: 1, @@ -295,15 +288,12 @@ describe('buildBulkBody', () => { test('bulk body builds original_event if it exists on the event to begin with with only kind information', () => { const ruleSO = sampleRuleSO(getQueryRuleParams()); const doc = sampleDocNoSortId(); - // @ts-expect-error @elastic/elasticsearch _source is optional delete doc._source.source; - // @ts-expect-error @elastic/elasticsearch _source is optional doc._source.event = { kind: 'event', }; - const fakeSignalSourceHit = buildBulkBody(ruleSO, doc); + const fakeSignalSourceHit: SignalHitOptionalTimestamp = buildBulkBody(ruleSO, doc); // Timestamp will potentially always be different so remove it for the test - // @ts-expect-error delete fakeSignalSourceHit['@timestamp']; const expected: Omit & { someKey: 'someValue' } = { someKey: 'someValue', @@ -339,7 +329,7 @@ describe('buildBulkBody', () => { depth: 0, }, ], - original_time: '2020-04-20T21:27:45+0000', + original_time: '2020-04-20T21:27:45.000Z', status: 'open', rule: expectedRule(), depth: 1, @@ -351,7 +341,6 @@ describe('buildBulkBody', () => { test('bulk body builds "original_signal" if it exists already as a numeric', () => { const ruleSO = sampleRuleSO(getQueryRuleParams()); const sampleDoc = sampleDocNoSortId(); - // @ts-expect-error @elastic/elasticsearch _source is optional delete sampleDoc._source.source; const doc = ({ ...sampleDoc, @@ -393,7 +382,7 @@ describe('buildBulkBody', () => { depth: 0, }, ], - original_time: '2020-04-20T21:27:45+0000', + original_time: '2020-04-20T21:27:45.000Z', status: 'open', rule: expectedRule(), depth: 1, @@ -405,7 +394,6 @@ describe('buildBulkBody', () => { test('bulk body builds "original_signal" if it exists already as an object', () => { const ruleSO = sampleRuleSO(getQueryRuleParams()); const sampleDoc = sampleDocNoSortId(); - // @ts-expect-error @elastic/elasticsearch _source is optional delete sampleDoc._source.source; const doc = ({ ...sampleDoc, @@ -447,7 +435,7 @@ describe('buildBulkBody', () => { depth: 0, }, ], - original_time: '2020-04-20T21:27:45+0000', + original_time: '2020-04-20T21:27:45.000Z', status: 'open', rule: expectedRule(), depth: 1, @@ -466,9 +454,8 @@ describe('buildSignalFromSequence', () => { block2._source.new_key = 'new_key_value'; const blocks = [block1, block2]; const ruleSO = sampleRuleSO(getQueryRuleParams()); - const signal = buildSignalFromSequence(blocks, ruleSO); + const signal: SignalHitOptionalTimestamp = buildSignalFromSequence(blocks, ruleSO); // Timestamp will potentially always be different so remove it for the test - // @ts-expect-error delete signal['@timestamp']; const expected: Omit & { new_key: string } = { new_key: 'new_key_value', @@ -552,9 +539,8 @@ describe('buildSignalFromSequence', () => { block2._source['@timestamp'] = '2021-05-20T22:28:46+0000'; block2._source.someKey = 'someOtherValue'; const ruleSO = sampleRuleSO(getQueryRuleParams()); - const signal = buildSignalFromSequence([block1, block2], ruleSO); + const signal: SignalHitOptionalTimestamp = buildSignalFromSequence([block1, block2], ruleSO); // Timestamp will potentially always be different so remove it for the test - // @ts-expect-error delete signal['@timestamp']; const expected: Omit = { event: { @@ -635,12 +621,13 @@ describe('buildSignalFromSequence', () => { describe('buildSignalFromEvent', () => { test('builds a basic signal from a single event', () => { const ancestor = sampleDocWithAncestors().hits.hits[0]; - // @ts-expect-error @elastic/elasticsearch _source is optional delete ancestor._source.source; const ruleSO = sampleRuleSO(getQueryRuleParams()); - const signal = buildSignalFromEvent(ancestor, ruleSO, true); + const signal: Omit & { + '@timestamp'?: SignalHit['@timestamp']; + } = buildSignalFromEvent(ancestor, ruleSO, true); + // Timestamp will potentially always be different so remove it for the test - // @ts-expect-error delete signal['@timestamp']; const expected: Omit & { someKey: 'someValue' } = { someKey: 'someValue', @@ -651,7 +638,7 @@ describe('buildSignalFromEvent', () => { _meta: { version: SIGNALS_TEMPLATE_VERSION, }, - original_time: '2020-04-20T21:27:45+0000', + original_time: '2020-04-20T21:27:45.000Z', parent: { id: sampleIdGuid, rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_event_type_signal.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_event_type_signal.test.ts index 185c165442921..0ae81770e83c2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_event_type_signal.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_event_type_signal.test.ts @@ -16,7 +16,6 @@ describe('buildEventTypeSignal', () => { test('it returns the event appended of kind signal if it does not exist', () => { const doc = sampleDocNoSortId(); - // @ts-expect-error @elastic/elasticsearch _source is optional delete doc._source.event; const eventType = buildEventTypeSignal(doc); const expected: object = { kind: 'signal' }; @@ -25,7 +24,6 @@ describe('buildEventTypeSignal', () => { test('it returns the event appended of kind signal if it is an empty object', () => { const doc = sampleDocNoSortId(); - // @ts-expect-error @elastic/elasticsearch _source is optional doc._source.event = {}; const eventType = buildEventTypeSignal(doc); const expected: object = { kind: 'signal' }; @@ -34,7 +32,6 @@ describe('buildEventTypeSignal', () => { test('it returns the event with kind signal and other properties if they exist', () => { const doc = sampleDocNoSortId(); - // @ts-expect-error @elastic/elasticsearch _source is optional doc._source.event = { action: 'socket_opened', module: 'system', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.test.ts index 3f4a17dc091ab..28cea9ea22b0d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.test.ts @@ -24,12 +24,6 @@ describe('create_signals', () => { size: 100, ignore_unavailable: true, body: { - docvalue_fields: [ - { - field: '@timestamp', - format: 'strict_date_optional_time', - }, - ], query: { bool: { filter: [ @@ -67,6 +61,10 @@ describe('create_signals', () => { field: '*', include_unmapped: true, }, + { + field: '@timestamp', + format: 'strict_date_optional_time', + }, ], sort: [ { @@ -96,16 +94,6 @@ describe('create_signals', () => { size: 100, ignore_unavailable: true, body: { - docvalue_fields: [ - { - field: 'event.ingested', - format: 'strict_date_optional_time', - }, - { - field: '@timestamp', - format: 'strict_date_optional_time', - }, - ], query: { bool: { filter: [ @@ -167,6 +155,14 @@ describe('create_signals', () => { field: '*', include_unmapped: true, }, + { + field: 'event.ingested', + format: 'strict_date_optional_time', + }, + { + field: '@timestamp', + format: 'strict_date_optional_time', + }, ], sort: [ { @@ -203,12 +199,6 @@ describe('create_signals', () => { size: 100, ignore_unavailable: true, body: { - docvalue_fields: [ - { - field: '@timestamp', - format: 'strict_date_optional_time', - }, - ], query: { bool: { filter: [ @@ -246,6 +236,10 @@ describe('create_signals', () => { field: '*', include_unmapped: true, }, + { + field: '@timestamp', + format: 'strict_date_optional_time', + }, ], search_after: [fakeSortId], sort: [ @@ -276,12 +270,6 @@ describe('create_signals', () => { size: 100, ignore_unavailable: true, body: { - docvalue_fields: [ - { - field: '@timestamp', - format: 'strict_date_optional_time', - }, - ], query: { bool: { filter: [ @@ -319,6 +307,10 @@ describe('create_signals', () => { field: '*', include_unmapped: true, }, + { + field: '@timestamp', + format: 'strict_date_optional_time', + }, ], search_after: [fakeSortIdNumber], sort: [ @@ -348,12 +340,6 @@ describe('create_signals', () => { size: 100, ignore_unavailable: true, body: { - docvalue_fields: [ - { - field: '@timestamp', - format: 'strict_date_optional_time', - }, - ], query: { bool: { filter: [ @@ -391,6 +377,10 @@ describe('create_signals', () => { field: '*', include_unmapped: true, }, + { + field: '@timestamp', + format: 'strict_date_optional_time', + }, ], sort: [ { @@ -427,7 +417,6 @@ describe('create_signals', () => { size: 100, ignore_unavailable: true, body: { - docvalue_fields: [{ field: '@timestamp', format: 'strict_date_optional_time' }], query: { bool: { filter: [ @@ -465,6 +454,10 @@ describe('create_signals', () => { field: '*', include_unmapped: true, }, + { + field: '@timestamp', + format: 'strict_date_optional_time', + }, ], aggregations: { tags: { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts index 86fb51e4785ad..123b7cbb3b591 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts @@ -94,7 +94,7 @@ export const buildEventsSearchQuery = ({ ]; const filterWithTime: estypes.QueryContainer[] = [ - // but tests contain undefined, so I suppose it's desired behaviour + // but tests contain undefined, so I suppose it's desired behavior // @ts-expect-error undefined in not assignable to QueryContainer filter, { bool: { filter: [{ bool: { should: [...rangeFilter], minimum_should_match: 1 } }] } }, @@ -106,7 +106,6 @@ export const buildEventsSearchQuery = ({ size, ignore_unavailable: true, body: { - docvalue_fields: docFields, query: { bool: { filter: [ @@ -122,6 +121,7 @@ export const buildEventsSearchQuery = ({ field: '*', include_unmapped: true, }, + ...docFields, ], ...(aggregations ? { aggregations } : {}), sort: [ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts index 412ccf7a40e33..bd5444a325128 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts @@ -134,7 +134,6 @@ describe('buildRuleWithOverrides', () => { }, ]; const doc = sampleDocNoSortId(); - // @ts-expect-error @elastic/elasticsearch _source is optional doc._source.new_risk_score = newRiskScore; const rule = buildRuleWithOverrides(ruleSO, doc._source!); const expected = { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts index 6408b5fe9de10..3a30da170d3f2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts @@ -28,7 +28,6 @@ describe('buildSignal', () => { test('it builds a signal as expected without original_event if event does not exist', () => { const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); - // @ts-expect-error @elastic/elasticsearch _source is optional delete doc._source.event; const rule = getRulesSchemaMock(); const signal = { @@ -61,7 +60,7 @@ describe('buildSignal', () => { depth: 0, }, ], - original_time: '2020-04-20T21:27:45+0000', + original_time: '2020-04-20T21:27:45.000Z', status: 'open', rule: { author: [], @@ -105,7 +104,6 @@ describe('buildSignal', () => { test('it builds a signal as expected with original_event if is present', () => { const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); - // @ts-expect-error @elastic/elasticsearch _source is optional doc._source.event = { action: 'socket_opened', dataset: 'socket', @@ -143,7 +141,7 @@ describe('buildSignal', () => { depth: 0, }, ], - original_time: '2020-04-20T21:27:45+0000', + original_time: '2020-04-20T21:27:45.000Z', original_event: { action: 'socket_opened', dataset: 'socket', @@ -193,7 +191,6 @@ describe('buildSignal', () => { test('it builds a ancestor correctly if the parent does not exist', () => { const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); - // @ts-expect-error @elastic/elasticsearch _source is optional doc._source.event = { action: 'socket_opened', dataset: 'socket', @@ -212,14 +209,12 @@ describe('buildSignal', () => { test('it builds a ancestor correctly if the parent does exist', () => { const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); - // @ts-expect-error @elastic/elasticsearch _source is optional doc._source.event = { action: 'socket_opened', dataset: 'socket', kind: 'event', module: 'system', }; - // @ts-expect-error @elastic/elasticsearch _source is optional doc._source.signal = { parents: [ { @@ -255,7 +250,6 @@ describe('buildSignal', () => { test('it builds a signal ancestor correctly if the parent does not exist', () => { const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); - // @ts-expect-error @elastic/elasticsearch _source is optional doc._source.event = { action: 'socket_opened', dataset: 'socket', @@ -276,14 +270,12 @@ describe('buildSignal', () => { test('it builds a signal ancestor correctly if the parent does exist', () => { const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); - // @ts-expect-error @elastic/elasticsearch _source is optional doc._source.event = { action: 'socket_opened', dataset: 'socket', kind: 'event', module: 'system', }; - // @ts-expect-error @elastic/elasticsearch _source is optional doc._source.signal = { parents: [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts index 237536a99c0f0..a415c83e857c2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts @@ -10,6 +10,7 @@ import { RulesSchema } from '../../../../common/detection_engine/schemas/respons import { SIGNALS_TEMPLATE_VERSION } from '../routes/index/get_signals_template'; import { isEventTypeSignal } from './build_event_type_signal'; import { Signal, Ancestor, BaseSignalHit, ThresholdResult } from './types'; +import { getValidDateFromDoc } from './utils'; /** * Takes a parent signal or event document and extracts the information needed for the corresponding entry in the child @@ -103,6 +104,7 @@ const isThresholdResult = (thresholdResult: SearchTypes): thresholdResult is Thr /** * Creates signal fields that are only available in the special case where a signal has only 1 parent signal/event. + * We copy the original time from the document as "original_time" since we override the timestamp with the current date time. * @param doc The parent signal/event of the new signal to be built. */ export const additionalSignalFields = (doc: BaseSignalHit) => { @@ -110,10 +112,13 @@ export const additionalSignalFields = (doc: BaseSignalHit) => { if (thresholdResult != null && !isThresholdResult(thresholdResult)) { throw new Error(`threshold_result failed to validate: ${thresholdResult}`); } + const originalTime = getValidDateFromDoc({ + doc, + timestampOverride: undefined, + }); return { parent: buildParent(removeClashes(doc)), - // @ts-expect-error @elastic/elasticsearch _source is optional - original_time: doc._source['@timestamp'], // This field has already been replaced with timestampOverride, if provided. + original_time: originalTime != null ? originalTime.toISOString() : undefined, original_event: doc._source?.event ?? undefined, threshold_result: thresholdResult, original_signal: diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/mappings/build_rule_name_from_mapping.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/mappings/build_rule_name_from_mapping.test.ts index b6281b637d434..23e5aecc5c553 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/mappings/build_rule_name_from_mapping.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/mappings/build_rule_name_from_mapping.test.ts @@ -15,7 +15,6 @@ describe('buildRuleNameFromMapping', () => { test('rule name defaults to provided if mapping is incomplete', () => { const ruleName = buildRuleNameFromMapping({ - // @ts-expect-error @elastic/elasticsearch _source is optional eventSource: sampleDocNoSortId()._source, ruleName: 'rule-name', ruleNameMapping: 'message', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts index cc4ed6a45807b..079049c56037c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts @@ -584,23 +584,41 @@ export const lastValidDate = ({ return undefined; } else { const lastRecord = searchResult.hits.hits[searchResult.hits.hits.length - 1]; - const timestamp = timestampOverride ?? '@timestamp'; - const timestampValue = - lastRecord.fields != null && lastRecord.fields[timestamp] != null - ? lastRecord.fields[timestamp][0] - : // @ts-expect-error @elastic/elasticsearch _source is optional - lastRecord._source[timestamp]; - const lastTimestamp = - typeof timestampValue === 'string' || typeof timestampValue === 'number' - ? timestampValue - : undefined; - if (lastTimestamp != null) { - const tempMoment = moment(lastTimestamp); - if (tempMoment.isValid()) { - return tempMoment.toDate(); - } else { - return undefined; - } + return getValidDateFromDoc({ doc: lastRecord, timestampOverride }); + } +}; + +/** + * Given a search hit this will return a valid last date if it can find one, otherwise it + * will return undefined. This tries the "fields" first to get a formatted date time if it can, but if + * it cannot it will resort to using the "_source" fields second which can be problematic if the date time + * is not correctly ISO8601 or epoch milliseconds formatted. + * @param searchResult The result to try and parse out the timestamp. + * @param timestampOverride The timestamp override to use its values if we have it. + */ +export const getValidDateFromDoc = ({ + doc, + timestampOverride, +}: { + doc: estypes.Hit; + timestampOverride: TimestampOverrideOrUndefined; +}): Date | undefined => { + const timestamp = timestampOverride ?? '@timestamp'; + const timestampValue = + doc.fields != null && doc.fields[timestamp] != null + ? doc.fields[timestamp][0] + : // @ts-expect-error @elastic/elasticsearch _source is optional + doc._source[timestamp]; + const lastTimestamp = + typeof timestampValue === 'string' || typeof timestampValue === 'number' + ? timestampValue + : undefined; + if (lastTimestamp != null) { + const tempMoment = moment(lastTimestamp); + if (tempMoment.isValid()) { + return tempMoment.toDate(); + } else { + return undefined; } } }; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/generating_signals.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/generating_signals.ts index f9f378bc4bfa8..8638f6c1bd7ed 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/generating_signals.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/generating_signals.ts @@ -1616,119 +1616,6 @@ export default ({ getService }: FtrProviderContext) => { }); }); - describe('Signals generated from events with timestamp override field and ensures search_after continues to work when documents are missing timestamp override field', () => { - beforeEach(async () => { - await createSignalsIndex(supertest); - await esArchiver.load('auditbeat/hosts'); - }); - - afterEach(async () => { - await deleteSignalsIndex(supertest); - await deleteAllAlerts(supertest); - await esArchiver.unload('auditbeat/hosts'); - }); - - /** - * This represents our worst case scenario where this field is not mapped on any index - * We want to check that our logic continues to function within the constraints of search after - * Elasticsearch returns java's long.MAX_VALUE for unmapped date fields - * Javascript does not support numbers this large, but without passing in a number of this size - * The search_after will continue to return the same results and not iterate to the next set - * So to circumvent this limitation of javascript we return the stringified version of Java's - * Long.MAX_VALUE so that search_after does not enter into an infinite loop. - * - * ref: https://github.com/elastic/elasticsearch/issues/28806#issuecomment-369303620 - */ - it('should generate 200 signals when timestamp override does not exist', async () => { - const rule: QueryCreateSchema = { - ...getRuleForSignalTesting(['auditbeat-*']), - timestamp_override: 'event.fakeingested', - max_signals: 200, - }; - - const { id } = await createRule(supertest, rule); - await waitForRuleSuccessOrStatus(supertest, id, 'partial failure'); - await waitForSignalsToBePresent(supertest, 200, [id]); - const signalsResponse = await getSignalsByIds(supertest, [id], 200); - const signals = signalsResponse.hits.hits.map((hit) => hit._source); - - expect(signals.length).equal(200); - }); - }); - - /** - * Here we test the functionality of timestamp overrides. If the rule specifies a timestamp override, - * then the documents will be queried and sorted using the timestamp override field. - * If no timestamp override field exists in the indices but one was provided to the rule, - * the rule's query will additionally search for events using the `@timestamp` field - */ - describe('Signals generated from events with timestamp override field', async () => { - beforeEach(async () => { - await deleteSignalsIndex(supertest); - await createSignalsIndex(supertest); - await esArchiver.load('security_solution/timestamp_override_1'); - await esArchiver.load('security_solution/timestamp_override_2'); - await esArchiver.load('security_solution/timestamp_override_3'); - await esArchiver.load('security_solution/timestamp_override_4'); - }); - - afterEach(async () => { - await deleteSignalsIndex(supertest); - await deleteAllAlerts(supertest); - await esArchiver.unload('security_solution/timestamp_override_1'); - await esArchiver.unload('security_solution/timestamp_override_2'); - await esArchiver.unload('security_solution/timestamp_override_3'); - await esArchiver.unload('security_solution/timestamp_override_4'); - }); - - it('should generate signals with event.ingested, @timestamp and (event.ingested + timestamp)', async () => { - const rule: QueryCreateSchema = { - ...getRuleForSignalTesting(['myfa*']), - timestamp_override: 'event.ingested', - }; - - const { id } = await createRule(supertest, rule); - - await waitForRuleSuccessOrStatus(supertest, id, 'partial failure'); - await waitForSignalsToBePresent(supertest, 3, [id]); - const signalsResponse = await getSignalsByIds(supertest, [id], 3); - const signals = signalsResponse.hits.hits.map((hit) => hit._source); - const signalsOrderedByEventId = orderBy(signals, 'signal.parent.id', 'asc'); - - expect(signalsOrderedByEventId.length).equal(3); - }); - - it('should generate 2 signals with @timestamp', async () => { - const rule: QueryCreateSchema = getRuleForSignalTesting(['myfa*']); - - const { id } = await createRule(supertest, rule); - - await waitForRuleSuccessOrStatus(supertest, id, 'partial failure'); - await waitForSignalsToBePresent(supertest, 2, [id]); - const signalsResponse = await getSignalsByIds(supertest, [id]); - const signals = signalsResponse.hits.hits.map((hit) => hit._source); - const signalsOrderedByEventId = orderBy(signals, 'signal.parent.id', 'asc'); - - expect(signalsOrderedByEventId.length).equal(2); - }); - - it('should generate 2 signals when timestamp override does not exist', async () => { - const rule: QueryCreateSchema = { - ...getRuleForSignalTesting(['myfa*']), - timestamp_override: 'event.fakeingestfield', - }; - const { id } = await createRule(supertest, rule); - - await waitForRuleSuccessOrStatus(supertest, id, 'partial failure'); - await waitForSignalsToBePresent(supertest, 2, [id]); - const signalsResponse = await getSignalsByIds(supertest, [id, id]); - const signals = signalsResponse.hits.hits.map((hit) => hit._source); - const signalsOrderedByEventId = orderBy(signals, 'signal.parent.id', 'asc'); - - expect(signalsOrderedByEventId.length).equal(2); - }); - }); - describe('Signals generated from events with name override field', async () => { beforeEach(async () => { await deleteSignalsIndex(supertest); diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/timestamps.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/timestamps.ts index 29a2ed4c9f666..8f93d305371f2 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/timestamps.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/timestamps.ts @@ -6,45 +6,33 @@ */ import expect from '@kbn/expect'; -import { CreateRulesSchema } from '../../../../plugins/security_solution/common/detection_engine/schemas/request'; +import { orderBy } from 'lodash'; +import { QueryCreateSchema } from '../../../../plugins/security_solution/common/detection_engine/schemas/request'; -import { - DETECTION_ENGINE_RULES_URL, - DETECTION_ENGINE_RULES_STATUS_URL, -} from '../../../../plugins/security_solution/common/constants'; import { FtrProviderContext } from '../../common/ftr_provider_context'; import { createSignalsIndex, deleteAllAlerts, deleteSignalsIndex, createRule, - getSimpleRule, - getSimpleRuleOutput, - getSimpleRuleOutputWithoutRuleId, - getSimpleRuleWithoutRuleId, - removeServerGeneratedProperties, - removeServerGeneratedPropertiesIncludingRuleId, - getSimpleMlRule, - getSimpleMlRuleOutput, waitForRuleSuccessOrStatus, waitForSignalsToBePresent, - waitForAlertToComplete, getRuleForSignalTesting, getSignalsByIds, - getRuleForSignalTestingWithTimestampOverride, } from '../../utils'; -import { ROLES } from '../../../../plugins/security_solution/common/test'; -import { createUserAndRole, deleteUserAndRole } from '../roles_users_utils'; -import { RuleStatusResponse } from '../../../../plugins/security_solution/server/lib/detection_engine/rules/types'; // eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); - const supertestWithoutAuth = getService('supertestWithoutAuth'); const esArchiver = getService('esArchiver'); - describe('timestamps', () => { - describe('source index with timestamp in milliseconds', () => { + /** + * Tests around timestamps within signals such as the copying of timestamps correctly into + * the "signal.original_time" field, ensuring that timestamp overrides operate, and ensuring that + * partial errors happen correctly + */ + describe('timestamp tests', () => { + describe('Signals generated from events with a timestamp in milliseconds', () => { beforeEach(async () => { await createSignalsIndex(supertest); await esArchiver.load('security_solution/timestamp_in_seconds'); @@ -56,14 +44,149 @@ export default ({ getService }: FtrProviderContext) => { await esArchiver.unload('security_solution/timestamp_in_seconds'); }); - it('should convert a timestamp in epoch_seconds to the correct ISO format', async () => { + it('should convert the timestamp which is epoch_seconds into the correct ISO format', async () => { const rule = getRuleForSignalTesting(['timestamp_in_seconds']); const { id } = await createRule(supertest, rule); await waitForRuleSuccessOrStatus(supertest, id); await waitForSignalsToBePresent(supertest, 1, [id]); const signalsOpen = await getSignalsByIds(supertest, [id]); const hits = signalsOpen.hits.hits.map((hit) => hit._source.signal.original_time).sort(); - expect(hits).to.eql([]); + expect(hits).to.eql(['2021-06-02T23:33:15.000Z']); + }); + }); + + /** + * Here we test the functionality of timestamp overrides. If the rule specifies a timestamp override, + * then the documents will be queried and sorted using the timestamp override field. + * If no timestamp override field exists in the indices but one was provided to the rule, + * the rule's query will additionally search for events using the `@timestamp` field + */ + describe('Signals generated from events with timestamp override field', async () => { + beforeEach(async () => { + await deleteSignalsIndex(supertest); + await createSignalsIndex(supertest); + await esArchiver.load('security_solution/timestamp_override_1'); + await esArchiver.load('security_solution/timestamp_override_2'); + await esArchiver.load('security_solution/timestamp_override_3'); + await esArchiver.load('security_solution/timestamp_override_4'); + }); + + afterEach(async () => { + await deleteSignalsIndex(supertest); + await deleteAllAlerts(supertest); + await esArchiver.unload('security_solution/timestamp_override_1'); + await esArchiver.unload('security_solution/timestamp_override_2'); + await esArchiver.unload('security_solution/timestamp_override_3'); + await esArchiver.unload('security_solution/timestamp_override_4'); + }); + + it('should generate signals with event.ingested, @timestamp and (event.ingested + timestamp)', async () => { + const rule: QueryCreateSchema = { + ...getRuleForSignalTesting(['myfa*']), + timestamp_override: 'event.ingested', + }; + + const { id } = await createRule(supertest, rule); + + await waitForRuleSuccessOrStatus(supertest, id, 'partial failure'); + await waitForSignalsToBePresent(supertest, 3, [id]); + const signalsResponse = await getSignalsByIds(supertest, [id], 3); + const signals = signalsResponse.hits.hits.map((hit) => hit._source); + const signalsOrderedByEventId = orderBy(signals, 'signal.parent.id', 'asc'); + + expect(signalsOrderedByEventId.length).equal(3); + }); + + it('should generate 2 signals with @timestamp', async () => { + const rule: QueryCreateSchema = getRuleForSignalTesting(['myfa*']); + + const { id } = await createRule(supertest, rule); + + await waitForRuleSuccessOrStatus(supertest, id, 'partial failure'); + await waitForSignalsToBePresent(supertest, 2, [id]); + const signalsResponse = await getSignalsByIds(supertest, [id]); + const signals = signalsResponse.hits.hits.map((hit) => hit._source); + const signalsOrderedByEventId = orderBy(signals, 'signal.parent.id', 'asc'); + + expect(signalsOrderedByEventId.length).equal(2); + }); + + it('should generate 2 signals when timestamp override does not exist', async () => { + const rule: QueryCreateSchema = { + ...getRuleForSignalTesting(['myfa*']), + timestamp_override: 'event.fakeingestfield', + }; + const { id } = await createRule(supertest, rule); + + await waitForRuleSuccessOrStatus(supertest, id, 'partial failure'); + await waitForSignalsToBePresent(supertest, 2, [id]); + const signalsResponse = await getSignalsByIds(supertest, [id, id]); + const signals = signalsResponse.hits.hits.map((hit) => hit._source); + const signalsOrderedByEventId = orderBy(signals, 'signal.parent.id', 'asc'); + + expect(signalsOrderedByEventId.length).equal(2); + }); + + /** + * We should not use the timestamp override as the "original_time" as that can cause + * confusion if you have both a timestamp and an override in the source event. Instead the "original_time" + * field should only be overridden by the "timestamp" since when we generate a signal + * and we add a new timestamp to the signal. + */ + it('should NOT use the timestamp override as the "original_time"', async () => { + const rule: QueryCreateSchema = { + ...getRuleForSignalTesting(['myfakeindex-2']), + timestamp_override: 'event.ingested', + }; + const { id } = await createRule(supertest, rule); + + await waitForRuleSuccessOrStatus(supertest, id); + await waitForSignalsToBePresent(supertest, 1, [id]); + const signalsResponse = await getSignalsByIds(supertest, [id, id]); + const hits = signalsResponse.hits.hits + .map((hit) => hit._source.signal.original_time) + .sort(); + expect(hits).to.eql([undefined]); + }); + }); + + describe('Signals generated from events with timestamp override field and ensures search_after continues to work when documents are missing timestamp override field', () => { + beforeEach(async () => { + await createSignalsIndex(supertest); + await esArchiver.load('auditbeat/hosts'); + }); + + afterEach(async () => { + await deleteSignalsIndex(supertest); + await deleteAllAlerts(supertest); + await esArchiver.unload('auditbeat/hosts'); + }); + + /** + * This represents our worst case scenario where this field is not mapped on any index + * We want to check that our logic continues to function within the constraints of search after + * Elasticsearch returns java's long.MAX_VALUE for unmapped date fields + * Javascript does not support numbers this large, but without passing in a number of this size + * The search_after will continue to return the same results and not iterate to the next set + * So to circumvent this limitation of javascript we return the stringified version of Java's + * Long.MAX_VALUE so that search_after does not enter into an infinite loop. + * + * ref: https://github.com/elastic/elasticsearch/issues/28806#issuecomment-369303620 + */ + it('should generate 200 signals when timestamp override does not exist', async () => { + const rule: QueryCreateSchema = { + ...getRuleForSignalTesting(['auditbeat-*']), + timestamp_override: 'event.fakeingested', + max_signals: 200, + }; + + const { id } = await createRule(supertest, rule); + await waitForRuleSuccessOrStatus(supertest, id, 'partial failure'); + await waitForSignalsToBePresent(supertest, 200, [id]); + const signalsResponse = await getSignalsByIds(supertest, [id], 200); + const signals = signalsResponse.hits.hits.map((hit) => hit._source); + + expect(signals.length).equal(200); }); }); }); From b301d84b3eed1768cf73de6384ad0fd852187f88 Mon Sep 17 00:00:00 2001 From: FrankHassanabad Date: Fri, 4 Jun 2021 11:11:18 -0600 Subject: [PATCH 3/5] Removed optional parameter and fixed tests to operate with it _not_ optional --- .../lib/detection_engine/signals/build_events_query.ts | 4 +--- .../signals/single_search_after.test.ts | 10 +++++----- .../detection_engine/signals/single_search_after.ts | 2 +- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts index 123b7cbb3b591..0414439580361 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts @@ -17,7 +17,7 @@ interface BuildEventsSearchQuery { index: string[]; from: string; to: string; - filter?: estypes.QueryContainer; + filter: estypes.QueryContainer; size: number; sortOrder?: SortOrderOrUndefined; searchAfterSortIds: SortResults | undefined; @@ -94,8 +94,6 @@ export const buildEventsSearchQuery = ({ ]; const filterWithTime: estypes.QueryContainer[] = [ - // but tests contain undefined, so I suppose it's desired behavior - // @ts-expect-error undefined in not assignable to QueryContainer filter, { bool: { filter: [{ bool: { should: [...rangeFilter], minimum_should_match: 1 } }] } }, ]; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.test.ts index a40459d312b9f..a67016491aaef 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.test.ts @@ -41,7 +41,7 @@ describe('singleSearchAfter', () => { services: mockService, logger: mockLogger, pageSize: 1, - filter: undefined, + filter: {}, timestampOverride: undefined, buildRuleMessage, }); @@ -59,7 +59,7 @@ describe('singleSearchAfter', () => { services: mockService, logger: mockLogger, pageSize: 1, - filter: undefined, + filter: {}, timestampOverride: undefined, buildRuleMessage, }); @@ -109,7 +109,7 @@ describe('singleSearchAfter', () => { services: mockService, logger: mockLogger, pageSize: 1, - filter: undefined, + filter: {}, timestampOverride: undefined, buildRuleMessage, }); @@ -132,7 +132,7 @@ describe('singleSearchAfter', () => { services: mockService, logger: mockLogger, pageSize: 1, - filter: undefined, + filter: {}, timestampOverride: undefined, buildRuleMessage, }); @@ -152,7 +152,7 @@ describe('singleSearchAfter', () => { services: mockService, logger: mockLogger, pageSize: 1, - filter: undefined, + filter: {}, timestampOverride: undefined, buildRuleMessage, }) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts index 57ed05bcb27cf..ae22964eced92 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts @@ -32,7 +32,7 @@ interface SingleSearchAfterParams { logger: Logger; pageSize: number; sortOrder?: SortOrderOrUndefined; - filter?: estypes.QueryContainer; + filter: estypes.QueryContainer; timestampOverride: TimestampOverrideOrUndefined; buildRuleMessage: BuildRuleMessage; } From 02184dc0be65d61e1479bfa1411a750cc609a3a4 Mon Sep 17 00:00:00 2001 From: FrankHassanabad Date: Fri, 4 Jun 2021 11:36:00 -0600 Subject: [PATCH 4/5] Removes another ts-expect-error and fixes the typing around it --- .../detection_engine/signals/build_bulk_body.test.ts | 5 ++--- .../server/lib/detection_engine/signals/utils.ts | 12 +++++++----- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts index 6cb00e08e3f86..4d3ca26f5a71e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts @@ -24,6 +24,7 @@ import { SignalHit, SignalSourceHit } from './types'; import { SIGNALS_TEMPLATE_VERSION } from '../routes/index/get_signals_template'; import { getQueryRuleParams, getThresholdRuleParams } from '../schemas/rule_schemas.mock'; +// This allows us to not have to use ts-expect-error with delete in the code. type SignalHitOptionalTimestamp = Omit & { '@timestamp'?: SignalHit['@timestamp']; }; @@ -623,9 +624,7 @@ describe('buildSignalFromEvent', () => { const ancestor = sampleDocWithAncestors().hits.hits[0]; delete ancestor._source.source; const ruleSO = sampleRuleSO(getQueryRuleParams()); - const signal: Omit & { - '@timestamp'?: SignalHit['@timestamp']; - } = buildSignalFromEvent(ancestor, ruleSO, true); + const signal: SignalHitOptionalTimestamp = buildSignalFromEvent(ancestor, ruleSO, true); // Timestamp will potentially always be different so remove it for the test delete signal['@timestamp']; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts index 079049c56037c..dde9986e8bdf5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts @@ -38,6 +38,7 @@ import { Signal, WrappedSignalHit, RuleRangeTuple, + BaseSignalHit, } from './types'; import { BuildRuleMessage } from './rule_messages'; import { ShardError } from '../../types'; @@ -577,7 +578,7 @@ export const lastValidDate = ({ searchResult, timestampOverride, }: { - searchResult: estypes.SearchResponse; + searchResult: SignalSearchResponse; timestampOverride: TimestampOverrideOrUndefined; }): Date | undefined => { if (searchResult.hits.hits.length === 0) { @@ -600,15 +601,16 @@ export const getValidDateFromDoc = ({ doc, timestampOverride, }: { - doc: estypes.Hit; + doc: BaseSignalHit; timestampOverride: TimestampOverrideOrUndefined; }): Date | undefined => { const timestamp = timestampOverride ?? '@timestamp'; const timestampValue = doc.fields != null && doc.fields[timestamp] != null ? doc.fields[timestamp][0] - : // @ts-expect-error @elastic/elasticsearch _source is optional - doc._source[timestamp]; + : doc._source != null + ? doc._source[timestamp] + : undefined; const lastTimestamp = typeof timestampValue === 'string' || typeof timestampValue === 'number' ? timestampValue @@ -627,7 +629,7 @@ export const createSearchAfterReturnTypeFromResponse = ({ searchResult, timestampOverride, }: { - searchResult: estypes.SearchResponse; + searchResult: SignalSearchResponse; timestampOverride: TimestampOverrideOrUndefined; }): SearchAfterAndBulkCreateReturnType => { return createSearchAfterReturnType({ From 88d2e1e5281734362cc5b3d9d58a1dc7454252a8 Mon Sep 17 00:00:00 2001 From: FrankHassanabad Date: Fri, 4 Jun 2021 12:21:52 -0600 Subject: [PATCH 5/5] Added more tests and fixed more types and removed more ts-expect-error --- .../signals/__mocks__/es_results.ts | 4 +- .../detection_engine/signals/utils.test.ts | 86 +++++++++++++++++-- .../security_and_spaces/tests/timestamps.ts | 19 +++- .../timestamp_override/mappings.json | 1 + .../timestamp_override_1/mappings.json | 1 + .../timestamp_override_2/mappings.json | 1 + .../timestamp_override_3/mappings.json | 1 + .../timestamp_override_4/data.json | 2 +- .../timestamp_override_4/mappings.json | 1 + .../timestamp_override_5/data.json | 14 +++ .../timestamp_override_5/mappings.json | 39 +++++++++ 11 files changed, 159 insertions(+), 10 deletions(-) create mode 100644 x-pack/test/functional/es_archives/security_solution/timestamp_override_5/data.json create mode 100644 x-pack/test/functional/es_archives/security_solution/timestamp_override_5/mappings.json diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts index 33a6878fcfb9c..0fed141ca4dbc 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts @@ -562,7 +562,9 @@ export const sampleBulkCreateErrorResult = { export const sampleDocSearchResultsNoSortId = ( someUuid: string = sampleIdGuid -): SignalSearchResponse => ({ +): SignalSearchResponse & { + hits: { hits: Array> }; +} => ({ took: 10, timed_out: false, _shards: { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts index f49492939eeb1..60bf0ec337f3d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts @@ -40,6 +40,7 @@ import { lastValidDate, calculateThresholdSignalUuid, buildChunkedOrFilter, + getValidDateFromDoc, } from './utils'; import { BulkResponseErrorAggregation, SearchAfterAndBulkCreateReturnType } from './types'; import { @@ -54,6 +55,7 @@ import { sampleDocSearchResultsNoSortIdNoHits, repeatedSearchResultsWithSortId, sampleDocSearchResultsNoSortId, + sampleDocNoSortId, } from './__mocks__/es_results'; import { ShardError } from '../../types'; @@ -1172,7 +1174,6 @@ describe('utils', () => { test('It will not set an invalid date time stamp from a non-existent @timestamp when the index is not 100% ECS compliant', () => { const searchResult = sampleDocSearchResultsNoSortId(); - // @ts-expect-error @elastic/elasticsearch _source is optional (searchResult.hits.hits[0]._source['@timestamp'] as unknown) = undefined; if (searchResult.hits.hits[0].fields != null) { (searchResult.hits.hits[0].fields['@timestamp'] as unknown) = undefined; @@ -1186,7 +1187,6 @@ describe('utils', () => { test('It will not set an invalid date time stamp from a null @timestamp when the index is not 100% ECS compliant', () => { const searchResult = sampleDocSearchResultsNoSortId(); - // @ts-expect-error @elastic/elasticsearch _source is optional (searchResult.hits.hits[0]._source['@timestamp'] as unknown) = null; if (searchResult.hits.hits[0].fields != null) { (searchResult.hits.hits[0].fields['@timestamp'] as unknown) = null; @@ -1200,7 +1200,6 @@ describe('utils', () => { test('It will not set an invalid date time stamp from an invalid @timestamp string', () => { const searchResult = sampleDocSearchResultsNoSortId(); - // @ts-expect-error @elastic/elasticsearch _source is optional (searchResult.hits.hits[0]._source['@timestamp'] as unknown) = 'invalid'; if (searchResult.hits.hits[0].fields != null) { (searchResult.hits.hits[0].fields['@timestamp'] as unknown) = ['invalid']; @@ -1216,7 +1215,6 @@ describe('utils', () => { describe('lastValidDate', () => { test('It returns undefined if the search result contains a null timestamp', () => { const searchResult = sampleDocSearchResultsNoSortId(); - // @ts-expect-error @elastic/elasticsearch _source is optional (searchResult.hits.hits[0]._source['@timestamp'] as unknown) = null; if (searchResult.hits.hits[0].fields != null) { (searchResult.hits.hits[0].fields['@timestamp'] as unknown) = null; @@ -1227,7 +1225,6 @@ describe('utils', () => { test('It returns undefined if the search result contains a undefined timestamp', () => { const searchResult = sampleDocSearchResultsNoSortId(); - // @ts-expect-error @elastic/elasticsearch _source is optional (searchResult.hits.hits[0]._source['@timestamp'] as unknown) = undefined; if (searchResult.hits.hits[0].fields != null) { (searchResult.hits.hits[0].fields['@timestamp'] as unknown) = undefined; @@ -1238,7 +1235,6 @@ describe('utils', () => { test('It returns undefined if the search result contains an invalid string value', () => { const searchResult = sampleDocSearchResultsNoSortId(); - // @ts-expect-error @elastic/elasticsearch _source is optional (searchResult.hits.hits[0]._source['@timestamp'] as unknown) = 'invalid value'; if (searchResult.hits.hits[0].fields != null) { (searchResult.hits.hits[0].fields['@timestamp'] as unknown) = ['invalid value']; @@ -1294,6 +1290,84 @@ describe('utils', () => { }); }); + describe('getValidDateFromDoc', () => { + test('It returns undefined if the search result contains a null timestamp', () => { + const doc = sampleDocNoSortId(); + (doc._source['@timestamp'] as unknown) = null; + if (doc.fields != null) { + (doc.fields['@timestamp'] as unknown) = null; + } + const date = getValidDateFromDoc({ doc, timestampOverride: undefined }); + expect(date).toEqual(undefined); + }); + + test('It returns undefined if the search result contains a undefined timestamp', () => { + const doc = sampleDocNoSortId(); + (doc._source['@timestamp'] as unknown) = undefined; + if (doc.fields != null) { + (doc.fields['@timestamp'] as unknown) = undefined; + } + const date = getValidDateFromDoc({ doc, timestampOverride: undefined }); + expect(date).toEqual(undefined); + }); + + test('It returns undefined if the search result contains an invalid string value', () => { + const doc = sampleDocNoSortId(); + (doc._source['@timestamp'] as unknown) = 'invalid value'; + if (doc.fields != null) { + (doc.fields['@timestamp'] as unknown) = ['invalid value']; + } + const date = getValidDateFromDoc({ doc, timestampOverride: undefined }); + expect(date).toEqual(undefined); + }); + + test('It returns normal date time if set', () => { + const doc = sampleDocNoSortId(); + const date = getValidDateFromDoc({ doc, timestampOverride: undefined }); + expect(date?.toISOString()).toEqual('2020-04-20T21:27:45.000Z'); + }); + + test('It returns date time from field if set there', () => { + const timestamp = '2020-10-07T19:27:19.136Z'; + let doc = sampleDocNoSortId(); + if (doc == null) { + throw new TypeError('Test requires one element'); + } + doc = { + ...doc, + fields: { + '@timestamp': [timestamp], + }, + }; + const date = getValidDateFromDoc({ doc, timestampOverride: undefined }); + expect(date?.toISOString()).toEqual(timestamp); + }); + + test('It returns timestampOverride date time if set', () => { + const override = '2020-10-07T19:20:28.049Z'; + const doc = sampleDocNoSortId(); + doc._source.different_timestamp = new Date(override).toISOString(); + const date = getValidDateFromDoc({ doc, timestampOverride: 'different_timestamp' }); + expect(date?.toISOString()).toEqual(override); + }); + + test('It returns timestampOverride date time from fields if set on it', () => { + const override = '2020-10-07T19:36:31.110Z'; + let doc = sampleDocNoSortId(); + if (doc == null) { + throw new TypeError('Test requires one element'); + } + doc = { + ...doc, + fields: { + different_timestamp: [override], + }, + }; + const date = getValidDateFromDoc({ doc, timestampOverride: 'different_timestamp' }); + expect(date?.toISOString()).toEqual(override); + }); + }); + describe('createSearchAfterReturnType', () => { test('createSearchAfterReturnType will return full object when nothing is passed', () => { const searchAfterReturnType = createSearchAfterReturnType(); diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/timestamps.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/timestamps.ts index 8f93d305371f2..16610e6a44915 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/timestamps.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/timestamps.ts @@ -32,19 +32,21 @@ export default ({ getService }: FtrProviderContext) => { * partial errors happen correctly */ describe('timestamp tests', () => { - describe('Signals generated from events with a timestamp in milliseconds', () => { + describe('Signals generated from events with a timestamp in seconds is converted correctly into the forced ISO8601 format when copying', () => { beforeEach(async () => { await createSignalsIndex(supertest); await esArchiver.load('security_solution/timestamp_in_seconds'); + await esArchiver.load('security_solution/timestamp_override_5'); }); afterEach(async () => { await deleteSignalsIndex(supertest); await deleteAllAlerts(supertest); await esArchiver.unload('security_solution/timestamp_in_seconds'); + await esArchiver.unload('security_solution/timestamp_override_5'); }); - it('should convert the timestamp which is epoch_seconds into the correct ISO format', async () => { + it('should convert the @timestamp which is epoch_seconds into the correct ISO format', async () => { const rule = getRuleForSignalTesting(['timestamp_in_seconds']); const { id } = await createRule(supertest, rule); await waitForRuleSuccessOrStatus(supertest, id); @@ -53,6 +55,19 @@ export default ({ getService }: FtrProviderContext) => { const hits = signalsOpen.hits.hits.map((hit) => hit._source.signal.original_time).sort(); expect(hits).to.eql(['2021-06-02T23:33:15.000Z']); }); + + it('should still use the @timestamp field even with an override field. It should never use the override field', async () => { + const rule: QueryCreateSchema = { + ...getRuleForSignalTesting(['myfakeindex-5']), + timestamp_override: 'event.ingested', + }; + const { id } = await createRule(supertest, rule); + await waitForRuleSuccessOrStatus(supertest, id); + await waitForSignalsToBePresent(supertest, 1, [id]); + const signalsOpen = await getSignalsByIds(supertest, [id]); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.signal.original_time).sort(); + expect(hits).to.eql(['2020-12-16T15:16:18.000Z']); + }); }); /** diff --git a/x-pack/test/functional/es_archives/security_solution/timestamp_override/mappings.json b/x-pack/test/functional/es_archives/security_solution/timestamp_override/mappings.json index 085ab34a3d58a..092519a792863 100644 --- a/x-pack/test/functional/es_archives/security_solution/timestamp_override/mappings.json +++ b/x-pack/test/functional/es_archives/security_solution/timestamp_override/mappings.json @@ -3,6 +3,7 @@ "value": { "index": "myfakeindex-1", "mappings": { + "dynamic": "strict", "properties": { "message": { "type": "text", diff --git a/x-pack/test/functional/es_archives/security_solution/timestamp_override_1/mappings.json b/x-pack/test/functional/es_archives/security_solution/timestamp_override_1/mappings.json index 085ab34a3d58a..092519a792863 100644 --- a/x-pack/test/functional/es_archives/security_solution/timestamp_override_1/mappings.json +++ b/x-pack/test/functional/es_archives/security_solution/timestamp_override_1/mappings.json @@ -3,6 +3,7 @@ "value": { "index": "myfakeindex-1", "mappings": { + "dynamic": "strict", "properties": { "message": { "type": "text", diff --git a/x-pack/test/functional/es_archives/security_solution/timestamp_override_2/mappings.json b/x-pack/test/functional/es_archives/security_solution/timestamp_override_2/mappings.json index 49a27a423cdaa..1f1c1673fe1a2 100644 --- a/x-pack/test/functional/es_archives/security_solution/timestamp_override_2/mappings.json +++ b/x-pack/test/functional/es_archives/security_solution/timestamp_override_2/mappings.json @@ -3,6 +3,7 @@ "value": { "index": "myfakeindex-2", "mappings": { + "dynamic": "strict", "properties": { "message": { "type": "text", diff --git a/x-pack/test/functional/es_archives/security_solution/timestamp_override_3/mappings.json b/x-pack/test/functional/es_archives/security_solution/timestamp_override_3/mappings.json index 736584386a705..a0409280c34eb 100644 --- a/x-pack/test/functional/es_archives/security_solution/timestamp_override_3/mappings.json +++ b/x-pack/test/functional/es_archives/security_solution/timestamp_override_3/mappings.json @@ -3,6 +3,7 @@ "value": { "index": "myfakeindex-3", "mappings": { + "dynamic": "strict", "properties": { "message": { "type": "text", diff --git a/x-pack/test/functional/es_archives/security_solution/timestamp_override_4/data.json b/x-pack/test/functional/es_archives/security_solution/timestamp_override_4/data.json index ca7025b36154c..ad0e7cbab7d2b 100644 --- a/x-pack/test/functional/es_archives/security_solution/timestamp_override_4/data.json +++ b/x-pack/test/functional/es_archives/security_solution/timestamp_override_4/data.json @@ -6,7 +6,7 @@ "message": "hello world 4", "@timestamp": "2020-12-16T15:16:18.570Z", "event": { - "ingested": "2020-12-16T15:16:18.570Z" + "ingested": "2020-12-16T16:16:18.570Z" } }, "type": "_doc" diff --git a/x-pack/test/functional/es_archives/security_solution/timestamp_override_4/mappings.json b/x-pack/test/functional/es_archives/security_solution/timestamp_override_4/mappings.json index ab4edc9f300e1..a4e021e45ff9e 100644 --- a/x-pack/test/functional/es_archives/security_solution/timestamp_override_4/mappings.json +++ b/x-pack/test/functional/es_archives/security_solution/timestamp_override_4/mappings.json @@ -3,6 +3,7 @@ "value": { "index": "myfakeindex-4", "mappings": { + "dynamic": "strict", "properties": { "@timestamp": { "type": "date" diff --git a/x-pack/test/functional/es_archives/security_solution/timestamp_override_5/data.json b/x-pack/test/functional/es_archives/security_solution/timestamp_override_5/data.json new file mode 100644 index 0000000000000..f2c81e9b5e45e --- /dev/null +++ b/x-pack/test/functional/es_archives/security_solution/timestamp_override_5/data.json @@ -0,0 +1,14 @@ +{ + "type": "doc", + "value": { + "index": "myfakeindex-5", + "source": { + "@timestamp": 1608131778, + "message": "hello world 4", + "event": { + "ingested": 1622676795 + } + }, + "type": "_doc" + } +} diff --git a/x-pack/test/functional/es_archives/security_solution/timestamp_override_5/mappings.json b/x-pack/test/functional/es_archives/security_solution/timestamp_override_5/mappings.json new file mode 100644 index 0000000000000..a9735aaeca8ef --- /dev/null +++ b/x-pack/test/functional/es_archives/security_solution/timestamp_override_5/mappings.json @@ -0,0 +1,39 @@ +{ + "type": "index", + "value": { + "index": "myfakeindex-5", + "mappings": { + "dynamic": "strict", + "properties": { + "@timestamp": { + "type": "date", + "format": "epoch_second" + }, + "message": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "event": { + "properties": { + "ingested": { + "type": "date", + "format": "epoch_second" + } + } + } + } + }, + "settings": { + "index": { + "refresh_interval": "1s", + "number_of_replicas": "1", + "number_of_shards": "1" + } + } + } +}