diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts index 1590a4f0fbb04..0fed141ca4dbc 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts @@ -146,7 +146,7 @@ export const sampleDocWithSortId = ( export const sampleDocNoSortId = ( someUuid: string = sampleIdGuid, ip?: string -): SignalSourceHit => ({ +): SignalSourceHit & { _source: Required['_source'] } => ({ _index: 'myFakeSignalIndex', _type: 'doc', _score: 100, @@ -225,12 +225,12 @@ export const sampleWrappedSignalHit = (): WrappedSignalHit => { }; }; -export const sampleDocWithAncestors = (): SignalSearchResponse => { +export const sampleDocWithAncestors = (): SignalSearchResponse & { + hits: { hits: Array> }; +} => { const sampleDoc = sampleDocNoSortId(); delete sampleDoc.sort; - // @ts-expect-error @elastic/elasticsearch _source is optional delete sampleDoc._source.source; - // @ts-expect-error @elastic/elasticsearch _source is optional sampleDoc._source.signal = { parent: { id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', @@ -562,7 +562,9 @@ export const sampleBulkCreateErrorResult = { export const sampleDocSearchResultsNoSortId = ( someUuid: string = sampleIdGuid -): SignalSearchResponse => ({ +): SignalSearchResponse & { + hits: { hits: Array> }; +} => ({ took: 10, timed_out: false, _shards: { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts index 743d9580218a3..4d3ca26f5a71e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts @@ -24,6 +24,11 @@ import { SignalHit, SignalSourceHit } from './types'; import { SIGNALS_TEMPLATE_VERSION } from '../routes/index/get_signals_template'; import { getQueryRuleParams, getThresholdRuleParams } from '../schemas/rule_schemas.mock'; +// This allows us to not have to use ts-expect-error with delete in the code. +type SignalHitOptionalTimestamp = Omit & { + '@timestamp'?: SignalHit['@timestamp']; +}; + describe('buildBulkBody', () => { beforeEach(() => { jest.clearAllMocks(); @@ -32,11 +37,9 @@ describe('buildBulkBody', () => { test('bulk body builds well-defined body', () => { const ruleSO = sampleRuleSO(getQueryRuleParams()); const doc = sampleDocNoSortId(); - // @ts-expect-error @elastic/elasticsearch _source is optional delete doc._source.source; - const fakeSignalSourceHit = buildBulkBody(ruleSO, doc); + const fakeSignalSourceHit: SignalHitOptionalTimestamp = buildBulkBody(ruleSO, doc); // Timestamp will potentially always be different so remove it for the test - // @ts-expect-error delete fakeSignalSourceHit['@timestamp']; const expected: Omit & { someKey: 'someValue' } = { someKey: 'someValue', @@ -69,7 +72,7 @@ describe('buildBulkBody', () => { depth: 0, }, ], - original_time: '2020-04-20T21:27:45+0000', + original_time: '2020-04-20T21:27:45.000Z', status: 'open', rule: expectedRule(), depth: 1, @@ -81,9 +84,8 @@ describe('buildBulkBody', () => { test('bulk body builds well-defined body with threshold results', () => { const ruleSO = sampleRuleSO(getThresholdRuleParams()); const baseDoc = sampleDocNoSortId(); - const doc: SignalSourceHit = { + const doc: SignalSourceHit & { _source: Required['_source'] } = { ...baseDoc, - // @ts-expect-error @elastic/elasticsearch _source is optional _source: { ...baseDoc._source, threshold_result: { @@ -96,11 +98,9 @@ describe('buildBulkBody', () => { }, }, }; - // @ts-expect-error @elastic/elasticsearch _source is optional delete doc._source.source; - const fakeSignalSourceHit = buildBulkBody(ruleSO, doc); + const fakeSignalSourceHit: SignalHitOptionalTimestamp = buildBulkBody(ruleSO, doc); // Timestamp will potentially always be different so remove it for the test - // @ts-expect-error delete fakeSignalSourceHit['@timestamp']; const expected: Omit & { someKey: 'someValue' } = { someKey: 'someValue', @@ -133,7 +133,7 @@ describe('buildBulkBody', () => { depth: 0, }, ], - original_time: '2020-04-20T21:27:45+0000', + original_time: '2020-04-20T21:27:45.000Z', status: 'open', rule: { ...expectedRule(), @@ -167,18 +167,15 @@ describe('buildBulkBody', () => { test('bulk body builds original_event if it exists on the event to begin with', () => { const ruleSO = sampleRuleSO(getQueryRuleParams()); const doc = sampleDocNoSortId(); - // @ts-expect-error @elastic/elasticsearch _source is optional delete doc._source.source; - // @ts-expect-error @elastic/elasticsearch _source is optional doc._source.event = { action: 'socket_opened', module: 'system', dataset: 'socket', kind: 'event', }; - const fakeSignalSourceHit = buildBulkBody(ruleSO, doc); + const fakeSignalSourceHit: SignalHitOptionalTimestamp = buildBulkBody(ruleSO, doc); // Timestamp will potentially always be different so remove it for the test - // @ts-expect-error delete fakeSignalSourceHit['@timestamp']; const expected: Omit & { someKey: 'someValue' } = { someKey: 'someValue', @@ -220,7 +217,7 @@ describe('buildBulkBody', () => { depth: 0, }, ], - original_time: '2020-04-20T21:27:45+0000', + original_time: '2020-04-20T21:27:45.000Z', status: 'open', rule: expectedRule(), depth: 1, @@ -232,17 +229,14 @@ describe('buildBulkBody', () => { test('bulk body builds original_event if it exists on the event to begin with but no kind information', () => { const ruleSO = sampleRuleSO(getQueryRuleParams()); const doc = sampleDocNoSortId(); - // @ts-expect-error @elastic/elasticsearch _source is optional delete doc._source.source; - // @ts-expect-error @elastic/elasticsearch _source is optional doc._source.event = { action: 'socket_opened', module: 'system', dataset: 'socket', }; - const fakeSignalSourceHit = buildBulkBody(ruleSO, doc); + const fakeSignalSourceHit: SignalHitOptionalTimestamp = buildBulkBody(ruleSO, doc); // Timestamp will potentially always be different so remove it for the test - // @ts-expect-error delete fakeSignalSourceHit['@timestamp']; const expected: Omit & { someKey: 'someValue' } = { someKey: 'someValue', @@ -283,7 +277,7 @@ describe('buildBulkBody', () => { depth: 0, }, ], - original_time: '2020-04-20T21:27:45+0000', + original_time: '2020-04-20T21:27:45.000Z', status: 'open', rule: expectedRule(), depth: 1, @@ -295,15 +289,12 @@ describe('buildBulkBody', () => { test('bulk body builds original_event if it exists on the event to begin with with only kind information', () => { const ruleSO = sampleRuleSO(getQueryRuleParams()); const doc = sampleDocNoSortId(); - // @ts-expect-error @elastic/elasticsearch _source is optional delete doc._source.source; - // @ts-expect-error @elastic/elasticsearch _source is optional doc._source.event = { kind: 'event', }; - const fakeSignalSourceHit = buildBulkBody(ruleSO, doc); + const fakeSignalSourceHit: SignalHitOptionalTimestamp = buildBulkBody(ruleSO, doc); // Timestamp will potentially always be different so remove it for the test - // @ts-expect-error delete fakeSignalSourceHit['@timestamp']; const expected: Omit & { someKey: 'someValue' } = { someKey: 'someValue', @@ -339,7 +330,7 @@ describe('buildBulkBody', () => { depth: 0, }, ], - original_time: '2020-04-20T21:27:45+0000', + original_time: '2020-04-20T21:27:45.000Z', status: 'open', rule: expectedRule(), depth: 1, @@ -351,7 +342,6 @@ describe('buildBulkBody', () => { test('bulk body builds "original_signal" if it exists already as a numeric', () => { const ruleSO = sampleRuleSO(getQueryRuleParams()); const sampleDoc = sampleDocNoSortId(); - // @ts-expect-error @elastic/elasticsearch _source is optional delete sampleDoc._source.source; const doc = ({ ...sampleDoc, @@ -393,7 +383,7 @@ describe('buildBulkBody', () => { depth: 0, }, ], - original_time: '2020-04-20T21:27:45+0000', + original_time: '2020-04-20T21:27:45.000Z', status: 'open', rule: expectedRule(), depth: 1, @@ -405,7 +395,6 @@ describe('buildBulkBody', () => { test('bulk body builds "original_signal" if it exists already as an object', () => { const ruleSO = sampleRuleSO(getQueryRuleParams()); const sampleDoc = sampleDocNoSortId(); - // @ts-expect-error @elastic/elasticsearch _source is optional delete sampleDoc._source.source; const doc = ({ ...sampleDoc, @@ -447,7 +436,7 @@ describe('buildBulkBody', () => { depth: 0, }, ], - original_time: '2020-04-20T21:27:45+0000', + original_time: '2020-04-20T21:27:45.000Z', status: 'open', rule: expectedRule(), depth: 1, @@ -466,9 +455,8 @@ describe('buildSignalFromSequence', () => { block2._source.new_key = 'new_key_value'; const blocks = [block1, block2]; const ruleSO = sampleRuleSO(getQueryRuleParams()); - const signal = buildSignalFromSequence(blocks, ruleSO); + const signal: SignalHitOptionalTimestamp = buildSignalFromSequence(blocks, ruleSO); // Timestamp will potentially always be different so remove it for the test - // @ts-expect-error delete signal['@timestamp']; const expected: Omit & { new_key: string } = { new_key: 'new_key_value', @@ -552,9 +540,8 @@ describe('buildSignalFromSequence', () => { block2._source['@timestamp'] = '2021-05-20T22:28:46+0000'; block2._source.someKey = 'someOtherValue'; const ruleSO = sampleRuleSO(getQueryRuleParams()); - const signal = buildSignalFromSequence([block1, block2], ruleSO); + const signal: SignalHitOptionalTimestamp = buildSignalFromSequence([block1, block2], ruleSO); // Timestamp will potentially always be different so remove it for the test - // @ts-expect-error delete signal['@timestamp']; const expected: Omit = { event: { @@ -635,12 +622,11 @@ describe('buildSignalFromSequence', () => { describe('buildSignalFromEvent', () => { test('builds a basic signal from a single event', () => { const ancestor = sampleDocWithAncestors().hits.hits[0]; - // @ts-expect-error @elastic/elasticsearch _source is optional delete ancestor._source.source; const ruleSO = sampleRuleSO(getQueryRuleParams()); - const signal = buildSignalFromEvent(ancestor, ruleSO, true); + const signal: SignalHitOptionalTimestamp = buildSignalFromEvent(ancestor, ruleSO, true); + // Timestamp will potentially always be different so remove it for the test - // @ts-expect-error delete signal['@timestamp']; const expected: Omit & { someKey: 'someValue' } = { someKey: 'someValue', @@ -651,7 +637,7 @@ describe('buildSignalFromEvent', () => { _meta: { version: SIGNALS_TEMPLATE_VERSION, }, - original_time: '2020-04-20T21:27:45+0000', + original_time: '2020-04-20T21:27:45.000Z', parent: { id: sampleIdGuid, rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_event_type_signal.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_event_type_signal.test.ts index 185c165442921..0ae81770e83c2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_event_type_signal.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_event_type_signal.test.ts @@ -16,7 +16,6 @@ describe('buildEventTypeSignal', () => { test('it returns the event appended of kind signal if it does not exist', () => { const doc = sampleDocNoSortId(); - // @ts-expect-error @elastic/elasticsearch _source is optional delete doc._source.event; const eventType = buildEventTypeSignal(doc); const expected: object = { kind: 'signal' }; @@ -25,7 +24,6 @@ describe('buildEventTypeSignal', () => { test('it returns the event appended of kind signal if it is an empty object', () => { const doc = sampleDocNoSortId(); - // @ts-expect-error @elastic/elasticsearch _source is optional doc._source.event = {}; const eventType = buildEventTypeSignal(doc); const expected: object = { kind: 'signal' }; @@ -34,7 +32,6 @@ describe('buildEventTypeSignal', () => { test('it returns the event with kind signal and other properties if they exist', () => { const doc = sampleDocNoSortId(); - // @ts-expect-error @elastic/elasticsearch _source is optional doc._source.event = { action: 'socket_opened', module: 'system', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.test.ts index 3f4a17dc091ab..28cea9ea22b0d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.test.ts @@ -24,12 +24,6 @@ describe('create_signals', () => { size: 100, ignore_unavailable: true, body: { - docvalue_fields: [ - { - field: '@timestamp', - format: 'strict_date_optional_time', - }, - ], query: { bool: { filter: [ @@ -67,6 +61,10 @@ describe('create_signals', () => { field: '*', include_unmapped: true, }, + { + field: '@timestamp', + format: 'strict_date_optional_time', + }, ], sort: [ { @@ -96,16 +94,6 @@ describe('create_signals', () => { size: 100, ignore_unavailable: true, body: { - docvalue_fields: [ - { - field: 'event.ingested', - format: 'strict_date_optional_time', - }, - { - field: '@timestamp', - format: 'strict_date_optional_time', - }, - ], query: { bool: { filter: [ @@ -167,6 +155,14 @@ describe('create_signals', () => { field: '*', include_unmapped: true, }, + { + field: 'event.ingested', + format: 'strict_date_optional_time', + }, + { + field: '@timestamp', + format: 'strict_date_optional_time', + }, ], sort: [ { @@ -203,12 +199,6 @@ describe('create_signals', () => { size: 100, ignore_unavailable: true, body: { - docvalue_fields: [ - { - field: '@timestamp', - format: 'strict_date_optional_time', - }, - ], query: { bool: { filter: [ @@ -246,6 +236,10 @@ describe('create_signals', () => { field: '*', include_unmapped: true, }, + { + field: '@timestamp', + format: 'strict_date_optional_time', + }, ], search_after: [fakeSortId], sort: [ @@ -276,12 +270,6 @@ describe('create_signals', () => { size: 100, ignore_unavailable: true, body: { - docvalue_fields: [ - { - field: '@timestamp', - format: 'strict_date_optional_time', - }, - ], query: { bool: { filter: [ @@ -319,6 +307,10 @@ describe('create_signals', () => { field: '*', include_unmapped: true, }, + { + field: '@timestamp', + format: 'strict_date_optional_time', + }, ], search_after: [fakeSortIdNumber], sort: [ @@ -348,12 +340,6 @@ describe('create_signals', () => { size: 100, ignore_unavailable: true, body: { - docvalue_fields: [ - { - field: '@timestamp', - format: 'strict_date_optional_time', - }, - ], query: { bool: { filter: [ @@ -391,6 +377,10 @@ describe('create_signals', () => { field: '*', include_unmapped: true, }, + { + field: '@timestamp', + format: 'strict_date_optional_time', + }, ], sort: [ { @@ -427,7 +417,6 @@ describe('create_signals', () => { size: 100, ignore_unavailable: true, body: { - docvalue_fields: [{ field: '@timestamp', format: 'strict_date_optional_time' }], query: { bool: { filter: [ @@ -465,6 +454,10 @@ describe('create_signals', () => { field: '*', include_unmapped: true, }, + { + field: '@timestamp', + format: 'strict_date_optional_time', + }, ], aggregations: { tags: { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts index 86fb51e4785ad..0414439580361 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts @@ -17,7 +17,7 @@ interface BuildEventsSearchQuery { index: string[]; from: string; to: string; - filter?: estypes.QueryContainer; + filter: estypes.QueryContainer; size: number; sortOrder?: SortOrderOrUndefined; searchAfterSortIds: SortResults | undefined; @@ -94,8 +94,6 @@ export const buildEventsSearchQuery = ({ ]; const filterWithTime: estypes.QueryContainer[] = [ - // but tests contain undefined, so I suppose it's desired behaviour - // @ts-expect-error undefined in not assignable to QueryContainer filter, { bool: { filter: [{ bool: { should: [...rangeFilter], minimum_should_match: 1 } }] } }, ]; @@ -106,7 +104,6 @@ export const buildEventsSearchQuery = ({ size, ignore_unavailable: true, body: { - docvalue_fields: docFields, query: { bool: { filter: [ @@ -122,6 +119,7 @@ export const buildEventsSearchQuery = ({ field: '*', include_unmapped: true, }, + ...docFields, ], ...(aggregations ? { aggregations } : {}), sort: [ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts index 412ccf7a40e33..bd5444a325128 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts @@ -134,7 +134,6 @@ describe('buildRuleWithOverrides', () => { }, ]; const doc = sampleDocNoSortId(); - // @ts-expect-error @elastic/elasticsearch _source is optional doc._source.new_risk_score = newRiskScore; const rule = buildRuleWithOverrides(ruleSO, doc._source!); const expected = { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts index 6408b5fe9de10..3a30da170d3f2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts @@ -28,7 +28,6 @@ describe('buildSignal', () => { test('it builds a signal as expected without original_event if event does not exist', () => { const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); - // @ts-expect-error @elastic/elasticsearch _source is optional delete doc._source.event; const rule = getRulesSchemaMock(); const signal = { @@ -61,7 +60,7 @@ describe('buildSignal', () => { depth: 0, }, ], - original_time: '2020-04-20T21:27:45+0000', + original_time: '2020-04-20T21:27:45.000Z', status: 'open', rule: { author: [], @@ -105,7 +104,6 @@ describe('buildSignal', () => { test('it builds a signal as expected with original_event if is present', () => { const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); - // @ts-expect-error @elastic/elasticsearch _source is optional doc._source.event = { action: 'socket_opened', dataset: 'socket', @@ -143,7 +141,7 @@ describe('buildSignal', () => { depth: 0, }, ], - original_time: '2020-04-20T21:27:45+0000', + original_time: '2020-04-20T21:27:45.000Z', original_event: { action: 'socket_opened', dataset: 'socket', @@ -193,7 +191,6 @@ describe('buildSignal', () => { test('it builds a ancestor correctly if the parent does not exist', () => { const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); - // @ts-expect-error @elastic/elasticsearch _source is optional doc._source.event = { action: 'socket_opened', dataset: 'socket', @@ -212,14 +209,12 @@ describe('buildSignal', () => { test('it builds a ancestor correctly if the parent does exist', () => { const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); - // @ts-expect-error @elastic/elasticsearch _source is optional doc._source.event = { action: 'socket_opened', dataset: 'socket', kind: 'event', module: 'system', }; - // @ts-expect-error @elastic/elasticsearch _source is optional doc._source.signal = { parents: [ { @@ -255,7 +250,6 @@ describe('buildSignal', () => { test('it builds a signal ancestor correctly if the parent does not exist', () => { const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); - // @ts-expect-error @elastic/elasticsearch _source is optional doc._source.event = { action: 'socket_opened', dataset: 'socket', @@ -276,14 +270,12 @@ describe('buildSignal', () => { test('it builds a signal ancestor correctly if the parent does exist', () => { const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); - // @ts-expect-error @elastic/elasticsearch _source is optional doc._source.event = { action: 'socket_opened', dataset: 'socket', kind: 'event', module: 'system', }; - // @ts-expect-error @elastic/elasticsearch _source is optional doc._source.signal = { parents: [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts index 237536a99c0f0..a415c83e857c2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts @@ -10,6 +10,7 @@ import { RulesSchema } from '../../../../common/detection_engine/schemas/respons import { SIGNALS_TEMPLATE_VERSION } from '../routes/index/get_signals_template'; import { isEventTypeSignal } from './build_event_type_signal'; import { Signal, Ancestor, BaseSignalHit, ThresholdResult } from './types'; +import { getValidDateFromDoc } from './utils'; /** * Takes a parent signal or event document and extracts the information needed for the corresponding entry in the child @@ -103,6 +104,7 @@ const isThresholdResult = (thresholdResult: SearchTypes): thresholdResult is Thr /** * Creates signal fields that are only available in the special case where a signal has only 1 parent signal/event. + * We copy the original time from the document as "original_time" since we override the timestamp with the current date time. * @param doc The parent signal/event of the new signal to be built. */ export const additionalSignalFields = (doc: BaseSignalHit) => { @@ -110,10 +112,13 @@ export const additionalSignalFields = (doc: BaseSignalHit) => { if (thresholdResult != null && !isThresholdResult(thresholdResult)) { throw new Error(`threshold_result failed to validate: ${thresholdResult}`); } + const originalTime = getValidDateFromDoc({ + doc, + timestampOverride: undefined, + }); return { parent: buildParent(removeClashes(doc)), - // @ts-expect-error @elastic/elasticsearch _source is optional - original_time: doc._source['@timestamp'], // This field has already been replaced with timestampOverride, if provided. + original_time: originalTime != null ? originalTime.toISOString() : undefined, original_event: doc._source?.event ?? undefined, threshold_result: thresholdResult, original_signal: diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/mappings/build_rule_name_from_mapping.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/mappings/build_rule_name_from_mapping.test.ts index b6281b637d434..23e5aecc5c553 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/mappings/build_rule_name_from_mapping.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/mappings/build_rule_name_from_mapping.test.ts @@ -15,7 +15,6 @@ describe('buildRuleNameFromMapping', () => { test('rule name defaults to provided if mapping is incomplete', () => { const ruleName = buildRuleNameFromMapping({ - // @ts-expect-error @elastic/elasticsearch _source is optional eventSource: sampleDocNoSortId()._source, ruleName: 'rule-name', ruleNameMapping: 'message', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.test.ts index a40459d312b9f..a67016491aaef 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.test.ts @@ -41,7 +41,7 @@ describe('singleSearchAfter', () => { services: mockService, logger: mockLogger, pageSize: 1, - filter: undefined, + filter: {}, timestampOverride: undefined, buildRuleMessage, }); @@ -59,7 +59,7 @@ describe('singleSearchAfter', () => { services: mockService, logger: mockLogger, pageSize: 1, - filter: undefined, + filter: {}, timestampOverride: undefined, buildRuleMessage, }); @@ -109,7 +109,7 @@ describe('singleSearchAfter', () => { services: mockService, logger: mockLogger, pageSize: 1, - filter: undefined, + filter: {}, timestampOverride: undefined, buildRuleMessage, }); @@ -132,7 +132,7 @@ describe('singleSearchAfter', () => { services: mockService, logger: mockLogger, pageSize: 1, - filter: undefined, + filter: {}, timestampOverride: undefined, buildRuleMessage, }); @@ -152,7 +152,7 @@ describe('singleSearchAfter', () => { services: mockService, logger: mockLogger, pageSize: 1, - filter: undefined, + filter: {}, timestampOverride: undefined, buildRuleMessage, }) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts index 57ed05bcb27cf..ae22964eced92 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts @@ -32,7 +32,7 @@ interface SingleSearchAfterParams { logger: Logger; pageSize: number; sortOrder?: SortOrderOrUndefined; - filter?: estypes.QueryContainer; + filter: estypes.QueryContainer; timestampOverride: TimestampOverrideOrUndefined; buildRuleMessage: BuildRuleMessage; } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts index f49492939eeb1..60bf0ec337f3d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts @@ -40,6 +40,7 @@ import { lastValidDate, calculateThresholdSignalUuid, buildChunkedOrFilter, + getValidDateFromDoc, } from './utils'; import { BulkResponseErrorAggregation, SearchAfterAndBulkCreateReturnType } from './types'; import { @@ -54,6 +55,7 @@ import { sampleDocSearchResultsNoSortIdNoHits, repeatedSearchResultsWithSortId, sampleDocSearchResultsNoSortId, + sampleDocNoSortId, } from './__mocks__/es_results'; import { ShardError } from '../../types'; @@ -1172,7 +1174,6 @@ describe('utils', () => { test('It will not set an invalid date time stamp from a non-existent @timestamp when the index is not 100% ECS compliant', () => { const searchResult = sampleDocSearchResultsNoSortId(); - // @ts-expect-error @elastic/elasticsearch _source is optional (searchResult.hits.hits[0]._source['@timestamp'] as unknown) = undefined; if (searchResult.hits.hits[0].fields != null) { (searchResult.hits.hits[0].fields['@timestamp'] as unknown) = undefined; @@ -1186,7 +1187,6 @@ describe('utils', () => { test('It will not set an invalid date time stamp from a null @timestamp when the index is not 100% ECS compliant', () => { const searchResult = sampleDocSearchResultsNoSortId(); - // @ts-expect-error @elastic/elasticsearch _source is optional (searchResult.hits.hits[0]._source['@timestamp'] as unknown) = null; if (searchResult.hits.hits[0].fields != null) { (searchResult.hits.hits[0].fields['@timestamp'] as unknown) = null; @@ -1200,7 +1200,6 @@ describe('utils', () => { test('It will not set an invalid date time stamp from an invalid @timestamp string', () => { const searchResult = sampleDocSearchResultsNoSortId(); - // @ts-expect-error @elastic/elasticsearch _source is optional (searchResult.hits.hits[0]._source['@timestamp'] as unknown) = 'invalid'; if (searchResult.hits.hits[0].fields != null) { (searchResult.hits.hits[0].fields['@timestamp'] as unknown) = ['invalid']; @@ -1216,7 +1215,6 @@ describe('utils', () => { describe('lastValidDate', () => { test('It returns undefined if the search result contains a null timestamp', () => { const searchResult = sampleDocSearchResultsNoSortId(); - // @ts-expect-error @elastic/elasticsearch _source is optional (searchResult.hits.hits[0]._source['@timestamp'] as unknown) = null; if (searchResult.hits.hits[0].fields != null) { (searchResult.hits.hits[0].fields['@timestamp'] as unknown) = null; @@ -1227,7 +1225,6 @@ describe('utils', () => { test('It returns undefined if the search result contains a undefined timestamp', () => { const searchResult = sampleDocSearchResultsNoSortId(); - // @ts-expect-error @elastic/elasticsearch _source is optional (searchResult.hits.hits[0]._source['@timestamp'] as unknown) = undefined; if (searchResult.hits.hits[0].fields != null) { (searchResult.hits.hits[0].fields['@timestamp'] as unknown) = undefined; @@ -1238,7 +1235,6 @@ describe('utils', () => { test('It returns undefined if the search result contains an invalid string value', () => { const searchResult = sampleDocSearchResultsNoSortId(); - // @ts-expect-error @elastic/elasticsearch _source is optional (searchResult.hits.hits[0]._source['@timestamp'] as unknown) = 'invalid value'; if (searchResult.hits.hits[0].fields != null) { (searchResult.hits.hits[0].fields['@timestamp'] as unknown) = ['invalid value']; @@ -1294,6 +1290,84 @@ describe('utils', () => { }); }); + describe('getValidDateFromDoc', () => { + test('It returns undefined if the search result contains a null timestamp', () => { + const doc = sampleDocNoSortId(); + (doc._source['@timestamp'] as unknown) = null; + if (doc.fields != null) { + (doc.fields['@timestamp'] as unknown) = null; + } + const date = getValidDateFromDoc({ doc, timestampOverride: undefined }); + expect(date).toEqual(undefined); + }); + + test('It returns undefined if the search result contains a undefined timestamp', () => { + const doc = sampleDocNoSortId(); + (doc._source['@timestamp'] as unknown) = undefined; + if (doc.fields != null) { + (doc.fields['@timestamp'] as unknown) = undefined; + } + const date = getValidDateFromDoc({ doc, timestampOverride: undefined }); + expect(date).toEqual(undefined); + }); + + test('It returns undefined if the search result contains an invalid string value', () => { + const doc = sampleDocNoSortId(); + (doc._source['@timestamp'] as unknown) = 'invalid value'; + if (doc.fields != null) { + (doc.fields['@timestamp'] as unknown) = ['invalid value']; + } + const date = getValidDateFromDoc({ doc, timestampOverride: undefined }); + expect(date).toEqual(undefined); + }); + + test('It returns normal date time if set', () => { + const doc = sampleDocNoSortId(); + const date = getValidDateFromDoc({ doc, timestampOverride: undefined }); + expect(date?.toISOString()).toEqual('2020-04-20T21:27:45.000Z'); + }); + + test('It returns date time from field if set there', () => { + const timestamp = '2020-10-07T19:27:19.136Z'; + let doc = sampleDocNoSortId(); + if (doc == null) { + throw new TypeError('Test requires one element'); + } + doc = { + ...doc, + fields: { + '@timestamp': [timestamp], + }, + }; + const date = getValidDateFromDoc({ doc, timestampOverride: undefined }); + expect(date?.toISOString()).toEqual(timestamp); + }); + + test('It returns timestampOverride date time if set', () => { + const override = '2020-10-07T19:20:28.049Z'; + const doc = sampleDocNoSortId(); + doc._source.different_timestamp = new Date(override).toISOString(); + const date = getValidDateFromDoc({ doc, timestampOverride: 'different_timestamp' }); + expect(date?.toISOString()).toEqual(override); + }); + + test('It returns timestampOverride date time from fields if set on it', () => { + const override = '2020-10-07T19:36:31.110Z'; + let doc = sampleDocNoSortId(); + if (doc == null) { + throw new TypeError('Test requires one element'); + } + doc = { + ...doc, + fields: { + different_timestamp: [override], + }, + }; + const date = getValidDateFromDoc({ doc, timestampOverride: 'different_timestamp' }); + expect(date?.toISOString()).toEqual(override); + }); + }); + describe('createSearchAfterReturnType', () => { test('createSearchAfterReturnType will return full object when nothing is passed', () => { const searchAfterReturnType = createSearchAfterReturnType(); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts index cc4ed6a45807b..dde9986e8bdf5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts @@ -38,6 +38,7 @@ import { Signal, WrappedSignalHit, RuleRangeTuple, + BaseSignalHit, } from './types'; import { BuildRuleMessage } from './rule_messages'; import { ShardError } from '../../types'; @@ -577,30 +578,49 @@ export const lastValidDate = ({ searchResult, timestampOverride, }: { - searchResult: estypes.SearchResponse; + searchResult: SignalSearchResponse; timestampOverride: TimestampOverrideOrUndefined; }): Date | undefined => { if (searchResult.hits.hits.length === 0) { return undefined; } else { const lastRecord = searchResult.hits.hits[searchResult.hits.hits.length - 1]; - const timestamp = timestampOverride ?? '@timestamp'; - const timestampValue = - lastRecord.fields != null && lastRecord.fields[timestamp] != null - ? lastRecord.fields[timestamp][0] - : // @ts-expect-error @elastic/elasticsearch _source is optional - lastRecord._source[timestamp]; - const lastTimestamp = - typeof timestampValue === 'string' || typeof timestampValue === 'number' - ? timestampValue - : undefined; - if (lastTimestamp != null) { - const tempMoment = moment(lastTimestamp); - if (tempMoment.isValid()) { - return tempMoment.toDate(); - } else { - return undefined; - } + return getValidDateFromDoc({ doc: lastRecord, timestampOverride }); + } +}; + +/** + * Given a search hit this will return a valid last date if it can find one, otherwise it + * will return undefined. This tries the "fields" first to get a formatted date time if it can, but if + * it cannot it will resort to using the "_source" fields second which can be problematic if the date time + * is not correctly ISO8601 or epoch milliseconds formatted. + * @param searchResult The result to try and parse out the timestamp. + * @param timestampOverride The timestamp override to use its values if we have it. + */ +export const getValidDateFromDoc = ({ + doc, + timestampOverride, +}: { + doc: BaseSignalHit; + timestampOverride: TimestampOverrideOrUndefined; +}): Date | undefined => { + const timestamp = timestampOverride ?? '@timestamp'; + const timestampValue = + doc.fields != null && doc.fields[timestamp] != null + ? doc.fields[timestamp][0] + : doc._source != null + ? doc._source[timestamp] + : undefined; + const lastTimestamp = + typeof timestampValue === 'string' || typeof timestampValue === 'number' + ? timestampValue + : undefined; + if (lastTimestamp != null) { + const tempMoment = moment(lastTimestamp); + if (tempMoment.isValid()) { + return tempMoment.toDate(); + } else { + return undefined; } } }; @@ -609,7 +629,7 @@ export const createSearchAfterReturnTypeFromResponse = ({ searchResult, timestampOverride, }: { - searchResult: estypes.SearchResponse; + searchResult: SignalSearchResponse; timestampOverride: TimestampOverrideOrUndefined; }): SearchAfterAndBulkCreateReturnType => { return createSearchAfterReturnType({ diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/generating_signals.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/generating_signals.ts index f9f378bc4bfa8..8638f6c1bd7ed 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/generating_signals.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/generating_signals.ts @@ -1616,119 +1616,6 @@ export default ({ getService }: FtrProviderContext) => { }); }); - describe('Signals generated from events with timestamp override field and ensures search_after continues to work when documents are missing timestamp override field', () => { - beforeEach(async () => { - await createSignalsIndex(supertest); - await esArchiver.load('auditbeat/hosts'); - }); - - afterEach(async () => { - await deleteSignalsIndex(supertest); - await deleteAllAlerts(supertest); - await esArchiver.unload('auditbeat/hosts'); - }); - - /** - * This represents our worst case scenario where this field is not mapped on any index - * We want to check that our logic continues to function within the constraints of search after - * Elasticsearch returns java's long.MAX_VALUE for unmapped date fields - * Javascript does not support numbers this large, but without passing in a number of this size - * The search_after will continue to return the same results and not iterate to the next set - * So to circumvent this limitation of javascript we return the stringified version of Java's - * Long.MAX_VALUE so that search_after does not enter into an infinite loop. - * - * ref: https://github.com/elastic/elasticsearch/issues/28806#issuecomment-369303620 - */ - it('should generate 200 signals when timestamp override does not exist', async () => { - const rule: QueryCreateSchema = { - ...getRuleForSignalTesting(['auditbeat-*']), - timestamp_override: 'event.fakeingested', - max_signals: 200, - }; - - const { id } = await createRule(supertest, rule); - await waitForRuleSuccessOrStatus(supertest, id, 'partial failure'); - await waitForSignalsToBePresent(supertest, 200, [id]); - const signalsResponse = await getSignalsByIds(supertest, [id], 200); - const signals = signalsResponse.hits.hits.map((hit) => hit._source); - - expect(signals.length).equal(200); - }); - }); - - /** - * Here we test the functionality of timestamp overrides. If the rule specifies a timestamp override, - * then the documents will be queried and sorted using the timestamp override field. - * If no timestamp override field exists in the indices but one was provided to the rule, - * the rule's query will additionally search for events using the `@timestamp` field - */ - describe('Signals generated from events with timestamp override field', async () => { - beforeEach(async () => { - await deleteSignalsIndex(supertest); - await createSignalsIndex(supertest); - await esArchiver.load('security_solution/timestamp_override_1'); - await esArchiver.load('security_solution/timestamp_override_2'); - await esArchiver.load('security_solution/timestamp_override_3'); - await esArchiver.load('security_solution/timestamp_override_4'); - }); - - afterEach(async () => { - await deleteSignalsIndex(supertest); - await deleteAllAlerts(supertest); - await esArchiver.unload('security_solution/timestamp_override_1'); - await esArchiver.unload('security_solution/timestamp_override_2'); - await esArchiver.unload('security_solution/timestamp_override_3'); - await esArchiver.unload('security_solution/timestamp_override_4'); - }); - - it('should generate signals with event.ingested, @timestamp and (event.ingested + timestamp)', async () => { - const rule: QueryCreateSchema = { - ...getRuleForSignalTesting(['myfa*']), - timestamp_override: 'event.ingested', - }; - - const { id } = await createRule(supertest, rule); - - await waitForRuleSuccessOrStatus(supertest, id, 'partial failure'); - await waitForSignalsToBePresent(supertest, 3, [id]); - const signalsResponse = await getSignalsByIds(supertest, [id], 3); - const signals = signalsResponse.hits.hits.map((hit) => hit._source); - const signalsOrderedByEventId = orderBy(signals, 'signal.parent.id', 'asc'); - - expect(signalsOrderedByEventId.length).equal(3); - }); - - it('should generate 2 signals with @timestamp', async () => { - const rule: QueryCreateSchema = getRuleForSignalTesting(['myfa*']); - - const { id } = await createRule(supertest, rule); - - await waitForRuleSuccessOrStatus(supertest, id, 'partial failure'); - await waitForSignalsToBePresent(supertest, 2, [id]); - const signalsResponse = await getSignalsByIds(supertest, [id]); - const signals = signalsResponse.hits.hits.map((hit) => hit._source); - const signalsOrderedByEventId = orderBy(signals, 'signal.parent.id', 'asc'); - - expect(signalsOrderedByEventId.length).equal(2); - }); - - it('should generate 2 signals when timestamp override does not exist', async () => { - const rule: QueryCreateSchema = { - ...getRuleForSignalTesting(['myfa*']), - timestamp_override: 'event.fakeingestfield', - }; - const { id } = await createRule(supertest, rule); - - await waitForRuleSuccessOrStatus(supertest, id, 'partial failure'); - await waitForSignalsToBePresent(supertest, 2, [id]); - const signalsResponse = await getSignalsByIds(supertest, [id, id]); - const signals = signalsResponse.hits.hits.map((hit) => hit._source); - const signalsOrderedByEventId = orderBy(signals, 'signal.parent.id', 'asc'); - - expect(signalsOrderedByEventId.length).equal(2); - }); - }); - describe('Signals generated from events with name override field', async () => { beforeEach(async () => { await deleteSignalsIndex(supertest); diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts index 57b24f6de2a48..477f1f37f50d2 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts @@ -42,6 +42,7 @@ export default ({ loadTestFile }: FtrProviderContext): void => { loadTestFile(require.resolve('./create_signals_migrations')); loadTestFile(require.resolve('./finalize_signals_migrations')); loadTestFile(require.resolve('./delete_signals_migrations')); + loadTestFile(require.resolve('./timestamps')); }); // That split here enable us on using a different ciGroup to run the tests diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/timestamps.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/timestamps.ts new file mode 100644 index 0000000000000..16610e6a44915 --- /dev/null +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/timestamps.ts @@ -0,0 +1,208 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { orderBy } from 'lodash'; +import { QueryCreateSchema } from '../../../../plugins/security_solution/common/detection_engine/schemas/request'; + +import { FtrProviderContext } from '../../common/ftr_provider_context'; +import { + createSignalsIndex, + deleteAllAlerts, + deleteSignalsIndex, + createRule, + waitForRuleSuccessOrStatus, + waitForSignalsToBePresent, + getRuleForSignalTesting, + getSignalsByIds, +} from '../../utils'; + +// eslint-disable-next-line import/no-default-export +export default ({ getService }: FtrProviderContext) => { + const supertest = getService('supertest'); + const esArchiver = getService('esArchiver'); + + /** + * Tests around timestamps within signals such as the copying of timestamps correctly into + * the "signal.original_time" field, ensuring that timestamp overrides operate, and ensuring that + * partial errors happen correctly + */ + describe('timestamp tests', () => { + describe('Signals generated from events with a timestamp in seconds is converted correctly into the forced ISO8601 format when copying', () => { + beforeEach(async () => { + await createSignalsIndex(supertest); + await esArchiver.load('security_solution/timestamp_in_seconds'); + await esArchiver.load('security_solution/timestamp_override_5'); + }); + + afterEach(async () => { + await deleteSignalsIndex(supertest); + await deleteAllAlerts(supertest); + await esArchiver.unload('security_solution/timestamp_in_seconds'); + await esArchiver.unload('security_solution/timestamp_override_5'); + }); + + it('should convert the @timestamp which is epoch_seconds into the correct ISO format', async () => { + const rule = getRuleForSignalTesting(['timestamp_in_seconds']); + const { id } = await createRule(supertest, rule); + await waitForRuleSuccessOrStatus(supertest, id); + await waitForSignalsToBePresent(supertest, 1, [id]); + const signalsOpen = await getSignalsByIds(supertest, [id]); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.signal.original_time).sort(); + expect(hits).to.eql(['2021-06-02T23:33:15.000Z']); + }); + + it('should still use the @timestamp field even with an override field. It should never use the override field', async () => { + const rule: QueryCreateSchema = { + ...getRuleForSignalTesting(['myfakeindex-5']), + timestamp_override: 'event.ingested', + }; + const { id } = await createRule(supertest, rule); + await waitForRuleSuccessOrStatus(supertest, id); + await waitForSignalsToBePresent(supertest, 1, [id]); + const signalsOpen = await getSignalsByIds(supertest, [id]); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.signal.original_time).sort(); + expect(hits).to.eql(['2020-12-16T15:16:18.000Z']); + }); + }); + + /** + * Here we test the functionality of timestamp overrides. If the rule specifies a timestamp override, + * then the documents will be queried and sorted using the timestamp override field. + * If no timestamp override field exists in the indices but one was provided to the rule, + * the rule's query will additionally search for events using the `@timestamp` field + */ + describe('Signals generated from events with timestamp override field', async () => { + beforeEach(async () => { + await deleteSignalsIndex(supertest); + await createSignalsIndex(supertest); + await esArchiver.load('security_solution/timestamp_override_1'); + await esArchiver.load('security_solution/timestamp_override_2'); + await esArchiver.load('security_solution/timestamp_override_3'); + await esArchiver.load('security_solution/timestamp_override_4'); + }); + + afterEach(async () => { + await deleteSignalsIndex(supertest); + await deleteAllAlerts(supertest); + await esArchiver.unload('security_solution/timestamp_override_1'); + await esArchiver.unload('security_solution/timestamp_override_2'); + await esArchiver.unload('security_solution/timestamp_override_3'); + await esArchiver.unload('security_solution/timestamp_override_4'); + }); + + it('should generate signals with event.ingested, @timestamp and (event.ingested + timestamp)', async () => { + const rule: QueryCreateSchema = { + ...getRuleForSignalTesting(['myfa*']), + timestamp_override: 'event.ingested', + }; + + const { id } = await createRule(supertest, rule); + + await waitForRuleSuccessOrStatus(supertest, id, 'partial failure'); + await waitForSignalsToBePresent(supertest, 3, [id]); + const signalsResponse = await getSignalsByIds(supertest, [id], 3); + const signals = signalsResponse.hits.hits.map((hit) => hit._source); + const signalsOrderedByEventId = orderBy(signals, 'signal.parent.id', 'asc'); + + expect(signalsOrderedByEventId.length).equal(3); + }); + + it('should generate 2 signals with @timestamp', async () => { + const rule: QueryCreateSchema = getRuleForSignalTesting(['myfa*']); + + const { id } = await createRule(supertest, rule); + + await waitForRuleSuccessOrStatus(supertest, id, 'partial failure'); + await waitForSignalsToBePresent(supertest, 2, [id]); + const signalsResponse = await getSignalsByIds(supertest, [id]); + const signals = signalsResponse.hits.hits.map((hit) => hit._source); + const signalsOrderedByEventId = orderBy(signals, 'signal.parent.id', 'asc'); + + expect(signalsOrderedByEventId.length).equal(2); + }); + + it('should generate 2 signals when timestamp override does not exist', async () => { + const rule: QueryCreateSchema = { + ...getRuleForSignalTesting(['myfa*']), + timestamp_override: 'event.fakeingestfield', + }; + const { id } = await createRule(supertest, rule); + + await waitForRuleSuccessOrStatus(supertest, id, 'partial failure'); + await waitForSignalsToBePresent(supertest, 2, [id]); + const signalsResponse = await getSignalsByIds(supertest, [id, id]); + const signals = signalsResponse.hits.hits.map((hit) => hit._source); + const signalsOrderedByEventId = orderBy(signals, 'signal.parent.id', 'asc'); + + expect(signalsOrderedByEventId.length).equal(2); + }); + + /** + * We should not use the timestamp override as the "original_time" as that can cause + * confusion if you have both a timestamp and an override in the source event. Instead the "original_time" + * field should only be overridden by the "timestamp" since when we generate a signal + * and we add a new timestamp to the signal. + */ + it('should NOT use the timestamp override as the "original_time"', async () => { + const rule: QueryCreateSchema = { + ...getRuleForSignalTesting(['myfakeindex-2']), + timestamp_override: 'event.ingested', + }; + const { id } = await createRule(supertest, rule); + + await waitForRuleSuccessOrStatus(supertest, id); + await waitForSignalsToBePresent(supertest, 1, [id]); + const signalsResponse = await getSignalsByIds(supertest, [id, id]); + const hits = signalsResponse.hits.hits + .map((hit) => hit._source.signal.original_time) + .sort(); + expect(hits).to.eql([undefined]); + }); + }); + + describe('Signals generated from events with timestamp override field and ensures search_after continues to work when documents are missing timestamp override field', () => { + beforeEach(async () => { + await createSignalsIndex(supertest); + await esArchiver.load('auditbeat/hosts'); + }); + + afterEach(async () => { + await deleteSignalsIndex(supertest); + await deleteAllAlerts(supertest); + await esArchiver.unload('auditbeat/hosts'); + }); + + /** + * This represents our worst case scenario where this field is not mapped on any index + * We want to check that our logic continues to function within the constraints of search after + * Elasticsearch returns java's long.MAX_VALUE for unmapped date fields + * Javascript does not support numbers this large, but without passing in a number of this size + * The search_after will continue to return the same results and not iterate to the next set + * So to circumvent this limitation of javascript we return the stringified version of Java's + * Long.MAX_VALUE so that search_after does not enter into an infinite loop. + * + * ref: https://github.com/elastic/elasticsearch/issues/28806#issuecomment-369303620 + */ + it('should generate 200 signals when timestamp override does not exist', async () => { + const rule: QueryCreateSchema = { + ...getRuleForSignalTesting(['auditbeat-*']), + timestamp_override: 'event.fakeingested', + max_signals: 200, + }; + + const { id } = await createRule(supertest, rule); + await waitForRuleSuccessOrStatus(supertest, id, 'partial failure'); + await waitForSignalsToBePresent(supertest, 200, [id]); + const signalsResponse = await getSignalsByIds(supertest, [id], 200); + const signals = signalsResponse.hits.hits.map((hit) => hit._source); + + expect(signals.length).equal(200); + }); + }); + }); +}; diff --git a/x-pack/test/functional/es_archives/security_solution/README.md b/x-pack/test/functional/es_archives/security_solution/README.md new file mode 100644 index 0000000000000..c832e0835bbbc --- /dev/null +++ b/x-pack/test/functional/es_archives/security_solution/README.md @@ -0,0 +1,11 @@ +Collection of data sets for use within various tests. Most of the tests to these live in either: + +``` +x-pack/test/detection_engine_api_integrations/security_and_spaces/tests +``` + +or + +``` +x-pack/test/api_integration/apis/security_solution +``` diff --git a/x-pack/test/functional/es_archives/security_solution/timestamp_in_seconds/data.json b/x-pack/test/functional/es_archives/security_solution/timestamp_in_seconds/data.json new file mode 100644 index 0000000000000..46b30b239bbc7 --- /dev/null +++ b/x-pack/test/functional/es_archives/security_solution/timestamp_in_seconds/data.json @@ -0,0 +1,10 @@ +{ + "type": "doc", + "value": { + "index": "timestamp_in_seconds", + "source": { + "@timestamp": 1622676795 + }, + "type": "_doc" + } +} diff --git a/x-pack/test/functional/es_archives/security_solution/timestamp_in_seconds/mappings.json b/x-pack/test/functional/es_archives/security_solution/timestamp_in_seconds/mappings.json new file mode 100644 index 0000000000000..fd8880fe0bc49 --- /dev/null +++ b/x-pack/test/functional/es_archives/security_solution/timestamp_in_seconds/mappings.json @@ -0,0 +1,22 @@ +{ + "type": "index", + "value": { + "index": "timestamp_in_seconds", + "mappings": { + "dynamic": "strict", + "properties": { + "@timestamp": { + "type": "date", + "format": "epoch_second" + } + } + }, + "settings": { + "index": { + "refresh_interval": "1s", + "number_of_replicas": "1", + "number_of_shards": "1" + } + } + } +} diff --git a/x-pack/test/functional/es_archives/security_solution/timestamp_override/mappings.json b/x-pack/test/functional/es_archives/security_solution/timestamp_override/mappings.json index 085ab34a3d58a..092519a792863 100644 --- a/x-pack/test/functional/es_archives/security_solution/timestamp_override/mappings.json +++ b/x-pack/test/functional/es_archives/security_solution/timestamp_override/mappings.json @@ -3,6 +3,7 @@ "value": { "index": "myfakeindex-1", "mappings": { + "dynamic": "strict", "properties": { "message": { "type": "text", diff --git a/x-pack/test/functional/es_archives/security_solution/timestamp_override_1/mappings.json b/x-pack/test/functional/es_archives/security_solution/timestamp_override_1/mappings.json index 085ab34a3d58a..092519a792863 100644 --- a/x-pack/test/functional/es_archives/security_solution/timestamp_override_1/mappings.json +++ b/x-pack/test/functional/es_archives/security_solution/timestamp_override_1/mappings.json @@ -3,6 +3,7 @@ "value": { "index": "myfakeindex-1", "mappings": { + "dynamic": "strict", "properties": { "message": { "type": "text", diff --git a/x-pack/test/functional/es_archives/security_solution/timestamp_override_2/mappings.json b/x-pack/test/functional/es_archives/security_solution/timestamp_override_2/mappings.json index 49a27a423cdaa..1f1c1673fe1a2 100644 --- a/x-pack/test/functional/es_archives/security_solution/timestamp_override_2/mappings.json +++ b/x-pack/test/functional/es_archives/security_solution/timestamp_override_2/mappings.json @@ -3,6 +3,7 @@ "value": { "index": "myfakeindex-2", "mappings": { + "dynamic": "strict", "properties": { "message": { "type": "text", diff --git a/x-pack/test/functional/es_archives/security_solution/timestamp_override_3/mappings.json b/x-pack/test/functional/es_archives/security_solution/timestamp_override_3/mappings.json index 736584386a705..a0409280c34eb 100644 --- a/x-pack/test/functional/es_archives/security_solution/timestamp_override_3/mappings.json +++ b/x-pack/test/functional/es_archives/security_solution/timestamp_override_3/mappings.json @@ -3,6 +3,7 @@ "value": { "index": "myfakeindex-3", "mappings": { + "dynamic": "strict", "properties": { "message": { "type": "text", diff --git a/x-pack/test/functional/es_archives/security_solution/timestamp_override_4/data.json b/x-pack/test/functional/es_archives/security_solution/timestamp_override_4/data.json index ca7025b36154c..ad0e7cbab7d2b 100644 --- a/x-pack/test/functional/es_archives/security_solution/timestamp_override_4/data.json +++ b/x-pack/test/functional/es_archives/security_solution/timestamp_override_4/data.json @@ -6,7 +6,7 @@ "message": "hello world 4", "@timestamp": "2020-12-16T15:16:18.570Z", "event": { - "ingested": "2020-12-16T15:16:18.570Z" + "ingested": "2020-12-16T16:16:18.570Z" } }, "type": "_doc" diff --git a/x-pack/test/functional/es_archives/security_solution/timestamp_override_4/mappings.json b/x-pack/test/functional/es_archives/security_solution/timestamp_override_4/mappings.json index ab4edc9f300e1..a4e021e45ff9e 100644 --- a/x-pack/test/functional/es_archives/security_solution/timestamp_override_4/mappings.json +++ b/x-pack/test/functional/es_archives/security_solution/timestamp_override_4/mappings.json @@ -3,6 +3,7 @@ "value": { "index": "myfakeindex-4", "mappings": { + "dynamic": "strict", "properties": { "@timestamp": { "type": "date" diff --git a/x-pack/test/functional/es_archives/security_solution/timestamp_override_5/data.json b/x-pack/test/functional/es_archives/security_solution/timestamp_override_5/data.json new file mode 100644 index 0000000000000..f2c81e9b5e45e --- /dev/null +++ b/x-pack/test/functional/es_archives/security_solution/timestamp_override_5/data.json @@ -0,0 +1,14 @@ +{ + "type": "doc", + "value": { + "index": "myfakeindex-5", + "source": { + "@timestamp": 1608131778, + "message": "hello world 4", + "event": { + "ingested": 1622676795 + } + }, + "type": "_doc" + } +} diff --git a/x-pack/test/functional/es_archives/security_solution/timestamp_override_5/mappings.json b/x-pack/test/functional/es_archives/security_solution/timestamp_override_5/mappings.json new file mode 100644 index 0000000000000..a9735aaeca8ef --- /dev/null +++ b/x-pack/test/functional/es_archives/security_solution/timestamp_override_5/mappings.json @@ -0,0 +1,39 @@ +{ + "type": "index", + "value": { + "index": "myfakeindex-5", + "mappings": { + "dynamic": "strict", + "properties": { + "@timestamp": { + "type": "date", + "format": "epoch_second" + }, + "message": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "event": { + "properties": { + "ingested": { + "type": "date", + "format": "epoch_second" + } + } + } + } + }, + "settings": { + "index": { + "refresh_interval": "1s", + "number_of_replicas": "1", + "number_of_shards": "1" + } + } + } +}