From b5fb78bfce877eba9ed6308151b807ddf53aed7c Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Tue, 28 Apr 2020 20:45:57 +0100 Subject: [PATCH 1/2] [ML] Fix new job wizard with multiple indices (#64567) Co-authored-by: Elastic Machine --- x-pack/plugins/ml/common/util/job_utils.d.ts | 2 ++ x-pack/plugins/ml/common/util/job_utils.js | 6 ++++++ .../new_job/common/job_creator/util/default_configs.ts | 3 ++- .../index_or_search/preconfigured_job_redirect.ts | 2 +- .../server/models/data_recognizer/data_recognizer.ts | 10 ++++++---- 5 files changed, 17 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/ml/common/util/job_utils.d.ts b/x-pack/plugins/ml/common/util/job_utils.d.ts index bfad422e0ab48..4528fbfbb774d 100644 --- a/x-pack/plugins/ml/common/util/job_utils.d.ts +++ b/x-pack/plugins/ml/common/util/job_utils.d.ts @@ -52,3 +52,5 @@ export function getLatestDataOrBucketTimestamp( ): number; export function prefixDatafeedId(datafeedId: string, prefix: string): string; + +export function splitIndexPatternNames(indexPatternName: string): string[]; diff --git a/x-pack/plugins/ml/common/util/job_utils.js b/x-pack/plugins/ml/common/util/job_utils.js index de0aa4b886629..8fe5733ce67bd 100644 --- a/x-pack/plugins/ml/common/util/job_utils.js +++ b/x-pack/plugins/ml/common/util/job_utils.js @@ -588,3 +588,9 @@ export function processCreatedBy(customSettings) { delete customSettings.created_by; } } + +export function splitIndexPatternNames(indexPatternName) { + return indexPatternName.includes(',') + ? indexPatternName.split(',').map(i => i.trim()) + : [indexPatternName]; +} diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/util/default_configs.ts b/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/util/default_configs.ts index 306fd82dc8758..9dda8eec206e4 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/util/default_configs.ts +++ b/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/util/default_configs.ts @@ -7,6 +7,7 @@ import { IndexPatternTitle } from '../../../../../../../common/types/kibana'; import { Field, Aggregation, EVENT_RATE_FIELD_ID } from '../../../../../../../common/types/fields'; import { Job, Datafeed, Detector } from '../../../../../../../common/types/anomaly_detection_jobs'; +import { splitIndexPatternNames } from '../../../../../../../common/util/job_utils'; export function createEmptyJob(): Job { return { @@ -28,7 +29,7 @@ export function createEmptyDatafeed(indexPatternTitle: IndexPatternTitle): Dataf return { datafeed_id: '', job_id: '', - indices: [indexPatternTitle], + indices: splitIndexPatternNames(indexPatternTitle), query: {}, }; } diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/index_or_search/preconfigured_job_redirect.ts b/x-pack/plugins/ml/public/application/jobs/new_job/pages/index_or_search/preconfigured_job_redirect.ts index 50a84eb3d11cb..69df2773f9f8d 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/pages/index_or_search/preconfigured_job_redirect.ts +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/index_or_search/preconfigured_job_redirect.ts @@ -50,7 +50,7 @@ function getWizardUrlFromCloningJob(job: CombinedJob) { break; } - const indexPatternId = getIndexPatternIdFromName(job.datafeed_config.indices[0]); + const indexPatternId = getIndexPatternIdFromName(job.datafeed_config.indices.join()); return `jobs/new_job/${page}?index=${indexPatternId}&_g=()`; } diff --git a/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts b/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts index 84f81a30f36b8..40b2a524151b3 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts +++ b/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts @@ -29,7 +29,11 @@ import { JobSpecificOverride, isGeneralJobOverride, } from '../../../common/types/modules'; -import { getLatestDataOrBucketTimestamp, prefixDatafeedId } from '../../../common/util/job_utils'; +import { + getLatestDataOrBucketTimestamp, + prefixDatafeedId, + splitIndexPatternNames, +} from '../../../common/util/job_utils'; import { mlLog } from '../../client/log'; import { calculateModelMemoryLimitProvider } from '../calculate_model_memory_limit'; import { fieldsServiceProvider } from '../fields_service'; @@ -828,9 +832,7 @@ export class DataRecognizer { updateDatafeedIndices(moduleConfig: Module) { // if the supplied index pattern contains a comma, split into multiple indices and // add each one to the datafeed - const indexPatternNames = this.indexPatternName.includes(',') - ? this.indexPatternName.split(',').map(i => i.trim()) - : [this.indexPatternName]; + const indexPatternNames = splitIndexPatternNames(this.indexPatternName); moduleConfig.datafeeds.forEach(df => { const newIndices: string[] = []; From 8d880ffb772efefceffccbd3bb7aa0290677325e Mon Sep 17 00:00:00 2001 From: kqualters-elastic <56408403+kqualters-elastic@users.noreply.github.com> Date: Tue, 28 Apr 2020 16:11:01 -0400 Subject: [PATCH 2/2] [Endpoint] Recursive resolver children (#61914) --- .../plugins/endpoint/common/generate_data.ts | 15 +- .../plugins/endpoint/common/models/event.ts | 33 ++- .../endpoint/common/schema/resolver.ts | 59 +++++ x-pack/plugins/endpoint/common/types.ts | 25 ++ .../endpoint/view/alerts/resolver.tsx | 3 +- .../embeddables/resolver/store/data/action.ts | 14 +- .../resolver/store/data/graphing.test.ts | 54 ++-- .../resolver/store/data/reducer.ts | 15 +- .../resolver/store/data/selectors.ts | 6 +- .../embeddables/resolver/store/middleware.ts | 104 ++++---- .../embeddables/resolver/store/selectors.ts | 5 + .../public/embeddables/resolver/types.ts | 1 + .../embeddables/resolver/view/index.tsx | 12 + .../resolver/view/use_camera.test.tsx | 8 +- .../endpoint/server/routes/resolver.ts | 32 ++- .../server/routes/resolver/ancestry.ts | 39 +++ .../server/routes/resolver/children.ts | 71 +----- .../endpoint/server/routes/resolver/events.ts | 38 +++ .../server/routes/resolver/lifecycle.ts | 96 -------- .../server/routes/resolver/queries/base.ts | 27 +- .../routes/resolver/queries/children.test.ts | 42 ++-- .../routes/resolver/queries/children.ts | 42 ++-- ...{related_events.test.ts => events.test.ts} | 37 ++- .../queries/{related_events.ts => events.ts} | 25 +- .../routes/resolver/queries/lifecycle.test.ts | 19 +- .../routes/resolver/queries/lifecycle.ts | 20 +- .../routes/resolver/queries/stats.test.ts | 188 ++++++++++++++ .../server/routes/resolver/queries/stats.ts | 147 +++++++++++ .../server/routes/resolver/related_events.ts | 77 ------ .../endpoint/server/routes/resolver/tree.ts | 53 ++++ .../server/routes/resolver/utils/fetch.ts | 123 ++++++++++ .../server/routes/resolver/utils/normalize.ts | 30 --- .../routes/resolver/utils/pagination.ts | 58 +++-- .../server/routes/resolver/utils/tree.ts | 230 ++++++++++++++++++ .../api_integration/apis/endpoint/resolver.ts | 119 +++++---- 35 files changed, 1293 insertions(+), 574 deletions(-) create mode 100644 x-pack/plugins/endpoint/common/schema/resolver.ts create mode 100644 x-pack/plugins/endpoint/server/routes/resolver/ancestry.ts create mode 100644 x-pack/plugins/endpoint/server/routes/resolver/events.ts delete mode 100644 x-pack/plugins/endpoint/server/routes/resolver/lifecycle.ts rename x-pack/plugins/endpoint/server/routes/resolver/queries/{related_events.test.ts => events.test.ts} (75%) rename x-pack/plugins/endpoint/server/routes/resolver/queries/{related_events.ts => events.ts} (74%) create mode 100644 x-pack/plugins/endpoint/server/routes/resolver/queries/stats.test.ts create mode 100644 x-pack/plugins/endpoint/server/routes/resolver/queries/stats.ts delete mode 100644 x-pack/plugins/endpoint/server/routes/resolver/related_events.ts create mode 100644 x-pack/plugins/endpoint/server/routes/resolver/tree.ts create mode 100644 x-pack/plugins/endpoint/server/routes/resolver/utils/fetch.ts delete mode 100644 x-pack/plugins/endpoint/server/routes/resolver/utils/normalize.ts create mode 100644 x-pack/plugins/endpoint/server/routes/resolver/utils/tree.ts diff --git a/x-pack/plugins/endpoint/common/generate_data.ts b/x-pack/plugins/endpoint/common/generate_data.ts index 61fbb779986dc..e40fc3e386bc8 100644 --- a/x-pack/plugins/endpoint/common/generate_data.ts +++ b/x-pack/plugins/endpoint/common/generate_data.ts @@ -307,7 +307,7 @@ export class EndpointDocGenerator { process: { entity_id: options.entityID ? options.entityID : this.randomString(10), parent: options.parentEntityID ? { entity_id: options.parentEntityID } : undefined, - name: options.processName ? options.processName : 'powershell.exe', + name: options.processName ? options.processName : randomProcessName(), }, }; } @@ -645,3 +645,16 @@ export class EndpointDocGenerator { return uuid.v4({ random: [...this.randomNGenerator(255, 16)] }); } } + +const fakeProcessNames = [ + 'lsass.exe', + 'notepad.exe', + 'mimikatz.exe', + 'powershell.exe', + 'iexlorer.exe', + 'explorer.exe', +]; +/** Return a random fake process name */ +function randomProcessName(): string { + return fakeProcessNames[Math.floor(Math.random() * fakeProcessNames.length)]; +} diff --git a/x-pack/plugins/endpoint/common/models/event.ts b/x-pack/plugins/endpoint/common/models/event.ts index 650486f3c3858..47f39d2d11797 100644 --- a/x-pack/plugins/endpoint/common/models/event.ts +++ b/x-pack/plugins/endpoint/common/models/event.ts @@ -4,17 +4,13 @@ * you may not use this file except in compliance with the Elastic License. */ -import { EndpointEvent, LegacyEndpointEvent } from '../types'; +import { LegacyEndpointEvent, ResolverEvent } from '../types'; -export function isLegacyEvent( - event: EndpointEvent | LegacyEndpointEvent -): event is LegacyEndpointEvent { +export function isLegacyEvent(event: ResolverEvent): event is LegacyEndpointEvent { return (event as LegacyEndpointEvent).endgame !== undefined; } -export function eventTimestamp( - event: EndpointEvent | LegacyEndpointEvent -): string | undefined | number { +export function eventTimestamp(event: ResolverEvent): string | undefined | number { if (isLegacyEvent(event)) { return event.endgame.timestamp_utc; } else { @@ -22,10 +18,31 @@ export function eventTimestamp( } } -export function eventName(event: EndpointEvent | LegacyEndpointEvent): string { +export function eventName(event: ResolverEvent): string { if (isLegacyEvent(event)) { return event.endgame.process_name ? event.endgame.process_name : ''; } else { return event.process.name; } } + +export function eventId(event: ResolverEvent): string { + if (isLegacyEvent(event)) { + return event.endgame.serial_event_id ? String(event.endgame.serial_event_id) : ''; + } + return event.event.id; +} + +export function entityId(event: ResolverEvent): string { + if (isLegacyEvent(event)) { + return event.endgame.unique_pid ? String(event.endgame.unique_pid) : ''; + } + return event.process.entity_id; +} + +export function parentEntityId(event: ResolverEvent): string | undefined { + if (isLegacyEvent(event)) { + return event.endgame.unique_ppid ? String(event.endgame.unique_ppid) : undefined; + } + return event.process.parent?.entity_id; +} diff --git a/x-pack/plugins/endpoint/common/schema/resolver.ts b/x-pack/plugins/endpoint/common/schema/resolver.ts new file mode 100644 index 0000000000000..f21307e407fd0 --- /dev/null +++ b/x-pack/plugins/endpoint/common/schema/resolver.ts @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { schema } from '@kbn/config-schema'; + +/** + * Used to validate GET requests for a complete resolver tree. + */ +export const validateTree = { + params: schema.object({ id: schema.string() }), + query: schema.object({ + children: schema.number({ defaultValue: 10, min: 0, max: 100 }), + generations: schema.number({ defaultValue: 3, min: 0, max: 3 }), + ancestors: schema.number({ defaultValue: 3, min: 0, max: 5 }), + events: schema.number({ defaultValue: 100, min: 0, max: 1000 }), + afterEvent: schema.maybe(schema.string()), + afterChild: schema.maybe(schema.string()), + legacyEndpointID: schema.maybe(schema.string()), + }), +}; + +/** + * Used to validate GET requests for non process events for a specific event. + */ +export const validateEvents = { + params: schema.object({ id: schema.string() }), + query: schema.object({ + events: schema.number({ defaultValue: 100, min: 1, max: 1000 }), + afterEvent: schema.maybe(schema.string()), + legacyEndpointID: schema.maybe(schema.string()), + }), +}; + +/** + * Used to validate GET requests for the ancestors of a process event. + */ +export const validateAncestry = { + params: schema.object({ id: schema.string() }), + query: schema.object({ + ancestors: schema.number({ defaultValue: 0, min: 0, max: 10 }), + legacyEndpointID: schema.maybe(schema.string()), + }), +}; + +/** + * Used to validate GET requests for children of a specified process event. + */ +export const validateChildren = { + params: schema.object({ id: schema.string() }), + query: schema.object({ + children: schema.number({ defaultValue: 10, min: 10, max: 100 }), + generations: schema.number({ defaultValue: 3, min: 0, max: 3 }), + afterChild: schema.maybe(schema.string()), + legacyEndpointID: schema.maybe(schema.string()), + }), +}; diff --git a/x-pack/plugins/endpoint/common/types.ts b/x-pack/plugins/endpoint/common/types.ts index 8773c5441860a..8fce15d1c794c 100644 --- a/x-pack/plugins/endpoint/common/types.ts +++ b/x-pack/plugins/endpoint/common/types.ts @@ -35,6 +35,31 @@ type ImmutableObject = { readonly [K in keyof T]: Immutable }; */ export type AlertAPIOrdering = 'asc' | 'desc'; +export interface ResolverNodeStats { + totalEvents: number; + totalAlerts: number; +} + +export interface ResolverNodePagination { + nextChild?: string | null; + nextEvent?: string | null; + nextAncestor?: string | null; + nextAlert?: string | null; +} + +/** + * A node that contains pointers to other nodes, arrrays of resolver events, and any metadata associated with resolver specific data + */ +export interface ResolverNode { + id: string; + children: ResolverNode[]; + events: ResolverEvent[]; + lifecycle: ResolverEvent[]; + ancestors?: ResolverNode[]; + pagination: ResolverNodePagination; + stats?: ResolverNodeStats; +} + /** * Returned by 'api/endpoint/alerts' */ diff --git a/x-pack/plugins/endpoint/public/applications/endpoint/view/alerts/resolver.tsx b/x-pack/plugins/endpoint/public/applications/endpoint/view/alerts/resolver.tsx index d18bc59a35f52..d32ad4dd9defc 100644 --- a/x-pack/plugins/endpoint/public/applications/endpoint/view/alerts/resolver.tsx +++ b/x-pack/plugins/endpoint/public/applications/endpoint/view/alerts/resolver.tsx @@ -33,5 +33,6 @@ export const AlertDetailResolver = styled( width: 100%; display: flex; flex-grow: 1; - min-height: 500px; + /* gross demo hack */ + min-height: calc(100vh - 505px); `; diff --git a/x-pack/plugins/endpoint/public/embeddables/resolver/store/data/action.ts b/x-pack/plugins/endpoint/public/embeddables/resolver/store/data/action.ts index 373afa89921dc..3ec15f2f1985d 100644 --- a/x-pack/plugins/endpoint/public/embeddables/resolver/store/data/action.ts +++ b/x-pack/plugins/endpoint/public/embeddables/resolver/store/data/action.ts @@ -8,13 +8,11 @@ import { ResolverEvent } from '../../../../../common/types'; interface ServerReturnedResolverData { readonly type: 'serverReturnedResolverData'; - readonly payload: { - readonly data: { - readonly result: { - readonly search_results: readonly ResolverEvent[]; - }; - }; - }; + readonly payload: ResolverEvent[]; } -export type DataAction = ServerReturnedResolverData; +interface ServerFailedToReturnResolverData { + readonly type: 'serverFailedToReturnResolverData'; +} + +export type DataAction = ServerReturnedResolverData | ServerFailedToReturnResolverData; diff --git a/x-pack/plugins/endpoint/public/embeddables/resolver/store/data/graphing.test.ts b/x-pack/plugins/endpoint/public/embeddables/resolver/store/data/graphing.test.ts index f01136fe20ebf..f95ecc63d2a66 100644 --- a/x-pack/plugins/endpoint/public/embeddables/resolver/store/data/graphing.test.ts +++ b/x-pack/plugins/endpoint/public/embeddables/resolver/store/data/graphing.test.ts @@ -8,7 +8,7 @@ import { Store, createStore } from 'redux'; import { DataAction } from './action'; import { dataReducer } from './reducer'; import { DataState } from '../../types'; -import { LegacyEndpointEvent } from '../../../../../common/types'; +import { LegacyEndpointEvent, ResolverEvent } from '../../../../../common/types'; import { graphableProcesses, processNodePositionsAndEdgeLineSegments } from './selectors'; import { mockProcessEvent } from '../../models/process_event_test_helpers'; @@ -113,13 +113,7 @@ describe('resolver graph layout', () => { }); describe('when rendering no nodes', () => { beforeEach(() => { - const payload = { - data: { - result: { - search_results: [], - }, - }, - }; + const payload: ResolverEvent[] = []; const action: DataAction = { type: 'serverReturnedResolverData', payload }; store.dispatch(action); }); @@ -133,13 +127,7 @@ describe('resolver graph layout', () => { }); describe('when rendering one node', () => { beforeEach(() => { - const payload = { - data: { - result: { - search_results: [processA], - }, - }, - }; + const payload = [processA]; const action: DataAction = { type: 'serverReturnedResolverData', payload }; store.dispatch(action); }); @@ -153,13 +141,7 @@ describe('resolver graph layout', () => { }); describe('when rendering two nodes, one being the parent of the other', () => { beforeEach(() => { - const payload = { - data: { - result: { - search_results: [processA, processB], - }, - }, - }; + const payload = [processA, processB]; const action: DataAction = { type: 'serverReturnedResolverData', payload }; store.dispatch(action); }); @@ -173,23 +155,17 @@ describe('resolver graph layout', () => { }); describe('when rendering two forks, and one fork has an extra long tine', () => { beforeEach(() => { - const payload = { - data: { - result: { - search_results: [ - processA, - processB, - processC, - processD, - processE, - processF, - processG, - processH, - processI, - ], - }, - }, - }; + const payload = [ + processA, + processB, + processC, + processD, + processE, + processF, + processG, + processH, + processI, + ]; const action: DataAction = { type: 'serverReturnedResolverData', payload }; store.dispatch(action); }); diff --git a/x-pack/plugins/endpoint/public/embeddables/resolver/store/data/reducer.ts b/x-pack/plugins/endpoint/public/embeddables/resolver/store/data/reducer.ts index a3184389a794e..fc307002819a9 100644 --- a/x-pack/plugins/endpoint/public/embeddables/resolver/store/data/reducer.ts +++ b/x-pack/plugins/endpoint/public/embeddables/resolver/store/data/reducer.ts @@ -11,25 +11,28 @@ function initialState(): DataState { return { results: [], isLoading: false, + hasError: false, }; } export const dataReducer: Reducer = (state = initialState(), action) => { if (action.type === 'serverReturnedResolverData') { - const { - data: { - result: { search_results }, - }, - } = action.payload; return { ...state, - results: search_results, + results: action.payload, isLoading: false, + hasError: false, }; } else if (action.type === 'appRequestedResolverData') { return { ...state, isLoading: true, + hasError: false, + }; + } else if (action.type === 'serverFailedToReturnResolverData') { + return { + ...state, + hasError: true, }; } else { return state; diff --git a/x-pack/plugins/endpoint/public/embeddables/resolver/store/data/selectors.ts b/x-pack/plugins/endpoint/public/embeddables/resolver/store/data/selectors.ts index 5dda54d4ed029..59ee4b3b87505 100644 --- a/x-pack/plugins/endpoint/public/embeddables/resolver/store/data/selectors.ts +++ b/x-pack/plugins/endpoint/public/embeddables/resolver/store/data/selectors.ts @@ -34,6 +34,10 @@ export function isLoading(state: DataState) { return state.isLoading; } +export function hasError(state: DataState) { + return state.hasError; +} + /** * An isometric projection is a method for representing three dimensional objects in 2 dimensions. * More information about isometric projections can be found here https://en.wikipedia.org/wiki/Isometric_projection. @@ -293,7 +297,7 @@ function* levelOrderWithWidths( metadata.firstChildWidth = width; } else { const firstChildWidth = widths.get(siblings[0]); - const lastChildWidth = widths.get(siblings[0]); + const lastChildWidth = widths.get(siblings[siblings.length - 1]); if (firstChildWidth === undefined || lastChildWidth === undefined) { /** * All widths have been precalcluated, so this will not happen. diff --git a/x-pack/plugins/endpoint/public/embeddables/resolver/store/middleware.ts b/x-pack/plugins/endpoint/public/embeddables/resolver/store/middleware.ts index 4e57212e5c0c2..c7177c6387e7a 100644 --- a/x-pack/plugins/endpoint/public/embeddables/resolver/store/middleware.ts +++ b/x-pack/plugins/endpoint/public/embeddables/resolver/store/middleware.ts @@ -8,7 +8,7 @@ import { Dispatch, MiddlewareAPI } from 'redux'; import { KibanaReactContextValue } from '../../../../../../../src/plugins/kibana_react/public'; import { EndpointPluginServices } from '../../../plugin'; import { ResolverState, ResolverAction } from '../types'; -import { ResolverEvent } from '../../../../common/types'; +import { ResolverEvent, ResolverNode } from '../../../../common/types'; import * as event from '../../../../common/models/event'; type MiddlewareFactory = ( @@ -16,18 +16,18 @@ type MiddlewareFactory = ( ) => ( api: MiddlewareAPI, S> ) => (next: Dispatch) => (action: ResolverAction) => unknown; -interface Lifecycle { - lifecycle: ResolverEvent[]; -} -type ChildResponse = [Lifecycle]; -function flattenEvents(events: ChildResponse): ResolverEvent[] { - return events - .map((child: Lifecycle) => child.lifecycle) - .reduce( - (accumulator: ResolverEvent[], value: ResolverEvent[]) => accumulator.concat(value), - [] - ); +function flattenEvents(children: ResolverNode[], events: ResolverEvent[] = []): ResolverEvent[] { + return children.reduce((flattenedEvents, currentNode) => { + if (currentNode.lifecycle && currentNode.lifecycle.length > 0) { + flattenedEvents.push(...currentNode.lifecycle); + } + if (currentNode.children && currentNode.children.length > 0) { + return flattenEvents(currentNode.children, events); + } else { + return flattenedEvents; + } + }, events); } export const resolverMiddlewareFactory: MiddlewareFactory = context => { @@ -39,53 +39,43 @@ export const resolverMiddlewareFactory: MiddlewareFactory = context => { */ if (context?.services.http && action.payload.selectedEvent) { api.dispatch({ type: 'appRequestedResolverData' }); - let response = []; - let lifecycle: ResolverEvent[]; - let childEvents: ResolverEvent[]; - let relatedEvents: ResolverEvent[]; - let children = []; - const ancestors: ResolverEvent[] = []; - const maxAncestors = 5; - if (event.isLegacyEvent(action.payload.selectedEvent)) { - const uniquePid = action.payload.selectedEvent?.endgame?.unique_pid; - const legacyEndpointID = action.payload.selectedEvent?.agent?.id; - [{ lifecycle }, { children }, { events: relatedEvents }] = await Promise.all([ - context.services.http.get(`/api/endpoint/resolver/${uniquePid}`, { - query: { legacyEndpointID }, - }), - context.services.http.get(`/api/endpoint/resolver/${uniquePid}/children`, { - query: { legacyEndpointID }, - }), - context.services.http.get(`/api/endpoint/resolver/${uniquePid}/related`, { - query: { legacyEndpointID }, - }), - ]); - childEvents = children.length > 0 ? flattenEvents(children) : []; - } else { - const uniquePid = action.payload.selectedEvent.process.entity_id; - const ppid = action.payload.selectedEvent.process.parent?.entity_id; - async function getAncestors(pid: string | undefined) { - if (ancestors.length < maxAncestors && pid !== undefined) { - const parent = await context?.services.http.get(`/api/endpoint/resolver/${pid}`); - ancestors.push(parent.lifecycle[0]); - if (parent.lifecycle[0].process?.parent?.entity_id) { - await getAncestors(parent.lifecycle[0].process.parent.entity_id); - } - } + try { + let lifecycle: ResolverEvent[]; + let children: ResolverNode[]; + let ancestors: ResolverNode[]; + if (event.isLegacyEvent(action.payload.selectedEvent)) { + const entityId = action.payload.selectedEvent?.endgame?.unique_pid; + const legacyEndpointID = action.payload.selectedEvent?.agent?.id; + [{ lifecycle, children, ancestors }] = await Promise.all([ + context.services.http.get(`/api/endpoint/resolver/${entityId}`, { + query: { legacyEndpointID, children: 5, ancestors: 5 }, + }), + ]); + } else { + const entityId = action.payload.selectedEvent.process.entity_id; + [{ lifecycle, children, ancestors }] = await Promise.all([ + context.services.http.get(`/api/endpoint/resolver/${entityId}`, { + query: { + children: 5, + ancestors: 5, + }, + }), + ]); } - [{ lifecycle }, { children }, { events: relatedEvents }] = await Promise.all([ - context.services.http.get(`/api/endpoint/resolver/${uniquePid}`), - context.services.http.get(`/api/endpoint/resolver/${uniquePid}/children`), - context.services.http.get(`/api/endpoint/resolver/${uniquePid}/related`), - getAncestors(ppid), - ]); + const response: ResolverEvent[] = [ + ...lifecycle, + ...flattenEvents(children), + ...flattenEvents(ancestors), + ]; + api.dispatch({ + type: 'serverReturnedResolverData', + payload: response, + }); + } catch (error) { + api.dispatch({ + type: 'serverFailedToReturnResolverData', + }); } - childEvents = children.length > 0 ? flattenEvents(children) : []; - response = [...lifecycle, ...childEvents, ...relatedEvents, ...ancestors]; - api.dispatch({ - type: 'serverReturnedResolverData', - payload: { data: { result: { search_results: response } } }, - }); } } }; diff --git a/x-pack/plugins/endpoint/public/embeddables/resolver/store/selectors.ts b/x-pack/plugins/endpoint/public/embeddables/resolver/store/selectors.ts index e8ae3d08e5cb6..7d09d90881da9 100644 --- a/x-pack/plugins/endpoint/public/embeddables/resolver/store/selectors.ts +++ b/x-pack/plugins/endpoint/public/embeddables/resolver/store/selectors.ts @@ -102,6 +102,11 @@ function uiStateSelector(state: ResolverState) { */ export const isLoading = composeSelectors(dataStateSelector, dataSelectors.isLoading); +/** + * Whether or not the resolver encountered an error while fetching data + */ +export const hasError = composeSelectors(dataStateSelector, dataSelectors.hasError); + /** * Calls the `secondSelector` with the result of the `selector`. Use this when re-exporting a * concern-specific selector. `selector` should return the concern-specific state. diff --git a/x-pack/plugins/endpoint/public/embeddables/resolver/types.ts b/x-pack/plugins/endpoint/public/embeddables/resolver/types.ts index d370bda0d1842..17aa598720c59 100644 --- a/x-pack/plugins/endpoint/public/embeddables/resolver/types.ts +++ b/x-pack/plugins/endpoint/public/embeddables/resolver/types.ts @@ -136,6 +136,7 @@ export type CameraState = { export interface DataState { readonly results: readonly ResolverEvent[]; isLoading: boolean; + hasError: boolean; } export type Vector2 = readonly [number, number]; diff --git a/x-pack/plugins/endpoint/public/embeddables/resolver/view/index.tsx b/x-pack/plugins/endpoint/public/embeddables/resolver/view/index.tsx index 36155ece57a9c..2e7ca65c92dc1 100644 --- a/x-pack/plugins/endpoint/public/embeddables/resolver/view/index.tsx +++ b/x-pack/plugins/endpoint/public/embeddables/resolver/view/index.tsx @@ -8,6 +8,7 @@ import React, { useLayoutEffect } from 'react'; import { useSelector, useDispatch } from 'react-redux'; import styled from 'styled-components'; import { EuiLoadingSpinner } from '@elastic/eui'; +import { FormattedMessage } from '@kbn/i18n/react'; import * as selectors from '../store/selectors'; import { EdgeLine } from './edge_line'; import { Panel } from './panel'; @@ -59,6 +60,7 @@ export const Resolver = styled( const { projectionMatrix, ref, onMouseDown } = useCamera(); const isLoading = useSelector(selectors.isLoading); + const hasError = useSelector(selectors.hasError); const activeDescendantId = useSelector(selectors.uiActiveDescendantId); useLayoutEffect(() => { @@ -74,6 +76,16 @@ export const Resolver = styled(
+ ) : hasError ? ( +
+
+ {' '} + +
+
) : ( { } const serverResponseAction: ResolverAction = { type: 'serverReturnedResolverData', - payload: { - data: { - result: { - search_results: events, - }, - }, - }, + payload: events, }; act(() => { store.dispatch(serverResponseAction); diff --git a/x-pack/plugins/endpoint/server/routes/resolver.ts b/x-pack/plugins/endpoint/server/routes/resolver.ts index a96d431225b15..3599acacb4f59 100644 --- a/x-pack/plugins/endpoint/server/routes/resolver.ts +++ b/x-pack/plugins/endpoint/server/routes/resolver.ts @@ -6,20 +6,27 @@ import { IRouter } from 'kibana/server'; import { EndpointAppContext } from '../types'; -import { handleRelatedEvents, validateRelatedEvents } from './resolver/related_events'; -import { handleChildren, validateChildren } from './resolver/children'; -import { handleLifecycle, validateLifecycle } from './resolver/lifecycle'; +import { + validateTree, + validateEvents, + validateChildren, + validateAncestry, +} from '../../common/schema/resolver'; +import { handleEvents } from './resolver/events'; +import { handleChildren } from './resolver/children'; +import { handleAncestry } from './resolver/ancestry'; +import { handleTree } from './resolver/tree'; export function registerResolverRoutes(router: IRouter, endpointAppContext: EndpointAppContext) { const log = endpointAppContext.logFactory.get('resolver'); router.get( { - path: '/api/endpoint/resolver/{id}/related', - validate: validateRelatedEvents, + path: '/api/endpoint/resolver/{id}/events', + validate: validateEvents, options: { authRequired: true }, }, - handleRelatedEvents(log, endpointAppContext) + handleEvents(log, endpointAppContext) ); router.get( @@ -31,12 +38,21 @@ export function registerResolverRoutes(router: IRouter, endpointAppContext: Endp handleChildren(log, endpointAppContext) ); + router.get( + { + path: '/api/endpoint/resolver/{id}/ancestry', + validate: validateAncestry, + options: { authRequired: true }, + }, + handleAncestry(log, endpointAppContext) + ); + router.get( { path: '/api/endpoint/resolver/{id}', - validate: validateLifecycle, + validate: validateTree, options: { authRequired: true }, }, - handleLifecycle(log, endpointAppContext) + handleTree(log, endpointAppContext) ); } diff --git a/x-pack/plugins/endpoint/server/routes/resolver/ancestry.ts b/x-pack/plugins/endpoint/server/routes/resolver/ancestry.ts new file mode 100644 index 0000000000000..6648dc5b9e493 --- /dev/null +++ b/x-pack/plugins/endpoint/server/routes/resolver/ancestry.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { RequestHandler, Logger } from 'kibana/server'; +import { TypeOf } from '@kbn/config-schema'; +import { validateAncestry } from '../../../common/schema/resolver'; +import { Fetcher } from './utils/fetch'; +import { EndpointAppContext } from '../../types'; + +export function handleAncestry( + log: Logger, + endpointAppContext: EndpointAppContext +): RequestHandler, TypeOf> { + return async (context, req, res) => { + const { + params: { id }, + query: { ancestors, legacyEndpointID: endpointID }, + } = req; + try { + const indexRetriever = endpointAppContext.service.getIndexPatternRetriever(); + + const client = context.core.elasticsearch.dataClient; + const indexPattern = await indexRetriever.getEventIndexPattern(context); + + const fetcher = new Fetcher(client, id, indexPattern, endpointID); + const tree = await fetcher.ancestors(ancestors + 1); + + return res.ok({ + body: tree.render(), + }); + } catch (err) { + log.warn(err); + return res.internalError({ body: err }); + } + }; +} diff --git a/x-pack/plugins/endpoint/server/routes/resolver/children.ts b/x-pack/plugins/endpoint/server/routes/resolver/children.ts index c3b19b6c912b6..bb18b29a4b947 100644 --- a/x-pack/plugins/endpoint/server/routes/resolver/children.ts +++ b/x-pack/plugins/endpoint/server/routes/resolver/children.ts @@ -4,87 +4,32 @@ * you may not use this file except in compliance with the Elastic License. */ -import _ from 'lodash'; -import { schema } from '@kbn/config-schema'; import { RequestHandler, Logger } from 'kibana/server'; -import { extractEntityID } from './utils/normalize'; -import { getPaginationParams } from './utils/pagination'; -import { LifecycleQuery } from './queries/lifecycle'; -import { ChildrenQuery } from './queries/children'; +import { TypeOf } from '@kbn/config-schema'; +import { validateChildren } from '../../../common/schema/resolver'; +import { Fetcher } from './utils/fetch'; import { EndpointAppContext } from '../../types'; -interface ChildrenQueryParams { - after?: string; - limit: number; - /** - * legacyEndpointID is optional because there are two different types of identifiers: - * - * Legacy - * A legacy Entity ID is made up of the agent.id and unique_pid fields. The client will need to identify if - * it's looking at a legacy event and use those fields when making requests to the backend. The - * request would be /resolver/{id}?legacyEndpointID=and the {id} would be the unique_pid. - * - * Elastic Endpoint - * When interacting with the new form of data the client doesn't need the legacyEndpointID because it's already a - * part of the entityID in the new type of event. So for the same request the client would just hit resolver/{id} - * and the {id} would be entityID stored in the event's process.entity_id field. - */ - legacyEndpointID?: string; -} - -interface ChildrenPathParams { - id: string; -} - -export const validateChildren = { - params: schema.object({ id: schema.string() }), - query: schema.object({ - after: schema.maybe(schema.string()), - limit: schema.number({ defaultValue: 10, min: 1, max: 100 }), - legacyEndpointID: schema.maybe(schema.string()), - }), -}; - export function handleChildren( log: Logger, endpointAppContext: EndpointAppContext -): RequestHandler { +): RequestHandler, TypeOf> { return async (context, req, res) => { const { params: { id }, - query: { limit, after, legacyEndpointID }, + query: { children, generations, afterChild, legacyEndpointID: endpointID }, } = req; try { const indexRetriever = endpointAppContext.service.getIndexPatternRetriever(); - const pagination = getPaginationParams(limit, after); const indexPattern = await indexRetriever.getEventIndexPattern(context); const client = context.core.elasticsearch.dataClient; - const childrenQuery = new ChildrenQuery(indexPattern, legacyEndpointID, pagination); - const lifecycleQuery = new LifecycleQuery(indexPattern, legacyEndpointID); - - // Retrieve the related child process events for a given process - const { total, results: events, nextCursor } = await childrenQuery.search(client, id); - const childIDs = events.map(extractEntityID); - - // Retrieve the lifecycle events for the child processes (e.g. started, terminated etc) - // this needs to fire after the above since we don't yet have the entity ids until we - // run the first query - const { results: lifecycleEvents } = await lifecycleQuery.search(client, ...childIDs); - // group all of the lifecycle events by the child process id - const lifecycleGroups = Object.values(_.groupBy(lifecycleEvents, extractEntityID)); - const children = lifecycleGroups.map(group => ({ lifecycle: group })); + const fetcher = new Fetcher(client, id, indexPattern, endpointID); + const tree = await fetcher.children(children, generations, afterChild); return res.ok({ - body: { - children, - pagination: { - total, - next: nextCursor, - limit, - }, - }, + body: tree.render(), }); } catch (err) { log.warn(err); diff --git a/x-pack/plugins/endpoint/server/routes/resolver/events.ts b/x-pack/plugins/endpoint/server/routes/resolver/events.ts new file mode 100644 index 0000000000000..a70a6e8d097d0 --- /dev/null +++ b/x-pack/plugins/endpoint/server/routes/resolver/events.ts @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { TypeOf } from '@kbn/config-schema'; +import { RequestHandler, Logger } from 'kibana/server'; +import { validateEvents } from '../../../common/schema/resolver'; +import { Fetcher } from './utils/fetch'; +import { EndpointAppContext } from '../../types'; + +export function handleEvents( + log: Logger, + endpointAppContext: EndpointAppContext +): RequestHandler, TypeOf> { + return async (context, req, res) => { + const { + params: { id }, + query: { events, afterEvent, legacyEndpointID: endpointID }, + } = req; + try { + const indexRetriever = endpointAppContext.service.getIndexPatternRetriever(); + const client = context.core.elasticsearch.dataClient; + const indexPattern = await indexRetriever.getEventIndexPattern(context); + + const fetcher = new Fetcher(client, id, indexPattern, endpointID); + const tree = await fetcher.events(events, afterEvent); + + return res.ok({ + body: tree.render(), + }); + } catch (err) { + log.warn(err); + return res.internalError({ body: err }); + } + }; +} diff --git a/x-pack/plugins/endpoint/server/routes/resolver/lifecycle.ts b/x-pack/plugins/endpoint/server/routes/resolver/lifecycle.ts deleted file mode 100644 index 91a4c5d49bc54..0000000000000 --- a/x-pack/plugins/endpoint/server/routes/resolver/lifecycle.ts +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { schema } from '@kbn/config-schema'; -import { RequestHandler, Logger } from 'kibana/server'; -import { extractParentEntityID } from './utils/normalize'; -import { LifecycleQuery } from './queries/lifecycle'; -import { ResolverEvent } from '../../../common/types'; -import { EndpointAppContext } from '../../types'; - -interface LifecycleQueryParams { - ancestors: number; - /** - * legacyEndpointID is optional because there are two different types of identifiers: - * - * Legacy - * A legacy Entity ID is made up of the agent.id and unique_pid fields. The client will need to identify if - * it's looking at a legacy event and use those fields when making requests to the backend. The - * request would be /resolver/{id}?legacyEndpointID=and the {id} would be the unique_pid. - * - * Elastic Endpoint - * When interacting with the new form of data the client doesn't need the legacyEndpointID because it's already a - * part of the entityID in the new type of event. So for the same request the client would just hit resolver/{id} - * and the {id} would be entityID stored in the event's process.entity_id field. - */ - legacyEndpointID?: string; -} - -interface LifecyclePathParams { - id: string; -} - -export const validateLifecycle = { - params: schema.object({ id: schema.string() }), - query: schema.object({ - ancestors: schema.number({ defaultValue: 0, min: 0, max: 10 }), - legacyEndpointID: schema.maybe(schema.string()), - }), -}; - -function getParentEntityID(results: ResolverEvent[]) { - return results.length === 0 ? undefined : extractParentEntityID(results[0]); -} - -export function handleLifecycle( - log: Logger, - endpointAppContext: EndpointAppContext -): RequestHandler { - return async (context, req, res) => { - const { - params: { id }, - query: { ancestors, legacyEndpointID }, - } = req; - try { - const indexRetriever = endpointAppContext.service.getIndexPatternRetriever(); - const ancestorLifecycles = []; - const client = context.core.elasticsearch.dataClient; - const indexPattern = await indexRetriever.getEventIndexPattern(context); - const lifecycleQuery = new LifecycleQuery(indexPattern, legacyEndpointID); - const { results: processLifecycle } = await lifecycleQuery.search(client, id); - let nextParentID = getParentEntityID(processLifecycle); - - if (nextParentID) { - for (let i = 0; i < ancestors; i++) { - const { results: lifecycle } = await lifecycleQuery.search(client, nextParentID); - nextParentID = getParentEntityID(lifecycle); - - if (!nextParentID) { - break; - } - - ancestorLifecycles.push({ - lifecycle, - }); - } - } - - return res.ok({ - body: { - lifecycle: processLifecycle, - ancestors: ancestorLifecycles, - pagination: { - next: nextParentID || null, - ancestors, - }, - }, - }); - } catch (err) { - log.warn(err); - return res.internalError({ body: err }); - } - }; -} diff --git a/x-pack/plugins/endpoint/server/routes/resolver/queries/base.ts b/x-pack/plugins/endpoint/server/routes/resolver/queries/base.ts index 3b3b4b0c9e8ac..eba4e5581c136 100644 --- a/x-pack/plugins/endpoint/server/routes/resolver/queries/base.ts +++ b/x-pack/plugins/endpoint/server/routes/resolver/queries/base.ts @@ -4,8 +4,15 @@ * you may not use this file except in compliance with the Elastic License. */ +import { SearchResponse } from 'elasticsearch'; import { IScopedClusterClient } from 'kibana/server'; -import { paginate, paginatedResults, PaginationParams } from '../utils/pagination'; +import { ResolverEvent } from '../../../../common/types'; +import { + paginate, + paginatedResults, + PaginationParams, + PaginatedResults, +} from '../utils/pagination'; import { JsonObject } from '../../../../../../../src/plugins/kibana_utils/public'; import { legacyEventIndexPattern } from './legacy_event_index_pattern'; @@ -16,11 +23,13 @@ export abstract class ResolverQuery { private readonly pagination?: PaginationParams ) {} - protected paginateBy(field: string, query: JsonObject) { - if (!this.pagination) { - return query; - } - return paginate(this.pagination, field, query); + protected paginateBy(tiebreaker: string, aggregator: string) { + return (query: JsonObject) => { + if (!this.pagination) { + return query; + } + return paginate(this.pagination, tiebreaker, aggregator, query); + }; } build(...ids: string[]) { @@ -31,7 +40,11 @@ export abstract class ResolverQuery { } async search(client: IScopedClusterClient, ...ids: string[]) { - return paginatedResults(await client.callAsCurrentUser('search', this.build(...ids))); + return this.postSearch(await client.callAsCurrentUser('search', this.build(...ids))); + } + + protected postSearch(response: SearchResponse): PaginatedResults { + return paginatedResults(response); } protected abstract legacyQuery( diff --git a/x-pack/plugins/endpoint/server/routes/resolver/queries/children.test.ts b/x-pack/plugins/endpoint/server/routes/resolver/queries/children.test.ts index d1d090579f0cd..2a097e87c38b2 100644 --- a/x-pack/plugins/endpoint/server/routes/resolver/queries/children.test.ts +++ b/x-pack/plugins/endpoint/server/routes/resolver/queries/children.test.ts @@ -32,15 +32,28 @@ describe('children events query', () => { term: { 'event.category': 'process' }, }, { - term: { 'event.type': 'process_start' }, + term: { 'event.kind': 'event' }, + }, + { + bool: { + should: [ + { + term: { 'event.type': 'process_start' }, + }, + { + term: { 'event.action': 'fork_event' }, + }, + ], + }, }, ], }, }, aggs: { - total: { - value_count: { - field: 'endgame.serial_event_id', + totals: { + terms: { + field: 'endgame.unique_ppid', + size: 1, }, }, }, @@ -67,20 +80,14 @@ describe('children events query', () => { bool: { filter: [ { - bool: { - should: [ - { - terms: { 'endpoint.process.parent.entity_id': ['baz'] }, - }, - { - terms: { 'process.parent.entity_id': ['baz'] }, - }, - ], - }, + terms: { 'process.parent.entity_id': ['baz'] }, }, { term: { 'event.category': 'process' }, }, + { + term: { 'event.kind': 'event' }, + }, { term: { 'event.type': 'start' }, }, @@ -88,9 +95,10 @@ describe('children events query', () => { }, }, aggs: { - total: { - value_count: { - field: 'event.id', + totals: { + terms: { + field: 'process.parent.entity_id', + size: 1, }, }, }, diff --git a/x-pack/plugins/endpoint/server/routes/resolver/queries/children.ts b/x-pack/plugins/endpoint/server/routes/resolver/queries/children.ts index 6d084a0cf20e5..690c926d7e6d6 100644 --- a/x-pack/plugins/endpoint/server/routes/resolver/queries/children.ts +++ b/x-pack/plugins/endpoint/server/routes/resolver/queries/children.ts @@ -7,8 +7,9 @@ import { ResolverQuery } from './base'; export class ChildrenQuery extends ResolverQuery { protected legacyQuery(endpointID: string, uniquePIDs: string[], index: string) { + const paginator = this.paginateBy('endgame.serial_event_id', 'endgame.unique_ppid'); return { - body: this.paginateBy('endgame.serial_event_id', { + body: paginator({ query: { bool: { filter: [ @@ -22,11 +23,19 @@ export class ChildrenQuery extends ResolverQuery { term: { 'event.category': 'process' }, }, { - // Corner case, we could only have a process_running or process_terminated - // so to solve this we'll probably want to either search for all of them and only return one if that's - // possible in elastic search or in memory pull out a single event to return - // https://github.com/elastic/endpoint-app-team/issues/168 - term: { 'event.type': 'process_start' }, + term: { 'event.kind': 'event' }, + }, + { + bool: { + should: [ + { + term: { 'event.type': 'process_start' }, + }, + { + term: { 'event.action': 'fork_event' }, + }, + ], + }, }, ], }, @@ -37,31 +46,22 @@ export class ChildrenQuery extends ResolverQuery { } protected query(entityIDs: string[], index: string) { + const paginator = this.paginateBy('event.id', 'process.parent.entity_id'); return { - body: this.paginateBy('event.id', { + body: paginator({ query: { bool: { filter: [ { - bool: { - should: [ - { - terms: { 'endpoint.process.parent.entity_id': entityIDs }, - }, - { - terms: { 'process.parent.entity_id': entityIDs }, - }, - ], - }, + terms: { 'process.parent.entity_id': entityIDs }, }, { term: { 'event.category': 'process' }, }, { - // Corner case, we could only have a process_running or process_terminated - // so to solve this we'll probably want to either search for all of them and only return one if that's - // possible in elastic search or in memory pull out a single event to return - // https://github.com/elastic/endpoint-app-team/issues/168 + term: { 'event.kind': 'event' }, + }, + { term: { 'event.type': 'start' }, }, ], diff --git a/x-pack/plugins/endpoint/server/routes/resolver/queries/related_events.test.ts b/x-pack/plugins/endpoint/server/routes/resolver/queries/events.test.ts similarity index 75% rename from x-pack/plugins/endpoint/server/routes/resolver/queries/related_events.test.ts rename to x-pack/plugins/endpoint/server/routes/resolver/queries/events.test.ts index 26205e9db6e45..78e5ee9226581 100644 --- a/x-pack/plugins/endpoint/server/routes/resolver/queries/related_events.test.ts +++ b/x-pack/plugins/endpoint/server/routes/resolver/queries/events.test.ts @@ -3,7 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { RelatedEventsQuery } from './related_events'; +import { EventsQuery } from './events'; import { fakeEventIndexPattern } from './children.test'; import { legacyEventIndexPattern } from './legacy_event_index_pattern'; @@ -11,7 +11,7 @@ describe('related events query', () => { it('generates the correct legacy queries', () => { const timestamp = new Date().getTime(); expect( - new RelatedEventsQuery(legacyEventIndexPattern, 'awesome-id', { + new EventsQuery(legacyEventIndexPattern, 'awesome-id', { size: 1, timestamp, eventID: 'foo', @@ -27,6 +27,9 @@ describe('related events query', () => { { term: { 'agent.id': 'awesome-id' }, }, + { + term: { 'event.kind': 'event' }, + }, { bool: { must_not: { @@ -38,9 +41,10 @@ describe('related events query', () => { }, }, aggs: { - total: { - value_count: { - field: 'endgame.serial_event_id', + totals: { + terms: { + field: 'endgame.unique_pid', + size: 1, }, }, }, @@ -56,7 +60,7 @@ describe('related events query', () => { const timestamp = new Date().getTime(); expect( - new RelatedEventsQuery(fakeEventIndexPattern, undefined, { + new EventsQuery(fakeEventIndexPattern, undefined, { size: 1, timestamp, eventID: 'bar', @@ -67,16 +71,10 @@ describe('related events query', () => { bool: { filter: [ { - bool: { - should: [ - { - terms: { 'endpoint.process.entity_id': ['baz'] }, - }, - { - terms: { 'process.entity_id': ['baz'] }, - }, - ], - }, + terms: { 'process.entity_id': ['baz'] }, + }, + { + term: { 'event.kind': 'event' }, }, { bool: { @@ -89,9 +87,10 @@ describe('related events query', () => { }, }, aggs: { - total: { - value_count: { - field: 'event.id', + totals: { + terms: { + field: 'process.entity_id', + size: 1, }, }, }, diff --git a/x-pack/plugins/endpoint/server/routes/resolver/queries/related_events.ts b/x-pack/plugins/endpoint/server/routes/resolver/queries/events.ts similarity index 74% rename from x-pack/plugins/endpoint/server/routes/resolver/queries/related_events.ts rename to x-pack/plugins/endpoint/server/routes/resolver/queries/events.ts index cc5afe8face8d..b622cb8a21111 100644 --- a/x-pack/plugins/endpoint/server/routes/resolver/queries/related_events.ts +++ b/x-pack/plugins/endpoint/server/routes/resolver/queries/events.ts @@ -6,10 +6,11 @@ import { ResolverQuery } from './base'; import { JsonObject } from '../../../../../../../src/plugins/kibana_utils/public'; -export class RelatedEventsQuery extends ResolverQuery { +export class EventsQuery extends ResolverQuery { protected legacyQuery(endpointID: string, uniquePIDs: string[], index: string): JsonObject { + const paginator = this.paginateBy('endgame.serial_event_id', 'endgame.unique_pid'); return { - body: this.paginateBy('endgame.serial_event_id', { + body: paginator({ query: { bool: { filter: [ @@ -19,6 +20,9 @@ export class RelatedEventsQuery extends ResolverQuery { { term: { 'agent.id': endpointID }, }, + { + term: { 'event.kind': 'event' }, + }, { bool: { must_not: { @@ -35,22 +39,17 @@ export class RelatedEventsQuery extends ResolverQuery { } protected query(entityIDs: string[], index: string): JsonObject { + const paginator = this.paginateBy('event.id', 'process.entity_id'); return { - body: this.paginateBy('event.id', { + body: paginator({ query: { bool: { filter: [ { - bool: { - should: [ - { - terms: { 'endpoint.process.entity_id': entityIDs }, - }, - { - terms: { 'process.entity_id': entityIDs }, - }, - ], - }, + terms: { 'process.entity_id': entityIDs }, + }, + { + term: { 'event.kind': 'event' }, }, { bool: { diff --git a/x-pack/plugins/endpoint/server/routes/resolver/queries/lifecycle.test.ts b/x-pack/plugins/endpoint/server/routes/resolver/queries/lifecycle.test.ts index 9d3c675ecc54c..296135af83b72 100644 --- a/x-pack/plugins/endpoint/server/routes/resolver/queries/lifecycle.test.ts +++ b/x-pack/plugins/endpoint/server/routes/resolver/queries/lifecycle.test.ts @@ -21,12 +21,16 @@ describe('lifecycle query', () => { { term: { 'agent.id': 'awesome-id' }, }, + { + term: { 'event.kind': 'event' }, + }, { term: { 'event.category': 'process' }, }, ], }, }, + size: 10000, sort: [{ '@timestamp': 'asc' }], }, index: legacyEventIndexPattern, @@ -40,16 +44,10 @@ describe('lifecycle query', () => { bool: { filter: [ { - bool: { - should: [ - { - terms: { 'endpoint.process.entity_id': ['baz'] }, - }, - { - terms: { 'process.entity_id': ['baz'] }, - }, - ], - }, + terms: { 'process.entity_id': ['baz'] }, + }, + { + term: { 'event.kind': 'event' }, }, { term: { 'event.category': 'process' }, @@ -57,6 +55,7 @@ describe('lifecycle query', () => { ], }, }, + size: 10000, sort: [{ '@timestamp': 'asc' }], }, index: fakeEventIndexPattern, diff --git a/x-pack/plugins/endpoint/server/routes/resolver/queries/lifecycle.ts b/x-pack/plugins/endpoint/server/routes/resolver/queries/lifecycle.ts index 290c601e0e9d8..e775b0cf9b6d2 100644 --- a/x-pack/plugins/endpoint/server/routes/resolver/queries/lifecycle.ts +++ b/x-pack/plugins/endpoint/server/routes/resolver/queries/lifecycle.ts @@ -6,7 +6,6 @@ import { ResolverQuery } from './base'; import { JsonObject } from '../../../../../../../src/plugins/kibana_utils/public'; -// consider limiting the response size to a reasonable value in case we have a bunch of lifecycle events export class LifecycleQuery extends ResolverQuery { protected legacyQuery(endpointID: string, uniquePIDs: string[], index: string): JsonObject { return { @@ -20,12 +19,16 @@ export class LifecycleQuery extends ResolverQuery { { term: { 'agent.id': endpointID }, }, + { + term: { 'event.kind': 'event' }, + }, { term: { 'event.category': 'process' }, }, ], }, }, + size: 10000, sort: [{ '@timestamp': 'asc' }], }, index, @@ -39,16 +42,10 @@ export class LifecycleQuery extends ResolverQuery { bool: { filter: [ { - bool: { - should: [ - { - terms: { 'endpoint.process.entity_id': entityIDs }, - }, - { - terms: { 'process.entity_id': entityIDs }, - }, - ], - }, + terms: { 'process.entity_id': entityIDs }, + }, + { + term: { 'event.kind': 'event' }, }, { term: { 'event.category': 'process' }, @@ -56,6 +53,7 @@ export class LifecycleQuery extends ResolverQuery { ], }, }, + size: 10000, sort: [{ '@timestamp': 'asc' }], }, index, diff --git a/x-pack/plugins/endpoint/server/routes/resolver/queries/stats.test.ts b/x-pack/plugins/endpoint/server/routes/resolver/queries/stats.test.ts new file mode 100644 index 0000000000000..17a158aec7cf5 --- /dev/null +++ b/x-pack/plugins/endpoint/server/routes/resolver/queries/stats.test.ts @@ -0,0 +1,188 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { legacyEventIndexPattern } from './legacy_event_index_pattern'; +import { StatsQuery } from './stats'; +import { fakeEventIndexPattern } from './children.test'; + +describe('stats query', () => { + it('generates the correct legacy queries', () => { + expect(new StatsQuery(legacyEventIndexPattern, 'awesome-id').build('5')).toStrictEqual({ + body: { + size: 0, + query: { + bool: { + filter: [ + { + term: { + 'agent.id': 'awesome-id', + }, + }, + { + bool: { + should: [ + { + bool: { + filter: [ + { + term: { + 'event.kind': 'event', + }, + }, + { + terms: { + 'endgame.unique_pid': ['5'], + }, + }, + { + bool: { + must_not: { + term: { + 'event.category': 'process', + }, + }, + }, + }, + ], + }, + }, + { + bool: { + filter: [ + { + term: { + 'event.kind': 'alert', + }, + }, + { + terms: { + 'endgame.data.alert_details.acting_process.unique_pid': ['5'], + }, + }, + ], + }, + }, + ], + }, + }, + ], + }, + }, + aggs: { + alerts: { + filter: { + term: { + 'event.kind': 'alert', + }, + }, + aggs: { + ids: { + terms: { + field: 'endgame.data.alert_details.acting_process.unique_pid', + }, + }, + }, + }, + events: { + filter: { + term: { + 'event.kind': 'event', + }, + }, + aggs: { + ids: { + terms: { + field: 'endgame.unique_pid', + }, + }, + }, + }, + }, + }, + index: legacyEventIndexPattern, + }); + }); + + it('generates the correct non-legacy queries', () => { + expect(new StatsQuery(fakeEventIndexPattern).build('baz')).toStrictEqual({ + body: { + size: 0, + query: { + bool: { + filter: [ + { + terms: { + 'process.entity_id': ['baz'], + }, + }, + { + bool: { + should: [ + { + bool: { + filter: [ + { + term: { + 'event.kind': 'event', + }, + }, + { + bool: { + must_not: { + term: { + 'event.category': 'process', + }, + }, + }, + }, + ], + }, + }, + { + term: { + 'event.kind': 'alert', + }, + }, + ], + }, + }, + ], + }, + }, + aggs: { + alerts: { + filter: { + term: { + 'event.kind': 'alert', + }, + }, + aggs: { + ids: { + terms: { + field: 'process.entity_id', + }, + }, + }, + }, + events: { + filter: { + term: { + 'event.kind': 'event', + }, + }, + aggs: { + ids: { + terms: { + field: 'process.entity_id', + }, + }, + }, + }, + }, + }, + index: fakeEventIndexPattern, + }); + }); +}); diff --git a/x-pack/plugins/endpoint/server/routes/resolver/queries/stats.ts b/x-pack/plugins/endpoint/server/routes/resolver/queries/stats.ts new file mode 100644 index 0000000000000..7db3ab2b0cb1f --- /dev/null +++ b/x-pack/plugins/endpoint/server/routes/resolver/queries/stats.ts @@ -0,0 +1,147 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { SearchResponse } from 'elasticsearch'; +import { ResolverQuery } from './base'; +import { ResolverEvent } from '../../../../common/types'; +import { JsonObject } from '../../../../../../../src/plugins/kibana_utils/public'; +import { PaginatedResults } from '../utils/pagination'; + +export class StatsQuery extends ResolverQuery { + protected postSearch(response: SearchResponse): PaginatedResults { + const alerts = response.aggregations.alerts.ids.buckets.reduce( + (cummulative: any, bucket: any) => ({ ...cummulative, [bucket.key]: bucket.doc_count }), + {} + ); + const events = response.aggregations.events.ids.buckets.reduce( + (cummulative: any, bucket: any) => ({ ...cummulative, [bucket.key]: bucket.doc_count }), + {} + ); + return { + totals: {}, + results: [], + extras: { + alerts, + events, + }, + }; + } + + protected legacyQuery(endpointID: string, uniquePIDs: string[], index: string): JsonObject { + return { + body: { + size: 0, + query: { + bool: { + filter: [ + { + term: { 'agent.id': endpointID }, + }, + { + bool: { + should: [ + { + bool: { + filter: [ + { term: { 'event.kind': 'event' } }, + { terms: { 'endgame.unique_pid': uniquePIDs } }, + { + bool: { + must_not: { + term: { 'event.category': 'process' }, + }, + }, + }, + ], + }, + }, + { + bool: { + filter: [ + { term: { 'event.kind': 'alert' } }, + { + terms: { + 'endgame.data.alert_details.acting_process.unique_pid': uniquePIDs, + }, + }, + ], + }, + }, + ], + }, + }, + ], + }, + }, + aggs: { + alerts: { + filter: { term: { 'event.kind': 'alert' } }, + aggs: { + ids: { terms: { field: 'endgame.data.alert_details.acting_process.unique_pid' } }, + }, + }, + events: { + filter: { term: { 'event.kind': 'event' } }, + aggs: { + ids: { terms: { field: 'endgame.unique_pid' } }, + }, + }, + }, + }, + index, + }; + } + + protected query(entityIDs: string[], index: string): JsonObject { + return { + body: { + size: 0, + query: { + bool: { + filter: [ + { terms: { 'process.entity_id': entityIDs } }, + { + bool: { + should: [ + { + bool: { + filter: [ + { term: { 'event.kind': 'event' } }, + { + bool: { + must_not: { + term: { 'event.category': 'process' }, + }, + }, + }, + ], + }, + }, + { term: { 'event.kind': 'alert' } }, + ], + }, + }, + ], + }, + }, + aggs: { + alerts: { + filter: { term: { 'event.kind': 'alert' } }, + aggs: { + ids: { terms: { field: 'process.entity_id' } }, + }, + }, + events: { + filter: { term: { 'event.kind': 'event' } }, + aggs: { + ids: { terms: { field: 'process.entity_id' } }, + }, + }, + }, + }, + index, + }; + } +} diff --git a/x-pack/plugins/endpoint/server/routes/resolver/related_events.ts b/x-pack/plugins/endpoint/server/routes/resolver/related_events.ts deleted file mode 100644 index 83e111a1e62e6..0000000000000 --- a/x-pack/plugins/endpoint/server/routes/resolver/related_events.ts +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { schema } from '@kbn/config-schema'; -import { RequestHandler, Logger } from 'kibana/server'; -import { getPaginationParams } from './utils/pagination'; -import { RelatedEventsQuery } from './queries/related_events'; -import { EndpointAppContext } from '../../types'; - -interface RelatedEventsQueryParams { - after?: string; - limit: number; - /** - * legacyEndpointID is optional because there are two different types of identifiers: - * - * Legacy - * A legacy Entity ID is made up of the agent.id and unique_pid fields. The client will need to identify if - * it's looking at a legacy event and use those fields when making requests to the backend. The - * request would be /resolver/{id}?legacyEndpointID=and the {id} would be the unique_pid. - * - * Elastic Endpoint - * When interacting with the new form of data the client doesn't need the legacyEndpointID because it's already a - * part of the entityID in the new type of event. So for the same request the client would just hit resolver/{id} - * and the {id} would be entityID stored in the event's process.entity_id field. - */ - legacyEndpointID?: string; -} - -interface RelatedEventsPathParams { - id: string; -} - -export const validateRelatedEvents = { - params: schema.object({ id: schema.string() }), - query: schema.object({ - after: schema.maybe(schema.string()), - limit: schema.number({ defaultValue: 100, min: 1, max: 1000 }), - legacyEndpointID: schema.maybe(schema.string()), - }), -}; - -export function handleRelatedEvents( - log: Logger, - endpointAppContext: EndpointAppContext -): RequestHandler { - return async (context, req, res) => { - const { - params: { id }, - query: { limit, after, legacyEndpointID }, - } = req; - try { - const indexRetriever = endpointAppContext.service.getIndexPatternRetriever(); - const pagination = getPaginationParams(limit, after); - - const client = context.core.elasticsearch.dataClient; - const indexPattern = await indexRetriever.getEventIndexPattern(context); - // Retrieve the related non-process events for a given process - const relatedEventsQuery = new RelatedEventsQuery(indexPattern, legacyEndpointID, pagination); - const relatedEvents = await relatedEventsQuery.search(client, id); - - const { total, results: events, nextCursor } = relatedEvents; - - return res.ok({ - body: { - events, - pagination: { total, next: nextCursor, limit }, - }, - }); - } catch (err) { - log.warn(err); - return res.internalError({ body: err }); - } - }; -} diff --git a/x-pack/plugins/endpoint/server/routes/resolver/tree.ts b/x-pack/plugins/endpoint/server/routes/resolver/tree.ts new file mode 100644 index 0000000000000..25f15586341d5 --- /dev/null +++ b/x-pack/plugins/endpoint/server/routes/resolver/tree.ts @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { RequestHandler, Logger } from 'kibana/server'; +import { TypeOf } from '@kbn/config-schema'; +import { validateTree } from '../../../common/schema/resolver'; +import { Fetcher } from './utils/fetch'; +import { Tree } from './utils/tree'; +import { EndpointAppContext } from '../../types'; + +export function handleTree( + log: Logger, + endpointAppContext: EndpointAppContext +): RequestHandler, TypeOf> { + return async (context, req, res) => { + const { + params: { id }, + query: { + children, + generations, + ancestors, + events, + afterEvent, + afterChild, + legacyEndpointID: endpointID, + }, + } = req; + try { + const client = context.core.elasticsearch.dataClient; + const indexRetriever = endpointAppContext.service.getIndexPatternRetriever(); + const indexPattern = await indexRetriever.getEventIndexPattern(context); + + const fetcher = new Fetcher(client, id, indexPattern, endpointID); + const tree = await Tree.merge( + fetcher.children(children, generations, afterChild), + fetcher.ancestors(ancestors + 1), + fetcher.events(events, afterEvent) + ); + + const enrichedTree = await fetcher.stats(tree); + + return res.ok({ + body: enrichedTree.render(), + }); + } catch (err) { + log.warn(err); + return res.internalError({ body: 'Error retrieving tree.' }); + } + }; +} diff --git a/x-pack/plugins/endpoint/server/routes/resolver/utils/fetch.ts b/x-pack/plugins/endpoint/server/routes/resolver/utils/fetch.ts new file mode 100644 index 0000000000000..7315b4ee6c618 --- /dev/null +++ b/x-pack/plugins/endpoint/server/routes/resolver/utils/fetch.ts @@ -0,0 +1,123 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { IScopedClusterClient } from 'kibana/server'; +import { entityId, parentEntityId } from '../../../../common/models/event'; +import { getPaginationParams } from './pagination'; +import { Tree } from './tree'; +import { LifecycleQuery } from '../queries/lifecycle'; +import { ChildrenQuery } from '../queries/children'; +import { EventsQuery } from '../queries/events'; +import { StatsQuery } from '../queries/stats'; + +export class Fetcher { + constructor( + private readonly client: IScopedClusterClient, + private readonly id: string, + private readonly indexPattern: string, + private readonly endpointID?: string + ) {} + + public async ancestors(limit: number): Promise { + const tree = new Tree(this.id); + await this.doAncestors(tree, this.id, this.id, limit); + return tree; + } + + public async children(limit: number, generations: number, after?: string): Promise { + const tree = new Tree(this.id); + await this.doChildren(tree, [this.id], limit, generations, after); + return tree; + } + + public async events(limit: number, after?: string): Promise { + const tree = new Tree(this.id); + await this.doEvents(tree, limit, after); + return tree; + } + + public async stats(tree: Tree): Promise { + await this.doStats(tree); + return tree; + } + + private async doAncestors(tree: Tree, curNode: string, previousNode: string, levels: number) { + if (levels === 0) { + tree.setNextAncestor(curNode); + return; + } + + const query = new LifecycleQuery(this.indexPattern, this.endpointID); + const { results } = await query.search(this.client, curNode); + + if (results.length === 0) { + tree.setNextAncestor(null); + return; + } + tree.addAncestor(previousNode, ...results); + + const next = parentEntityId(results[0]); + if (next !== undefined) { + await this.doAncestors(tree, next, curNode, levels - 1); + } + } + + private async doEvents(tree: Tree, limit: number, after?: string) { + const query = new EventsQuery( + this.indexPattern, + this.endpointID, + getPaginationParams(limit, after) + ); + + const { totals, results } = await query.search(this.client, this.id); + tree.addEvent(...results); + tree.paginateEvents(totals, results); + if (results.length === 0) tree.setNextEvent(null); + } + + private async doChildren( + tree: Tree, + ids: string[], + limit: number, + levels: number, + after?: string + ) { + if (levels === 0 || ids.length === 0) return; + + const childrenQuery = new ChildrenQuery( + this.indexPattern, + this.endpointID, + getPaginationParams(limit, after) + ); + const lifecycleQuery = new LifecycleQuery(this.indexPattern, this.endpointID); + + const { totals, results } = await childrenQuery.search(this.client, ...ids); + if (results.length === 0) { + tree.markLeafNode(...ids); + return; + } + + const childIDs = results.map(entityId); + const children = (await lifecycleQuery.search(this.client, ...childIDs)).results; + + tree.addChild(...children); + tree.paginateChildren(totals, results); + tree.markLeafNode(...childIDs); + + await this.doChildren(tree, childIDs, limit * limit, levels - 1); + } + + private async doStats(tree: Tree) { + const statsQuery = new StatsQuery(this.indexPattern, this.endpointID); + const ids = tree.ids(); + const { extras } = await statsQuery.search(this.client, ...ids); + const alerts = extras?.alerts || {}; + const events = extras?.events || {}; + ids.forEach(id => { + tree.addStats(id, { totalAlerts: alerts[id] || 0, totalEvents: events[id] || 0 }); + }); + } +} diff --git a/x-pack/plugins/endpoint/server/routes/resolver/utils/normalize.ts b/x-pack/plugins/endpoint/server/routes/resolver/utils/normalize.ts deleted file mode 100644 index 6d5ac8efdc1da..0000000000000 --- a/x-pack/plugins/endpoint/server/routes/resolver/utils/normalize.ts +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { ResolverEvent } from '../../../../common/types'; -import { isLegacyEvent } from '../../../../common/models/event'; - -export function extractEventID(event: ResolverEvent) { - if (isLegacyEvent(event)) { - return String(event.endgame.serial_event_id); - } - return event.event.id; -} - -export function extractEntityID(event: ResolverEvent) { - if (isLegacyEvent(event)) { - return String(event.endgame.unique_pid); - } - return event.process.entity_id; -} - -export function extractParentEntityID(event: ResolverEvent) { - if (isLegacyEvent(event)) { - const ppid = event.endgame.unique_ppid; - return ppid && String(ppid); // if unique_ppid is undefined return undefined - } - return event.process.parent?.entity_id; -} diff --git a/x-pack/plugins/endpoint/server/routes/resolver/utils/pagination.ts b/x-pack/plugins/endpoint/server/routes/resolver/utils/pagination.ts index 5a64f3ff9ddb6..20249b81660bb 100644 --- a/x-pack/plugins/endpoint/server/routes/resolver/utils/pagination.ts +++ b/x-pack/plugins/endpoint/server/routes/resolver/utils/pagination.ts @@ -6,7 +6,7 @@ import { SearchResponse } from 'elasticsearch'; import { ResolverEvent } from '../../../../common/types'; -import { extractEventID } from './normalize'; +import { entityId } from '../../../../common/models/event'; import { JsonObject } from '../../../../../../../src/plugins/kibana_utils/public'; export interface PaginationParams { @@ -15,12 +15,19 @@ export interface PaginationParams { eventID?: string; } +export interface PaginatedResults { + totals: Record; + results: ResolverEvent[]; + // content holder for any other extra aggregation counts + extras?: Record>; +} + interface PaginationCursor { timestamp: number; eventID: string; } -function urlEncodeCursor(data: PaginationCursor) { +function urlEncodeCursor(data: PaginationCursor): string { const value = JSON.stringify(data); return Buffer.from(value, 'utf8') .toString('base64') @@ -56,10 +63,16 @@ export function getPaginationParams(limit: number, after?: string): PaginationPa return { size: limit }; } -export function paginate(pagination: PaginationParams, field: string, query: JsonObject) { +export function paginate( + pagination: PaginationParams, + tiebreaker: string, + aggregator: string, + query: JsonObject +): JsonObject { const { size, timestamp, eventID } = pagination; - query.sort = [{ '@timestamp': 'asc' }, { [field]: 'asc' }]; - query.aggs = { total: { value_count: { field } } }; + query.sort = [{ '@timestamp': 'asc' }, { [tiebreaker]: 'asc' }]; + query.aggs = query.aggs || {}; + query.aggs = Object.assign({}, query.aggs, { totals: { terms: { field: aggregator, size } } }); query.size = size; if (timestamp && eventID) { query.search_after = [timestamp, eventID] as Array; @@ -67,25 +80,28 @@ export function paginate(pagination: PaginationParams, field: string, query: Jso return query; } -export function paginatedResults( - response: SearchResponse -): { total: number; results: ResolverEvent[]; nextCursor: string | null } { - const total = response.aggregations?.total?.value || 0; - if (response.hits.hits.length === 0) { - return { total, results: [], nextCursor: null }; +export function buildPaginationCursor(total: number, results: ResolverEvent[]): string | null { + if (total > results.length && results.length > 0) { + const lastResult = results[results.length - 1]; + const cursor = { + timestamp: lastResult['@timestamp'], + eventID: entityId(lastResult), + }; + return urlEncodeCursor(cursor); } + return null; +} - const results: ResolverEvent[] = []; - for (const hit of response.hits.hits) { - results.push(hit._source); +export function paginatedResults(response: SearchResponse): PaginatedResults { + if (response.hits.hits.length === 0) { + return { totals: {}, results: [] }; } - // results will be at least 1 because of length check at the top of the function - const next = results[results.length - 1]; - const cursor = { - timestamp: next['@timestamp'], - eventID: extractEventID(next), - }; + const totals = response.aggregations?.totals?.buckets?.reduce( + (cummulative: any, bucket: any) => ({ ...cummulative, [bucket.key]: bucket.doc_count }), + {} + ); - return { total, results, nextCursor: urlEncodeCursor(cursor) }; + const results = response.hits.hits.map(hit => hit._source); + return { totals, results }; } diff --git a/x-pack/plugins/endpoint/server/routes/resolver/utils/tree.ts b/x-pack/plugins/endpoint/server/routes/resolver/utils/tree.ts new file mode 100644 index 0000000000000..5a55c23b90873 --- /dev/null +++ b/x-pack/plugins/endpoint/server/routes/resolver/utils/tree.ts @@ -0,0 +1,230 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import _ from 'lodash'; +import { + ResolverEvent, + ResolverNode, + ResolverNodeStats, + ResolverNodePagination, +} from '../../../../common/types'; +import { entityId, parentEntityId } from '../../../../common/models/event'; +import { buildPaginationCursor } from './pagination'; + +type ExtractFunction = (event: ResolverEvent) => string | undefined; + +function createNode(id: string): ResolverNode { + return { id, children: [], pagination: {}, events: [], lifecycle: [] }; +} +/** + * This class aids in constructing a tree of process events. It works in the following way: + * + * 1. We construct a tree structure starting with the root node for the event we're requesting. + * 2. We leverage the ability to pass hashes and arrays by reference to construct a fast cache of + * process identifiers that updates the tree structure as we push values into the cache. + * + * When we query a single level of results for child process events we have a flattened, sorted result + * list that we need to add into a constructed tree. We also need to signal in an API response whether + * or not there are more child processes events that we have not yet retrieved, and, if so, for what parent + * process. So, at the end of our tree construction we have a relational layout of the events with no + * pagination information for the given parent nodes. In order to actually construct both the tree and + * insert the pagination information we basically do the following: + * + * 1. Using a terms aggregation query, we return an approximate roll-up of the number of child process + * "creation" events, this gives us an estimation of the number of associated children per parent + * 2. We feed these child process creation event "unique identifiers" (basically a process.entity_id) + * into a second query to get the current state of the process via its "lifecycle" events. + * 3. We construct the tree above with the "lifecycle" events. + * 4. Using the terms query results, we mark each non-leaf node with the number of expected children, if our + * tree has less children than expected, we create a pagination cursor to indicate "we have a truncated set + * of values". + * 5. We mark each leaf node (the last level of the tree we're constructing) with a "null" for the expected + * number of children to indicate "we have not yet attempted to get any children". + * + * Following this scheme, we use exactly 2 queries per level of children that we return--one for the pagination + * and one for the lifecycle events of the processes. The downside to this is that we need to dynamically expand + * the number of documents we can retrieve per level due to the exponential fanout of child processes, + * what this means is that noisy neighbors for a given level may hide other child process events that occur later + * temporally in the same level--so, while a heavily forking process might get shown, maybe the actually malicious + * event doesn't show up in the tree at the beginning. + */ + +export class Tree { + protected cache: Map; + protected root: ResolverNode; + protected id: string; + + constructor(id: string) { + const root = createNode(id); + this.id = id; + this.cache = new Map(); + this.root = root; + this.cache.set(id, root); + } + + public render(): ResolverNode { + return this.root; + } + + public ids(): string[] { + return [...this.cache.keys()]; + } + + public static async merge( + childrenPromise: Promise, + ancestorsPromise: Promise, + eventsPromise: Promise + ): Promise { + const [children, ancestors, events] = await Promise.all([ + childrenPromise, + ancestorsPromise, + eventsPromise, + ]); + + /* + * we only allow for merging when we have partial trees that + * represent the same root node + */ + const rootID = children.id; + if (rootID !== ancestors.id || rootID !== events.id) { + throw new Error('cannot merge trees with different roots'); + } + + Object.entries(ancestors.cache).forEach(([id, node]) => { + if (rootID !== id) { + children.cache.set(id, node); + } + }); + + children.root.lifecycle = ancestors.root.lifecycle; + children.root.ancestors = ancestors.root.ancestors; + children.root.events = events.root.events; + + Object.assign(children.root.pagination, ancestors.root.pagination, events.root.pagination); + + return children; + } + + public addEvent(...events: ResolverEvent[]): void { + events.forEach(event => { + const id = entityId(event); + + this.ensureCache(id); + const currentNode = this.cache.get(id); + if (currentNode !== undefined) { + currentNode.events.push(event); + } + }); + } + + public addAncestor(id: string, ...events: ResolverEvent[]): void { + events.forEach(event => { + const ancestorID = entityId(event); + if (this.cache.get(ancestorID) === undefined) { + const newParent = createNode(ancestorID); + this.cache.set(ancestorID, newParent); + if (!this.root.ancestors) { + this.root.ancestors = []; + } + this.root.ancestors.push(newParent); + } + const currentAncestor = this.cache.get(ancestorID); + if (currentAncestor !== undefined) { + currentAncestor.lifecycle.push(event); + } + }); + } + + public addStats(id: string, stats: ResolverNodeStats): void { + this.ensureCache(id); + const currentNode = this.cache.get(id); + if (currentNode !== undefined) { + currentNode.stats = stats; + } + } + + public setNextAncestor(next: string | null): void { + this.root.pagination.nextAncestor = next; + } + + public setNextEvent(next: string | null): void { + this.root.pagination.nextEvent = next; + } + + public setNextAlert(next: string | null): void { + this.root.pagination.nextAlert = next; + } + + public addChild(...events: ResolverEvent[]): void { + events.forEach(event => { + const id = entityId(event); + const parentID = parentEntityId(event); + + this.ensureCache(parentID); + let currentNode = this.cache.get(id); + + if (currentNode === undefined) { + currentNode = createNode(id); + this.cache.set(id, currentNode); + if (parentID !== undefined) { + const parentNode = this.cache.get(parentID); + if (parentNode !== undefined) { + parentNode.children.push(currentNode); + } + } + } + currentNode.lifecycle.push(event); + }); + } + + public markLeafNode(...ids: string[]): void { + ids.forEach(id => { + this.ensureCache(id); + const currentNode = this.cache.get(id); + if (currentNode !== undefined && !currentNode.pagination.nextChild) { + currentNode.pagination.nextChild = null; + } + }); + } + + public paginateEvents(totals: Record, events: ResolverEvent[]): void { + return this.paginate(entityId, 'nextEvent', totals, events); + } + + public paginateChildren(totals: Record, children: ResolverEvent[]): void { + return this.paginate(parentEntityId, 'nextChild', totals, children); + } + + private paginate( + grouper: ExtractFunction, + attribute: keyof ResolverNodePagination, + totals: Record, + records: ResolverEvent[] + ): void { + const grouped = _.groupBy(records, grouper); + Object.entries(totals).forEach(([id, total]) => { + if (this.cache.get(id) !== undefined) { + if (grouped[id]) { + /* + * if we have any results, attempt to build a pagination cursor, the function + * below hands back a null value if no cursor is necessary because we have + * all of the records. + */ + const currentNode = this.cache.get(id); + if (currentNode !== undefined) { + currentNode.pagination[attribute] = buildPaginationCursor(total, grouped[id]); + } + } + } + }); + } + + private ensureCache(id: string | undefined): void { + if (id === undefined || this.cache.get(id) === undefined) { + throw new Error('dangling node'); + } + } +} diff --git a/x-pack/test/api_integration/apis/endpoint/resolver.ts b/x-pack/test/api_integration/apis/endpoint/resolver.ts index e8d336e875b99..73fe435764b74 100644 --- a/x-pack/test/api_integration/apis/endpoint/resolver.ts +++ b/x-pack/test/api_integration/apis/endpoint/resolver.ts @@ -27,72 +27,65 @@ export default function resolverAPIIntegrationTests({ getService }: FtrProviderC it('should return details for the root node', async () => { const { body } = await supertest - .get(`/api/endpoint/resolver/${entityID}/related?legacyEndpointID=${endpointID}`) + .get(`/api/endpoint/resolver/${entityID}/events?legacyEndpointID=${endpointID}`) .set(commonHeaders) .expect(200); expect(body.events.length).to.eql(1); - expect(body.pagination.next).to.eql(cursor); - expect(body.pagination.total).to.eql(1); - // default limit - expect(body.pagination.limit).to.eql(100); + expect(body.pagination.nextEvent).to.eql(null); }); it('returns no values when there is no more data', async () => { const { body } = await supertest // after is set to the document id of the last event so there shouldn't be any more after it .get( - `/api/endpoint/resolver/${entityID}/related?legacyEndpointID=${endpointID}&after=${cursor}` + `/api/endpoint/resolver/${entityID}/events?legacyEndpointID=${endpointID}&afterEvent=${cursor}` ) .set(commonHeaders) .expect(200); expect(body.events).be.empty(); - expect(body.pagination.next).to.eql(null); - expect(body.pagination.total).to.eql(1); + expect(body.pagination.nextEvent).to.eql(null); }); it('should return the first page of information when the cursor is invalid', async () => { const { body } = await supertest .get( - `/api/endpoint/resolver/${entityID}/related?legacyEndpointID=${endpointID}&after=blah` + `/api/endpoint/resolver/${entityID}/events?legacyEndpointID=${endpointID}&afterEvent=blah` ) .set(commonHeaders) .expect(200); - expect(body.pagination.total).to.eql(1); - expect(body.pagination.next).to.not.eql(null); + expect(body.pagination.nextEvent).to.eql(null); }); it('should error on invalid pagination values', async () => { await supertest - .get(`/api/endpoint/resolver/${entityID}/related?limit=0`) + .get(`/api/endpoint/resolver/${entityID}/events?events=0`) .set(commonHeaders) .expect(400); await supertest - .get(`/api/endpoint/resolver/${entityID}/related?limit=2000`) + .get(`/api/endpoint/resolver/${entityID}/events?events=2000`) .set(commonHeaders) .expect(400); await supertest - .get(`/api/endpoint/resolver/${entityID}/related?limit=-1`) + .get(`/api/endpoint/resolver/${entityID}/events?events=-1`) .set(commonHeaders) .expect(400); }); it('should not find any events', async () => { const { body } = await supertest - .get(`/api/endpoint/resolver/5555/related`) + .get(`/api/endpoint/resolver/5555/events`) .set(commonHeaders) .expect(200); - expect(body.pagination.total).to.eql(0); - expect(body.pagination.next).to.eql(null); + expect(body.pagination.nextEvent).to.eql(null); expect(body.events).to.be.empty(); }); it('should return no results for an invalid endpoint ID', async () => { const { body } = await supertest - .get(`/api/endpoint/resolver/${entityID}/related?legacyEndpointID=foo`) + .get(`/api/endpoint/resolver/${entityID}/events?legacyEndpointID=foo`) .set(commonHeaders) .expect(200); - expect(body.pagination.total).to.eql(0); - expect(body.pagination.next).to.eql(null); + expect(body.pagination.nextEvent).to.eql(null); expect(body.events).to.be.empty(); }); }); @@ -103,48 +96,46 @@ export default function resolverAPIIntegrationTests({ getService }: FtrProviderC it('should return details for the root node', async () => { const { body } = await supertest - .get(`/api/endpoint/resolver/${entityID}?legacyEndpointID=${endpointID}&ancestors=5`) + .get( + `/api/endpoint/resolver/${entityID}/ancestry?legacyEndpointID=${endpointID}&ancestors=5` + ) .set(commonHeaders) .expect(200); expect(body.lifecycle.length).to.eql(2); - expect(body.ancestors.length).to.eql(1); - expect(body.pagination.next).to.eql(null); - // 5 is default parameter - expect(body.pagination.ancestors).to.eql(5); + expect(body.parent).not.to.eql(null); + expect(body.pagination.nextAncestor).to.eql(null); }); it('should have a populated next parameter', async () => { const { body } = await supertest - .get(`/api/endpoint/resolver/${entityID}?legacyEndpointID=${endpointID}`) + .get(`/api/endpoint/resolver/${entityID}/ancestry?legacyEndpointID=${endpointID}`) .set(commonHeaders) .expect(200); - expect(body.pagination.next).to.eql('94041'); + expect(body.pagination.nextAncestor).to.eql('94041'); }); it('should handle an ancestors param request', async () => { let { body } = await supertest - .get(`/api/endpoint/resolver/${entityID}?legacyEndpointID=${endpointID}`) + .get(`/api/endpoint/resolver/${entityID}/ancestry?legacyEndpointID=${endpointID}`) .set(commonHeaders) .expect(200); - const next = body.pagination.next; + const next = body.pagination.nextAncestor; ({ body } = await supertest - .get(`/api/endpoint/resolver/${next}?legacyEndpointID=${endpointID}&ancestors=1`) + .get(`/api/endpoint/resolver/${next}/ancestry?legacyEndpointID=${endpointID}&ancestors=1`) .set(commonHeaders) .expect(200)); expect(body.lifecycle.length).to.eql(1); - expect(body.ancestors.length).to.eql(0); - expect(body.pagination.next).to.eql(null); + expect(body.pagination.nextAncestor).to.eql(null); }); it('should handle an invalid id', async () => { const { body } = await supertest - .get(`/api/endpoint/resolver/alskdjflasj`) + .get(`/api/endpoint/resolver/alskdjflasj/ancestry`) .set(commonHeaders) .expect(200); expect(body.lifecycle.length).to.eql(0); - expect(body.ancestors.length).to.eql(0); - expect(body.pagination.next).to.eql(null); + expect(body.pagination.nextAncestor).to.eql(null); }); }); @@ -158,51 +149,58 @@ export default function resolverAPIIntegrationTests({ getService }: FtrProviderC .get(`/api/endpoint/resolver/${entityID}/children?legacyEndpointID=${endpointID}`) .set(commonHeaders) .expect(200); - expect(body.pagination.total).to.eql(1); - expect(body.pagination.next).to.eql(cursor); - // default limit - expect(body.pagination.limit).to.eql(10); - expect(body.children.length).to.eql(1); expect(body.children[0].lifecycle.length).to.eql(2); expect(body.children[0].lifecycle[0].endgame.unique_pid).to.eql(94042); }); + it('returns multiple levels of child process lifecycle events', async () => { + const { body } = await supertest + .get(`/api/endpoint/resolver/93802/children?legacyEndpointID=${endpointID}&generations=3`) + .set(commonHeaders) + .expect(200); + expect(body.pagination.nextChild).to.be(null); + expect(body.children[0].pagination.nextChild).to.be(null); + + expect(body.children.length).to.eql(8); + expect(body.children[0].children[0].lifecycle.length).to.eql(2); + expect(body.children[0].lifecycle[0].endgame.unique_pid).to.eql(93932); + }); + it('returns no values when there is no more data', async () => { const { body } = await supertest // after is set to the document id of the last event so there shouldn't be any more after it .get( - `/api/endpoint/resolver/${entityID}/children?legacyEndpointID=${endpointID}&after=${cursor}` + `/api/endpoint/resolver/${entityID}/children?legacyEndpointID=${endpointID}&afterChild=${cursor}` ) .set(commonHeaders) .expect(200); expect(body.children).be.empty(); - expect(body.pagination.next).to.eql(null); - expect(body.pagination.total).to.eql(1); + expect(body.pagination.nextChild).to.eql(null); }); it('returns the first page of information when the cursor is invalid', async () => { const { body } = await supertest .get( - `/api/endpoint/resolver/${entityID}/children?legacyEndpointID=${endpointID}&after=blah` + `/api/endpoint/resolver/${entityID}/children?legacyEndpointID=${endpointID}&afterChild=blah` ) .set(commonHeaders) .expect(200); - expect(body.pagination.total).to.eql(1); - expect(body.pagination.next).to.not.eql(null); + expect(body.children.length).to.eql(1); + expect(body.pagination.nextChild).to.be(null); }); it('errors on invalid pagination values', async () => { await supertest - .get(`/api/endpoint/resolver/${entityID}/children?limit=0`) + .get(`/api/endpoint/resolver/${entityID}/children?children=0`) .set(commonHeaders) .expect(400); await supertest - .get(`/api/endpoint/resolver/${entityID}/children?limit=2000`) + .get(`/api/endpoint/resolver/${entityID}/children?children=2000`) .set(commonHeaders) .expect(400); await supertest - .get(`/api/endpoint/resolver/${entityID}/children?limit=-1`) + .get(`/api/endpoint/resolver/${entityID}/children?children=-1`) .set(commonHeaders) .expect(400); }); @@ -212,8 +210,7 @@ export default function resolverAPIIntegrationTests({ getService }: FtrProviderC .get(`/api/endpoint/resolver/5555/children`) .set(commonHeaders) .expect(200); - expect(body.pagination.total).to.eql(0); - expect(body.pagination.next).to.eql(null); + expect(body.pagination.nextChild).to.eql(null); expect(body.children).to.be.empty(); }); @@ -222,10 +219,26 @@ export default function resolverAPIIntegrationTests({ getService }: FtrProviderC .get(`/api/endpoint/resolver/${entityID}/children?legacyEndpointID=foo`) .set(commonHeaders) .expect(200); - expect(body.pagination.total).to.eql(0); - expect(body.pagination.next).to.eql(null); + expect(body.pagination.nextChild).to.eql(null); expect(body.children).to.be.empty(); }); }); + + describe('tree endpoint', () => { + const endpointID = '5a0c957f-b8e7-4538-965e-57e8bb86ad3a'; + + it('returns ancestors, events, children, and current process lifecycle', async () => { + const { body } = await supertest + .get(`/api/endpoint/resolver/93933?legacyEndpointID=${endpointID}`) + .set(commonHeaders) + .expect(200); + expect(body.pagination.nextAncestor).to.equal(null); + expect(body.pagination.nextEvent).to.equal(null); + expect(body.pagination.nextChild).to.equal(null); + expect(body.children.length).to.equal(0); + expect(body.events.length).to.equal(0); + expect(body.lifecycle.length).to.equal(2); + }); + }); }); }