[Security Solution][Resolver] Allow a configurable entity_id field (#…

…81679)

* Trying to flesh out new tree route

* Working on the descendants query

* Almost working descendants

* Possible solution for aggs

* Working aggregations extraction

* Working on the ancestry array for descendants

* Making changes to the unique id for  ancestr

* Implementing ancestry funcitonality

* Deleting the multiple edges

* Fleshing out the descendants loop for levels

* Writing tests for ancestors and descendants

* Fixing type errors and writing more tests

* Renaming validation variable and deprecating old tree routes

* Renaming tree integration test file

* Adding some integration tests

* Fixing ancestry to handle multiple nodes in the request and writing more tests

* Adding more tests

* Renaming new tree to handler file

* Renaming new tree directory

* Adding more unit tests

* Using doc value fields and working on types

* Adding comments and more tests

* Fixing timestamp test issue

* Adding more comments

* Fixing timestamp test issue take 2

* Adding id, parent, and name fields to the top level response

* Fixing generator start and end time generation

* Adding more comments

* Revert "Fixing generator start and end time generation"

This reverts commit 9e9abf6.

* Adding test for time

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

x-pack/plugins/security_solution/common/endpoint/generate_data.test.ts

@@ -27,7 +27,6 @@ interface Node {
 }
 
 describe('data generator data streams', () => {
-  // these tests cast the result of the generate methods so that we can specifically compare the `data_stream` fields
   it('creates a generator with default data streams', () => {
     const generator = new EndpointDocGenerator('seed');
     expect(generator.generateHostMetadata().data_stream).toEqual({
@@ -268,6 +267,31 @@ describe('data generator', () => {
       }
     };
 
+    it('sets the start and end times correctly', () => {
+      const startOfEpoch = new Date(0);
+      let startTime = new Date(timestampSafeVersion(tree.allEvents[0]) ?? startOfEpoch);
+      expect(startTime).not.toEqual(startOfEpoch);
+      let endTime = new Date(timestampSafeVersion(tree.allEvents[0]) ?? startOfEpoch);
+      expect(startTime).not.toEqual(startOfEpoch);
+
+      for (const event of tree.allEvents) {
+        const currentEventTime = new Date(timestampSafeVersion(event) ?? startOfEpoch);
+        expect(currentEventTime).not.toEqual(startOfEpoch);
+        expect(tree.startTime.getTime()).toBeLessThanOrEqual(currentEventTime.getTime());
+        expect(tree.endTime.getTime()).toBeGreaterThanOrEqual(currentEventTime.getTime());
+        if (currentEventTime < startTime) {
+          startTime = currentEventTime;
+        }
+
+        if (currentEventTime > endTime) {
+          endTime = currentEventTime;
+        }
+      }
+      expect(startTime).toEqual(tree.startTime);
+      expect(endTime).toEqual(tree.endTime);
+      expect(endTime.getTime() - startTime.getTime()).toBeGreaterThanOrEqual(0);
+    });
+
     it('creates related events in ascending order', () => {
       // the order should not change since it should already be in ascending order
       const relatedEventsAsc = _.cloneDeep(tree.origin.relatedEvents).sort(

x-pack/plugins/security_solution/common/endpoint/generate_data.ts

@@ -317,6 +317,8 @@ export interface Tree {
    * All events from children, ancestry, origin, and the alert in a single array
    */
   allEvents: Event[];
+  startTime: Date;
+  endTime: Date;
 }
 
 export interface TreeOptions {
@@ -718,6 +720,35 @@ export class EndpointDocGenerator {
     };
   }
 
+  private static getStartEndTimes(events: Event[]): { startTime: Date; endTime: Date } {
+    let startTime: number;
+    let endTime: number;
+    if (events.length > 0) {
+      startTime = timestampSafeVersion(events[0]) ?? new Date().getTime();
+      endTime = startTime;
+    } else {
+      startTime = new Date().getTime();
+      endTime = startTime;
+    }
+
+    for (const event of events) {
+      const eventTimestamp = timestampSafeVersion(event);
+      if (eventTimestamp !== undefined) {
+        if (eventTimestamp < startTime) {
+          startTime = eventTimestamp;
+        }
+
+        if (eventTimestamp > endTime) {
+          endTime = eventTimestamp;
+        }
+      }
+    }
+    return {
+      startTime: new Date(startTime),
+      endTime: new Date(endTime),
+    };
+  }
+
   /**
    * This generates a full resolver tree and keeps the entire tree in memory. This is useful for tests that want
    * to compare results from elasticsearch with the actual events created by this generator. Because all the events
@@ -815,12 +846,17 @@ export class EndpointDocGenerator {
     const childrenByParent = groupNodesByParent(childrenNodes);
     const levels = createLevels(childrenByParent, [], childrenByParent.get(origin.id));
 
+    const allEvents = [...ancestry, ...children];
+    const { startTime, endTime } = EndpointDocGenerator.getStartEndTimes(allEvents);
+
     return {
       children: childrenNodes,
       ancestry: ancestryNodes,
-      allEvents: [...ancestry, ...children],
+      allEvents,
       origin,
       childrenLevels: levels,
+      startTime,
+      endTime,
     };
   }
 

x-pack/plugins/security_solution/common/endpoint/schema/resolver.ts

@@ -7,9 +7,9 @@
 import { schema } from '@kbn/config-schema';
 
 /**
- * Used to validate GET requests for a complete resolver tree.
+ * Used to validate GET requests for a complete resolver tree centered around an entity_id.
  */
-export const validateTree = {
+export const validateTreeEntityID = {
   params: schema.object({ id: schema.string({ minLength: 1 }) }),
   query: schema.object({
     children: schema.number({ defaultValue: 200, min: 0, max: 10000 }),
@@ -23,6 +23,44 @@ export const validateTree = {
   }),
 };
 
+/**
+ * Used to validate GET requests for a complete resolver tree.
+ */
+export const validateTree = {
+  body: schema.object({
+    /**
+     * If the ancestry field is specified this field will be ignored
+     *
+     * If the ancestry field is specified we have a much more performant way of retrieving levels so let's not limit
+     * the number of levels that come back in that scenario. We could still limit it, but what we'd likely have to do
+     * is get all the levels back like we normally do with the ancestry array, bucket them together by level, and then
+     * remove the levels that exceeded the requested number which seems kind of wasteful.
+     */
+    descendantLevels: schema.number({ defaultValue: 20, min: 0, max: 1000 }),
+    descendants: schema.number({ defaultValue: 1000, min: 0, max: 10000 }),
+    // if the ancestry array isn't specified allowing 200 might be too high
+    ancestors: schema.number({ defaultValue: 200, min: 0, max: 10000 }),
+    timerange: schema.object({
+      from: schema.string(),
+      to: schema.string(),
+    }),
+    schema: schema.object({
+      // the ancestry field is optional
+      ancestry: schema.maybe(schema.string({ minLength: 1 })),
+      id: schema.string({ minLength: 1 }),
+      name: schema.maybe(schema.string({ minLength: 1 })),
+      parent: schema.string({ minLength: 1 }),
+    }),
+    // only allowing strings and numbers for node IDs because Elasticsearch only allows those types for collapsing:
+    // https://www.elastic.co/guide/en/elasticsearch/reference/current/collapse-search-results.html
+    // We use collapsing in our Elasticsearch queries for the tree api
+    nodes: schema.arrayOf(schema.oneOf([schema.string({ minLength: 1 }), schema.number()]), {
+      minSize: 1,
+    }),
+    indexPatterns: schema.arrayOf(schema.string(), { minSize: 1 }),
+  }),
+};
+
 /**
  * Used to validate POST requests for `/resolver/events` api.
  */

x-pack/plugins/security_solution/common/endpoint/types/index.ts

@@ -78,6 +78,56 @@ export interface EventStats {
   byCategory: Record<string, number>;
 }
 
+/**
+ * Represents the object structure of a returned document when using doc value fields to filter the fields
+ * returned in a document from an Elasticsearch query.
+ *
+ * Here is an example:
+ *
+ * {
+ *  "_index": ".ds-logs-endpoint.events.process-default-000001",
+ *  "_id": "bc7brnUBxO0aE7QcCVHo",
+ *  "_score": null,
+ *  "fields": { <----------- The FieldsObject represents this portion
+ *    "@timestamp": [
+ *      "2020-11-09T21:13:25.246Z"
+ *    ],
+ *    "process.name": "explorer.exe",
+ *    "process.parent.entity_id": [
+ *      "0i17c2m22c"
+ *    ],
+ *    "process.Ext.ancestry": [ <------------ Notice that the keys are flattened
+ *      "0i17c2m22c",
+ *      "2z9j8dlx72",
+ *      "oj61pr6g62",
+ *      "x0leonbrc9"
+ *    ],
+ *    "process.entity_id": [
+ *      "6k8waczi22"
+ *    ]
+ *  },
+ *  "sort": [
+ *    0,
+ *    1604956405246
+ *  ]
+ * }
+ */
+export interface FieldsObject {
+  [key: string]: ECSField<number | string>;
+}
+
+/**
+ * A node in a resolver graph.
+ */
+export interface ResolverNode {
+  data: FieldsObject;
+  id: string | number;
+  // the very root node might not have the parent field defined
+  parent?: string | number;
+  name?: string;
+  stats: EventStats;
+}
+
 /**
  * Statistical information for a node in a resolver tree.
  */

x-pack/plugins/security_solution/server/endpoint/routes/resolver.ts

@@ -7,23 +7,34 @@
 import { IRouter } from 'kibana/server';
 import { EndpointAppContext } from '../types';
 import {
-  validateTree,
+  validateTreeEntityID,
   validateEvents,
   validateChildren,
   validateAncestry,
   validateAlerts,
   validateEntities,
+  validateTree,
 } from '../../../common/endpoint/schema/resolver';
 import { handleChildren } from './resolver/children';
 import { handleAncestry } from './resolver/ancestry';
-import { handleTree } from './resolver/tree';
+import { handleTree as handleTreeEntityID } from './resolver/tree';
+import { handleTree } from './resolver/tree/handler';
 import { handleAlerts } from './resolver/alerts';
 import { handleEntities } from './resolver/entity';
 import { handleEvents } from './resolver/events';
 
 export function registerResolverRoutes(router: IRouter, endpointAppContext: EndpointAppContext) {
   const log = endpointAppContext.logFactory.get('resolver');
 
+  router.post(
+    {
+      path: '/api/endpoint/resolver/tree',
+      validate: validateTree,
+      options: { authRequired: true },
+    },
+    handleTree(log)
+  );
+
   router.post(
     {
       path: '/api/endpoint/resolver/events',
@@ -33,6 +44,9 @@ export function registerResolverRoutes(router: IRouter, endpointAppContext: Endp
     handleEvents(log)
   );
 
+  /**
+   * @deprecated will be removed because it is not used
+   */
   router.post(
     {
       path: '/api/endpoint/resolver/{id}/alerts',
@@ -42,6 +56,9 @@ export function registerResolverRoutes(router: IRouter, endpointAppContext: Endp
     handleAlerts(log, endpointAppContext)
   );
 
+  /**
+   * @deprecated use the /resolver/tree api instead
+   */
   router.get(
     {
       path: '/api/endpoint/resolver/{id}/children',
@@ -51,6 +68,9 @@ export function registerResolverRoutes(router: IRouter, endpointAppContext: Endp
     handleChildren(log, endpointAppContext)
   );
 
+  /**
+   * @deprecated use the /resolver/tree api instead
+   */
   router.get(
     {
       path: '/api/endpoint/resolver/{id}/ancestry',
@@ -60,13 +80,16 @@ export function registerResolverRoutes(router: IRouter, endpointAppContext: Endp
     handleAncestry(log, endpointAppContext)
   );
 
+  /**
+   * @deprecated use the /resolver/tree api instead
+   */
   router.get(
     {
       path: '/api/endpoint/resolver/{id}',
-      validate: validateTree,
+      validate: validateTreeEntityID,
       options: { authRequired: true },
     },
-    handleTree(log, endpointAppContext)
+    handleTreeEntityID(log, endpointAppContext)
   );
 
   /**

x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree.ts

@@ -7,14 +7,17 @@
 import { RequestHandler, Logger } from 'kibana/server';
 import { TypeOf } from '@kbn/config-schema';
 import { eventsIndexPattern, alertsIndexPattern } from '../../../../common/endpoint/constants';
-import { validateTree } from '../../../../common/endpoint/schema/resolver';
+import { validateTreeEntityID } from '../../../../common/endpoint/schema/resolver';
 import { Fetcher } from './utils/fetch';
 import { EndpointAppContext } from '../../types';
 
 export function handleTree(
   log: Logger,
   endpointAppContext: EndpointAppContext
-): RequestHandler<TypeOf<typeof validateTree.params>, TypeOf<typeof validateTree.query>> {
+): RequestHandler<
+  TypeOf<typeof validateTreeEntityID.params>,
+  TypeOf<typeof validateTreeEntityID.query>
+> {
   return async (context, req, res) => {
     try {
       const client = context.core.elasticsearch.legacy.client;

x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/handler.ts

@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { RequestHandler, Logger } from 'kibana/server';
+import { TypeOf } from '@kbn/config-schema';
+import { validateTree } from '../../../../../common/endpoint/schema/resolver';
+import { Fetcher } from './utils/fetch';
+
+export function handleTree(
+  log: Logger
+): RequestHandler<unknown, unknown, TypeOf<typeof validateTree.body>> {
+  return async (context, req, res) => {
+    try {
+      const client = context.core.elasticsearch.client;
+      const fetcher = new Fetcher(client);
+      const body = await fetcher.tree(req.body);
+      return res.ok({
+        body,
+      });
+    } catch (err) {
+      log.warn(err);
+      return res.internalError({ body: 'Error retrieving tree.' });
+    }
+  };
+}