[Security_Solution] Split up indices (#69589) (#69718)

* Fixing resolver alert generation

* Splitting indices up

* Removing tests that could randomly fail because of the generation code

* Adding support for multiple indices

* Updating archives with the new index names

* Removing alerts data stream

* Switching to process instead of fake

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

x-pack/plugins/security_solution/common/endpoint/constants.ts

@@ -4,7 +4,8 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-export const eventsIndexPattern = 'events-endpoint-*';
+export const eventsIndexPattern = 'logs-endpoint.events.*';
+export const alertsIndexPattern = 'logs-endpoint.alerts-*';
 export const metadataIndexPattern = 'metrics-endpoint.metadata-*';
 export const policyIndexPattern = 'metrics-endpoint.policy-*';
 export const telemetryIndexPattern = 'metrics-endpoint.telemetry-*';

x-pack/plugins/security_solution/common/endpoint/index_data.ts

@@ -16,14 +16,15 @@ export async function indexHostsAndAlerts(
  metadataIndex: string,
  policyIndex: string,
  eventIndex: string,
+ alertIndex: string,
  alertsPerHost: number,
  options: TreeOptions = {}
 ) {
  const random = seedrandom(seed);
  for (let i = 0; i < numHosts; i++) {
  const generator = new EndpointDocGenerator(random);
  await indexHostDocs(numDocs, client, metadataIndex, policyIndex, generator);
- await indexAlerts(client, eventIndex, generator, alertsPerHost, options);
+ await indexAlerts(client, eventIndex, alertIndex, generator, alertsPerHost, options);
  }
  await client.indices.refresh({
  index: eventIndex,
@@ -65,7 +66,8 @@ async function indexHostDocs(
 
 async function indexAlerts(
  client: Client,
- index: string,
+ eventIndex: string,
+ alertIndex: string,
  generator: EndpointDocGenerator,
  numAlerts: number,
  options: TreeOptions = {}
@@ -82,9 +84,14 @@ async function indexAlerts(
  }
  const body = resolverDocs.reduce(
  // eslint-disable-next-line @typescript-eslint/no-explicit-any
- (array: Array<Record<string, any>>, doc) => (
- array.push({ create: { _index: index } }, doc), array
- ),
+ (array: Array<Record<string, any>>, doc) => {
+ let index = eventIndex;
+ if (doc.event.kind === 'alert') {
+ index = alertIndex;
+ }
+ array.push({ create: { _index: index } }, doc);
+ return array;
+ },
  []
  );
  await client.bulk({ body, refresh: 'true' });

x-pack/plugins/security_solution/public/endpoint_alerts/store/middleware.ts

@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { eventsIndexPattern } from '../../../common/endpoint/constants';
+import { alertsIndexPattern } from '../../../common/endpoint/constants';
 import { IIndexPattern } from '../../../../../../src/plugins/data/public';
 import {
  AlertResultList,
@@ -49,10 +49,10 @@ export const alertMiddlewareFactory: ImmutableMiddlewareFactory<AlertListState>
  async function fetchIndexPatterns(): Promise<IIndexPattern[]> {
  const { indexPatterns } = depsStart.data;
  const fields = await indexPatterns.getFieldsForWildcard({
- pattern: eventsIndexPattern,
+ pattern: alertsIndexPattern,
  });
  const indexPattern: IIndexPattern = {
- title: eventsIndexPattern,
+ title: alertsIndexPattern,
  fields,
  };
 

x-pack/plugins/security_solution/scripts/endpoint/resolver_generator.ts

@@ -95,19 +95,25 @@ async function main() {
  eventIndex: {
  alias: 'ei',
  describe: 'index to store events in',
- default: 'events-endpoint-1',
+ default: 'logs-endpoint.events.process-default',
+ type: 'string',
+ },
+ alertIndex: {
+ alias: 'ai',
+ describe: 'index to store alerts in',
+ default: 'logs-endpoint.alerts-default',
  type: 'string',
  },
  metadataIndex: {
  alias: 'mi',
  describe: 'index to store host metadata in',
- default: 'metrics-endpoint.metadata-default-1',
+ default: 'metrics-endpoint.metadata-default',
  type: 'string',
  },
  policyIndex: {
  alias: 'pi',
  describe: 'index to store host policy in',
- default: 'metrics-endpoint.policy-default-1',
+ default: 'metrics-endpoint.policy-default',
  type: 'string',
  },
  ancestors: {
@@ -192,7 +198,10 @@ async function main() {
 
  const client = new Client(clientOptions);
  if (argv.delete) {
- await deleteIndices([argv.eventIndex, argv.metadataIndex, argv.policyIndex], client);
+ await deleteIndices(
+ [argv.eventIndex, argv.metadataIndex, argv.policyIndex, argv.alertIndex],
+ client
+ );
  }
 
  let seed = argv.seed;
@@ -209,6 +218,7 @@ async function main() {
  argv.metadataIndex,
  argv.policyIndex,
  argv.eventIndex,
+ argv.alertIndex,
  argv.alertsPerHost,
  {
  ancestors: argv.ancestors,

x-pack/plugins/security_solution/server/endpoint/alerts/handlers/details/index.ts

@@ -5,7 +5,7 @@
  */
 import { GetResponse } from 'elasticsearch';
 import { KibanaRequest, RequestHandler } from 'kibana/server';
-import { eventsIndexPattern } from '../../../../../common/endpoint/constants';
+import { alertsIndexPattern } from '../../../../../common/endpoint/constants';
 import { AlertEvent } from '../../../../../common/endpoint/types';
 import { EndpointAppContext } from '../../../types';
 import { AlertDetailsRequestParams } from '../../../../../common/endpoint_alerts/types';
@@ -34,7 +34,7 @@ export const alertDetailsHandlerWrapper = function (
  ctx,
  req.params,
  response,
- eventsIndexPattern
+ alertsIndexPattern
  );
 
  const currentHostInfo = await getHostData(

x-pack/plugins/security_solution/server/endpoint/alerts/handlers/list/index.ts

@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 import { RequestHandler } from 'kibana/server';
-import { eventsIndexPattern } from '../../../../../common/endpoint/constants';
+import { alertsIndexPattern } from '../../../../../common/endpoint/constants';
 import { EndpointAppContext } from '../../../types';
 import { searchESForAlerts } from '../lib';
 import { getRequestData, mapToAlertResultList } from './lib';
@@ -23,7 +23,7 @@ export const alertListHandlerWrapper = function (
  const response = await searchESForAlerts(
  ctx.core.elasticsearch.legacy.client,
  reqData,
- eventsIndexPattern
+ alertsIndexPattern
  );
  const mappedBody = await mapToAlertResultList(ctx, endpointAppContext, reqData, response);
  return res.ok({ body: mappedBody });

x-pack/plugins/security_solution/server/endpoint/routes/resolver/alerts.ts

@@ -7,7 +7,7 @@
 import { TypeOf } from '@kbn/config-schema';
 import { RequestHandler, Logger } from 'kibana/server';
 import { validateAlerts } from '../../../../common/endpoint/schema/resolver';
-import { eventsIndexPattern } from '../../../../common/endpoint/constants';
+import { alertsIndexPattern, eventsIndexPattern } from '../../../../common/endpoint/constants';
 import { Fetcher } from './utils/fetch';
 import { EndpointAppContext } from '../../types';
 
@@ -23,7 +23,7 @@ export function handleAlerts(
  try {
  const client = context.core.elasticsearch.legacy.client;
 
- const fetcher = new Fetcher(client, id, eventsIndexPattern, endpointID);
+ const fetcher = new Fetcher(client, id, eventsIndexPattern, alertsIndexPattern, endpointID);
 
  return res.ok({
  body: await fetcher.alerts(alerts, afterAlert),

x-pack/plugins/security_solution/server/endpoint/routes/resolver/ancestry.ts

@@ -6,7 +6,7 @@
 
 import { RequestHandler, Logger } from 'kibana/server';
 import { TypeOf } from '@kbn/config-schema';
-import { eventsIndexPattern } from '../../../../common/endpoint/constants';
+import { eventsIndexPattern, alertsIndexPattern } from '../../../../common/endpoint/constants';
 import { validateAncestry } from '../../../../common/endpoint/schema/resolver';
 import { Fetcher } from './utils/fetch';
 import { EndpointAppContext } from '../../types';
@@ -23,7 +23,7 @@ export function handleAncestry(
  try {
  const client = context.core.elasticsearch.legacy.client;
 
- const fetcher = new Fetcher(client, id, eventsIndexPattern, endpointID);
+ const fetcher = new Fetcher(client, id, eventsIndexPattern, alertsIndexPattern, endpointID);
  const ancestorInfo = await fetcher.ancestors(ancestors);
 
  return res.ok({

x-pack/plugins/security_solution/server/endpoint/routes/resolver/children.ts

@@ -6,7 +6,7 @@
 
 import { RequestHandler, Logger } from 'kibana/server';
 import { TypeOf } from '@kbn/config-schema';
-import { eventsIndexPattern } from '../../../../common/endpoint/constants';
+import { eventsIndexPattern, alertsIndexPattern } from '../../../../common/endpoint/constants';
 import { validateChildren } from '../../../../common/endpoint/schema/resolver';
 import { Fetcher } from './utils/fetch';
 import { EndpointAppContext } from '../../types';
@@ -22,7 +22,7 @@ export function handleChildren(
  } = req;
  try {
  const client = context.core.elasticsearch.legacy.client;
- const fetcher = new Fetcher(client, id, eventsIndexPattern, endpointID);
+ const fetcher = new Fetcher(client, id, eventsIndexPattern, alertsIndexPattern, endpointID);
 
  return res.ok({
  body: await fetcher.children(children, generations, afterChild),

x-pack/plugins/security_solution/server/endpoint/routes/resolver/events.ts

@@ -6,7 +6,7 @@
 
 import { TypeOf } from '@kbn/config-schema';
 import { RequestHandler, Logger } from 'kibana/server';
-import { eventsIndexPattern } from '../../../../common/endpoint/constants';
+import { eventsIndexPattern, alertsIndexPattern } from '../../../../common/endpoint/constants';
 import { validateEvents } from '../../../../common/endpoint/schema/resolver';
 import { Fetcher } from './utils/fetch';
 import { EndpointAppContext } from '../../types';
@@ -23,7 +23,7 @@ export function handleEvents(
  try {
  const client = context.core.elasticsearch.legacy.client;
 
- const fetcher = new Fetcher(client, id, eventsIndexPattern, endpointID);
+ const fetcher = new Fetcher(client, id, eventsIndexPattern, alertsIndexPattern, endpointID);
 
  return res.ok({
  body: await fetcher.events(events, afterEvent),

x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/alerts.ts

@@ -15,7 +15,7 @@ import { JsonObject } from '../../../../../../../../src/plugins/kibana_utils/com
 export class AlertsQuery extends ResolverQuery<PaginatedResults> {
  constructor(
  private readonly pagination: PaginationBuilder,
- indexPattern: string,
+ indexPattern: string | string[],
  endpointID?: string
  ) {
  super(indexPattern, endpointID);

x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/base.ts

@@ -25,13 +25,16 @@ export abstract class ResolverQuery<T> implements MSearchQuery {
  * we need `endpointID` for legacy data is because we don't have a cross endpoint unique identifier for process
  * events. Instead we use `unique_pid/ppid` and `endpointID` to uniquely identify a process event.
  */
- constructor(private readonly indexPattern: string, private readonly endpointID?: string) {}
+ constructor(
+ private readonly indexPattern: string | string[],
+ private readonly endpointID?: string
+ ) {}
 
  private static createIdsArray(ids: string | string[]): string[] {
  return Array.isArray(ids) ? ids : [ids];
  }
 
- private buildQuery(ids: string | string[]): { query: JsonObject; index: string } {
+ private buildQuery(ids: string | string[]): { query: JsonObject; index: string | string[] } {
  const idsArray = ResolverQuery.createIdsArray(ids);
  if (this.endpointID) {
  return { query: this.legacyQuery(this.endpointID, idsArray), index: legacyEventIndexPattern };

x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/children.ts

@@ -15,7 +15,7 @@ import { JsonObject } from '../../../../../../../../src/plugins/kibana_utils/com
 export class ChildrenQuery extends ResolverQuery<PaginatedResults> {
  constructor(
  private readonly pagination: PaginationBuilder,
- indexPattern: string,
+ indexPattern: string | string[],
  endpointID?: string
  ) {
  super(indexPattern, endpointID);

x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/events.ts

@@ -15,7 +15,7 @@ import { JsonObject } from '../../../../../../../../src/plugins/kibana_utils/com
 export class EventsQuery extends ResolverQuery<PaginatedResults> {
  constructor(
  private readonly pagination: PaginationBuilder,
- indexPattern: string,
+ indexPattern: string | string[],
  endpointID?: string
  ) {
  super(indexPattern, endpointID);

x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree.ts

@@ -6,7 +6,7 @@
 
 import { RequestHandler, Logger } from 'kibana/server';
 import { TypeOf } from '@kbn/config-schema';
-import { eventsIndexPattern } from '../../../../common/endpoint/constants';
+import { eventsIndexPattern, alertsIndexPattern } from '../../../../common/endpoint/constants';
 import { validateTree } from '../../../../common/endpoint/schema/resolver';
 import { Fetcher } from './utils/fetch';
 import { Tree } from './utils/tree';
@@ -34,7 +34,7 @@ export function handleTree(
  try {
  const client = context.core.elasticsearch.legacy.client;
 
- const fetcher = new Fetcher(client, id, eventsIndexPattern, endpointID);
+ const fetcher = new Fetcher(client, id, eventsIndexPattern, alertsIndexPattern, endpointID);
 
  const [childrenNodes, ancestry, relatedEvents, relatedAlerts] = await Promise.all([
  fetcher.children(children, generations, afterChild),

x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/fetch.ts

@@ -39,9 +39,13 @@ export class Fetcher {
  */
  private readonly id: string,
  /**
- * Index pattern for searching ES
+ * Index pattern for searching ES for events
  */
- private readonly indexPattern: string,
+ private readonly eventsIndexPattern: string,
+ /**
+ * Index pattern for searching ES for alerts
+ */
+ private readonly alertsIndexPattern: string,
  /**
  * This is used for searching legacy events
  */
@@ -109,7 +113,7 @@ export class Fetcher {
  public async alerts(limit: number, after?: string): Promise<ResolverRelatedAlerts> {
  const query = new AlertsQuery(
  PaginationBuilder.createBuilder(limit, after),
- this.indexPattern,
+ this.alertsIndexPattern,
  this.endpointID
  );
 
@@ -140,7 +144,7 @@ export class Fetcher {
  }
 
  private async getNode(entityID: string): Promise<LifecycleNode | undefined> {
- const query = new LifecycleQuery(this.indexPattern, this.endpointID);
+ const query = new LifecycleQuery(this.eventsIndexPattern, this.endpointID);
  const results = await query.search(this.client, entityID);
  if (results.length === 0) {
  return;
@@ -172,7 +176,7 @@ export class Fetcher {
  return;
  }
 
- const query = new LifecycleQuery(this.indexPattern, this.endpointID);
+ const query = new LifecycleQuery(this.eventsIndexPattern, this.endpointID);
  const results = await query.search(this.client, ancestors);
 
  if (results.length === 0) {
@@ -210,7 +214,7 @@ export class Fetcher {
  private async doEvents(limit: number, after?: string) {
  const query = new EventsQuery(
  PaginationBuilder.createBuilder(limit, after),
- this.indexPattern,
+ this.eventsIndexPattern,
  this.endpointID
  );
 
@@ -243,10 +247,10 @@ export class Fetcher {
 
  const childrenQuery = new ChildrenQuery(
  PaginationBuilder.createBuilder(limit, after),
- this.indexPattern,
+ this.eventsIndexPattern,
  this.endpointID
  );
- const lifecycleQuery = new LifecycleQuery(this.indexPattern, this.endpointID);
+ const lifecycleQuery = new LifecycleQuery(this.eventsIndexPattern, this.endpointID);
 
  const { totals, results } = await childrenQuery.search(this.client, ids);
  if (results.length === 0) {
@@ -262,7 +266,10 @@ export class Fetcher {
  }
 
  private async doStats(tree: Tree) {
- const statsQuery = new StatsQuery(this.indexPattern, this.endpointID);
+ const statsQuery = new StatsQuery(
+ [this.eventsIndexPattern, this.alertsIndexPattern],
+ this.endpointID
+ );
  const ids = tree.ids();
  const res = await statsQuery.search(this.client, ids);
  const alerts = res.alerts;