[Security] Alert Telemetry for the Security app (#77200)

This adds a `TelemetryEventsSender` component that can be used to publish Endpoint alerts to our Telemetry service. The alerts are filtered by a set of allowed fields (for PII) and batched in a queue to be sent once per minute. There is a cap of 100 alerts per minute to be sent. The component respects the telemetry opt-in status and enriches the alerts with the cluster ID and name.

The Detection Engine is slightly modified to send endpoint telemetry events via the `TelemetryEventsSender`. Only the "custom query" rule type is modified because that's the only one that can create Endpoint Alerts.

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
Co-authored-by: Garrett Spong <spong@users.noreply.github.com>

x-pack/plugins/security_solution/kibana.json

@@ -27,7 +27,8 @@
     "spaces",
     "usageCollection",
     "lists",
-    "home"
+    "home",
+    "telemetry"
   ],
   "server": true,
   "ui": true,

x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts

@@ -130,6 +130,7 @@ describe('searchAfterAndBulkCreate', () => {
       exceptionsList: [exceptionItem],
       services: mockService,
       logger: mockLogger,
+      eventsTelemetry: undefined,
       id: sampleRuleGuid,
       inputIndexPattern,
       signalsIndex: DEFAULT_SIGNALS_INDEX,
@@ -255,6 +256,7 @@ describe('searchAfterAndBulkCreate', () => {
       exceptionsList: [exceptionItem],
       services: mockService,
       logger: mockLogger,
+      eventsTelemetry: undefined,
       id: sampleRuleGuid,
       inputIndexPattern,
       signalsIndex: DEFAULT_SIGNALS_INDEX,
@@ -334,6 +336,7 @@ describe('searchAfterAndBulkCreate', () => {
       exceptionsList: [exceptionItem],
       services: mockService,
       logger: mockLogger,
+      eventsTelemetry: undefined,
       id: sampleRuleGuid,
       inputIndexPattern,
       signalsIndex: DEFAULT_SIGNALS_INDEX,
@@ -394,6 +397,7 @@ describe('searchAfterAndBulkCreate', () => {
       exceptionsList: [exceptionItem],
       services: mockService,
       logger: mockLogger,
+      eventsTelemetry: undefined,
       id: sampleRuleGuid,
       inputIndexPattern,
       signalsIndex: DEFAULT_SIGNALS_INDEX,
@@ -452,6 +456,7 @@ describe('searchAfterAndBulkCreate', () => {
       exceptionsList: [exceptionItem],
       services: mockService,
       logger: mockLogger,
+      eventsTelemetry: undefined,
       id: sampleRuleGuid,
       inputIndexPattern,
       signalsIndex: DEFAULT_SIGNALS_INDEX,
@@ -535,6 +540,7 @@ describe('searchAfterAndBulkCreate', () => {
       exceptionsList: [exceptionItem],
       services: mockService,
       logger: mockLogger,
+      eventsTelemetry: undefined,
       id: sampleRuleGuid,
       inputIndexPattern,
       signalsIndex: DEFAULT_SIGNALS_INDEX,
@@ -615,6 +621,7 @@ describe('searchAfterAndBulkCreate', () => {
       exceptionsList: [],
       services: mockService,
       logger: mockLogger,
+      eventsTelemetry: undefined,
       id: sampleRuleGuid,
       inputIndexPattern,
       signalsIndex: DEFAULT_SIGNALS_INDEX,
@@ -664,6 +671,7 @@ describe('searchAfterAndBulkCreate', () => {
       ruleParams: sampleParams,
       services: mockService,
       logger: mockLogger,
+      eventsTelemetry: undefined,
       id: sampleRuleGuid,
       inputIndexPattern,
       signalsIndex: DEFAULT_SIGNALS_INDEX,
@@ -719,6 +727,7 @@ describe('searchAfterAndBulkCreate', () => {
       ruleParams: sampleParams,
       services: mockService,
       logger: mockLogger,
+      eventsTelemetry: undefined,
       id: sampleRuleGuid,
       inputIndexPattern,
       signalsIndex: DEFAULT_SIGNALS_INDEX,
@@ -790,6 +799,7 @@ describe('searchAfterAndBulkCreate', () => {
       ruleParams: sampleParams,
       services: mockService,
       logger: mockLogger,
+      eventsTelemetry: undefined,
       id: sampleRuleGuid,
       inputIndexPattern,
       signalsIndex: DEFAULT_SIGNALS_INDEX,
@@ -898,6 +908,7 @@ describe('searchAfterAndBulkCreate', () => {
       exceptionsList: [],
       services: mockService,
       logger: mockLogger,
+      eventsTelemetry: undefined,
       id: sampleRuleGuid,
       inputIndexPattern,
       signalsIndex: DEFAULT_SIGNALS_INDEX,

x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts

@@ -7,6 +7,7 @@
 import { singleSearchAfter } from './single_search_after';
 import { singleBulkCreate } from './single_bulk_create';
 import { filterEventsAgainstList } from './filter_events_with_list';
+import { sendAlertTelemetryEvents } from './send_telemetry_events';
 import {
   createSearchAfterReturnType,
   createSearchAfterReturnTypeFromResponse,
@@ -25,6 +26,7 @@ export const searchAfterAndBulkCreate = async ({
   services,
   listClient,
   logger,
+  eventsTelemetry,
   id,
   inputIndexPattern,
   signalsIndex,
@@ -188,6 +190,14 @@ export const searchAfterAndBulkCreate = async ({
           logger.debug(
             buildRuleMessage(`filteredEvents.hits.hits: ${filteredEvents.hits.hits.length}`)
           );
+
+          sendAlertTelemetryEvents(
+            logger,
+            eventsTelemetry,
+            filteredEvents,
+            ruleParams,
+            buildRuleMessage
+          );
         }
 
         // we are guaranteed to have searchResult hits at this point

x-pack/plugins/security_solution/server/lib/detection_engine/signals/send_telemetry_events.test.ts

@@ -0,0 +1,78 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { selectEvents } from './send_telemetry_events';
+
+describe('sendAlertTelemetry', () => {
+  it('selectEvents', () => {
+    const filteredEvents = {
+      took: 0,
+      timed_out: false,
+      _shards: {
+        total: 1,
+        successful: 1,
+        failed: 0,
+        skipped: 0,
+      },
+      hits: {
+        total: 2,
+        max_score: 0,
+        hits: [
+          {
+            _index: 'x',
+            _type: 'x',
+            _id: 'x',
+            _score: 0,
+            _source: {
+              '@timestamp': 'x',
+              key1: 'hello',
+              data_stream: {
+                dataset: 'endpoint.events',
+              },
+            },
+          },
+          {
+            _index: 'x',
+            _type: 'x',
+            _id: 'x',
+            _score: 0,
+            _source: {
+              '@timestamp': 'x',
+              key2: 'hello',
+              data_stream: {
+                dataset: 'endpoint.alerts',
+                other: 'x',
+              },
+            },
+          },
+          {
+            _index: 'x',
+            _type: 'x',
+            _id: 'x',
+            _score: 0,
+            _source: {
+              '@timestamp': 'x',
+              key3: 'hello',
+              data_stream: {},
+            },
+          },
+        ],
+      },
+    };
+
+    const sources = selectEvents(filteredEvents);
+    expect(sources).toStrictEqual([
+      {
+        '@timestamp': 'x',
+        key2: 'hello',
+        data_stream: {
+          dataset: 'endpoint.alerts',
+          other: 'x',
+        },
+      },
+    ]);
+  });
+});

x-pack/plugins/security_solution/server/lib/detection_engine/signals/send_telemetry_events.ts

@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { TelemetryEventsSender, TelemetryEvent } from '../../telemetry/sender';
+import { RuleTypeParams } from '../types';
+import { BuildRuleMessage } from './rule_messages';
+import { SignalSearchResponse, SignalSource } from './types';
+import { Logger } from '../../../../../../../src/core/server';
+
+export interface SearchResultWithSource {
+  _source: SignalSource;
+}
+
+export function selectEvents(filteredEvents: SignalSearchResponse): TelemetryEvent[] {
+  const sources = filteredEvents.hits.hits.map(function (
+    obj: SearchResultWithSource
+  ): TelemetryEvent {
+    return obj._source;
+  });
+
+  // Filter out non-endpoint alerts
+  return sources.filter((obj: TelemetryEvent) => obj.data_stream?.dataset === 'endpoint.alerts');
+}
+
+export function sendAlertTelemetryEvents(
+  logger: Logger,
+  eventsTelemetry: TelemetryEventsSender | undefined,
+  filteredEvents: SignalSearchResponse,
+  ruleParams: RuleTypeParams,
+  buildRuleMessage: BuildRuleMessage
+) {
+  if (eventsTelemetry === undefined) {
+    return;
+  }
+
+  const sources = selectEvents(filteredEvents);
+
+  try {
+    eventsTelemetry.queueTelemetryEvents(sources);
+  } catch (exc) {
+    logger.error(buildRuleMessage(`[-] queing telemetry events failed ${exc}`));
+  }
+}

x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts

@@ -138,6 +138,7 @@ describe('rules_notification_alert_type', () => {
 
     alert = signalRulesAlertType({
       logger,
+      eventsTelemetry: undefined,
       version,
       ml: mlMock,
       lists: listMock.createSetup(),
@@ -344,6 +345,7 @@ describe('rules_notification_alert_type', () => {
         payload = getPayload(ruleAlert, alertServices) as jest.Mocked<RuleExecutorOptions>;
         alert = signalRulesAlertType({
           logger,
+          eventsTelemetry: undefined,
           version,
           ml: undefined,
           lists: undefined,

x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts

@@ -56,18 +56,21 @@ import { ruleStatusServiceFactory } from './rule_status_service';
 import { buildRuleMessageFactory } from './rule_messages';
 import { ruleStatusSavedObjectsClientFactory } from './rule_status_saved_objects_client';
 import { getNotificationResultsLink } from '../notifications/utils';
+import { TelemetryEventsSender } from '../../telemetry/sender';
 import { buildEqlSearchRequest } from '../../../../common/detection_engine/get_query_filter';
 import { bulkInsertSignals } from './single_bulk_create';
 import { buildSignalFromEvent, buildSignalGroupFromSequence } from './build_bulk_body';
 import { createThreatSignals } from './threat_mapping/create_threat_signals';
 
 export const signalRulesAlertType = ({
   logger,
+  eventsTelemetry,
   version,
   ml,
   lists,
 }: {
   logger: Logger;
+  eventsTelemetry: TelemetryEventsSender | undefined;
   version: string;
   ml: SetupPlugins['ml'];
   lists: SetupPlugins['lists'] | undefined;
@@ -369,6 +372,7 @@ export const signalRulesAlertType = ({
             previousStartedAt,
             listClient,
             logger,
+            eventsTelemetry,
             alertId,
             outputIndex,
             params,
@@ -409,6 +413,7 @@ export const signalRulesAlertType = ({
             ruleParams: params,
             services,
             logger,
+            eventsTelemetry,
             id: alertId,
             inputIndexPattern: inputIndex,
             signalsIndex: outputIndex,

x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts

@@ -28,6 +28,7 @@ export const createThreatSignal = async ({
   previousStartedAt,
   listClient,
   logger,
+  eventsTelemetry,
   alertId,
   outputIndex,
   params,
@@ -77,6 +78,7 @@ export const createThreatSignal = async ({
     ruleParams: params,
     services,
     logger,
+    eventsTelemetry,
     id: alertId,
     inputIndexPattern: inputIndex,
     signalsIndex: outputIndex,

x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts

@@ -24,6 +24,7 @@ export const createThreatSignals = async ({
   previousStartedAt,
   listClient,
   logger,
+  eventsTelemetry,
   alertId,
   outputIndex,
   params,
@@ -79,6 +80,7 @@ export const createThreatSignals = async ({
       previousStartedAt,
       listClient,
       logger,
+      eventsTelemetry,
       alertId,
       outputIndex,
       params,

x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts

@@ -21,6 +21,7 @@ import { AlertServices } from '../../../../../../alerts/server';
 import { ExceptionListItemSchema } from '../../../../../../lists/common/schemas';
 import { ILegacyScopedClusterClient, Logger } from '../../../../../../../../src/core/server';
 import { RuleAlertAction } from '../../../../../common/detection_engine/types';
+import { TelemetryEventsSender } from '../../../telemetry/sender';
 import { BuildRuleMessage } from '../rule_messages';
 import { SearchAfterAndBulkCreateReturnType } from '../types';
 
@@ -38,6 +39,7 @@ export interface CreateThreatSignalsOptions {
   previousStartedAt: Date | null;
   listClient: ListClient;
   logger: Logger;
+  eventsTelemetry: TelemetryEventsSender | undefined;
   alertId: string;
   outputIndex: string;
   params: RuleTypeParams;
@@ -73,6 +75,7 @@ export interface CreateThreatSignalOptions {
   previousStartedAt: Date | null;
   listClient: ListClient;
   logger: Logger;
+  eventsTelemetry: TelemetryEventsSender | undefined;
   alertId: string;
   outputIndex: string;
   params: RuleTypeParams;

x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts

@@ -21,6 +21,7 @@ import { ListClient } from '../../../../../lists/server';
 import { Logger } from '../../../../../../../src/core/server';
 import { ExceptionListItemSchema } from '../../../../../lists/common/schemas';
 import { BuildRuleMessage } from './rule_messages';
+import { TelemetryEventsSender } from '../../telemetry/sender';
 
 // used for gap detection code
 // eslint-disable-next-line @typescript-eslint/naming-convention
@@ -211,6 +212,7 @@ export interface SearchAfterAndBulkCreateParams {
   listClient: ListClient;
   exceptionsList: ExceptionListItemSchema[];
   logger: Logger;
+  eventsTelemetry: TelemetryEventsSender | undefined;
   id: string;
   inputIndexPattern: string[];
   signalsIndex: string;