[SIEM][Detection Engine] Consolidates types and makes incremental sig…

…nal schema changes (#48679) (#48701) ## Summary Schema changes --- * Changed `signal.severity` from `number` to `string` * Change the `@timestamp` to be the time the signal was detected. To describe this more, when the alert begins processing the rule, it creates a time stamp from `now` and then populates each signal with that value of `now`. The next time the alert re-starts processes, it repeats the same thing. * Changed `signal.time_detected` to be `original_time` for any UI that wants to filter or use that. * Added a new field to `signal` called `index` which contains the original index so that locating the originating signal is possible. * Removed `error` as we are going to not push errors for each signal at this point. We can be clever and create "maintenance" signals that we generate with our ID's though if we want and reserve a block of rule id's if we do that. * Updated the other ECS fields and ECS schema to be on 1.2 from [here](https://raw.githubusercontent.com/elastic/ecs/master/generated/elasticsearch/7/template.json ) Code fixes --- * Fixed more incorrect TypeScript type bugs. * Fixed a bug where we were creating strings from arrays rather than just pushing the arrays directly in some cases. * Removed the last any's from the detection engine folder by pushing the types down. * Removed more touch points with the types where I could find them which should make changing the schema and endpoints faster next time. * #47015 ### Checklist Use ~~strikethroughs~~ to remove checklist items you don't feel are applicable to this PR. ~~- [ ] This was checked for cross-browser compatibility, [including a check against IE11](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#cross-browser-compatibility)~~ ~~- [ ] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/master/packages/kbn-i18n/README.md)~~ ~~- [ ] [Documentation](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#writing-documentation) was added for features that require explanation or tutorials~~ ~~- [ ] [Unit or functional tests](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#cross-browser-compatibility) were updated or added to match the most common scenarios~~ ~~- [ ] This was checked for [keyboard-only and screenreader accessibility](https://developer.mozilla.org/en-US/docs/Learn/Tools_and_testing/Cross_browser_testing/Accessibility#Accessibility_testing_checklist)~~ ### For maintainers ~~- [ ] This was checked for breaking API changes and was [labeled appropriately](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#release-notes-process)~~ ~~- [ ] This includes a feature addition or change that requires a release note and was [labeled appropriately](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#release-notes-process)~~
elastic · Oct 19, 2019 · 2e0f598 · 2e0f598
1 parent 1a86af2
commit 2e0f598
Show file tree

Hide file tree

Showing 19 changed files with 1,078 additions and 800 deletions.
diff --git a/x-pack/legacy/plugins/siem/scripts/convert_saved_search_to_signals.js b/x-pack/legacy/plugins/siem/scripts/convert_saved_search_to_signals.js
@@ -29,7 +29,7 @@ const path = require('path');
 // doing a search to KQL conversion before posting it as a signal or checking it
 // into another repository.
 const INTERVAL = '24h';
-const SEVERITY = 1;
+const SEVERITY = 'low';
 const TYPE = 'kql';
 const FROM = 'now-24h';
 const TO = 'now';

diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/build_events_reindex.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/build_events_reindex.ts
@@ -20,9 +20,9 @@ interface BuildEventsReIndexParams {
   maxDocs: string;
   filter: Record<string, {}> | undefined;
   kql: string | undefined;
-  severity: number;
+  severity: string;
   name: string;
-  timeDetected: number;
+  timeDetected: string;
   ruleRevision: number;
   id: string;
   type: string;
@@ -131,6 +131,7 @@ export const buildEventsReIndex = ({
           def parent = [
             "id": ctx._id,
             "type": "event",
+            "index": ctx._index,
             "depth": 1
           ];
 
@@ -140,14 +141,15 @@ export const buildEventsReIndex = ({
             "rule_type": "${type}",
             "parent": parent,
             "name": "${name}",
-            "severity": ${severity},
+            "severity": "${severity}",
             "description": "${description}",
-            "time_detected": "${timeDetected}",
+            "original_time": ctx._source['@timestamp'],
             "index_patterns": indexPatterns,
             "references": references
           ];
 
           ctx._source.signal = signal;
+          ctx._source['@timestamp'] = "${timeDetected}";
         `,
         lang: 'painless',
       },

diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/signals_alert_type.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/signals_alert_type.ts
@@ -34,7 +34,7 @@ export const signalsAlertType = ({ logger }: { logger: Logger }): SignalAlertTyp
         kql: schema.nullable(schema.string()),
         maxSignals: schema.number({ defaultValue: 100 }),
         name: schema.string(),
-        severity: schema.number(),
+        severity: schema.string(),
         to: schema.string(),
         type: schema.string(),
         references: schema.arrayOf(schema.string(), { defaultValue: [] }),
@@ -87,7 +87,7 @@ export const signalsAlertType = ({ logger }: { logger: Logger }): SignalAlertTyp
         severity,
         description,
         name,
-        timeDetected: Date.now(),
+        timeDetected: new Date().toISOString(),
         filter,
         maxDocs: maxSignals,
         ruleRevision: 1,

diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/types.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/types.ts
@@ -16,6 +16,7 @@ import {
 } from '../../../../../alerting/server/types';
 import { AlertsClient } from '../../../../../alerting/server/alerts_client';
 import { ActionsClient } from '../../../../../actions/server/actions_client';
+import { SearchResponse } from '../../types';
 
 export interface SignalAlertParams {
   description: string;
@@ -28,7 +29,7 @@ export interface SignalAlertParams {
   kql: string | undefined;
   maxSignals: string;
   name: string;
-  severity: number;
+  severity: string;
   type: 'filter' | 'kql';
   to: string;
   references: string[];
@@ -78,26 +79,30 @@ export interface SignalsRequest extends Hapi.Request {
 }
 
 export type SignalExecutorOptions = Omit<AlertExecutorOptions, 'params'> & {
-  params: {
-    description: string;
-    from: string;
-    id: string;
-    index: string[];
-    interval: string;
-    enabled: boolean;
-    filter: Record<string, {}> | undefined;
-    kql: string | undefined;
-    maxSignals: string;
-    name: string;
-    severity: number;
-    type: 'filter' | 'kql';
-    to: string;
-    references: string[];
+  params: SignalAlertParams & {
     scrollSize: number;
     scrollLock: string;
   };
 };
 
+export type SearchTypes =
+  | string
+  | string[]
+  | number
+  | number[]
+  | boolean
+  | boolean[]
+  | object
+  | object[];
+
+export interface SignalSource {
+  [key: string]: SearchTypes;
+  '@timestamp': string;
+}
+
+export type SignalSearchResponse = SearchResponse<SignalSource>;
+export type SignalSourceHit = SignalSearchResponse['hits']['hits'][0];
+
 // This returns true because by default a SignalAlertTypeDefinition is an AlertType
 // since we are only increasing the strictness of params.
 export const isAlertExecutor = (obj: SignalAlertTypeDefinition): obj is AlertType => {

diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.ts
@@ -4,48 +4,48 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { SearchResponse, SearchHit, SignalHit } from '../../types';
+import { SignalHit } from '../../types';
 import { Logger } from '../../../../../../../../src/core/server';
 import { AlertServices } from '../../../../../alerting/server/types';
+import { SignalSourceHit, SignalSearchResponse, SignalAlertParams } from './types';
 
 // format scroll search result for signals index.
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
-export const buildBulkBody = (doc: SearchHit, signalParams: Record<string, any>): SignalHit => {
-  const indexPatterns = signalParams.index.map((element: string) => `"${element}"`).join(',');
-  const refs = signalParams.references.map((element: string) => `"${element}"`).join(',');
+export const buildBulkBody = (doc: SignalSourceHit, signalParams: SignalAlertParams): SignalHit => {
   return {
     ...doc._source,
     signal: {
+      '@timestamp': new Date().toISOString(),
       rule_revision: 1,
       rule_id: signalParams.id,
       rule_type: signalParams.type,
       parent: {
         id: doc._id,
         type: 'event',
+        index: doc._index,
         depth: 1,
       },
       name: signalParams.name,
       severity: signalParams.severity,
       description: signalParams.description,
-      time_detected: Date.now(),
-      index_patterns: indexPatterns,
-      references: refs,
+      original_time: doc._source['@timestamp'],
+      index_patterns: signalParams.index,
+      references: signalParams.references,
     },
   };
 };
 
 // Bulk Index documents.
 export const singleBulkIndex = async (
-  sr: SearchResponse<object>,
-  params: Record<string, any>, // eslint-disable-line @typescript-eslint/no-explicit-any
+  sr: SignalSearchResponse,
+  params: SignalAlertParams,
   service: AlertServices,
   logger: Logger
 ): Promise<boolean> => {
   if (sr.hits.hits.length === 0) {
     logger.warn('First search result yielded 0 documents');
     return false;
   }
-  const bulkBody = sr.hits.hits.flatMap((doc: SearchHit) => [
+  const bulkBody = sr.hits.hits.flatMap(doc => [
     {
       index: {
         _index: process.env.SIGNALS_INDEX || '.siem-signals-10-01-2019',
@@ -68,10 +68,10 @@ export const singleBulkIndex = async (
 // Given a scroll id, grab the next set of documents
 export const singleScroll = async (
   scrollId: string | undefined,
-  params: Record<string, any>, // eslint-disable-line @typescript-eslint/no-explicit-any
+  params: SignalAlertParams & { scrollLock?: number }, // TODO: Finish plumbing the scrollLock all the way to the REST endpoint if this algorithm continues to use it.
   service: AlertServices,
   logger: Logger
-): Promise<SearchResponse<object>> => {
+): Promise<SignalSearchResponse> => {
   const scroll = params.scrollLock ? params.scrollLock : '1m';
   try {
     const nextScrollResult = await service.callCluster('scroll', {
@@ -87,8 +87,8 @@ export const singleScroll = async (
 
 // scroll through documents and re-index using bulk endpoint.
 export const scrollAndBulkIndex = async (
-  someResult: SearchResponse<object>,
-  params: Record<string, any>, // eslint-disable-line @typescript-eslint/no-explicit-any
+  someResult: SignalSearchResponse,
+  params: SignalAlertParams,
   service: AlertServices,
   logger: Logger
 ): Promise<boolean> => {

diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/__mocks__/request_responses.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/__mocks__/request_responses.ts
@@ -16,7 +16,7 @@ export const getUpdateRequest = (): ServerInjectOptions => ({
     index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'],
     interval: '5m',
     name: 'Detect Root/Admin Users',
-    severity: 1,
+    severity: 'high',
     type: 'kql',
     from: 'now-6m',
     to: 'now',
@@ -55,7 +55,7 @@ export const getCreateRequest = (): ServerInjectOptions => ({
     index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'],
     interval: '5m',
     name: 'Detect Root/Admin Users',
-    severity: 1,
+    severity: 'high',
     type: 'kql',
     from: 'now-6m',
     to: 'now',
@@ -82,7 +82,7 @@ export const createAlertResult = () => ({
     kql: 'user.name: root or user.name: admin',
     maxSignals: 100,
     name: 'Detect Root/Admin Users',
-    severity: 1,
+    severity: 'high',
     to: 'now',
     type: 'kql',
     references: [],
@@ -136,7 +136,7 @@ export const updateAlertResult = () => ({
     kql: 'user.name: root or user.name: admin',
     maxSignals: 100,
     name: 'Detect Root/Admin Users',
-    severity: 1,
+    severity: 'high',
     to: 'now',
     type: 'kql',
     references: [],

diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/create_signals.route.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/create_signals.route.test.ts
@@ -78,7 +78,7 @@ describe('create_signals', () => {
           index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'],
           interval: '5m',
           name: 'Detect Root/Admin Users',
-          severity: 1,
+          severity: 'high',
           type: 'kql',
           from: 'now-6m',
           to: 'now',
@@ -103,7 +103,7 @@ describe('create_signals', () => {
           index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'],
           interval: '5m',
           name: 'Detect Root/Admin Users',
-          severity: 1,
+          severity: 'high',
           type: 'kql',
           from: 'now-6m',
           to: 'now',
@@ -128,7 +128,7 @@ describe('create_signals', () => {
           index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'],
           interval: '5m',
           name: 'Detect Root/Admin Users',
-          severity: 1,
+          severity: 'high',
           type: 'filter',
           from: 'now-6m',
           to: 'now',
@@ -153,7 +153,7 @@ describe('create_signals', () => {
           index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'],
           interval: '5m',
           name: 'Detect Root/Admin Users',
-          severity: 1,
+          severity: 'high',
           type: 'something-made-up', // This is a made up type that causes the 400
           from: 'now-6m',
           to: 'now',

diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/create_signals_route.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/create_signals_route.ts
@@ -30,7 +30,7 @@ export const createCreateSignalsRoute: Hapi.ServerRoute = {
         kql: Joi.string(),
         max_signals: Joi.number().default(100),
         name: Joi.string().required(),
-        severity: Joi.number().required(),
+        severity: Joi.string().required(),
         to: Joi.string().required(),
         type: Joi.string()
           .valid('filter', 'kql')

diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/update_signals_route.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/update_signals_route.test.ts
@@ -77,7 +77,7 @@ describe('update_signals', () => {
           index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'],
           interval: '5m',
           name: 'Detect Root/Admin Users',
-          severity: 1,
+          severity: 'high',
           type: 'kql',
           from: 'now-6m',
           to: 'now',
@@ -102,7 +102,7 @@ describe('update_signals', () => {
           index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'],
           interval: '5m',
           name: 'Detect Root/Admin Users',
-          severity: 1,
+          severity: 'high',
           type: 'kql',
           from: 'now-6m',
           to: 'now',
@@ -127,7 +127,7 @@ describe('update_signals', () => {
           index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'],
           interval: '5m',
           name: 'Detect Root/Admin Users',
-          severity: 1,
+          severity: 'high',
           type: 'filter',
           from: 'now-6m',
           to: 'now',
@@ -152,7 +152,7 @@ describe('update_signals', () => {
           index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'],
           interval: '5m',
           name: 'Detect Root/Admin Users',
-          severity: 1,
+          severity: 'high',
           type: 'something-made-up', // This is a made up type that causes the 400
           from: 'now-6m',
           to: 'now',
@@ -177,7 +177,7 @@ describe('update_signals', () => {
           index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'],
           interval: '5m',
           name: 'Detect Root/Admin Users',
-          severity: 1,
+          severity: 'high',
           type: 'kql',
           from: 'now-6m',
           to: 'now',

diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/update_signals_route.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/update_signals_route.ts
@@ -37,7 +37,7 @@ export const createUpdateSignalsRoute: Hapi.ServerRoute = {
         kql: Joi.string(),
         max_signals: Joi.number().default(100),
         name: Joi.string(),
-        severity: Joi.number(),
+        severity: Joi.string(),
         to: Joi.string(),
         type: Joi.string().valid('filter', 'kql'),
         references: Joi.array().default([]),

diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/post_x_signals.sh b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/post_x_signals.sh
@@ -27,7 +27,7 @@ do {
     \"index\": [\"auditbeat-*\", \"filebeat-*\", \"packetbeat-*\", \"winlogbeat-*\"],
     \"interval\": \"24h\",
     \"name\": \"Detect Root/Admin Users\",
-    \"severity\": 1,
+    \"severity\": \"high\",
     \"type\": \"kql\",
     \"from\": \"now-6m\",
     \"to\": \"now\",

diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signals/root_or_admin_1.json b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signals/root_or_admin_1.json
@@ -4,9 +4,10 @@
   "index": ["auditbeat-*", "filebeat-*", "packetbeat-*", "winlogbeat-*"],
   "interval": "5m",
   "name": "Detect Root/Admin Users",
-  "severity": 1,
+  "severity": "high",
   "type": "kql",
   "from": "now-6m",
   "to": "now",
-  "kql": "user.name: root or user.name: admin"
+  "kql": "user.name: root or user.name: admin",
+  "references": ["http://www.example.com", "https://ww.example.com"]
 }
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signals/root_or_admin_2.json b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signals/root_or_admin_2.json
@@ -4,7 +4,7 @@
   "index": ["auditbeat-*", "filebeat-*", "packetbeat-*", "winlogbeat-*"],
   "interval": "24h",
   "name": "Detect Root/Admin Users over a long period of time",
-  "severity": 1,
+  "severity": "high",
   "type": "kql",
   "from": "now-1y",
   "to": "now",

diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signals/root_or_admin_3.json b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signals/root_or_admin_3.json
@@ -4,7 +4,7 @@
   "index": ["auditbeat-*", "filebeat-*", "packetbeat-*", "winlogbeat-*"],
   "interval": "5m",
   "name": "Detect Root/Admin Users",
-  "severity": 1,
+  "severity": "high",
   "type": "kql",
   "from": "now-16y",
   "to": "now-15y",

diff --git a/...y/plugins/siem/server/lib/detection_engine/scripts/signals/root_or_admin_filter_9999.json b/...y/plugins/siem/server/lib/detection_engine/scripts/signals/root_or_admin_filter_9999.json
@@ -4,7 +4,7 @@
   "index": ["auditbeat-*", "filebeat-*", "packetbeat-*", "winlogbeat-*"],
   "interval": "5m",
   "name": "Detect Root/Admin Users",
-  "severity": 1,
+  "severity": "high",
   "type": "filter",
   "from": "now-6m",
   "to": "now",