[Security Solution] [Platform] Adds support for data views and runtime field mappings in rule creation, exceptions, and during execution

x-pack/plugins/security_solution/common/detection_engine/schemas/alerts/8.3.0/index.ts

@@ -0,0 +1,56 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ALERT_BUILDING_BLOCK_TYPE, ALERT_UUID } from '@kbn/rule-data-utils';
+import { AlertWithCommonFields800 } from '@kbn/rule-registry-plugin/common/schemas/8.0.0';
+import {
+  ALERT_GROUP_ID,
+  ALERT_GROUP_INDEX,
+  ALERT_RULE_INDICES,
+} from '../../../../field_maps/field_names';
+import type {
+  Ancestor800,
+  BaseFields800,
+  EqlBuildingBlockAlert800,
+  EqlShellAlert800,
+} from '../8.0.0';
+
+export type { Ancestor800 as Ancestor830 };
+
+/* DO NOT MODIFY THIS SCHEMA TO ADD NEW FIELDS. These types represent the alerts that shipped in 8.3.0.
+Any changes to these types should be bug fixes so the types more accurately represent the alerts from 8.3.0.
+If you are adding new fields for a new release of Kibana, create a new sibling folder to this one
+for the version to be released and add the field(s) to the schema in that folder.
+Then, update `../index.ts` to import from the new folder that has the latest schemas, add the
+new schemas to the union of all alert schemas, and re-export the new schemas as the `*Latest` schemas.
+*/
+export interface BaseFields830 extends BaseFields800 {
+  [ALERT_RULE_INDICES]: string[];
+}
+
+export interface WrappedFields830<T extends BaseFields830> {
+  _id: string;
+  _index: string;
+  _source: T & { [ALERT_UUID]: string };
+}
+
+export type GenericAlert830 = AlertWithCommonFields800<BaseFields830>;
+
+// This is the type of the final generated alert including base fields, common fields
+// added by the alertWithPersistence function, and arbitrary fields copied from source documents
+export type DetectionAlert830 = GenericAlert830 | EqlShellAlert800 | EqlBuildingBlockAlert800;
+
+export interface EqlShellFields830 extends BaseFields830 {
+  [ALERT_GROUP_ID]: string;
+  [ALERT_UUID]: string;
+}
+
+export interface EqlBuildingBlockFields830 extends BaseFields830 {
+  [ALERT_GROUP_ID]: string;
+  [ALERT_GROUP_INDEX]: number;
+  [ALERT_BUILDING_BLOCK_TYPE]: 'default';
+}

x-pack/plugins/security_solution/common/detection_engine/schemas/alerts/index.ts

@@ -6,23 +6,25 @@
  */
 
 import type {
-  Ancestor800,
-  BaseFields800,
-  DetectionAlert800,
-  WrappedFields800,
-  EqlBuildingBlockFields800,
-  EqlShellFields800,
-} from './8.0.0';
+  EqlBuildingBlockFields830,
+  EqlShellFields830,
+  WrappedFields830,
+  DetectionAlert830,
+  BaseFields830,
+  Ancestor830,
+} from './8.3.0';
+
+import type { DetectionAlert800 } from './8.0.0';
 
 // When new Alert schemas are created for new Kibana versions, add the DetectionAlert type from the new version
 // here, e.g. `export type DetectionAlert = DetectionAlert800 | DetectionAlert820` if a new schema is created in 8.2.0
-export type DetectionAlert = DetectionAlert800;
+export type DetectionAlert = DetectionAlert800 | DetectionAlert830;
 
 export type {
-  Ancestor800 as AncestorLatest,
-  BaseFields800 as BaseFieldsLatest,
-  DetectionAlert800 as DetectionAlertLatest,
-  WrappedFields800 as WrappedFieldsLatest,
-  EqlBuildingBlockFields800 as EqlBuildingBlockFieldsLatest,
-  EqlShellFields800 as EqlShellFieldsLatest,
+  Ancestor830 as AncestorLatest,
+  BaseFields830 as BaseFieldsLatest,
+  DetectionAlert830 as DetectionAlertLatest,
+  WrappedFields830 as WrappedFieldsLatest,
+  EqlBuildingBlockFields830 as EqlBuildingBlockFieldsLatest,
+  EqlShellFields830 as EqlShellFieldsLatest,
 };

x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts

@@ -123,6 +123,12 @@ export type IdOrUndefined = t.TypeOf<typeof idOrUndefined>;
 export const index = t.array(t.string);
 export type Index = t.TypeOf<typeof index>;
 
+export const data_view_id = t.string;
+export type DataViewId = t.TypeOf<typeof data_view_id>;
+
+export const dataViewIdOrUndefined = t.union([data_view_id, t.undefined]);
+export type DataViewIdOrUndefined = t.TypeOf<typeof dataViewIdOrUndefined>;
+
 export const indexOrUndefined = t.union([index, t.undefined]);
 export type IndexOrUndefined = t.TypeOf<typeof indexOrUndefined>;
 
@@ -462,14 +468,17 @@ const bulkActionEditPayloadTags = t.type({
 
 export type BulkActionEditPayloadTags = t.TypeOf<typeof bulkActionEditPayloadTags>;
 
-const bulkActionEditPayloadIndexPatterns = t.type({
-  type: t.union([
-    t.literal(BulkActionEditType.add_index_patterns),
-    t.literal(BulkActionEditType.delete_index_patterns),
-    t.literal(BulkActionEditType.set_index_patterns),
-  ]),
-  value: index,
-});
+const bulkActionEditPayloadIndexPatterns = t.intersection([
+  t.type({
+    type: t.union([
+      t.literal(BulkActionEditType.add_index_patterns),
+      t.literal(BulkActionEditType.delete_index_patterns),
+      t.literal(BulkActionEditType.set_index_patterns),
+    ]),
+    value: index,
+  }),
+  t.exact(t.partial({ overwriteDataViews: t.boolean })),
+]);
 
 export type BulkActionEditPayloadIndexPatterns = t.TypeOf<
   typeof bulkActionEditPayloadIndexPatterns

x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.ts

@@ -50,6 +50,7 @@ import {
   anomaly_threshold,
   filters,
   index,
+  data_view_id,
   saved_id,
   timeline_id,
   timeline_title,
@@ -114,6 +115,7 @@ export const addPrepackagedRulesSchema = t.intersection([
       filters, // defaults to undefined if not set during decode
       from: DefaultFromString, // defaults to "now-6m" if not set during decode
       index, // defaults to undefined if not set during decode
+      data_view_id, // defaults to undefined if not set during decode
       interval: DefaultIntervalString, // defaults to "5m" if not set during decode
       query, // defaults to undefined if not set during decode
       language, // defaults to undefined if not set during decode

x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.test.ts

@@ -1773,4 +1773,124 @@ describe('import rules schema', () => {
       expect(message.schema).toEqual(expected);
     });
   });
+
+  describe('data_view_id', () => {
+    test('Defined data_view_id and empty index does validate', () => {
+      const payload: ImportRulesSchema = {
+        rule_id: 'rule-1',
+        risk_score: 50,
+        description: 'some description',
+        from: 'now-5m',
+        to: 'now',
+        name: 'some-name',
+        severity: 'low',
+        type: 'query',
+        query: 'some query',
+        data_view_id: 'logs-*',
+        index: [],
+        interval: '5m',
+      };
+
+      const decoded = importRulesSchema.decode(payload);
+      const checked = exactCheck(payload, decoded);
+      const message = pipe(checked, foldLeftRight);
+      expect(getPaths(left(message.errors))).toEqual([]);
+      const expected: ImportRulesSchemaDecoded = {
+        author: [],
+        severity_mapping: [],
+        risk_score_mapping: [],
+        rule_id: 'rule-1',
+        risk_score: 50,
+        description: 'some description',
+        from: 'now-5m',
+        to: 'now',
+        name: 'some-name',
+        severity: 'low',
+        type: 'query',
+        query: 'some query',
+        index: [],
+        data_view_id: 'logs-*',
+        interval: '5m',
+        references: [],
+        actions: [],
+        enabled: true,
+        false_positives: [],
+        max_signals: DEFAULT_MAX_SIGNALS,
+        tags: [],
+        threat: [],
+        throttle: null,
+        version: 1,
+        exceptions_list: [],
+        immutable: false,
+      };
+      expect(message.schema).toEqual(expected);
+    });
+
+    // Both can be defined, but if a data_view_id is defined, rule will use that one
+    test('Defined data_view_id and index does validate', () => {
+      const payload: ImportRulesSchema = {
+        rule_id: 'rule-1',
+        risk_score: 50,
+        description: 'some description',
+        from: 'now-5m',
+        to: 'now',
+        name: 'some-name',
+        severity: 'low',
+        type: 'query',
+        query: 'some query',
+        data_view_id: 'logs-*',
+        index: ['auditbeat-*'],
+        interval: '5m',
+      };
+
+      const decoded = importRulesSchema.decode(payload);
+      const checked = exactCheck(payload, decoded);
+      const message = pipe(checked, foldLeftRight);
+      expect(getPaths(left(message.errors))).toEqual([]);
+      const expected: ImportRulesSchemaDecoded = {
+        author: [],
+        severity_mapping: [],
+        risk_score_mapping: [],
+        rule_id: 'rule-1',
+        risk_score: 50,
+        description: 'some description',
+        from: 'now-5m',
+        to: 'now',
+        name: 'some-name',
+        severity: 'low',
+        type: 'query',
+        query: 'some query',
+        index: ['auditbeat-*'],
+        data_view_id: 'logs-*',
+        interval: '5m',
+        references: [],
+        actions: [],
+        enabled: true,
+        false_positives: [],
+        max_signals: DEFAULT_MAX_SIGNALS,
+        tags: [],
+        threat: [],
+        throttle: null,
+        version: 1,
+        exceptions_list: [],
+        immutable: false,
+      };
+      expect(message.schema).toEqual(expected);
+    });
+
+    test('data_view_id cannot be a number', () => {
+      const payload: Omit<ImportRulesSchema, 'data_view_id'> & { data_view_id: number } = {
+        ...getImportRulesSchemaMock(),
+        data_view_id: 5,
+      };
+
+      const decoded = importRulesSchema.decode(payload);
+      const checked = exactCheck(payload, decoded);
+      const message = pipe(checked, foldLeftRight);
+      expect(getPaths(left(message.errors))).toEqual([
+        'Invalid value "5" supplied to "data_view_id"',
+      ]);
+      expect(message.schema).toEqual({});
+    });
+  });
 });

x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.ts

@@ -53,6 +53,7 @@ import {
   filters,
   RuleId,
   index,
+  data_view_id,
   output_index,
   saved_id,
   timeline_id,
@@ -123,6 +124,7 @@ export const importRulesSchema = t.intersection([
       filters, // defaults to undefined if not set during decode
       from: DefaultFromString, // defaults to "now-6m" if not set during decode
       index, // defaults to undefined if not set during decode
+      data_view_id, // defaults to undefined if not set during decode
       immutable: OnlyFalseAllowed, // defaults to "false" if not set during decode
       interval: DefaultIntervalString, // defaults to "5m" if not set during decode
       query, // defaults to undefined if not set during decode

x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_schema.ts

@@ -38,6 +38,7 @@ import {
   anomaly_threshold,
   filters,
   index,
+  data_view_id,
   output_index,
   saved_id,
   timeline_id,
@@ -89,6 +90,7 @@ export const patchRulesSchema = t.exact(
     from,
     rule_id,
     index,
+    data_view_id,
     interval,
     query,
     language,

x-pack/plugins/security_solution/common/detection_engine/schemas/request/rule_schemas.mock.ts

@@ -29,6 +29,18 @@ export const getCreateRulesSchemaMock = (ruleId = 'rule-1'): QueryCreateSchema =
   rule_id: ruleId,
 });
 
+export const getCreateRulesSchemaMockWithDataView = (ruleId = 'rule-1'): QueryCreateSchema => ({
+  data_view_id: 'logs-*',
+  description: 'Detecting root and admin users',
+  name: 'Query with a rule id',
+  query: 'user.name: root or user.name: admin',
+  severity: 'high',
+  type: 'query',
+  risk_score: 55,
+  language: 'kuery',
+  rule_id: ruleId,
+});
+
 export const getCreateSavedQueryRulesSchemaMock = (ruleId = 'rule-1'): SavedQueryCreateSchema => ({
   description: 'Detecting root and admin users',
   name: 'Query with a rule id',
@@ -56,7 +68,7 @@ export const getCreateThreatMatchRulesSchemaMock = (
   language: 'kuery',
   rule_id: ruleId,
   threat_query: '*:*',
-  threat_index: ['list-index'],
+  threat_index: ['auditbeat-*'],
   threat_indicator_path: DEFAULT_INDICATOR_SOURCE_PATH,
   interval: '5m',
   from: 'now-6m',