[Security Solution][Detections] Update detection alert mappings to EC…

…S v1.10.0 (#101680) (#101847)

## Summary

* Grabbed the ECS mappings from [v1.10.0 tag]( https://github.com/elastic/ecs/blob/v1.10.0/generated/elasticsearch/7/template.json)
* Updated the fields that had `constant_keyword` to `keyword` since we do many to 1 of source to signals index
* Wrote a unit tests which tests to ensure we don't have any `constant_keyword` fields
* Updated the `SIGNALS_TEMPLATE_VERSION` version by an increment of 10.

This should mostly fix:
#101572

Since agents add their data into `_source` even though they have a `constant_keyword`. When agents do not include the values in `_source` we will have to merge `fields` into `_source` before copying which are still planning on doing before release.

### Checklist

- [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios

Co-authored-by: Frank Hassanabad <frank.hassanabad@elastic.co>

x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/ecs_mapping.json

@@ -1,10 +1,8 @@
 {
-  "index_patterns": [
-    "try-ecs-*"
-  ],
+  "index_patterns": ["try-ecs-*"],
   "mappings": {
     "_meta": {
-      "version": "1.9.0"
+      "version": "1.10.0"
     },
     "date_detection": false,
     "dynamic_templates": [
@@ -331,6 +329,19 @@
           }
         }
       },
+      "data_stream": {
+        "properties": {
+          "dataset": {
+            "type": "keyword"
+          },
+          "namespace": {
+            "type": "keyword"
+          },
+          "type": {
+            "type": "keyword"
+          }
+        }
+      },
       "destination": {
         "properties": {
           "address": {
@@ -1802,6 +1813,54 @@
           }
         }
       },
+      "orchestrator": {
+        "properties": {
+          "api_version": {
+            "ignore_above": 1024,
+            "type": "keyword"
+          },
+          "cluster": {
+            "properties": {
+              "name": {
+                "ignore_above": 1024,
+                "type": "keyword"
+              },
+              "url": {
+                "ignore_above": 1024,
+                "type": "keyword"
+              },
+              "version": {
+                "ignore_above": 1024,
+                "type": "keyword"
+              }
+            }
+          },
+          "namespace": {
+            "ignore_above": 1024,
+            "type": "keyword"
+          },
+          "organization": {
+            "ignore_above": 1024,
+            "type": "keyword"
+          },
+          "resource": {
+            "properties": {
+              "name": {
+                "ignore_above": 1024,
+                "type": "keyword"
+              },
+              "type": {
+                "ignore_above": 1024,
+                "type": "keyword"
+              }
+            }
+          },
+          "type": {
+            "ignore_above": 1024,
+            "type": "keyword"
+          }
+        }
+      },
       "organization": {
         "properties": {
           "id": {

x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.test.ts

@@ -43,6 +43,49 @@ describe('get_signals_template', () => {
     expect(template.settings.mapping.total_fields.limit).toBeGreaterThanOrEqual(10000);
   });
 
+  // If you see this test fail, you should track down any and all "constant_keyword" in your ecs_mapping.json and replace
+  // those with "keyword". The paths that fail in the array below will be something like:
+  // - Expected  - 1
+  // + Received  + 5
+  //
+  // - Array []
+  // + Array [
+  // +   "mappings.properties.data_stream.properties.dataset",
+  // +   "mappings.properties.data_stream.properties.namespace",
+  // +   "mappings.properties.data_stream.properties.type",
+  // + ]
+  // which means that in your ecs_mapping you have paths such as "mappings.properties.data_stream.properties.dataset" which
+  // contain a constant_keyword which needs to be replaced with a normal keyword instead.
+  //
+  // The reason why we deviate from ECS standards here is because when you have a many to 1 relationship where you have
+  // several different indexes with different "constant_keyword" values you cannot copy them over into a single "constant_keyword".
+  // Instead you have to use "keyword". This test was first introduced when ECS 1.10 came out and data_stream.* values which had
+  // "constant_keyword" fields and we needed to change those to be "keyword" instead.
+  test('it should NOT have any "constant_keyword" and instead those should be replaced with regular "keyword" in the mapping', () => {
+    const template = getSignalsTemplate('test-index');
+
+    // Small recursive function to find any values of "constant_keyword" and mark which fields it was found on and then error on those fields
+    // The matchers from jest such as jest.toMatchObject do not support recursion, so I have to write it here:
+    // https://github.com/facebook/jest/issues/2506
+    const recursiveConstantKeywordFound = (path: string, inputTemplate: object): string[] =>
+      Object.entries(inputTemplate).reduce<string[]>((accum, [key, innerValue]) => {
+        if (typeof innerValue === 'object') {
+          return [
+            ...accum,
+            ...recursiveConstantKeywordFound(path !== '' ? `${path}.${key}` : key, innerValue),
+          ];
+        } else {
+          if (key === 'type' && innerValue === 'constant_keyword') {
+            return [...accum, path];
+          } else {
+            return accum;
+          }
+        }
+      }, []);
+    const constantKeywordsFound = recursiveConstantKeywordFound('', template);
+    expect(constantKeywordsFound).toEqual([]);
+  });
+
   test('it should match snapshot', () => {
     const template = getSignalsTemplate('test-index');
     expect(template).toMatchSnapshot();

x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.ts

@@ -22,7 +22,7 @@ import otherMapping from './other_mappings.json';
   incremented by 10 in order to add "room" for the aforementioned patch
   release
 */
-export const SIGNALS_TEMPLATE_VERSION = 35;
+export const SIGNALS_TEMPLATE_VERSION = 45;
 export const MIN_EQL_RULE_INDEX_VERSION = 2;
 
 export const getSignalsTemplate = (index: string) => {

x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_ml.ts

@@ -23,6 +23,7 @@ import {
   deleteListsIndex,
   importFile,
 } from '../../../lists_api_integration/utils';
+import { SIGNALS_TEMPLATE_VERSION } from '../../../../plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template';
 
 // eslint-disable-next-line import/no-default-export
 export default ({ getService }: FtrProviderContext) => {
@@ -127,7 +128,7 @@ export default ({ getService }: FtrProviderContext) => {
         host: { name: ['mothra'] },
         event: { kind: 'signal' },
         signal: {
-          _meta: { version: 35 },
+          _meta: { version: SIGNALS_TEMPLATE_VERSION },
           parents: [
             {
               id:

x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_threat_matching.ts

@@ -27,6 +27,7 @@ import {
 
 import { getCreateThreatMatchRulesSchemaMock } from '../../../../plugins/security_solution/common/detection_engine/schemas/request/rule_schemas.mock';
 import { getThreatMatchingSchemaPartialMock } from '../../../../plugins/security_solution/common/detection_engine/schemas/response/rules_schema.mocks';
+import { SIGNALS_TEMPLATE_VERSION } from '../../../../plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template';
 
 const format = (value: unknown): string => JSON.stringify(value, null, 2);
 
@@ -201,7 +202,7 @@ export default ({ getService }: FtrProviderContext) => {
           },
           signal: {
             _meta: {
-              version: 35,
+              version: SIGNALS_TEMPLATE_VERSION,
             },
             ancestors: [
               {

x-pack/test/detection_engine_api_integration/security_and_spaces/tests/generating_signals.ts

@@ -853,7 +853,7 @@ export default ({ getService }: FtrProviderContext) => {
             'host.id': '8cc95778cce5407c809480e8e32ad76b',
             event: { kind: 'signal' },
             signal: {
-              _meta: { version: 35 },
+              _meta: { version: SIGNALS_TEMPLATE_VERSION },
               parents: [
                 {
                   depth: 0,
@@ -1011,7 +1011,7 @@ export default ({ getService }: FtrProviderContext) => {
             'host.id': '8cc95778cce5407c809480e8e32ad76b',
             event: { kind: 'signal' },
             signal: {
-              _meta: { version: 35 },
+              _meta: { version: SIGNALS_TEMPLATE_VERSION },
               parents: [
                 {
                   depth: 0,
@@ -1101,7 +1101,7 @@ export default ({ getService }: FtrProviderContext) => {
             'process.name': 'sshd',
             event: { kind: 'signal' },
             signal: {
-              _meta: { version: 35 },
+              _meta: { version: SIGNALS_TEMPLATE_VERSION },
               parents: [
                 {
                   depth: 0,