[SIEM][Detection Engine] Adds a tags service and optimizes alert_id l…

…ookups

## Summary

* Adds a tags services for use by UI's that want to get a list of all the unique tags that are on all of the rules just like an aggregation
* Removes the horribly inefficient `alert_id` look up that was a full alert scan and instead uses an internal structure that it augments to the tags for fast `alert_id` look ups.
* Adds unit tests for the tags and internal structure tags
* Updates other unit tests

Usage for the UI:

```sh
GET /api/detection_engine/tags
```

or shell script:
```sh
./get_tags.sh
```

Returns:

```sh
[
  "tag_1",
  "tag_2"
]
```

Testing:
Ensure that the internal structure does not leak when doing any of these script/API calls
  * ./get_tags.sh
  * ./post_rule.sh ./rules/queries/query_with_tags.json
  * ./update_rule.sh ./rules/queries/query_with_tags.json
  * ./delete_rule.sh
  * ./find_rules.sh
  * ./find_rule_by_filter.sh "alert.attributes.enabled:%20true"
  * ./find_rule_by_filter.sh "alert.attributes.tags:tag_1"

Caveat:

You can do filter searches against tags that have the double underscore such as:

```sh
./find_rule_by_filter.sh "alert.attributes.tags:%20__*"
```

But that shouldn't be a big problem and more than likely no one will be naming something with double underscores.

### Checklist

Use ~~strikethroughs~~ to remove checklist items you don't feel are applicable to this PR.

~~- [ ] This was checked for cross-browser compatibility, [including a check against IE11](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#cross-browser-compatibility)~~

~~- [ ] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/master/packages/kbn-i18n/README.md)~~

~~- [ ] [Documentation](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#writing-documentation) was added for features that require explanation or tutorials~~

- [x] [Unit or functional tests](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#cross-browser-compatibility) were updated or added to match the most common scenarios

~~- [ ] This was checked for [keyboard-only and screenreader accessibility](https://developer.mozilla.org/en-US/docs/Learn/Tools_and_testing/Cross_browser_testing/Accessibility#Accessibility_testing_checklist)~~

### For maintainers

~~- [ ] This was checked for breaking API changes and was [labeled appropriately](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#release-notes-process)~~

- [x] This includes a feature addition or change that requires a release note and was [labeled appropriately](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#release-notes-process)

x-pack/legacy/plugins/siem/common/constants.ts

@@ -40,13 +40,21 @@ export const DEFAULT_TIMEPICKER_QUICK_RANGES = 'timepicker:quickRanges';
  */
 export const SIGNALS_ID = `${APP_ID}.signals`;
 
+/**
+ * Special internal structure for tags for signals. This is used
+ * to filter out tags that have internal structures within them.
+ */
+export const INTERNAL_IDENTIFIER = '__internal';
+export const INTERNAL_RULE_ID_KEY = `${INTERNAL_IDENTIFIER}_rule_id`;
+
 /**
  * Detection engine routes
  */
 export const DETECTION_ENGINE_URL = '/api/detection_engine';
 export const DETECTION_ENGINE_RULES_URL = `${DETECTION_ENGINE_URL}/rules`;
 export const DETECTION_ENGINE_PRIVILEGES_URL = `${DETECTION_ENGINE_URL}/privileges`;
 export const DETECTION_ENGINE_INDEX_URL = `${DETECTION_ENGINE_URL}/index`;
+export const DETECTION_ENGINE_TAGS_URL = `${DETECTION_ENGINE_URL}/tags`;
 
 /**
  * Default signals index key for kibana.dev.yml

x-pack/legacy/plugins/siem/server/kibana.index.ts

@@ -19,6 +19,7 @@ import { querySignalsRoute } from './lib/detection_engine/routes/signals/query_s
 import { ServerFacade } from './types';
 import { deleteIndexRoute } from './lib/detection_engine/routes/index/delete_index_route';
 import { isAlertExecutor } from './lib/detection_engine/signals/types';
+import { readTagsRoute } from './lib/detection_engine/routes/tags/read_tags_route';
 import { readPrivilegesRoute } from './lib/detection_engine/routes/privileges/read_privileges_route';
 
 const APP_ID = 'siem';
@@ -54,6 +55,8 @@ export const initServerWithKibana = (context: PluginInitializerContext, __legacy
   readIndexRoute(__legacy);
   deleteIndexRoute(__legacy);
 
+  // Detection Engine tags routes that have the REST endpoints of /api/detection_engine/tags
+  readTagsRoute(__legacy);
   // Privileges API to get the generic user privileges
   readPrivilegesRoute(__legacy);
 };

x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/__mocks__/request_responses.ts

@@ -12,6 +12,7 @@ import {
   DETECTION_ENGINE_SIGNALS_STATUS_URL,
   DETECTION_ENGINE_PRIVILEGES_URL,
   DETECTION_ENGINE_QUERY_SIGNALS_URL,
+  INTERNAL_RULE_ID_KEY,
 } from '../../../../../common/constants';
 import { RuleAlertType } from '../../rules/types';
 import { RuleAlertParamsRest } from '../../types';
@@ -171,7 +172,7 @@ export const createActionResult = (): ActionResult => ({
 export const getResult = (): RuleAlertType => ({
   id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd',
   name: 'Detect Root/Admin Users',
-  tags: [],
+  tags: [`${INTERNAL_RULE_ID_KEY}:rule-1`],
   alertTypeId: 'siem.signals',
   params: {
     description: 'Detecting root and admin users',

x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.test.ts

@@ -11,8 +11,10 @@ import {
   getIdError,
   transformFindAlertsOrError,
   transformOrError,
+  transformTags,
 } from './utils';
 import { getResult } from '../__mocks__/request_responses';
+import { INTERNAL_IDENTIFIER } from '../../../../../common/constants';
 
 describe('utils', () => {
   describe('transformAlertToRule', () => {
@@ -335,6 +337,53 @@ describe('utils', () => {
         type: 'query',
       });
     });
+
+    test('should work with tags but filter out any internal tags', () => {
+      const fullRule = getResult();
+      fullRule.tags = ['tag 1', 'tag 2', `${INTERNAL_IDENTIFIER}_some_other_value`];
+      const rule = transformAlertToRule(fullRule);
+      expect(rule).toEqual({
+        created_by: 'elastic',
+        description: 'Detecting root and admin users',
+        enabled: true,
+        false_positives: [],
+        from: 'now-6m',
+        id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd',
+        immutable: false,
+        index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'],
+        interval: '5m',
+        risk_score: 50,
+        rule_id: 'rule-1',
+        language: 'kuery',
+        max_signals: 100,
+        name: 'Detect Root/Admin Users',
+        output_index: '.siem-signals',
+        query: 'user.name: root or user.name: admin',
+        references: ['http://www.example.com', 'https://ww.example.com'],
+        severity: 'high',
+        updated_by: 'elastic',
+        tags: ['tag 1', 'tag 2'],
+        threats: [
+          {
+            framework: 'MITRE ATT&CK',
+            tactic: {
+              id: 'TA0040',
+              name: 'impact',
+              reference: 'https://attack.mitre.org/tactics/TA0040/',
+            },
+            techniques: [
+              {
+                id: 'T1499',
+                name: 'endpoint denial of service',
+                reference: 'https://attack.mitre.org/techniques/T1499/',
+              },
+            ],
+          },
+        ],
+        to: 'now',
+        type: 'query',
+      });
+    });
   });
 
   describe('getIdError', () => {
@@ -493,4 +542,21 @@ describe('utils', () => {
       expect((output as Boom).message).toEqual('Internal error transforming');
     });
   });
+
+  describe('transformTags', () => {
+    test('it returns tags that have no internal structures', () => {
+      expect(transformTags(['tag 1', 'tag 2'])).toEqual(['tag 1', 'tag 2']);
+    });
+
+    test('it returns empty tags given empty tags', () => {
+      expect(transformTags([])).toEqual([]);
+    });
+
+    test('it returns tags with internal tags stripped out', () => {
+      expect(transformTags(['tag 1', `${INTERNAL_IDENTIFIER}_some_value`, 'tag 2'])).toEqual([
+        'tag 1',
+        'tag 2',
+      ]);
+    });
+  });
 });

x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.ts

@@ -6,6 +6,7 @@
 
 import Boom from 'boom';
 import { pickBy } from 'lodash/fp';
+import { INTERNAL_IDENTIFIER } from '../../../../../common/constants';
 import { RuleAlertType, isAlertType, isAlertTypes } from '../../rules/types';
 import { OutputRuleAlertRest } from '../../types';
 
@@ -25,6 +26,10 @@ export const getIdError = ({
   }
 };
 
+export const transformTags = (tags: string[]): string[] => {
+  return tags.filter(tag => !tag.startsWith(INTERNAL_IDENTIFIER));
+};
+
 // Transforms the data but will remove any null or undefined it encounters and not include
 // those on the export
 export const transformAlertToRule = (alert: RuleAlertType): Partial<OutputRuleAlertRest> => {
@@ -51,7 +56,7 @@ export const transformAlertToRule = (alert: RuleAlertType): Partial<OutputRuleAl
     meta: alert.params.meta,
     severity: alert.params.severity,
     updated_by: alert.updatedBy,
-    tags: alert.tags,
+    tags: transformTags(alert.tags),
     to: alert.params.to,
     type: alert.params.type,
     threats: alert.params.threats,

x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/tags/read_tags_route.ts

@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import Hapi from 'hapi';
+import { isFunction } from 'lodash/fp';
+import { DETECTION_ENGINE_TAGS_URL } from '../../../../../common/constants';
+import { ServerFacade, RequestFacade } from '../../../../types';
+import { transformError } from '../utils';
+import { readTags } from '../../tags/read_tags';
+
+export const createReadTagsRoute: Hapi.ServerRoute = {
+  method: 'GET',
+  path: DETECTION_ENGINE_TAGS_URL,
+  options: {
+    tags: ['access:siem'],
+    validate: {
+      options: {
+        abortEarly: false,
+      },
+    },
+  },
+  async handler(request: RequestFacade, headers) {
+    const alertsClient = isFunction(request.getAlertsClient) ? request.getAlertsClient() : null;
+    const actionsClient = isFunction(request.getActionsClient) ? request.getActionsClient() : null;
+
+    if (!alertsClient || !actionsClient) {
+      return headers.response().code(404);
+    }
+
+    try {
+      const tags = await readTags({
+        alertsClient,
+      });
+      return tags;
+    } catch (err) {
+      return transformError(err);
+    }
+  },
+};
+
+export const readTagsRoute = (server: ServerFacade) => {
+  server.route(createReadTagsRoute);
+};

x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/add_rule_id_to_tags.test.ts

@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { addRuleIdToTags } from './add_rule_id_to_tags';
+import { INTERNAL_RULE_ID_KEY } from '../../../../common/constants';
+
+describe('add_rule_id_to_tags', () => {
+  test('it should add a rule id as an internal structure to a single tag', () => {
+    const tags = addRuleIdToTags(['tag 1'], 'rule-1');
+    expect(tags).toEqual(['tag 1', `${INTERNAL_RULE_ID_KEY}:rule-1`]);
+  });
+
+  test('it should add a rule id as an internal structure to two tags', () => {
+    const tags = addRuleIdToTags(['tag 1', 'tag 2'], 'rule-1');
+    expect(tags).toEqual(['tag 1', 'tag 2', `${INTERNAL_RULE_ID_KEY}:rule-1`]);
+  });
+
+  test('it should add a rule id as an internal structure with empty tags', () => {
+    const tags = addRuleIdToTags([], 'rule-1');
+    expect(tags).toEqual([`${INTERNAL_RULE_ID_KEY}:rule-1`]);
+  });
+
+  test('it should add not add an internal structure if rule id is undefined', () => {
+    const tags = addRuleIdToTags(['tag 1'], undefined);
+    expect(tags).toEqual(['tag 1']);
+  });
+
+  test('it should add not add an internal structure if rule id is null', () => {
+    const tags = addRuleIdToTags(['tag 1'], null);
+    expect(tags).toEqual(['tag 1']);
+  });
+});

x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/add_rule_id_to_tags.ts

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { INTERNAL_RULE_ID_KEY } from '../../../../common/constants';
+
+export const addRuleIdToTags = (tags: string[], ruleId: string | null | undefined): string[] => {
+  if (ruleId == null) {
+    return tags;
+  } else {
+    return [...tags, `${INTERNAL_RULE_ID_KEY}:${ruleId}`];
+  }
+};

x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/create_rules.ts

@@ -6,6 +6,7 @@
 
 import { SIGNALS_ID } from '../../../../common/constants';
 import { RuleParams } from './types';
+import { addRuleIdToTags } from './add_rule_id_to_tags';
 
 export const createRules = async ({
   alertsClient,
@@ -37,7 +38,7 @@ export const createRules = async ({
   return alertsClient.create({
     data: {
       name,
-      tags,
+      tags: addRuleIdToTags(tags, ruleId),
       alertTypeId: SIGNALS_ID,
       params: {
         description,