[Security Solution] Siem signals -> alerts as data field and index al…

…iases (elastic#106049)

* Add aliases mapping signal fields to alerts as data fields

* Add aliases mapping alerts as data fields to signal fields

* Replace siem signals templates per space and add AAD index aliases to siem signals indices

* Remove first version of new mapping json file

* Convert existing legacy siem-signals templates to new ES templates

* Catch 404 if siem signals templates were already updated

* Enhance error message when index exists but is not write index for alias

* Check if alias write index exists before creating new write index

* More robust write target creation logic

* Add RBAC required fields for AAD to siem signals indices

* Fix index name in index mapping update

* Throw errors if bulk retry fails or existing indices are not writeable

* Add new template to routes even without experimental rule registry flag enabled

* Check template version before updating template

* First pass at modifying routes to handle inserting field aliases

* Always insert field aliases when create_index_route is called

* Update snapshot test

* Remove template update logic from plugin setup

* Use aliases_version field to detect if aliases need update

* Fix bugs

* oops update snapshot

* Use internal user for PUT alias to fix perms issue

* Update comment

* Disable new resource creation if ruleRegistryEnabled

* Only attempt to add aliases if siem-signals index already exists

* Fix types, add aliases to aad indices, use package field names

* Undo adding aliases to AAD indices

* Remove unused import

* Update test and snapshot oops

* Filter out kibana.* fields from generated signals

* Update cypress test to account for new fields in table

* Properly handle space ids with dashes in them

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

packages/kbn-rule-data-utils/src/technical_field_names.ts

@@ -34,6 +34,7 @@ const ALERT_EVALUATION_THRESHOLD = `${ALERT_NAMESPACE}.evaluation.threshold` as
 const ALERT_EVALUATION_VALUE = `${ALERT_NAMESPACE}.evaluation.value` as const;
 const ALERT_ID = `${ALERT_NAMESPACE}.id` as const;
 const ALERT_OWNER = `${ALERT_NAMESPACE}.owner` as const;
+const ALERT_CONSUMERS = `${ALERT_NAMESPACE}.consumers` as const;
 const ALERT_PRODUCER = `${ALERT_NAMESPACE}.producer` as const;
 const ALERT_REASON = `${ALERT_NAMESPACE}.reason` as const;
 const ALERT_RISK_SCORE = `${ALERT_NAMESPACE}.risk_score` as const;
@@ -70,6 +71,7 @@ const ALERT_RULE_SEVERITY_MAPPING = `${ALERT_RULE_NAMESPACE}.severity_mapping` a
 const ALERT_RULE_TAGS = `${ALERT_RULE_NAMESPACE}.tags` as const;
 const ALERT_RULE_TO = `${ALERT_RULE_NAMESPACE}.to` as const;
 const ALERT_RULE_TYPE = `${ALERT_RULE_NAMESPACE}.type` as const;
+const ALERT_RULE_TYPE_ID = `${ALERT_RULE_NAMESPACE}.rule_type_id` as const;
 const ALERT_RULE_UPDATED_AT = `${ALERT_RULE_NAMESPACE}.updated_at` as const;
 const ALERT_RULE_UPDATED_BY = `${ALERT_RULE_NAMESPACE}.updated_by` as const;
 const ALERT_RULE_VERSION = `${ALERT_RULE_NAMESPACE}.version` as const;
@@ -99,6 +101,7 @@ const fields = {
   ALERT_EVALUATION_VALUE,
   ALERT_ID,
   ALERT_OWNER,
+  ALERT_CONSUMERS,
   ALERT_PRODUCER,
   ALERT_REASON,
   ALERT_RISK_SCORE,
@@ -124,6 +127,7 @@ const fields = {
   ALERT_RULE_TAGS,
   ALERT_RULE_TO,
   ALERT_RULE_TYPE,
+  ALERT_RULE_TYPE_ID,
   ALERT_RULE_UPDATED_AT,
   ALERT_RULE_UPDATED_BY,
   ALERT_RULE_VERSION,
@@ -151,6 +155,7 @@ export {
   ALERT_NAMESPACE,
   ALERT_RULE_NAMESPACE,
   ALERT_OWNER,
+  ALERT_CONSUMERS,
   ALERT_PRODUCER,
   ALERT_REASON,
   ALERT_RISK_SCORE,
@@ -179,6 +184,7 @@ export {
   ALERT_RULE_TAGS,
   ALERT_RULE_TO,
   ALERT_RULE_TYPE,
+  ALERT_RULE_TYPE_ID,
   ALERT_RULE_UPDATED_AT,
   ALERT_RULE_UPDATED_BY,
   ALERT_RULE_VERSION,

x-pack/plugins/rule_registry/server/index.ts

@@ -11,11 +11,11 @@ import { RuleRegistryPlugin } from './plugin';
 export * from './config';
 export type { RuleRegistryPluginSetupContract, RuleRegistryPluginStartContract } from './plugin';
 export type { RacRequestHandlerContext, RacApiRequestHandlerContext } from './types';
+export { RuleDataPluginService } from './rule_data_plugin_service';
 export { RuleDataClient } from './rule_data_client';
 export { IRuleDataClient } from './rule_data_client/types';
 export { getRuleData, RuleExecutorData } from './utils/get_rule_executor_data';
 export { createLifecycleRuleTypeFactory } from './utils/create_lifecycle_rule_type_factory';
-export { RuleDataPluginService } from './rule_data_plugin_service';
 export {
   LifecycleRuleExecutor,
   LifecycleAlertService,

x-pack/plugins/rule_registry/server/rule_data_client/index.ts

@@ -96,10 +96,20 @@ export class RuleDataClient implements IRuleDataClient {
           if (response.body.errors) {
             if (
               response.body.items.length > 0 &&
-              response.body.items?.[0]?.index?.error?.type === 'index_not_found_exception'
+              (response.body.items.every(
+                (item) => item.index?.error?.type === 'index_not_found_exception'
+              ) ||
+                response.body.items.every(
+                  (item) => item.index?.error?.type === 'illegal_argument_exception'
+                ))
             ) {
               return this.createWriteTargetIfNeeded({ namespace }).then(() => {
-                return clusterClient.bulk(requestWithDefaultParameters);
+                return clusterClient.bulk(requestWithDefaultParameters).then((retryResponse) => {
+                  if (retryResponse.body.errors) {
+                    throw new ResponseError(retryResponse);
+                  }
+                  return retryResponse;
+                });
               });
             }
             const error = new ResponseError(response);
@@ -116,13 +126,14 @@ export class RuleDataClient implements IRuleDataClient {
 
     const clusterClient = await this.getClusterClient();
 
-    const { body: aliasExists } = await clusterClient.indices.existsAlias({
-      name: alias,
+    const { body: indicesExist } = await clusterClient.indices.exists({
+      index: `${alias}-*`,
+      allow_no_indices: false,
     });
 
     const concreteIndexName = `${alias}-000001`;
 
-    if (!aliasExists) {
+    if (!indicesExist) {
       try {
         await clusterClient.indices.create({
           index: concreteIndexName,
@@ -135,11 +146,37 @@ export class RuleDataClient implements IRuleDataClient {
           },
         });
       } catch (err) {
-        // something might have created the index already, that sounds OK
-        if (err?.meta?.body?.error?.type !== 'resource_already_exists_exception') {
+        // If the index already exists and it's the write index for the alias,
+        // something else created it so suppress the error. If it's not the write
+        // index, that's bad, throw an error.
+        if (err?.meta?.body?.error?.type === 'resource_already_exists_exception') {
+          const { body: existingIndices } = await clusterClient.indices.get({
+            index: concreteIndexName,
+          });
+          if (!existingIndices[concreteIndexName]?.aliases?.[alias]?.is_write_index) {
+            throw Error(
+              `Attempted to create index: ${concreteIndexName} as the write index for alias: ${alias}, but the index already exists and is not the write index for the alias`
+            );
+          }
+        } else {
           throw err;
         }
       }
+    } else {
+      // If we find indices matching the pattern, then we expect one of them to be the write index for the alias.
+      // Throw an error if none of them are the write index.
+      const { body: aliasesResponse } = await clusterClient.indices.getAlias({
+        index: `${alias}-*`,
+      });
+      if (
+        !Object.entries(aliasesResponse).some(
+          ([_, aliasesObject]) => aliasesObject.aliases[alias]?.is_write_index
+        )
+      ) {
+        throw Error(
+          `Indices matching pattern ${alias}-* exist but none are set as the write index for alias ${alias}`
+        );
+      }
     }
   }
 }

x-pack/plugins/security_solution/cypress/ccs_integration/detection_alerts/alerts_details.spec.ts

@@ -58,7 +58,7 @@ describe('Alert details with unmapped fields', () => {
 
   it('Displays the unmapped field on the table', () => {
     const expectedUnmmappedField = {
-      row: 55,
+      row: 88,
       field: 'unmapped',
       text: 'This is the unmapped field',
     };

x-pack/plugins/security_solution/cypress/integration/detection_alerts/alerts_details.spec.ts

@@ -54,7 +54,7 @@ describe('Alert details with unmapped fields', () => {
 
   it('Displays the unmapped field on the table', () => {
     const expectedUnmmappedField = {
-      row: 55,
+      row: 88,
       field: 'unmapped',
       text: 'This is the unmapped field',
     };

x-pack/plugins/security_solution/server/client/client.ts

@@ -9,12 +9,15 @@ import { ConfigType } from '../config';
 
 export class AppClient {
   private readonly signalsIndex: string;
+  private readonly spaceId: string;
 
-  constructor(private spaceId: string, private config: ConfigType) {
+  constructor(_spaceId: string, private config: ConfigType) {
     const configuredSignalsIndex = this.config.signalsIndex;
 
-    this.signalsIndex = `${configuredSignalsIndex}-${this.spaceId}`;
+    this.signalsIndex = `${configuredSignalsIndex}-${_spaceId}`;
+    this.spaceId = _spaceId;
   }
 
   public getSignalsIndex = (): string => this.signalsIndex;
+  public getSpaceId = (): string => this.spaceId;
 }