[Draft][Rule Registry] Create index template per namespace

x-pack/plugins/apm/server/plugin.ts

@@ -113,15 +113,11 @@ export class APMPlugin
     const getCoreStart = () =>
       core.getStartServices().then(([coreStart]) => coreStart);
 
-    const alertsIndexPattern = ruleDataService.getFullAssetName(
-      'observability-apm*'
+    const assetName = 'observability-apm';
+    const componentTemplateName = ruleDataService.getFullAssetName(
+      'apm-mappings'
     );
-
     const initializeRuleDataTemplates = once(async () => {
-      const componentTemplateName = ruleDataService.getFullAssetName(
-        'apm-mappings'
-      );
-
       if (!ruleDataService.isWriteEnabled()) {
         return;
       }
@@ -154,19 +150,7 @@ export class APMPlugin
         },
       });
 
-      await ruleDataService.createOrUpdateIndexTemplate({
-        name: ruleDataService.getFullAssetName('apm-index-template'),
-        body: {
-          index_patterns: [alertsIndexPattern],
-          composed_of: [
-            ruleDataService.getFullAssetName(TECHNICAL_COMPONENT_TEMPLATE_NAME),
-            componentTemplateName,
-          ],
-        },
-      });
-      await ruleDataService.updateIndexMappingsMatchingPattern(
-        alertsIndexPattern
-      );
+      await ruleDataService.updateIndexMappingsForAsset(assetName);
     });
 
     // initialize eagerly
@@ -178,8 +162,12 @@ export class APMPlugin
 
     const ruleDataClient = ruleDataService.getRuleDataClient(
       APM_SERVER_FEATURE_ID,
-      ruleDataService.getFullAssetName('observability-apm'),
-      () => initializeRuleDataTemplatesPromise
+      assetName,
+      () => initializeRuleDataTemplatesPromise,
+      [
+        ruleDataService.getFullAssetName(TECHNICAL_COMPONENT_TEMPLATE_NAME),
+        componentTemplateName,
+      ]
     );
 
     const resourcePlugins = mapValues(plugins, (value, key) => {

x-pack/plugins/infra/server/services/rules/rule_data_client.ts

@@ -25,13 +25,10 @@ export const createRuleDataClient = ({
   logger: Logger;
   ruleDataService: RuleRegistryPluginSetupContract['ruleDataService'];
 }) => {
+  const componentTemplateName = ruleDataService.getFullAssetName(
+    `${registrationContext}-mappings`
+  );
   const initializeRuleDataTemplates = once(async () => {
-    const componentTemplateName = ruleDataService.getFullAssetName(
-      `${registrationContext}-mappings`
-    );
-
-    const indexNamePattern = ruleDataService.getFullAssetName(`${registrationContext}*`);
-
     if (!ruleDataService.isWriteEnabled()) {
       return;
     }
@@ -48,7 +45,7 @@ export const createRuleDataClient = ({
       },
     });
 
-    await ruleDataService.createOrUpdateIndexTemplate({
+    /*await ruleDataService.createOrUpdateIndexTemplate({
       name: ruleDataService.getFullAssetName(registrationContext),
       body: {
         index_patterns: [indexNamePattern],
@@ -57,9 +54,9 @@ export const createRuleDataClient = ({
           componentTemplateName,
         ],
       },
-    });
+    });*/
 
-    await ruleDataService.updateIndexMappingsMatchingPattern(indexNamePattern);
+    await ruleDataService.updateIndexMappingsForAsset(registrationContext);
   });
 
   // initialize eagerly
@@ -69,7 +66,11 @@ export const createRuleDataClient = ({
 
   return ruleDataService.getRuleDataClient(
     ownerFeatureId,
-    ruleDataService.getFullAssetName(registrationContext),
-    () => initializeRuleDataTemplatesPromise
+    registrationContext,
+    () => initializeRuleDataTemplatesPromise,
+    [
+      ruleDataService.getFullAssetName(TECHNICAL_COMPONENT_TEMPLATE_NAME),
+      componentTemplateName,
+    ]
   );
 };

x-pack/plugins/observability/server/plugin.ts

@@ -100,10 +100,12 @@ export class ObservabilityPlugin implements Plugin<ObservabilityPluginSetup> {
 
     const start = () => core.getStartServices().then(([coreStart]) => coreStart);
 
+    // ???? This appears to be a stub
     const ruleDataClient = plugins.ruleRegistry.ruleDataService.getRuleDataClient(
       'observability',
-      plugins.ruleRegistry.ruleDataService.getFullAssetName(),
-      () => Promise.resolve()
+      '',
+      () => Promise.resolve(),
+      [],
     );
 
     registerRoutes({

x-pack/plugins/rule_registry/server/rule_data_client/index.ts

@@ -5,8 +5,10 @@
  * 2.0.
  */
 
+import { IndicesPutIndexTemplateRequest } from '@elastic/elasticsearch/api/types';
 import { ResponseError } from '@elastic/elasticsearch/lib/errors';
 import { IndexPatternsFetcher } from '../../../../../src/plugins/data/server';
+import { DEFAULT_ILM_POLICY_ID } from '../../common/assets';
 import { RuleDataWriteDisabledError } from '../rule_data_plugin_service/errors';
 import {
   IRuleDataClient,
@@ -111,24 +113,86 @@ export class RuleDataClient implements IRuleDataClient {
     };
   }
 
+  createNamespacedIndexTemplate({
+    primaryNamespacedAlias,
+    secondaryNamespacedAlias,
+    namespace,
+  }: {
+    primaryNamespacedAlias: string;
+    secondaryNamespacedAlias?: string;
+    namespace?: string;
+  }): IndicesPutIndexTemplateRequest {
+    return {
+      name: primaryNamespacedAlias,
+      body: {
+        index_patterns: [`${primaryNamespacedAlias}-*`],
+        composed_of: [...this.options.componentTemplateNames],
+        template: {
+          aliases:
+            secondaryNamespacedAlias != null
+              ? {
+                  [secondaryNamespacedAlias]: {
+                    is_write_index: false,
+                  },
+                }
+              : undefined,
+          settings: {
+            'index.lifecycle': {
+              name: DEFAULT_ILM_POLICY_ID,
+              // TODO: fix the types in the ES package, they don't include rollover_alias???
+              // @ts-expect-error
+              rollover_alias: primaryNamespacedAlias,
+            },
+          },
+        },
+        _meta: {
+          namespace,
+        },
+        // By setting the priority to namespace.length, we ensure that if one namespace is a prefix of another namespace
+        // then newly created indices will use the matching template with the *longest* namespace
+        priority: namespace?.length,
+      },
+    };
+  }
+
   async createWriteTargetIfNeeded({ namespace }: { namespace?: string }) {
-    const alias = getNamespacedAlias({ alias: this.options.alias, namespace });
+    const primaryNamespacedAlias = getNamespacedAlias({ alias: this.options.alias, namespace });
 
     const clusterClient = await this.getClusterClient();
 
     const { body: aliasExists } = await clusterClient.indices.existsAlias({
-      name: alias,
+      name: primaryNamespacedAlias,
     });
 
-    const concreteIndexName = `${alias}-000001`;
+    const concreteIndexName = `${primaryNamespacedAlias}-000001`;
 
     if (!aliasExists) {
+      const secondaryNamespacedAlias =
+        this.options.secondaryAlias != null
+          ? getNamespacedAlias({ alias: this.options.secondaryAlias, namespace })
+          : undefined;
+      const template = this.createNamespacedIndexTemplate({
+        primaryNamespacedAlias,
+        secondaryNamespacedAlias,
+        namespace,
+      });
+      // TODO: need a way to update this template if/when we decide to make changes to the
+      // built in index template. Probably do it as part of updateIndexMappingsForAsset?
+      // (Before upgrading any indices, find and upgrade all namespaced index templates - component templates
+      // will already have been upgraded by solutions or rule registry, in the case of technical/ECS templates)
+      // With the current structure, it's tricky because the index template creation
+      // depends on both the namespace and secondary alias, both of which are not currently available
+      // to updateIndexMappingsForAsset. We can make the secondary alias available since
+      // it's known at plugin startup time, but
+      // the namespace values can really only come from the existing templates that we're trying to update
+      // - maybe we want to store the namespace as a _meta field on the index template for easy retrieval
+      await clusterClient.indices.putIndexTemplate(template);
       try {
         await clusterClient.indices.create({
           index: concreteIndexName,
           body: {
             aliases: {
-              [alias]: {
+              [primaryNamespacedAlias]: {
                 is_write_index: true,
               },
             },

x-pack/plugins/rule_registry/server/rule_data_client/types.ts

@@ -51,6 +51,8 @@ export interface RuleDataClientConstructorOptions {
   getClusterClient: () => Promise<ElasticsearchClient>;
   isWriteEnabled: boolean;
   ready: () => Promise<void>;
+  componentTemplateNames: string[];
   alias: string;
   feature: ValidFeatureId;
+  secondaryAlias?: string;
 }

x-pack/plugins/rule_registry/server/rule_data_plugin_service/index.ts

@@ -195,10 +195,11 @@ export class RuleDataPluginService {
     return this._createOrUpdateLifecyclePolicy(policy);
   }
 
-  async updateIndexMappingsMatchingPattern(pattern: string) {
+  async updateIndexMappingsForAsset(assetName: string) {
     await this.wait();
     const clusterClient = await this.getClusterClient();
-    const { body: aliasesResponse } = await clusterClient.indices.getAlias({ index: pattern });
+    const pattern = `${this.getFullAssetName(assetName)}-*`;
+    const { body: aliasesResponse } = await clusterClient.indices.getAlias({ name: pattern });
     const writeIndicesAndAliases: Array<{ index: string; alias: string }> = [];
     Object.entries(aliasesResponse).forEach(([index, aliases]) => {
       Object.entries(aliases.aliases).forEach(([aliasName, aliasProperties]) => {
@@ -235,13 +236,21 @@ export class RuleDataPluginService {
     return [this.options.index, assetName].filter(Boolean).join('-');
   }
 
-  getRuleDataClient(feature: ValidFeatureId, alias: string, initialize: () => Promise<void>) {
+  getRuleDataClient(
+    feature: ValidFeatureId,
+    assetName: string,
+    initialize: () => Promise<void>,
+    componentTemplateNames: string[],
+    secondaryAlias?: string
+  ) {
     return new RuleDataClient({
-      alias,
+      alias: this.getFullAssetName(assetName),
       feature,
       getClusterClient: () => this.getClusterClient(),
       isWriteEnabled: this.isWriteEnabled(),
       ready: initialize,
+      componentTemplateNames,
+      secondaryAlias,
     });
   }
 }

x-pack/plugins/rule_registry/server/rule_data_plugin_service/rule_data_plugin_service.mock.ts

@@ -21,7 +21,7 @@ const createRuleDataPluginService = () => {
     createOrUpdateIndexTemplate: jest.fn(),
     createOrUpdateLifecyclePolicy: jest.fn(),
     getRuleDataClient: jest.fn(),
-    updateIndexMappingsMatchingPattern: jest.fn(),
+    updateIndexMappingsForAsset: jest.fn(),
   };
   return mocked;
 };

x-pack/plugins/security_solution/server/plugin.ts

@@ -203,14 +203,12 @@ export class Plugin implements IPlugin<PluginSetup, PluginStart, SetupPlugins, S
     let ruleDataClient: RuleDataClient | null = null;
     if (isRuleRegistryEnabled) {
       const { ruleDataService } = plugins.ruleRegistry;
-
-      const alertsIndexPattern = ruleDataService.getFullAssetName('security.alerts*');
-
+      const assetName = 'security.alerts';
+      const componentTemplateName = ruleDataService.getFullAssetName('security.alerts-mappings');
       const initializeRuleDataTemplates = once(async () => {
         if (!ruleDataService.isWriteEnabled()) {
           return;
         }
-        const componentTemplateName = ruleDataService.getFullAssetName('security.alerts-mappings');
 
         await ruleDataService.createOrUpdateComponentTemplate({
           name: componentTemplateName,
@@ -224,7 +222,7 @@ export class Plugin implements IPlugin<PluginSetup, PluginStart, SetupPlugins, S
           },
         });
 
-        await ruleDataService.createOrUpdateIndexTemplate({
+        /* await ruleDataService.createOrUpdateIndexTemplate({
           name: ruleDataService.getFullAssetName('security.alerts-index-template'),
           body: {
             index_patterns: [alertsIndexPattern],
@@ -234,8 +232,8 @@ export class Plugin implements IPlugin<PluginSetup, PluginStart, SetupPlugins, S
               componentTemplateName,
             ],
           },
-        });
-        await ruleDataService.updateIndexMappingsMatchingPattern(alertsIndexPattern);
+        });*/
+        await ruleDataService.updateIndexMappingsForAsset(assetName);
       });
 
       // initialize eagerly
@@ -247,8 +245,14 @@ export class Plugin implements IPlugin<PluginSetup, PluginStart, SetupPlugins, S
 
       ruleDataClient = ruleDataService.getRuleDataClient(
         SERVER_APP_ID,
-        indexAlias,
-        () => initializeRuleDataTemplatesPromise
+        assetName,
+        () => initializeRuleDataTemplatesPromise,
+        [
+          ruleDataService.getFullAssetName(TECHNICAL_COMPONENT_TEMPLATE_NAME),
+          ruleDataService.getFullAssetName(ECS_COMPONENT_TEMPLATE_NAME),
+          componentTemplateName,
+        ],
+        config.signalsIndex
       );
 
       // Register rule types via rule-registry

x-pack/plugins/uptime/server/plugin.ts

@@ -34,11 +34,9 @@ export class Plugin implements PluginType {
   public setup(core: CoreSetup, plugins: UptimeCorePlugins) {
     this.logger = this.initContext.logger.get();
     const { ruleDataService } = plugins.ruleRegistry;
-
+    const assetName = 'observability.synthetics';
+    const componentTemplateName = ruleDataService.getFullAssetName('synthetics-mappings');
     const ready = once(async () => {
-      const componentTemplateName = ruleDataService.getFullAssetName('synthetics-mappings');
-      const alertsIndexPattern = ruleDataService.getFullAssetName('observability.synthetics*');
-
       if (!ruleDataService.isWriteEnabled()) {
         return;
       }
@@ -55,18 +53,7 @@ export class Plugin implements PluginType {
         },
       });
 
-      await ruleDataService.createOrUpdateIndexTemplate({
-        name: ruleDataService.getFullAssetName('synthetics-index-template'),
-        body: {
-          index_patterns: [alertsIndexPattern],
-          composed_of: [
-            ruleDataService.getFullAssetName(TECHNICAL_COMPONENT_TEMPLATE_NAME),
-            componentTemplateName,
-          ],
-        },
-      });
-
-      await ruleDataService.updateIndexMappingsMatchingPattern(alertsIndexPattern);
+      await ruleDataService.updateIndexMappingsForAsset(assetName);
     });
 
     // initialize eagerly
@@ -76,8 +63,9 @@ export class Plugin implements PluginType {
 
     const ruleDataClient = ruleDataService.getRuleDataClient(
       'synthetics',
-      ruleDataService.getFullAssetName('observability.synthetics'),
-      () => initializeRuleDataTemplatesPromise
+      assetName,
+      () => initializeRuleDataTemplatesPromise,
+      [ruleDataService.getFullAssetName(TECHNICAL_COMPONENT_TEMPLATE_NAME), componentTemplateName]
     );
 
     initServerWithKibana(