diff --git a/x-pack/plugins/apm/server/plugin.ts b/x-pack/plugins/apm/server/plugin.ts index e617ed0510a8d..647330eade1f5 100644 --- a/x-pack/plugins/apm/server/plugin.ts +++ b/x-pack/plugins/apm/server/plugin.ts @@ -127,6 +127,10 @@ export class APMPlugin const getCoreStart = () => core.getStartServices().then(([coreStart]) => coreStart); + const alertsIndexPattern = ruleDataService.getFullAssetName( + 'observability-apm*' + ); + const initializeRuleDataTemplates = once(async () => { const componentTemplateName = ruleDataService.getFullAssetName( 'apm-mappings' @@ -164,15 +168,16 @@ export class APMPlugin await ruleDataService.createOrUpdateIndexTemplate({ name: ruleDataService.getFullAssetName('apm-index-template'), body: { - index_patterns: [ - ruleDataService.getFullAssetName('observability-apm*'), - ], + index_patterns: [alertsIndexPattern], composed_of: [ ruleDataService.getFullAssetName(TECHNICAL_COMPONENT_TEMPLATE_NAME), componentTemplateName, ], }, }); + await ruleDataService.updateIndexMappingsMatchingPattern( + alertsIndexPattern + ); }); // initialize eagerly diff --git a/x-pack/plugins/rule_registry/server/rule_data_client/index.ts b/x-pack/plugins/rule_registry/server/rule_data_client/index.ts index ffc926fc74b56..a9e559a6b1932 100644 --- a/x-pack/plugins/rule_registry/server/rule_data_client/index.ts +++ b/x-pack/plugins/rule_registry/server/rule_data_client/index.ts @@ -5,8 +5,6 @@ * 2.0. */ -import { isEmpty } from 'lodash'; -import type { estypes } from '@elastic/elasticsearch'; import { ResponseError } from '@elastic/elasticsearch/lib/errors'; import { IndexPatternsFetcher } from '../../../../../src/plugins/data/server'; import { RuleDataWriteDisabledError } from '../rule_data_plugin_service/errors'; @@ -100,7 +98,7 @@ export class RuleDataClient implements IRuleDataClient { response.body.items.length > 0 && response.body.items?.[0]?.index?.error?.type === 'index_not_found_exception' ) { - return this.createOrUpdateWriteTarget({ namespace }).then(() => { + return this.createWriteTargetIfNeeded({ namespace }).then(() => { return clusterClient.bulk(requestWithDefaultParameters); }); } @@ -113,7 +111,7 @@ export class RuleDataClient implements IRuleDataClient { }; } - async createOrUpdateWriteTarget({ namespace }: { namespace?: string }) { + async createWriteTargetIfNeeded({ namespace }: { namespace?: string }) { const alias = getNamespacedAlias({ alias: this.options.alias, namespace }); const clusterClient = await this.getClusterClient(); @@ -138,25 +136,10 @@ export class RuleDataClient implements IRuleDataClient { }); } catch (err) { // something might have created the index already, that sounds OK - if (err?.meta?.body?.type !== 'resource_already_exists_exception') { + if (err?.meta?.body?.error?.type !== 'resource_already_exists_exception') { throw err; } } } - - const { body: simulateResponse } = await clusterClient.transport.request({ - method: 'POST', - path: `/_index_template/_simulate_index/${concreteIndexName}`, - }); - - const mappings: estypes.MappingTypeMapping = simulateResponse.template.mappings; - - if (isEmpty(mappings)) { - throw new Error( - 'No mappings would be generated for this index, possibly due to failed/misconfigured bootstrapping' - ); - } - - await clusterClient.indices.putMapping({ index: `${alias}*`, body: mappings }); } } diff --git a/x-pack/plugins/rule_registry/server/rule_data_client/types.ts b/x-pack/plugins/rule_registry/server/rule_data_client/types.ts index 46a37abcd1ffc..3b90079ec5238 100644 --- a/x-pack/plugins/rule_registry/server/rule_data_client/types.ts +++ b/x-pack/plugins/rule_registry/server/rule_data_client/types.ts @@ -34,7 +34,7 @@ export interface RuleDataWriter { export interface IRuleDataClient { getReader(options?: { namespace?: string }): RuleDataReader; getWriter(options?: { namespace?: string }): RuleDataWriter; - createOrUpdateWriteTarget(options: { namespace?: string }): Promise; + createWriteTargetIfNeeded(options: { namespace?: string }): Promise; } export interface RuleDataClientConstructorOptions { diff --git a/x-pack/plugins/rule_registry/server/rule_data_plugin_service/index.ts b/x-pack/plugins/rule_registry/server/rule_data_plugin_service/index.ts index abb56f3102a4a..33ff5281147e1 100644 --- a/x-pack/plugins/rule_registry/server/rule_data_plugin_service/index.ts +++ b/x-pack/plugins/rule_registry/server/rule_data_plugin_service/index.ts @@ -7,6 +7,7 @@ import { ClusterPutComponentTemplate } from '@elastic/elasticsearch/api/requestParams'; import { estypes } from '@elastic/elasticsearch'; import { ElasticsearchClient, Logger } from 'kibana/server'; +import { get, isEmpty } from 'lodash'; import { technicalComponentTemplate } from '../../common/assets/component_templates/technical_component_template'; import { DEFAULT_ILM_POLICY_ID, @@ -18,6 +19,7 @@ import { defaultLifecyclePolicy } from '../../common/assets/lifecycle_policies/d import { ClusterPutComponentTemplateBody, PutIndexTemplateRequest } from '../../common/types'; import { RuleDataClient } from '../rule_data_client'; import { RuleDataWriteDisabledError } from './errors'; +import { incrementIndexName } from './utils'; const BOOTSTRAP_TIMEOUT = 60000; @@ -109,6 +111,14 @@ export class RuleDataPluginService { const clusterClient = await this.getClusterClient(); this.options.logger.debug(`Installing index template ${template.name}`); + const { body: simulateResponse } = await clusterClient.indices.simulateTemplate(template); + const mappings: estypes.MappingTypeMapping = simulateResponse.template.mappings; + + if (isEmpty(mappings)) { + throw new Error( + 'No mappings would be generated for this index, possibly due to failed/misconfigured bootstrapping' + ); + } return clusterClient.indices.putIndexTemplate(template); } @@ -120,6 +130,42 @@ export class RuleDataPluginService { return clusterClient.ilm.putLifecycle(policy); } + private async updateAliasWriteIndexMapping({ index, alias }: { index: string; alias: string }) { + const clusterClient = await this.getClusterClient(); + + const simulatedIndexMapping = await clusterClient.indices.simulateIndexTemplate({ + name: index, + }); + const simulatedMapping = get(simulatedIndexMapping, ['body', 'template', 'mappings']); + try { + await clusterClient.indices.putMapping({ + index, + body: simulatedMapping, + }); + return; + } catch (err) { + if (err.meta?.body?.error?.type !== 'illegal_argument_exception') { + this.options.logger.error(`Failed to PUT mapping for alias ${alias}: ${err.message}`); + return; + } + const newIndexName = incrementIndexName(index); + if (newIndexName == null) { + this.options.logger.error(`Failed to increment write index name for alias: ${alias}`); + return; + } + try { + await clusterClient.indices.rollover({ + alias, + new_index: newIndexName, + }); + } catch (e) { + if (e?.meta?.body?.error?.type !== 'resource_already_exists_exception') { + this.options.logger.error(`Failed to rollover index for alias ${alias}: ${e.message}`); + } + } + } + } + async createOrUpdateComponentTemplate( template: ClusterPutComponentTemplate ) { @@ -137,6 +183,25 @@ export class RuleDataPluginService { return this._createOrUpdateLifecyclePolicy(policy); } + async updateIndexMappingsMatchingPattern(pattern: string) { + await this.wait(); + const clusterClient = await this.getClusterClient(); + const { body: aliasesResponse } = await clusterClient.indices.getAlias({ index: pattern }); + const writeIndicesAndAliases: Array<{ index: string; alias: string }> = []; + Object.entries(aliasesResponse).forEach(([index, aliases]) => { + Object.entries(aliases.aliases).forEach(([aliasName, aliasProperties]) => { + if (aliasProperties.is_write_index) { + writeIndicesAndAliases.push({ index, alias: aliasName }); + } + }); + }); + await Promise.all( + writeIndicesAndAliases.map((indexAndAlias) => + this.updateAliasWriteIndexMapping(indexAndAlias) + ) + ); + } + isReady() { return this.signal.isReady(); } diff --git a/x-pack/plugins/rule_registry/server/rule_data_plugin_service/utils.test.ts b/x-pack/plugins/rule_registry/server/rule_data_plugin_service/utils.test.ts new file mode 100644 index 0000000000000..37212c46d46a0 --- /dev/null +++ b/x-pack/plugins/rule_registry/server/rule_data_plugin_service/utils.test.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { incrementIndexName } from './utils'; + +describe('incrementIndexName', () => { + it('should increment 000001 to 000002', () => { + const oldIndex = '.alerts-mock-000001'; + const newIndex = incrementIndexName(oldIndex); + expect(newIndex).toEqual('.alerts-mock-000002'); + }); + + it('should increment 000010 to 000011', () => { + const oldIndex = '.alerts-mock-000010'; + const newIndex = incrementIndexName(oldIndex); + expect(newIndex).toEqual('.alerts-mock-000011'); + }); + + it('should return undefined if oldIndex does not end in a number', () => { + const oldIndex = '.alerts-mock-string'; + const newIndex = incrementIndexName(oldIndex); + expect(newIndex).toEqual(undefined); + }); +}); diff --git a/x-pack/plugins/rule_registry/server/rule_data_plugin_service/utils.ts b/x-pack/plugins/rule_registry/server/rule_data_plugin_service/utils.ts new file mode 100644 index 0000000000000..aaab338ff858a --- /dev/null +++ b/x-pack/plugins/rule_registry/server/rule_data_plugin_service/utils.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export function incrementIndexName(oldIndex: string) { + const baseIndexString = oldIndex.slice(0, -6); + const newIndexNumber = Number(oldIndex.slice(-6)) + 1; + if (isNaN(newIndexNumber)) { + return undefined; + } + return baseIndexString + String(newIndexNumber).padStart(6, '0'); +} diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index 2f3850ff49f4c..4389b22611748 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -197,10 +197,10 @@ export class Plugin implements IPlugin { - const componentTemplateName = ruleDataService.getFullAssetName( - 'security-solution-mappings' - ); + const componentTemplateName = ruleDataService.getFullAssetName('security.alerts-mappings'); if (!ruleDataService.isWriteEnabled()) { return; @@ -219,9 +219,9 @@ export class Plugin implements IPlugin initializeRuleDataTemplatesPromise );