From 6a76e342d5a0b2a1c202ba6dc7fc8bf76ed1d350 Mon Sep 17 00:00:00 2001 From: machadoum Date: Mon, 15 Jul 2024 16:11:49 +0200 Subject: [PATCH 1/5] Add support for extending mappings and ingest pipeline --- .../common/constants_entities.ts | 4 ---- .../entity_manager/common/helpers.ts | 19 +++++++++++++++ .../create_and_install_ingest_pipeline.ts | 4 ++++ .../lib/entities/delete_index_template.ts | 23 +++++++++++++++++++ .../generate_history_processors.ts | 12 ++++++++++ .../generate_latest_processors.ts | 12 ++++++++++ .../lib/entities/install_entity_definition.ts | 16 +++++++++++++ .../entities/uninstall_entity_definition.ts | 8 +++++++ .../server/lib/manage_index_templates.ts | 15 ++++++++++++ .../entity_manager/server/plugin.ts | 21 ++--------------- .../server/templates/components/helpers.ts | 16 +++++++++++++ .../templates/entities_history_template.ts | 13 +++++++---- .../templates/entities_latest_template.ts | 13 +++++++---- 13 files changed, 145 insertions(+), 31 deletions(-) create mode 100644 x-pack/plugins/observability_solution/entity_manager/common/helpers.ts create mode 100644 x-pack/plugins/observability_solution/entity_manager/server/lib/entities/delete_index_template.ts create mode 100644 x-pack/plugins/observability_solution/entity_manager/server/templates/components/helpers.ts diff --git a/x-pack/plugins/observability_solution/entity_manager/common/constants_entities.ts b/x-pack/plugins/observability_solution/entity_manager/common/constants_entities.ts index 9892c757fad3d..4366dd848438b 100644 --- a/x-pack/plugins/observability_solution/entity_manager/common/constants_entities.ts +++ b/x-pack/plugins/observability_solution/entity_manager/common/constants_entities.ts @@ -18,8 +18,6 @@ export const ENTITY_EVENT_COMPONENT_TEMPLATE_V1 = // History constants export const ENTITY_HISTORY = 'history' as const; -export const ENTITY_HISTORY_INDEX_TEMPLATE_V1 = - `${ENTITY_BASE_PREFIX}_${ENTITY_SCHEMA_VERSION_V1}_${ENTITY_HISTORY}_index_template` as const; export const ENTITY_HISTORY_BASE_COMPONENT_TEMPLATE_V1 = `${ENTITY_BASE_PREFIX}_${ENTITY_SCHEMA_VERSION_V1}_${ENTITY_HISTORY}_base` as const; export const ENTITY_HISTORY_PREFIX_V1 = @@ -29,8 +27,6 @@ export const ENTITY_HISTORY_INDEX_PREFIX_V1 = // Latest constants export const ENTITY_LATEST = 'latest' as const; -export const ENTITY_LATEST_INDEX_TEMPLATE_V1 = - `${ENTITY_BASE_PREFIX}_${ENTITY_SCHEMA_VERSION_V1}_${ENTITY_LATEST}_index_template` as const; export const ENTITY_LATEST_BASE_COMPONENT_TEMPLATE_V1 = `${ENTITY_BASE_PREFIX}_${ENTITY_SCHEMA_VERSION_V1}_${ENTITY_LATEST}_base` as const; export const ENTITY_LATEST_PREFIX_V1 = diff --git a/x-pack/plugins/observability_solution/entity_manager/common/helpers.ts b/x-pack/plugins/observability_solution/entity_manager/common/helpers.ts new file mode 100644 index 0000000000000..97a6317fee283 --- /dev/null +++ b/x-pack/plugins/observability_solution/entity_manager/common/helpers.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + ENTITY_BASE_PREFIX, + ENTITY_HISTORY, + ENTITY_LATEST, + ENTITY_SCHEMA_VERSION_V1, +} from './constants_entities'; + +export const getEntityHistoryIndexTemplateV1 = (definitionId: string) => + `${ENTITY_BASE_PREFIX}_${ENTITY_SCHEMA_VERSION_V1}_${ENTITY_HISTORY}_${definitionId}_index_template` as const; + +export const getEntityLatestIndexTemplateV1 = (definitionId: string) => + `${ENTITY_BASE_PREFIX}_${ENTITY_SCHEMA_VERSION_V1}_${ENTITY_LATEST}_${definitionId}_index_template` as const; diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/create_and_install_ingest_pipeline.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/create_and_install_ingest_pipeline.ts index 360f416cd5a00..9908149f624c6 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/create_and_install_ingest_pipeline.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/create_and_install_ingest_pipeline.ts @@ -27,6 +27,8 @@ export async function createAndInstallHistoryIngestPipeline( () => esClient.ingest.putPipeline({ id: historyId, + // TODO The ingest pipelines include a pipeline processor that calls out the pipelines named @custom and -history@custom or -latest@custom if they exists + // ignore_missing_pipeline is set to true to avoid errors when the pipeline does not exist processors: historyProcessors, _meta: { definitionVersion: definition.version, @@ -54,6 +56,8 @@ export async function createAndInstallLatestIngestPipeline( () => esClient.ingest.putPipeline({ id: latestId, + // TODO The ingest pipelines include a pipeline processor that calls out the pipelines named @custom and -history@custom or -latest@custom if they exists + // ignore_missing_pipeline is set to true to avoid errors when the pipeline does not exist processors: latestProcessors, _meta: { definitionVersion: definition.version, diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/delete_index_template.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/delete_index_template.ts new file mode 100644 index 0000000000000..ed55af0353183 --- /dev/null +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/delete_index_template.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient, Logger } from '@kbn/core/server'; +import { EntityDefinition } from '@kbn/entities-schema'; + +export async function deleteIndices( + esClient: ElasticsearchClient, + definition: EntityDefinition, + logger: Logger +) { + try { + const latestPipelineId = generateLatestIngestPipelineId(definition); + await retryTransientEsErrors(() => esClient.searchTemplate.delete({})); + } catch (e) { + logger.error(`Unable to delete latest ingest pipeline [${definition.id}].`); + throw e; + } +} diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.ts index 45ee008f9c6b5..56766b7f8ef21 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.ts @@ -163,5 +163,17 @@ export function generateHistoryProcessors(definition: EntityDefinition) { date_formats: ['UNIX_MS', 'ISO8601', "yyyy-MM-dd'T'HH:mm:ss.SSSXX"], }, }, + { + pipeline: { + ignore_missing_pipeline: true, + name: `${definition.id}@custom`, + }, + }, + { + pipeline: { + ignore_missing_pipeline: true, + name: `${definition.id}-history@custom`, + }, + }, ]; } diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_latest_processors.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_latest_processors.ts index 22b2ac19775a1..2dc2d3795ec5d 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_latest_processors.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_latest_processors.ts @@ -122,5 +122,17 @@ export function generateLatestProcessors(definition: EntityDefinition) { value: `${generateLatestIndexName(definition)}`, }, }, + { + pipeline: { + ignore_missing_pipeline: true, + name: `${definition.id}@custom`, + }, + }, + { + pipeline: { + ignore_missing_pipeline: true, + name: `${definition.id}-latest@custom`, + }, + }, ]; } diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/install_entity_definition.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/install_entity_definition.ts index 980c743575fe2..bfe41de0f62f6 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/install_entity_definition.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/install_entity_definition.ts @@ -28,6 +28,9 @@ import { stopAndDeleteLatestTransform, } from './stop_and_delete_transform'; import { uninstallEntityDefinition } from './uninstall_entity_definition'; +import { upsertTemplate } from '../manage_index_templates'; +import { getEntitiesLatestIndexTemplateConfig } from '../../templates/entities_latest_template'; +import { getEntitiesHistoryIndexTemplateConfig } from '../../templates/entities_history_template'; export interface InstallDefinitionParams { esClient: ElasticsearchClient; @@ -62,6 +65,19 @@ export async function installEntityDefinition({ const entityDefinition = await saveEntityDefinition(soClient, definition); installState.definition = true; + // create scoped index template + await upsertTemplate({ + esClient, + logger, + template: getEntitiesHistoryIndexTemplateConfig(definition.id), + }); + + await upsertTemplate({ + esClient, + logger, + template: getEntitiesLatestIndexTemplateConfig(definition.id), + }); + // install ingest pipelines logger.debug(`Installing ingest pipelines for definition ${definition.id}`); await createAndInstallHistoryIngestPipeline(esClient, entityDefinition, logger); diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/uninstall_entity_definition.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/uninstall_entity_definition.ts index 8642ebafa904b..9b8685031642a 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/uninstall_entity_definition.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/uninstall_entity_definition.ts @@ -9,6 +9,10 @@ import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; import { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server'; import { EntityDefinition } from '@kbn/entities-schema'; import { Logger } from '@kbn/logging'; +import { + getEntityHistoryIndexTemplateV1, + getEntityLatestIndexTemplateV1, +} from '../../../common/helpers'; import { deleteEntityDefinition } from './delete_entity_definition'; import { deleteIndices } from './delete_index'; import { deleteHistoryIngestPipeline, deleteLatestIngestPipeline } from './delete_ingest_pipeline'; @@ -17,6 +21,7 @@ import { stopAndDeleteHistoryTransform, stopAndDeleteLatestTransform, } from './stop_and_delete_transform'; +import { deleteTemplate } from '../manage_index_templates'; export async function uninstallEntityDefinition({ definition, @@ -36,6 +41,9 @@ export async function uninstallEntityDefinition({ await deleteHistoryIngestPipeline(esClient, definition, logger); await deleteLatestIngestPipeline(esClient, definition, logger); await deleteEntityDefinition(soClient, definition, logger); + await deleteTemplate({ esClient, logger, name: getEntityHistoryIndexTemplateV1(definition.id) }); + await deleteTemplate({ esClient, logger, name: getEntityLatestIndexTemplateV1(definition.id) }); + if (deleteData) { await deleteIndices(esClient, definition, logger); } diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/manage_index_templates.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/manage_index_templates.ts index 0f73ba7715bfd..a87fc8afba69a 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/manage_index_templates.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/manage_index_templates.ts @@ -23,6 +23,12 @@ interface ComponentManagementOptions { logger: Logger; } +interface DeleteTemplateOptions { + esClient: ElasticsearchClient; + name: string; + logger: Logger; +} + export async function upsertTemplate({ esClient, template, logger }: TemplateManagementOptions) { try { await esClient.indices.putIndexTemplate(template); @@ -37,6 +43,15 @@ export async function upsertTemplate({ esClient, template, logger }: TemplateMan logger.debug(() => `Entity manager index template: ${JSON.stringify(template)}`); } +export async function deleteTemplate({ esClient, name, logger }: DeleteTemplateOptions) { + try { + await esClient.indices.deleteIndexTemplate({ name }); + } catch (error: any) { + logger.error(`Error deleting entity manager index template: ${error.message}`); + return; + } +} + export async function upsertComponent({ esClient, component, logger }: ComponentManagementOptions) { try { await esClient.cluster.putComponentTemplate(component); diff --git a/x-pack/plugins/observability_solution/entity_manager/server/plugin.ts b/x-pack/plugins/observability_solution/entity_manager/server/plugin.ts index 3a51988841766..80154149e2402 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/plugin.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/plugin.ts @@ -14,7 +14,7 @@ import { PluginConfigDescriptor, Logger, } from '@kbn/core/server'; -import { upsertComponent, upsertTemplate } from './lib/manage_index_templates'; +import { upsertComponent } from './lib/manage_index_templates'; import { setupRoutes } from './routes'; import { EntityManagerPluginSetupDependencies, @@ -27,8 +27,6 @@ import { entityDefinition, EntityDiscoveryApiKeyType } from './saved_objects'; import { entitiesEntityComponentTemplateConfig } from './templates/components/entity'; import { entitiesLatestBaseComponentTemplateConfig } from './templates/components/base_latest'; import { entitiesHistoryBaseComponentTemplateConfig } from './templates/components/base_history'; -import { entitiesHistoryIndexTemplateConfig } from './templates/entities_history_template'; -import { entitiesLatestIndexTemplateConfig } from './templates/entities_latest_template'; export type EntityManagerServerPluginSetup = ReturnType; export type EntityManagerServerPluginStart = ReturnType; @@ -113,22 +111,7 @@ export class EntityManagerServerPlugin logger: this.logger, component: entitiesEntityComponentTemplateConfig, }), - ]) - .then(() => - upsertTemplate({ - esClient, - logger: this.logger, - template: entitiesHistoryIndexTemplateConfig, - }) - ) - .then(() => - upsertTemplate({ - esClient, - logger: this.logger, - template: entitiesLatestIndexTemplateConfig, - }) - ) - .catch(() => {}); + ]).catch(() => {}); return {}; } diff --git a/x-pack/plugins/observability_solution/entity_manager/server/templates/components/helpers.ts b/x-pack/plugins/observability_solution/entity_manager/server/templates/components/helpers.ts new file mode 100644 index 0000000000000..76cc837b0f7bf --- /dev/null +++ b/x-pack/plugins/observability_solution/entity_manager/server/templates/components/helpers.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const getCustomLatestTemplateComponents = (definitionId: string) => [ + `${definitionId}@custom`, + `${definitionId}-history@custom`, +]; + +export const getCustomHistoryTemplateComponents = (definitionId: string) => [ + `${definitionId}@custom`, + `${definitionId}-latest@custom`, +]; diff --git a/x-pack/plugins/observability_solution/entity_manager/server/templates/entities_history_template.ts b/x-pack/plugins/observability_solution/entity_manager/server/templates/entities_history_template.ts index d5ceeecd44828..ab2bac9e492eb 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/templates/entities_history_template.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/templates/entities_history_template.ts @@ -6,26 +6,31 @@ */ import { IndicesPutIndexTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; +import { getEntityHistoryIndexTemplateV1 } from '../../common/helpers'; import { ENTITY_ENTITY_COMPONENT_TEMPLATE_V1, ENTITY_EVENT_COMPONENT_TEMPLATE_V1, ENTITY_HISTORY_BASE_COMPONENT_TEMPLATE_V1, ENTITY_HISTORY_INDEX_PREFIX_V1, - ENTITY_HISTORY_INDEX_TEMPLATE_V1, } from '../../common/constants_entities'; +import { getCustomHistoryTemplateComponents } from './components/helpers'; -export const entitiesHistoryIndexTemplateConfig: IndicesPutIndexTemplateRequest = { - name: ENTITY_HISTORY_INDEX_TEMPLATE_V1, +export const getEntitiesHistoryIndexTemplateConfig = ( + definitionId: string +): IndicesPutIndexTemplateRequest => ({ + name: getEntityHistoryIndexTemplateV1(definitionId), _meta: { description: "Index template for indices managed by the Elastic Entity Model's entity discovery framework for the history dataset", ecs_version: '8.0.0', managed: true, }, + ignore_missing_component_templates: getCustomHistoryTemplateComponents(definitionId), composed_of: [ ENTITY_HISTORY_BASE_COMPONENT_TEMPLATE_V1, ENTITY_ENTITY_COMPONENT_TEMPLATE_V1, ENTITY_EVENT_COMPONENT_TEMPLATE_V1, + ...getCustomHistoryTemplateComponents(definitionId), ], index_patterns: [`${ENTITY_HISTORY_INDEX_PREFIX_V1}.*`], priority: 1, @@ -72,4 +77,4 @@ export const entitiesHistoryIndexTemplateConfig: IndicesPutIndexTemplateRequest }, }, }, -}; +}); diff --git a/x-pack/plugins/observability_solution/entity_manager/server/templates/entities_latest_template.ts b/x-pack/plugins/observability_solution/entity_manager/server/templates/entities_latest_template.ts index f601c3aa9d57d..b6911f39bf75a 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/templates/entities_latest_template.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/templates/entities_latest_template.ts @@ -6,26 +6,31 @@ */ import { IndicesPutIndexTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; +import { getEntityLatestIndexTemplateV1 } from '../../common/helpers'; import { ENTITY_ENTITY_COMPONENT_TEMPLATE_V1, ENTITY_EVENT_COMPONENT_TEMPLATE_V1, ENTITY_LATEST_BASE_COMPONENT_TEMPLATE_V1, ENTITY_LATEST_INDEX_PREFIX_V1, - ENTITY_LATEST_INDEX_TEMPLATE_V1, } from '../../common/constants_entities'; +import { getCustomLatestTemplateComponents } from './components/helpers'; -export const entitiesLatestIndexTemplateConfig: IndicesPutIndexTemplateRequest = { - name: ENTITY_LATEST_INDEX_TEMPLATE_V1, +export const getEntitiesLatestIndexTemplateConfig = ( + definitionId: string +): IndicesPutIndexTemplateRequest => ({ + name: getEntityLatestIndexTemplateV1(definitionId), _meta: { description: "Index template for indices managed by the Elastic Entity Model's entity discovery framework for the latest dataset", ecs_version: '8.0.0', managed: true, }, + ignore_missing_component_templates: getCustomLatestTemplateComponents(definitionId), composed_of: [ ENTITY_LATEST_BASE_COMPONENT_TEMPLATE_V1, ENTITY_ENTITY_COMPONENT_TEMPLATE_V1, ENTITY_EVENT_COMPONENT_TEMPLATE_V1, + ...getCustomLatestTemplateComponents(definitionId), ], index_patterns: [`${ENTITY_LATEST_INDEX_PREFIX_V1}.*`], priority: 1, @@ -72,4 +77,4 @@ export const entitiesLatestIndexTemplateConfig: IndicesPutIndexTemplateRequest = }, }, }, -}; +}); From 3ffcb515b6722cbfaa1e4be77b96b92b60672515 Mon Sep 17 00:00:00 2001 From: machadoum Date: Tue, 16 Jul 2024 11:09:15 +0200 Subject: [PATCH 2/5] Add support for @platfrom ingest pipeline and template component --- .../entities/create_and_install_ingest_pipeline.ts | 4 ---- .../ingest_pipeline/generate_history_processors.ts | 12 ++++++++++++ .../ingest_pipeline/generate_latest_processors.ts | 12 ++++++++++++ .../server/templates/components/helpers.ts | 4 ++++ 4 files changed, 28 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/create_and_install_ingest_pipeline.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/create_and_install_ingest_pipeline.ts index 9908149f624c6..360f416cd5a00 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/create_and_install_ingest_pipeline.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/create_and_install_ingest_pipeline.ts @@ -27,8 +27,6 @@ export async function createAndInstallHistoryIngestPipeline( () => esClient.ingest.putPipeline({ id: historyId, - // TODO The ingest pipelines include a pipeline processor that calls out the pipelines named @custom and -history@custom or -latest@custom if they exists - // ignore_missing_pipeline is set to true to avoid errors when the pipeline does not exist processors: historyProcessors, _meta: { definitionVersion: definition.version, @@ -56,8 +54,6 @@ export async function createAndInstallLatestIngestPipeline( () => esClient.ingest.putPipeline({ id: latestId, - // TODO The ingest pipelines include a pipeline processor that calls out the pipelines named @custom and -history@custom or -latest@custom if they exists - // ignore_missing_pipeline is set to true to avoid errors when the pipeline does not exist processors: latestProcessors, _meta: { definitionVersion: definition.version, diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.ts index 56766b7f8ef21..b2a284604f9a8 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.ts @@ -169,11 +169,23 @@ export function generateHistoryProcessors(definition: EntityDefinition) { name: `${definition.id}@custom`, }, }, + { + pipeline: { + ignore_missing_pipeline: true, + name: `${definition.id}@platform`, + }, + }, { pipeline: { ignore_missing_pipeline: true, name: `${definition.id}-history@custom`, }, }, + { + pipeline: { + ignore_missing_pipeline: true, + name: `${definition.id}-history@platform`, + }, + }, ]; } diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_latest_processors.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_latest_processors.ts index 2dc2d3795ec5d..c3c074814ffa1 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_latest_processors.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_latest_processors.ts @@ -128,11 +128,23 @@ export function generateLatestProcessors(definition: EntityDefinition) { name: `${definition.id}@custom`, }, }, + { + pipeline: { + ignore_missing_pipeline: true, + name: `${definition.id}@platform`, + }, + }, { pipeline: { ignore_missing_pipeline: true, name: `${definition.id}-latest@custom`, }, }, + { + pipeline: { + ignore_missing_pipeline: true, + name: `${definition.id}-latest@platform`, + }, + }, ]; } diff --git a/x-pack/plugins/observability_solution/entity_manager/server/templates/components/helpers.ts b/x-pack/plugins/observability_solution/entity_manager/server/templates/components/helpers.ts index 76cc837b0f7bf..e26828faacd05 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/templates/components/helpers.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/templates/components/helpers.ts @@ -7,10 +7,14 @@ export const getCustomLatestTemplateComponents = (definitionId: string) => [ `${definitionId}@custom`, + `${definitionId}@platform`, `${definitionId}-history@custom`, + `${definitionId}-history@platform`, ]; export const getCustomHistoryTemplateComponents = (definitionId: string) => [ `${definitionId}@custom`, + `${definitionId}@platform`, `${definitionId}-latest@custom`, + `${definitionId}-latest@platform`, ]; From d37db783d0958e5dab72bb7babcc3262903883db Mon Sep 17 00:00:00 2001 From: machadoum Date: Tue, 16 Jul 2024 11:40:51 +0200 Subject: [PATCH 3/5] Add proposed acceptance criteria --- .../ingest_pipeline/generate_history_processors.ts | 9 +++++---- .../ingest_pipeline/generate_latest_processors.ts | 9 +++++---- .../server/templates/components/helpers.ts | 8 ++++---- .../server/templates/entities_history_template.ts | 3 ++- 4 files changed, 16 insertions(+), 13 deletions(-) diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.ts index b2a284604f9a8..43f18b2b81bf0 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.ts @@ -166,25 +166,26 @@ export function generateHistoryProcessors(definition: EntityDefinition) { { pipeline: { ignore_missing_pipeline: true, - name: `${definition.id}@custom`, + name: `${definition.id}@platform`, }, }, { pipeline: { ignore_missing_pipeline: true, - name: `${definition.id}@platform`, + name: `${definition.id}-history@platform`, }, }, + { pipeline: { ignore_missing_pipeline: true, - name: `${definition.id}-history@custom`, + name: `${definition.id}@custom`, }, }, { pipeline: { ignore_missing_pipeline: true, - name: `${definition.id}-history@platform`, + name: `${definition.id}-history@custom`, }, }, ]; diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_latest_processors.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_latest_processors.ts index c3c074814ffa1..b9a18e8b7a2b6 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_latest_processors.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_latest_processors.ts @@ -125,25 +125,26 @@ export function generateLatestProcessors(definition: EntityDefinition) { { pipeline: { ignore_missing_pipeline: true, - name: `${definition.id}@custom`, + name: `${definition.id}@platform`, }, }, { pipeline: { ignore_missing_pipeline: true, - name: `${definition.id}@platform`, + name: `${definition.id}-latest@platform`, }, }, { pipeline: { ignore_missing_pipeline: true, - name: `${definition.id}-latest@custom`, + name: `${definition.id}@custom`, }, }, + { pipeline: { ignore_missing_pipeline: true, - name: `${definition.id}-latest@platform`, + name: `${definition.id}-latest@custom`, }, }, ]; diff --git a/x-pack/plugins/observability_solution/entity_manager/server/templates/components/helpers.ts b/x-pack/plugins/observability_solution/entity_manager/server/templates/components/helpers.ts index e26828faacd05..ae2c16f234685 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/templates/components/helpers.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/templates/components/helpers.ts @@ -6,15 +6,15 @@ */ export const getCustomLatestTemplateComponents = (definitionId: string) => [ - `${definitionId}@custom`, `${definitionId}@platform`, - `${definitionId}-history@custom`, `${definitionId}-history@platform`, + `${definitionId}@custom`, + `${definitionId}-history@custom`, ]; export const getCustomHistoryTemplateComponents = (definitionId: string) => [ - `${definitionId}@custom`, `${definitionId}@platform`, - `${definitionId}-latest@custom`, `${definitionId}-latest@platform`, + `${definitionId}@custom`, + `${definitionId}-latest@custom`, ]; diff --git a/x-pack/plugins/observability_solution/entity_manager/server/templates/entities_history_template.ts b/x-pack/plugins/observability_solution/entity_manager/server/templates/entities_history_template.ts index ab2bac9e492eb..63d589bfaa754 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/templates/entities_history_template.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/templates/entities_history_template.ts @@ -24,6 +24,7 @@ export const getEntitiesHistoryIndexTemplateConfig = ( "Index template for indices managed by the Elastic Entity Model's entity discovery framework for the history dataset", ecs_version: '8.0.0', managed: true, + managed_by: 'elastic_entity_model', }, ignore_missing_component_templates: getCustomHistoryTemplateComponents(definitionId), composed_of: [ @@ -33,7 +34,7 @@ export const getEntitiesHistoryIndexTemplateConfig = ( ...getCustomHistoryTemplateComponents(definitionId), ], index_patterns: [`${ENTITY_HISTORY_INDEX_PREFIX_V1}.*`], - priority: 1, + priority: 200, template: { mappings: { _meta: { From c4f52f69dc85673be5e78888c5f8422846d221cd Mon Sep 17 00:00:00 2001 From: machadoum Date: Tue, 16 Jul 2024 15:20:35 +0200 Subject: [PATCH 4/5] Make PR ready for review --- .../entity_manager/common/helpers.test.ts | 22 +++++++++++++ .../lib/entities/delete_index_template.ts | 23 ------------- .../generate_history_processors.test.ts.snap | 24 ++++++++++++++ .../generate_latest_processors.test.ts.snap | 24 ++++++++++++++ .../install_entity_definition.test.ts | 20 ++++++++++++ .../lib/entities/install_entity_definition.ts | 30 +++++++++++++++-- .../templates/components/helpers.test.ts | 32 +++++++++++++++++++ .../server/templates/components/helpers.ts | 4 +-- 8 files changed, 151 insertions(+), 28 deletions(-) create mode 100644 x-pack/plugins/observability_solution/entity_manager/common/helpers.test.ts delete mode 100644 x-pack/plugins/observability_solution/entity_manager/server/lib/entities/delete_index_template.ts create mode 100644 x-pack/plugins/observability_solution/entity_manager/server/templates/components/helpers.test.ts diff --git a/x-pack/plugins/observability_solution/entity_manager/common/helpers.test.ts b/x-pack/plugins/observability_solution/entity_manager/common/helpers.test.ts new file mode 100644 index 0000000000000..50ce0caeba0a3 --- /dev/null +++ b/x-pack/plugins/observability_solution/entity_manager/common/helpers.test.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getEntityHistoryIndexTemplateV1, getEntityLatestIndexTemplateV1 } from './helpers'; + +describe('helpers', () => { + it('getEntityHistoryIndexTemplateV1 should return the correct value', () => { + const definitionId = 'test'; + const result = getEntityHistoryIndexTemplateV1(definitionId); + expect(result).toEqual('entities_v1_history_test_index_template'); + }); + + it('getEntityLatestIndexTemplateV1 should return the correct value', () => { + const definitionId = 'test'; + const result = getEntityLatestIndexTemplateV1(definitionId); + expect(result).toEqual('entities_v1_latest_test_index_template'); + }); +}); diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/delete_index_template.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/delete_index_template.ts deleted file mode 100644 index ed55af0353183..0000000000000 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/delete_index_template.ts +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ElasticsearchClient, Logger } from '@kbn/core/server'; -import { EntityDefinition } from '@kbn/entities-schema'; - -export async function deleteIndices( - esClient: ElasticsearchClient, - definition: EntityDefinition, - logger: Logger -) { - try { - const latestPipelineId = generateLatestIngestPipelineId(definition); - await retryTransientEsErrors(() => esClient.searchTemplate.delete({})); - } catch (e) { - logger.error(`Unable to delete latest ingest pipeline [${definition.id}].`); - throw e; - } -} diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/__snapshots__/generate_history_processors.test.ts.snap b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/__snapshots__/generate_history_processors.test.ts.snap index c0482cb4f87e2..81669bc8fdf55 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/__snapshots__/generate_history_processors.test.ts.snap +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/__snapshots__/generate_history_processors.test.ts.snap @@ -160,5 +160,29 @@ if (ctx.entity?.metadata?.sourceIndex != null) { "index_name_prefix": ".entities.v1.history.admin-console-services.", }, }, + Object { + "pipeline": Object { + "ignore_missing_pipeline": true, + "name": "admin-console-services@platform", + }, + }, + Object { + "pipeline": Object { + "ignore_missing_pipeline": true, + "name": "admin-console-services-history@platform", + }, + }, + Object { + "pipeline": Object { + "ignore_missing_pipeline": true, + "name": "admin-console-services@custom", + }, + }, + Object { + "pipeline": Object { + "ignore_missing_pipeline": true, + "name": "admin-console-services-history@custom", + }, + }, ] `; diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/__snapshots__/generate_latest_processors.test.ts.snap b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/__snapshots__/generate_latest_processors.test.ts.snap index aed7fe8f4bf09..ff2e22b23dde8 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/__snapshots__/generate_latest_processors.test.ts.snap +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/__snapshots__/generate_latest_processors.test.ts.snap @@ -126,5 +126,29 @@ ctx.event.category = ctx.entity.identity.event.category.keySet().toArray()[0];", "value": ".entities.v1.latest.admin-console-services", }, }, + Object { + "pipeline": Object { + "ignore_missing_pipeline": true, + "name": "admin-console-services@platform", + }, + }, + Object { + "pipeline": Object { + "ignore_missing_pipeline": true, + "name": "admin-console-services-latest@platform", + }, + }, + Object { + "pipeline": Object { + "ignore_missing_pipeline": true, + "name": "admin-console-services@custom", + }, + }, + Object { + "pipeline": Object { + "ignore_missing_pipeline": true, + "name": "admin-console-services-latest@custom", + }, + }, ] `; diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/install_entity_definition.test.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/install_entity_definition.test.ts index 7df61543f227f..9326716609f43 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/install_entity_definition.test.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/install_entity_definition.test.ts @@ -34,6 +34,18 @@ const assertHasCreatedDefinition = ( overwrite: true, }); + expect(esClient.indices.putIndexTemplate).toBeCalledTimes(2); + expect(esClient.indices.putIndexTemplate).toBeCalledWith( + expect.objectContaining({ + name: `entities_v1_history_${definition.id}_index_template`, + }) + ); + expect(esClient.indices.putIndexTemplate).toBeCalledWith( + expect.objectContaining({ + name: `entities_v1_latest_${definition.id}_index_template`, + }) + ); + expect(esClient.ingest.putPipeline).toBeCalledTimes(2); expect(esClient.ingest.putPipeline).toBeCalledWith({ id: generateHistoryIngestPipelineId(builtInServicesEntityDefinition), @@ -111,6 +123,14 @@ const assertHasUninstalledDefinition = ( expect(esClient.transform.deleteTransform).toBeCalledTimes(2); expect(esClient.ingest.deletePipeline).toBeCalledTimes(2); expect(soClient.delete).toBeCalledTimes(1); + + expect(esClient.indices.deleteIndexTemplate).toBeCalledTimes(2); + expect(esClient.indices.deleteIndexTemplate).toBeCalledWith({ + name: `entities_v1_history_${definition.id}_index_template`, + }); + expect(esClient.indices.deleteIndexTemplate).toBeCalledWith({ + name: `entities_v1_latest_${definition.id}_index_template`, + }); }; describe('install_entity_definition', () => { diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/install_entity_definition.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/install_entity_definition.ts index bfe41de0f62f6..6ddea1e13bf98 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/install_entity_definition.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/install_entity_definition.ts @@ -9,6 +9,10 @@ import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; import { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server'; import { EntityDefinition } from '@kbn/entities-schema'; import { Logger } from '@kbn/logging'; +import { + getEntityHistoryIndexTemplateV1, + getEntityLatestIndexTemplateV1, +} from '../../../common/helpers'; import { createAndInstallHistoryIngestPipeline, createAndInstallLatestIngestPipeline, @@ -28,7 +32,7 @@ import { stopAndDeleteLatestTransform, } from './stop_and_delete_transform'; import { uninstallEntityDefinition } from './uninstall_entity_definition'; -import { upsertTemplate } from '../manage_index_templates'; +import { deleteTemplate, upsertTemplate } from '../manage_index_templates'; import { getEntitiesLatestIndexTemplateConfig } from '../../templates/entities_latest_template'; import { getEntitiesHistoryIndexTemplateConfig } from '../../templates/entities_history_template'; @@ -55,6 +59,10 @@ export async function installEntityDefinition({ latest: false, }, definition: false, + indexTemplates: { + history: false, + latest: false, + }, }; try { @@ -65,18 +73,19 @@ export async function installEntityDefinition({ const entityDefinition = await saveEntityDefinition(soClient, definition); installState.definition = true; - // create scoped index template + // install scoped index template await upsertTemplate({ esClient, logger, template: getEntitiesHistoryIndexTemplateConfig(definition.id), }); - + installState.indexTemplates.history = true; await upsertTemplate({ esClient, logger, template: getEntitiesLatestIndexTemplateConfig(definition.id), }); + installState.indexTemplates.latest = true; // install ingest pipelines logger.debug(`Installing ingest pipelines for definition ${definition.id}`); @@ -100,6 +109,21 @@ export async function installEntityDefinition({ await deleteEntityDefinition(soClient, definition, logger); } + if (installState.indexTemplates.history) { + await deleteTemplate({ + esClient, + logger, + name: getEntityHistoryIndexTemplateV1(definition.id), + }); + } + if (installState.indexTemplates.latest) { + await deleteTemplate({ + esClient, + logger, + name: getEntityLatestIndexTemplateV1(definition.id), + }); + } + if (installState.ingestPipelines.history) { await deleteHistoryIngestPipeline(esClient, definition, logger); } diff --git a/x-pack/plugins/observability_solution/entity_manager/server/templates/components/helpers.test.ts b/x-pack/plugins/observability_solution/entity_manager/server/templates/components/helpers.test.ts new file mode 100644 index 0000000000000..4e67f099de71a --- /dev/null +++ b/x-pack/plugins/observability_solution/entity_manager/server/templates/components/helpers.test.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getCustomHistoryTemplateComponents, getCustomLatestTemplateComponents } from './helpers'; + +describe('helpers', () => { + it('getCustomLatestTemplateComponents should return template component in the right sort order', () => { + const definitionId = 'test'; + const result = getCustomLatestTemplateComponents(definitionId); + expect(result).toEqual([ + 'test@platform', + 'test-history@platform', + 'test@custom', + 'test-history@custom', + ]); + }); + + it('getCustomHistoryTemplateComponents should return template component in the right sort order', () => { + const definitionId = 'test'; + const result = getCustomHistoryTemplateComponents(definitionId); + expect(result).toEqual([ + 'test@platform', + 'test-latest@platform', + 'test@custom', + 'test-latest@custom', + ]); + }); +}); diff --git a/x-pack/plugins/observability_solution/entity_manager/server/templates/components/helpers.ts b/x-pack/plugins/observability_solution/entity_manager/server/templates/components/helpers.ts index ae2c16f234685..c38381fdd3e1a 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/templates/components/helpers.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/templates/components/helpers.ts @@ -6,14 +6,14 @@ */ export const getCustomLatestTemplateComponents = (definitionId: string) => [ - `${definitionId}@platform`, + `${definitionId}@platform`, // @platform goes before so it can be overwritten by custom `${definitionId}-history@platform`, `${definitionId}@custom`, `${definitionId}-history@custom`, ]; export const getCustomHistoryTemplateComponents = (definitionId: string) => [ - `${definitionId}@platform`, + `${definitionId}@platform`, // @platform goes before so it can be overwritten by custom `${definitionId}-latest@platform`, `${definitionId}@custom`, `${definitionId}-latest@custom`, From d863b82f55bf08f1fbbf412f5e142c6cc31cbfb0 Mon Sep 17 00:00:00 2001 From: machadoum Date: Fri, 19 Jul 2024 10:57:08 +0200 Subject: [PATCH 5/5] Improved code based on code review feedback --- .../server/lib/auth/privileges.ts | 8 ++++- .../install_entity_definition.test.ts | 18 +++++++---- .../lib/entities/install_entity_definition.ts | 30 +++++++++---------- .../server/lib/manage_index_templates.ts | 18 +++++++---- .../templates/components/helpers.test.ts | 8 ++--- .../server/templates/components/helpers.ts | 8 ++--- .../templates/entities_latest_template.ts | 1 + 7 files changed, 55 insertions(+), 36 deletions(-) diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/auth/privileges.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/auth/privileges.ts index 00f09209fb3b6..3bc88127a5964 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/auth/privileges.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/auth/privileges.ts @@ -21,7 +21,13 @@ export const requiredRunTimePrivileges = { privileges: ['read', 'view_index_metadata'], }, ], - cluster: ['manage_transform', 'monitor_transform', 'manage_ingest_pipelines', 'monitor'], + cluster: [ + 'manage_transform', + 'monitor_transform', + 'manage_ingest_pipelines', + 'monitor', + 'manage_index_templates', + ], application: [ { application: 'kibana-.kibana', diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/install_entity_definition.test.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/install_entity_definition.test.ts index 60f50bc1a6589..95eb63253f40c 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/install_entity_definition.test.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/install_entity_definition.test.ts @@ -125,12 +125,18 @@ const assertHasUninstalledDefinition = ( expect(soClient.delete).toBeCalledTimes(1); expect(esClient.indices.deleteIndexTemplate).toBeCalledTimes(2); - expect(esClient.indices.deleteIndexTemplate).toBeCalledWith({ - name: `entities_v1_history_${definition.id}_index_template`, - }); - expect(esClient.indices.deleteIndexTemplate).toBeCalledWith({ - name: `entities_v1_latest_${definition.id}_index_template`, - }); + expect(esClient.indices.deleteIndexTemplate).toBeCalledWith( + { + name: `entities_v1_history_${definition.id}_index_template`, + }, + { ignore: [404] } + ); + expect(esClient.indices.deleteIndexTemplate).toBeCalledWith( + { + name: `entities_v1_latest_${definition.id}_index_template`, + }, + { ignore: [404] } + ); }; describe('install_entity_definition', () => { diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/install_entity_definition.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/install_entity_definition.ts index 6ddea1e13bf98..b47f17b6b00fa 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/install_entity_definition.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/install_entity_definition.ts @@ -109,21 +109,6 @@ export async function installEntityDefinition({ await deleteEntityDefinition(soClient, definition, logger); } - if (installState.indexTemplates.history) { - await deleteTemplate({ - esClient, - logger, - name: getEntityHistoryIndexTemplateV1(definition.id), - }); - } - if (installState.indexTemplates.latest) { - await deleteTemplate({ - esClient, - logger, - name: getEntityLatestIndexTemplateV1(definition.id), - }); - } - if (installState.ingestPipelines.history) { await deleteHistoryIngestPipeline(esClient, definition, logger); } @@ -139,6 +124,21 @@ export async function installEntityDefinition({ await stopAndDeleteLatestTransform(esClient, definition, logger); } + if (installState.indexTemplates.history) { + await deleteTemplate({ + esClient, + logger, + name: getEntityHistoryIndexTemplateV1(definition.id), + }); + } + if (installState.indexTemplates.latest) { + await deleteTemplate({ + esClient, + logger, + name: getEntityLatestIndexTemplateV1(definition.id), + }); + } + throw e; } } diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/manage_index_templates.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/manage_index_templates.ts index a87fc8afba69a..f300df4a92c1d 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/manage_index_templates.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/manage_index_templates.ts @@ -10,6 +10,7 @@ import { IndicesPutIndexTemplateRequest, } from '@elastic/elasticsearch/lib/api/types'; import { ElasticsearchClient, Logger } from '@kbn/core/server'; +import { retryTransientEsErrors } from './entities/helpers/retry'; interface TemplateManagementOptions { esClient: ElasticsearchClient; @@ -31,10 +32,10 @@ interface DeleteTemplateOptions { export async function upsertTemplate({ esClient, template, logger }: TemplateManagementOptions) { try { - await esClient.indices.putIndexTemplate(template); + await retryTransientEsErrors(() => esClient.indices.putIndexTemplate(template), { logger }); } catch (error: any) { logger.error(`Error updating entity manager index template: ${error.message}`); - return; + throw error; } logger.info( @@ -45,19 +46,24 @@ export async function upsertTemplate({ esClient, template, logger }: TemplateMan export async function deleteTemplate({ esClient, name, logger }: DeleteTemplateOptions) { try { - await esClient.indices.deleteIndexTemplate({ name }); + await retryTransientEsErrors( + () => esClient.indices.deleteIndexTemplate({ name }, { ignore: [404] }), + { logger } + ); } catch (error: any) { logger.error(`Error deleting entity manager index template: ${error.message}`); - return; + throw error; } } export async function upsertComponent({ esClient, component, logger }: ComponentManagementOptions) { try { - await esClient.cluster.putComponentTemplate(component); + await retryTransientEsErrors(() => esClient.cluster.putComponentTemplate(component), { + logger, + }); } catch (error: any) { logger.error(`Error updating entity manager component template: ${error.message}`); - return; + throw error; } logger.info( diff --git a/x-pack/plugins/observability_solution/entity_manager/server/templates/components/helpers.test.ts b/x-pack/plugins/observability_solution/entity_manager/server/templates/components/helpers.test.ts index 4e67f099de71a..3321ee39edeb4 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/templates/components/helpers.test.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/templates/components/helpers.test.ts @@ -13,9 +13,9 @@ describe('helpers', () => { const result = getCustomLatestTemplateComponents(definitionId); expect(result).toEqual([ 'test@platform', - 'test-history@platform', + 'test-latest@platform', 'test@custom', - 'test-history@custom', + 'test-latest@custom', ]); }); @@ -24,9 +24,9 @@ describe('helpers', () => { const result = getCustomHistoryTemplateComponents(definitionId); expect(result).toEqual([ 'test@platform', - 'test-latest@platform', + 'test-history@platform', 'test@custom', - 'test-latest@custom', + 'test-history@custom', ]); }); }); diff --git a/x-pack/plugins/observability_solution/entity_manager/server/templates/components/helpers.ts b/x-pack/plugins/observability_solution/entity_manager/server/templates/components/helpers.ts index c38381fdd3e1a..e976a216da97b 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/templates/components/helpers.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/templates/components/helpers.ts @@ -7,14 +7,14 @@ export const getCustomLatestTemplateComponents = (definitionId: string) => [ `${definitionId}@platform`, // @platform goes before so it can be overwritten by custom - `${definitionId}-history@platform`, + `${definitionId}-latest@platform`, `${definitionId}@custom`, - `${definitionId}-history@custom`, + `${definitionId}-latest@custom`, ]; export const getCustomHistoryTemplateComponents = (definitionId: string) => [ `${definitionId}@platform`, // @platform goes before so it can be overwritten by custom - `${definitionId}-latest@platform`, + `${definitionId}-history@platform`, `${definitionId}@custom`, - `${definitionId}-latest@custom`, + `${definitionId}-history@custom`, ]; diff --git a/x-pack/plugins/observability_solution/entity_manager/server/templates/entities_latest_template.ts b/x-pack/plugins/observability_solution/entity_manager/server/templates/entities_latest_template.ts index b6911f39bf75a..3ad09e7257a1a 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/templates/entities_latest_template.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/templates/entities_latest_template.ts @@ -24,6 +24,7 @@ export const getEntitiesLatestIndexTemplateConfig = ( "Index template for indices managed by the Elastic Entity Model's entity discovery framework for the latest dataset", ecs_version: '8.0.0', managed: true, + managed_by: 'elastic_entity_model', }, ignore_missing_component_templates: getCustomLatestTemplateComponents(definitionId), composed_of: [