diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts index 654ce7ea8ed814..e44e70b85efe0a 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts @@ -39,7 +39,7 @@ import { retryTransientEsErrors } from '../retry'; import { PackageESError, PackageInvalidArchiveError } from '../../../../errors'; import { getDefaultProperties, histogram, keyword, scaledFloat } from './mappings'; -import { isUserSettingsTemplate } from './utils'; +import { isUserSettingsTemplate, fillConstantKeywordValues } from './utils'; interface Properties { [key: string]: any; @@ -986,7 +986,7 @@ const updateAllDataStreams = async ( }); }, { - // Limit concurrent putMapping/rollover requests to avoid overhwhelming ES cluster + // Limit concurrent putMapping/rollover requests to avoid overwhelming ES cluster concurrency: 20, } ); @@ -1017,19 +1017,23 @@ const updateExistingDataStream = async ({ const currentSourceType = currentBackingIndexConfig.mappings?._source?.mode; let settings: IndicesIndexSettings; - let mappings: MappingTypeMapping; + let mappings: MappingTypeMapping = {}; let lifecycle: any; let subobjectsFieldChanged: boolean = false; + let simulateResult: any = {}; try { - const simulateResult = await retryTransientEsErrors(async () => + simulateResult = await retryTransientEsErrors(async () => esClient.indices.simulateTemplate({ name: await getIndexTemplate(esClient, dataStreamName), }) ); settings = simulateResult.template.settings; - mappings = simulateResult.template.mappings; - // @ts-expect-error template is not yet typed with DLM + mappings = fillConstantKeywordValues( + currentBackingIndexConfig?.mappings || {}, + simulateResult.template.mappings + ); + lifecycle = simulateResult.template.lifecycle; // for now, remove from object so as not to update stream or data stream properties of the index until type and name @@ -1063,6 +1067,7 @@ const updateExistingDataStream = async ({ subobjectsFieldChanged ) { logger.info(`Mappings update for ${dataStreamName} failed due to ${err}`); + logger.trace(`Attempted mappings: ${mappings}`); if (options?.skipDataStreamRollover === true) { logger.info( `Skipping rollover for ${dataStreamName} as "skipDataStreamRollover" is enabled` @@ -1075,6 +1080,7 @@ const updateExistingDataStream = async ({ } } logger.error(`Mappings update for ${dataStreamName} failed due to unexpected error: ${err}`); + logger.trace(`Attempted mappings: ${mappings}`); if (options?.ignoreMappingUpdateErrors === true) { logger.info(`Ignore mapping update errors as "ignoreMappingUpdateErrors" is enabled`); return; diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/utils.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/utils.test.ts new file mode 100644 index 00000000000000..a411ba32d2954d --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/utils.test.ts @@ -0,0 +1,226 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { fillConstantKeywordValues } from './utils'; + +describe('fillConstantKeywordValues', () => { + const oldMappings = { + dynamic: false, + _meta: { + managed_by: 'fleet', + managed: true, + package: { + name: 'elastic_agent', + }, + }, + dynamic_templates: [ + { + ecs_timestamp: { + match: '@timestamp', + mapping: { + ignore_malformed: false, + type: 'date', + }, + }, + }, + ], + date_detection: false, + properties: { + '@timestamp': { + type: 'date', + ignore_malformed: false, + }, + load: { + properties: { + '1': { + type: 'double', + }, + '5': { + type: 'double', + }, + '15': { + type: 'double', + }, + }, + }, + event: { + properties: { + agent_id_status: { + type: 'keyword', + ignore_above: 1024, + }, + dataset: { + type: 'constant_keyword', + value: 'elastic_agent.metricbeat', + }, + ingested: { + type: 'date', + format: 'strict_date_time_no_millis||strict_date_optional_time||epoch_millis', + ignore_malformed: false, + }, + }, + }, + message: { + type: 'match_only_text', + }, + 'dot.field': { + type: 'keyword', + }, + constant_keyword_without_value: { + type: 'constant_keyword', + }, + }, + }; + + const newMappings = { + dynamic: false, + _meta: { + managed_by: 'fleet', + managed: true, + package: { + name: 'elastic_agent', + }, + }, + dynamic_templates: [ + { + ecs_timestamp: { + match: '@timestamp', + mapping: { + ignore_malformed: false, + type: 'date', + }, + }, + }, + ], + date_detection: false, + properties: { + '@timestamp': { + type: 'date', + ignore_malformed: false, + }, + load: { + properties: { + '1': { + type: 'double', + }, + '5': { + type: 'double', + }, + '15': { + type: 'double', + }, + }, + }, + event: { + properties: { + agent_id_status: { + type: 'keyword', + ignore_above: 1024, + }, + dataset: { + type: 'constant_keyword', + }, + ingested: { + type: 'date', + format: 'strict_date_time_no_millis||strict_date_optional_time||epoch_millis', + ignore_malformed: false, + }, + }, + }, + message: { + type: 'match_only_text', + }, + 'dot.field': { + type: 'keyword', + }, + some_new_field: { + type: 'keyword', + }, + constant_keyword_without_value: { + type: 'constant_keyword', + }, + }, + }; + + it('should fill in missing constant_keyword values from old mappings correctly', () => { + // @ts-ignore + expect(fillConstantKeywordValues(oldMappings, newMappings)).toEqual({ + dynamic: false, + _meta: { + managed_by: 'fleet', + managed: true, + package: { + name: 'elastic_agent', + }, + }, + dynamic_templates: [ + { + ecs_timestamp: { + match: '@timestamp', + mapping: { + ignore_malformed: false, + type: 'date', + }, + }, + }, + ], + date_detection: false, + properties: { + '@timestamp': { + type: 'date', + ignore_malformed: false, + }, + load: { + properties: { + '1': { + type: 'double', + }, + '5': { + type: 'double', + }, + '15': { + type: 'double', + }, + }, + }, + event: { + properties: { + agent_id_status: { + type: 'keyword', + ignore_above: 1024, + }, + dataset: { + type: 'constant_keyword', + value: 'elastic_agent.metricbeat', + }, + ingested: { + type: 'date', + format: 'strict_date_time_no_millis||strict_date_optional_time||epoch_millis', + ignore_malformed: false, + }, + }, + }, + message: { + type: 'match_only_text', + }, + 'dot.field': { + type: 'keyword', + }, + some_new_field: { + type: 'keyword', + }, + constant_keyword_without_value: { + type: 'constant_keyword', + }, + }, + }); + }); + + it('should return the same mappings if old mappings are not provided', () => { + // @ts-ignore + expect(fillConstantKeywordValues({}, newMappings)).toMatchObject(newMappings); + }); +}); diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/utils.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/utils.ts index e435e54a828dff..6a34beb3710829 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/utils.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/utils.ts @@ -4,6 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ +import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { USER_SETTINGS_TEMPLATE_SUFFIX } from '../../../../constants'; @@ -12,3 +13,34 @@ type UserSettingsTemplateName = `${TemplateBaseName}${typeof USER_SETTINGS_TEMPL export const isUserSettingsTemplate = (name: string): name is UserSettingsTemplateName => name.endsWith(USER_SETTINGS_TEMPLATE_SUFFIX); + +// For any `constant_keyword` fields in `newMappings` that don't have a `value`, access the same field in +// the `oldMappings` and fill in the value from there +export const fillConstantKeywordValues = ( + oldMappings: MappingTypeMapping, + newMappings: MappingTypeMapping +) => { + const filledMappings = JSON.parse(JSON.stringify(newMappings)) as MappingTypeMapping; + const deepGet = (obj: any, keys: string[]) => keys.reduce((xs, x) => xs?.[x] ?? undefined, obj); + + const fillEmptyConstantKeywordFields = (mappings: unknown, currentPath: string[] = []) => { + if (!mappings) return; + for (const [key, potentialField] of Object.entries(mappings)) { + const path = [...currentPath, key]; + if (typeof potentialField === 'object') { + if (potentialField.type === 'constant_keyword' && potentialField.value === undefined) { + const valueFromOldMappings = deepGet(oldMappings.properties, [...path, 'value']); + if (valueFromOldMappings !== undefined) { + potentialField.value = valueFromOldMappings; + } + } else if (potentialField.properties && typeof potentialField.properties === 'object') { + fillEmptyConstantKeywordFields(potentialField.properties, [...path, 'properties']); + } + } + } + }; + + fillEmptyConstantKeywordFields(filledMappings.properties); + + return filledMappings; +};