[UII] Fill in empty values for constant_keyword fields from existing mappings

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts

@@ -39,7 +39,7 @@ import { retryTransientEsErrors } from '../retry';
 import { PackageESError, PackageInvalidArchiveError } from '../../../../errors';
 
 import { getDefaultProperties, histogram, keyword, scaledFloat } from './mappings';
-import { isUserSettingsTemplate } from './utils';
+import { isUserSettingsTemplate, fillConstantKeywordValues } from './utils';
 
 interface Properties {
  [key: string]: any;
@@ -986,7 +986,7 @@ const updateAllDataStreams = async (
  });
  },
  {
- // Limit concurrent putMapping/rollover requests to avoid overhwhelming ES cluster
+ // Limit concurrent putMapping/rollover requests to avoid overwhelming ES cluster
  concurrency: 20,
  }
  );
@@ -1017,19 +1017,23 @@ const updateExistingDataStream = async ({
  const currentSourceType = currentBackingIndexConfig.mappings?._source?.mode;
 
  let settings: IndicesIndexSettings;
- let mappings: MappingTypeMapping;
+ let mappings: MappingTypeMapping = {};
  let lifecycle: any;
  let subobjectsFieldChanged: boolean = false;
+ let simulateResult: any = {};
  try {
- const simulateResult = await retryTransientEsErrors(async () =>
+ simulateResult = await retryTransientEsErrors(async () =>
  esClient.indices.simulateTemplate({
  name: await getIndexTemplate(esClient, dataStreamName),
  })
  );
 
  settings = simulateResult.template.settings;
- mappings = simulateResult.template.mappings;
- // @ts-expect-error template is not yet typed with DLM
+ mappings = fillConstantKeywordValues(
+ currentBackingIndexConfig?.mappings || {},
+ simulateResult.template.mappings
+ );
+
  lifecycle = simulateResult.template.lifecycle;
 
  // for now, remove from object so as not to update stream or data stream properties of the index until type and name
@@ -1063,6 +1067,7 @@ const updateExistingDataStream = async ({
  subobjectsFieldChanged
  ) {
  logger.info(`Mappings update for ${dataStreamName} failed due to ${err}`);
+ logger.trace(`Attempted mappings: ${mappings}`);
  if (options?.skipDataStreamRollover === true) {
  logger.info(
  `Skipping rollover for ${dataStreamName} as "skipDataStreamRollover" is enabled`
@@ -1075,6 +1080,7 @@ const updateExistingDataStream = async ({
  }
  }
  logger.error(`Mappings update for ${dataStreamName} failed due to unexpected error: ${err}`);
+ logger.trace(`Attempted mappings: ${mappings}`);
  if (options?.ignoreMappingUpdateErrors === true) {
  logger.info(`Ignore mapping update errors as "ignoreMappingUpdateErrors" is enabled`);
  return;

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/utils.test.ts

@@ -0,0 +1,226 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { fillConstantKeywordValues } from './utils';
+
+describe('fillConstantKeywordValues', () => {
+ const oldMappings = {
+ dynamic: false,
+ _meta: {
+ managed_by: 'fleet',
+ managed: true,
+ package: {
+ name: 'elastic_agent',
+ },
+ },
+ dynamic_templates: [
+ {
+ ecs_timestamp: {
+ match: '@timestamp',
+ mapping: {
+ ignore_malformed: false,
+ type: 'date',
+ },
+ },
+ },
+ ],
+ date_detection: false,
+ properties: {
+ '@timestamp': {
+ type: 'date',
+ ignore_malformed: false,
+ },
+ load: {
+ properties: {
+ '1': {
+ type: 'double',
+ },
+ '5': {
+ type: 'double',
+ },
+ '15': {
+ type: 'double',
+ },
+ },
+ },
+ event: {
+ properties: {
+ agent_id_status: {
+ type: 'keyword',
+ ignore_above: 1024,
+ },
+ dataset: {
+ type: 'constant_keyword',
+ value: 'elastic_agent.metricbeat',
+ },
+ ingested: {
+ type: 'date',
+ format: 'strict_date_time_no_millis||strict_date_optional_time||epoch_millis',
+ ignore_malformed: false,
+ },
+ },
+ },
+ message: {
+ type: 'match_only_text',
+ },
+ 'dot.field': {
+ type: 'keyword',
+ },
+ constant_keyword_without_value: {
+ type: 'constant_keyword',
+ },
+ },
+ };
+
+ const newMappings = {
+ dynamic: false,
+ _meta: {
+ managed_by: 'fleet',
+ managed: true,
+ package: {
+ name: 'elastic_agent',
+ },
+ },
+ dynamic_templates: [
+ {
+ ecs_timestamp: {
+ match: '@timestamp',
+ mapping: {
+ ignore_malformed: false,
+ type: 'date',
+ },
+ },
+ },
+ ],
+ date_detection: false,
+ properties: {
+ '@timestamp': {
+ type: 'date',
+ ignore_malformed: false,
+ },
+ load: {
+ properties: {
+ '1': {
+ type: 'double',
+ },
+ '5': {
+ type: 'double',
+ },
+ '15': {
+ type: 'double',
+ },
+ },
+ },
+ event: {
+ properties: {
+ agent_id_status: {
+ type: 'keyword',
+ ignore_above: 1024,
+ },
+ dataset: {
+ type: 'constant_keyword',
+ },
+ ingested: {
+ type: 'date',
+ format: 'strict_date_time_no_millis||strict_date_optional_time||epoch_millis',
+ ignore_malformed: false,
+ },
+ },
+ },
+ message: {
+ type: 'match_only_text',
+ },
+ 'dot.field': {
+ type: 'keyword',
+ },
+ some_new_field: {
+ type: 'keyword',
+ },
+ constant_keyword_without_value: {
+ type: 'constant_keyword',
+ },
+ },
+ };
+
+ it('should fill in missing constant_keyword values from old mappings correctly', () => {
+ // @ts-ignore
+ expect(fillConstantKeywordValues(oldMappings, newMappings)).toEqual({
+ dynamic: false,
+ _meta: {
+ managed_by: 'fleet',
+ managed: true,
+ package: {
+ name: 'elastic_agent',
+ },
+ },
+ dynamic_templates: [
+ {
+ ecs_timestamp: {
+ match: '@timestamp',
+ mapping: {
+ ignore_malformed: false,
+ type: 'date',
+ },
+ },
+ },
+ ],
+ date_detection: false,
+ properties: {
+ '@timestamp': {
+ type: 'date',
+ ignore_malformed: false,
+ },
+ load: {
+ properties: {
+ '1': {
+ type: 'double',
+ },
+ '5': {
+ type: 'double',
+ },
+ '15': {
+ type: 'double',
+ },
+ },
+ },
+ event: {
+ properties: {
+ agent_id_status: {
+ type: 'keyword',
+ ignore_above: 1024,
+ },
+ dataset: {
+ type: 'constant_keyword',
+ value: 'elastic_agent.metricbeat',
+ },
+ ingested: {
+ type: 'date',
+ format: 'strict_date_time_no_millis||strict_date_optional_time||epoch_millis',
+ ignore_malformed: false,
+ },
+ },
+ },
+ message: {
+ type: 'match_only_text',
+ },
+ 'dot.field': {
+ type: 'keyword',
+ },
+ some_new_field: {
+ type: 'keyword',
+ },
+ constant_keyword_without_value: {
+ type: 'constant_keyword',
+ },
+ },
+ });
+ });
+
+ it('should return the same mappings if old mappings are not provided', () => {
+ // @ts-ignore
+ expect(fillConstantKeywordValues({}, newMappings)).toMatchObject(newMappings);
+ });
+});

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/utils.ts

@@ -4,6 +4,7 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
+import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 
 import { USER_SETTINGS_TEMPLATE_SUFFIX } from '../../../../constants';
 
@@ -12,3 +13,34 @@ type UserSettingsTemplateName = `${TemplateBaseName}${typeof USER_SETTINGS_TEMPL
 
 export const isUserSettingsTemplate = (name: string): name is UserSettingsTemplateName =>
  name.endsWith(USER_SETTINGS_TEMPLATE_SUFFIX);
+
+// For any `constant_keyword` fields in `newMappings` that don't have a `value`, access the same field in
+// the `oldMappings` and fill in the value from there
+export const fillConstantKeywordValues = (
+ oldMappings: MappingTypeMapping,
+ newMappings: MappingTypeMapping
+) => {
+ const filledMappings = JSON.parse(JSON.stringify(newMappings)) as MappingTypeMapping;
+ const deepGet = (obj: any, keys: string[]) => keys.reduce((xs, x) => xs?.[x] ?? undefined, obj);
+
+ const fillEmptyConstantKeywordFields = (mappings: unknown, currentPath: string[] = []) => {
+ if (!mappings) return;
+ for (const [key, potentialField] of Object.entries(mappings)) {
+ const path = [...currentPath, key];
+ if (typeof potentialField === 'object') {
+ if (potentialField.type === 'constant_keyword' && potentialField.value === undefined) {
+ const valueFromOldMappings = deepGet(oldMappings.properties, [...path, 'value']);
+ if (valueFromOldMappings !== undefined) {
+ potentialField.value = valueFromOldMappings;
+ }
+ } else if (potentialField.properties && typeof potentialField.properties === 'object') {
+ fillEmptyConstantKeywordFields(potentialField.properties, [...path, 'properties']);
+ }
+ }
+ }
+ };
+
+ fillEmptyConstantKeywordFields(filledMappings.properties);
+
+ return filledMappings;
+};