[UII] Fill in empty values for constant_keyword fields from existing mappings

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts

@@ -39,7 +39,7 @@ import { retryTransientEsErrors } from '../retry';
 import { PackageESError, PackageInvalidArchiveError } from '../../../../errors';
 
 import { getDefaultProperties, histogram, keyword, scaledFloat } from './mappings';
-import { isUserSettingsTemplate } from './utils';
+import { isUserSettingsTemplate, fillConstantKeywordValues } from './utils';
 
 interface Properties {
   [key: string]: any;
@@ -986,7 +986,7 @@ const updateAllDataStreams = async (
       });
     },
     {
-      // Limit concurrent putMapping/rollover requests to avoid overhwhelming ES cluster
+      // Limit concurrent putMapping/rollover requests to avoid overwhelming ES cluster
       concurrency: 20,
     }
   );
@@ -1017,19 +1017,23 @@ const updateExistingDataStream = async ({
   const currentSourceType = currentBackingIndexConfig.mappings?._source?.mode;
 
   let settings: IndicesIndexSettings;
-  let mappings: MappingTypeMapping;
+  let mappings: MappingTypeMapping = {};
   let lifecycle: any;
   let subobjectsFieldChanged: boolean = false;
+  let simulateResult: any = {};
   try {
-    const simulateResult = await retryTransientEsErrors(async () =>
+    simulateResult = await retryTransientEsErrors(async () =>
       esClient.indices.simulateTemplate({
         name: await getIndexTemplate(esClient, dataStreamName),
       })
     );
 
     settings = simulateResult.template.settings;
-    mappings = simulateResult.template.mappings;
-    // @ts-expect-error template is not yet typed with DLM
+    mappings = fillConstantKeywordValues(
+      currentBackingIndexConfig?.mappings || {},
+      simulateResult.template.mappings
+    );
+
     lifecycle = simulateResult.template.lifecycle;
 
     // for now, remove from object so as not to update stream or data stream properties of the index until type and name
@@ -1063,6 +1067,7 @@ const updateExistingDataStream = async ({
       subobjectsFieldChanged
     ) {
       logger.info(`Mappings update for ${dataStreamName} failed due to ${err}`);
+      logger.trace(`Attempted mappings: ${mappings}`);
       if (options?.skipDataStreamRollover === true) {
         logger.info(
           `Skipping rollover for ${dataStreamName} as "skipDataStreamRollover" is enabled`
@@ -1075,6 +1080,7 @@ const updateExistingDataStream = async ({
       }
     }
     logger.error(`Mappings update for ${dataStreamName} failed due to unexpected error: ${err}`);
+    logger.trace(`Attempted mappings: ${mappings}`);
     if (options?.ignoreMappingUpdateErrors === true) {
       logger.info(`Ignore mapping update errors as "ignoreMappingUpdateErrors" is enabled`);
       return;

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/utils.test.ts

@@ -0,0 +1,217 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { fillConstantKeywordValues } from './utils';
+
+describe('fillConstantKeywordValues', () => {
+  const oldMappings = {
+    dynamic: false,
+    _meta: {
+      managed_by: 'fleet',
+      managed: true,
+      package: {
+        name: 'elastic_agent',
+      },
+    },
+    dynamic_templates: [
+      {
+        ecs_timestamp: {
+          match: '@timestamp',
+          mapping: {
+            ignore_malformed: false,
+            type: 'date',
+          },
+        },
+      },
+    ],
+    date_detection: false,
+    properties: {
+      '@timestamp': {
+        type: 'date',
+        ignore_malformed: false,
+      },
+      load: {
+        properties: {
+          '1': {
+            type: 'double',
+          },
+          '5': {
+            type: 'double',
+          },
+          '15': {
+            type: 'double',
+          },
+        },
+      },
+      event: {
+        properties: {
+          agent_id_status: {
+            type: 'keyword',
+            ignore_above: 1024,
+          },
+          dataset: {
+            type: 'constant_keyword',
+            value: 'elastic_agent.metricbeat',
+          },
+          ingested: {
+            type: 'date',
+            format: 'strict_date_time_no_millis||strict_date_optional_time||epoch_millis',
+            ignore_malformed: false,
+          },
+        },
+      },
+      message: {
+        type: 'match_only_text',
+      },
+      'dot.field': {
+        type: 'keyword',
+      },
+    },
+  };
+
+  const newMappings = {
+    dynamic: false,
+    _meta: {
+      managed_by: 'fleet',
+      managed: true,
+      package: {
+        name: 'elastic_agent',
+      },
+    },
+    dynamic_templates: [
+      {
+        ecs_timestamp: {
+          match: '@timestamp',
+          mapping: {
+            ignore_malformed: false,
+            type: 'date',
+          },
+        },
+      },
+    ],
+    date_detection: false,
+    properties: {
+      '@timestamp': {
+        type: 'date',
+        ignore_malformed: false,
+      },
+      load: {
+        properties: {
+          '1': {
+            type: 'double',
+          },
+          '5': {
+            type: 'double',
+          },
+          '15': {
+            type: 'double',
+          },
+        },
+      },
+      event: {
+        properties: {
+          agent_id_status: {
+            type: 'keyword',
+            ignore_above: 1024,
+          },
+          dataset: {
+            type: 'constant_keyword',
+          },
+          ingested: {
+            type: 'date',
+            format: 'strict_date_time_no_millis||strict_date_optional_time||epoch_millis',
+            ignore_malformed: false,
+          },
+        },
+      },
+      message: {
+        type: 'match_only_text',
+      },
+      'dot.field': {
+        type: 'keyword',
+      },
+      some_new_field: {
+        type: 'keyword',
+      },
+    },
+  };
+
+  it('should fill in missing constant_keyword values from old mappings correctly', () => {
+    // @ts-ignore
+    expect(fillConstantKeywordValues(oldMappings, newMappings)).toMatchObject({
+      dynamic: false,
+      _meta: {
+        managed_by: 'fleet',
+        managed: true,
+        package: {
+          name: 'elastic_agent',
+        },
+      },
+      dynamic_templates: [
+        {
+          ecs_timestamp: {
+            match: '@timestamp',
+            mapping: {
+              ignore_malformed: false,
+              type: 'date',
+            },
+          },
+        },
+      ],
+      date_detection: false,
+      properties: {
+        '@timestamp': {
+          type: 'date',
+          ignore_malformed: false,
+        },
+        load: {
+          properties: {
+            '1': {
+              type: 'double',
+            },
+            '5': {
+              type: 'double',
+            },
+            '15': {
+              type: 'double',
+            },
+          },
+        },
+        event: {
+          properties: {
+            agent_id_status: {
+              type: 'keyword',
+              ignore_above: 1024,
+            },
+            dataset: {
+              type: 'constant_keyword',
+              value: 'elastic_agent.metricbeat',
+            },
+            ingested: {
+              type: 'date',
+              format: 'strict_date_time_no_millis||strict_date_optional_time||epoch_millis',
+              ignore_malformed: false,
+            },
+          },
+        },
+        message: {
+          type: 'match_only_text',
+        },
+        'dot.field': {
+          type: 'keyword',
+        },
+        some_new_field: {
+          type: 'keyword',
+        },
+      },
+    });
+  });
+
+  it('should return the same mappings if old mappings are not provided', () => {
+    // @ts-ignore
+    expect(fillConstantKeywordValues({}, newMappings)).toMatchObject(newMappings);
+  });
+});

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/utils.ts

@@ -4,6 +4,7 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
+import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 
 import { USER_SETTINGS_TEMPLATE_SUFFIX } from '../../../../constants';
 
@@ -12,3 +13,33 @@ type UserSettingsTemplateName = `${TemplateBaseName}${typeof USER_SETTINGS_TEMPL
 
 export const isUserSettingsTemplate = (name: string): name is UserSettingsTemplateName =>
   name.endsWith(USER_SETTINGS_TEMPLATE_SUFFIX);
+
+// For any `constant_keyword` fields in `newMappings` that don't have a `value`, access the same field in
+// the `oldMappings` and fill in the value from there
+export const fillConstantKeywordValues = (
+  oldMappings: MappingTypeMapping,
+  newMappings: MappingTypeMapping
+) => {
+  const filledMappings = JSON.parse(JSON.stringify(newMappings)) as MappingTypeMapping;
+  const deepGet = (obj: any, keys: string[]) => keys.reduce((xs, x) => xs?.[x] ?? null, obj);
+
+  const fillEmptyConstantKeywordFields = (
+    mappings: MappingTypeMapping,
+    currentPath: string[] = []
+  ) => {
+    for (const [key, potentialField] of Object.entries(mappings)) {
+      const path = [...currentPath, key];
+      if (typeof potentialField === 'object') {
+        if (potentialField.type === 'constant_keyword' && potentialField.value === undefined) {
+          potentialField.value = deepGet(oldMappings, [...path, 'value']);
+        } else {
+          fillEmptyConstantKeywordFields(potentialField, path);
+        }
+      }
+    }
+  };
+
+  fillEmptyConstantKeywordFields(filledMappings);
+
+  return filledMappings;
+};