[Fleet] Move mappings from index template to component template (elas…

…tic#124013)

* cherry pick from old branch

* use global template

* re-add _meta to index template

* simplify merge

* fix component_template test

* add settings to mapping component template

* put mapping settings on @mapping component template

* fix snapshot

* fix integration tests

* fix component template order and test

* use mapping component template for updating backing indices

* fix tests

* move code to util functions

* add comment

* split global templates

* update snaphsot

* rename variables

* fix tests again

* reinstall bundled packages when component templates change

* combine compoised_of generation statements

* remove duplicated global fields

* remove unnecessary rollover call

* use rollover API

* use simulate API to get template content

* remove unused parameters

* fix unit test

* improve simulate error handling

* re-add removed mapping assertions

* fix test

x-pack/plugins/fleet/common/types/models/epm.ts

@@ -483,6 +483,25 @@ export interface IndexTemplate {
   _meta: object;
 }
 
+export interface ESAssetMetadata {
+  package?: {
+    name: string;
+  };
+  managed_by: string;
+  managed: boolean;
+}
+export interface TemplateMapEntry {
+  _meta: ESAssetMetadata;
+  template:
+    | {
+        mappings: NonNullable<RegistryElasticsearch['index_template.mappings']>;
+      }
+    | {
+        settings: NonNullable<RegistryElasticsearch['index_template.settings']> | object;
+      };
+}
+
+export type TemplateMap = Record<string, TemplateMapEntry>;
 export interface IndexTemplateEntry {
   templateName: string;
   indexTemplate: IndexTemplate;

x-pack/plugins/fleet/server/constants/fleet_es_assets.ts

@@ -8,12 +8,44 @@
 import { getESAssetMetadata } from '../services/epm/elasticsearch/meta';
 
 const meta = getESAssetMetadata();
+export const MAPPINGS_TEMPLATE_SUFFIX = '@mappings';
+
+export const SETTINGS_TEMPLATE_SUFFIX = '@settings';
+
+export const USER_SETTINGS_TEMPLATE_SUFFIX = '@custom';
 
 export const FLEET_FINAL_PIPELINE_ID = '.fleet_final_pipeline-1';
 
-export const FLEET_GLOBAL_COMPONENT_TEMPLATE_NAME = '.fleet_component_template-1';
+export const FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME = '.fleet_globals-1';
+
+export const FLEET_GLOBALS_COMPONENT_TEMPLATE_CONTENT = {
+  _meta: meta,
+  template: {
+    settings: {},
+    mappings: {
+      _meta: meta,
+      // All the dynamic field mappings
+      dynamic_templates: [
+        // This makes sure all mappings are keywords by default
+        {
+          strings_as_keyword: {
+            mapping: {
+              ignore_above: 1024,
+              type: 'keyword',
+            },
+            match_mapping_type: 'string',
+          },
+        },
+      ],
+      // As we define fields ahead, we don't need any automatic field detection
+      // This makes sure all the fields are mapped to keyword by default to prevent mapping conflicts
+      date_detection: false,
+    },
+  },
+};
+export const FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_NAME = '.fleet_agent_id_verification-1';
 
-export const FLEET_GLOBAL_COMPONENT_TEMPLATE_CONTENT = {
+export const FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_CONTENT = {
   _meta: meta,
   template: {
     settings: {
@@ -40,6 +72,14 @@ export const FLEET_GLOBAL_COMPONENT_TEMPLATE_CONTENT = {
   },
 };
 
+export const FLEET_COMPONENT_TEMPLATES = [
+  { name: FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME, body: FLEET_GLOBALS_COMPONENT_TEMPLATE_CONTENT },
+  {
+    name: FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_NAME,
+    body: FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_CONTENT,
+  },
+];
+
 export const FLEET_FINAL_PIPELINE_VERSION = 2;
 
 // If the content is updated you probably need to update the FLEET_FINAL_PIPELINE_VERSION too to allow upgrade of the pipeline

x-pack/plugins/fleet/server/constants/index.ts

@@ -58,9 +58,15 @@ export {
 } from '../../common';
 
 export {
-  FLEET_GLOBAL_COMPONENT_TEMPLATE_NAME,
-  FLEET_GLOBAL_COMPONENT_TEMPLATE_CONTENT,
+  FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME,
+  FLEET_GLOBALS_COMPONENT_TEMPLATE_CONTENT,
+  FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_NAME,
+  FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_CONTENT,
+  FLEET_COMPONENT_TEMPLATES,
   FLEET_FINAL_PIPELINE_ID,
   FLEET_FINAL_PIPELINE_CONTENT,
   FLEET_FINAL_PIPELINE_VERSION,
+  MAPPINGS_TEMPLATE_SUFFIX,
+  SETTINGS_TEMPLATE_SUFFIX,
+  USER_SETTINGS_TEMPLATE_SUFFIX,
 } from './fleet_es_assets';

x-pack/plugins/fleet/server/services/epm/elasticsearch/meta.ts

@@ -7,15 +7,9 @@
 
 import { safeLoad, safeDump } from 'js-yaml';
 
-const MANAGED_BY_DEFAULT = 'fleet';
+import type { ESAssetMetadata } from '../../../../common/types';
 
-export interface ESAssetMetadata {
-  package?: {
-    name: string;
-  };
-  managed_by: string;
-  managed: boolean;
-}
+const MANAGED_BY_DEFAULT = 'fleet';
 
 /**
  * Build common metadata object for Elasticsearch assets installed by Fleet. Result should be