diff --git a/x-pack/plugins/fleet/common/types/models/epm.ts b/x-pack/plugins/fleet/common/types/models/epm.ts index 39650bfed5da8..dcff9f503bfe0 100644 --- a/x-pack/plugins/fleet/common/types/models/epm.ts +++ b/x-pack/plugins/fleet/common/types/models/epm.ts @@ -483,6 +483,25 @@ export interface IndexTemplate { _meta: object; } +export interface ESAssetMetadata { + package?: { + name: string; + }; + managed_by: string; + managed: boolean; +} +export interface TemplateMapEntry { + _meta: ESAssetMetadata; + template: + | { + mappings: NonNullable; + } + | { + settings: NonNullable | object; + }; +} + +export type TemplateMap = Record; export interface IndexTemplateEntry { templateName: string; indexTemplate: IndexTemplate; diff --git a/x-pack/plugins/fleet/server/constants/fleet_es_assets.ts b/x-pack/plugins/fleet/server/constants/fleet_es_assets.ts index 8cdb93be28148..859a25a0ec7c7 100644 --- a/x-pack/plugins/fleet/server/constants/fleet_es_assets.ts +++ b/x-pack/plugins/fleet/server/constants/fleet_es_assets.ts @@ -8,12 +8,44 @@ import { getESAssetMetadata } from '../services/epm/elasticsearch/meta'; const meta = getESAssetMetadata(); +export const MAPPINGS_TEMPLATE_SUFFIX = '@mappings'; + +export const SETTINGS_TEMPLATE_SUFFIX = '@settings'; + +export const USER_SETTINGS_TEMPLATE_SUFFIX = '@custom'; export const FLEET_FINAL_PIPELINE_ID = '.fleet_final_pipeline-1'; -export const FLEET_GLOBAL_COMPONENT_TEMPLATE_NAME = '.fleet_component_template-1'; +export const FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME = '.fleet_globals-1'; + +export const FLEET_GLOBALS_COMPONENT_TEMPLATE_CONTENT = { + _meta: meta, + template: { + settings: {}, + mappings: { + _meta: meta, + // All the dynamic field mappings + dynamic_templates: [ + // This makes sure all mappings are keywords by default + { + strings_as_keyword: { + mapping: { + ignore_above: 1024, + type: 'keyword', + }, + match_mapping_type: 'string', + }, + }, + ], + // As we define fields ahead, we don't need any automatic field detection + // This makes sure all the fields are mapped to keyword by default to prevent mapping conflicts + date_detection: false, + }, + }, +}; +export const FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_NAME = '.fleet_agent_id_verification-1'; -export const FLEET_GLOBAL_COMPONENT_TEMPLATE_CONTENT = { +export const FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_CONTENT = { _meta: meta, template: { settings: { @@ -40,6 +72,14 @@ export const FLEET_GLOBAL_COMPONENT_TEMPLATE_CONTENT = { }, }; +export const FLEET_COMPONENT_TEMPLATES = [ + { name: FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME, body: FLEET_GLOBALS_COMPONENT_TEMPLATE_CONTENT }, + { + name: FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_NAME, + body: FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_CONTENT, + }, +]; + export const FLEET_FINAL_PIPELINE_VERSION = 2; // If the content is updated you probably need to update the FLEET_FINAL_PIPELINE_VERSION too to allow upgrade of the pipeline diff --git a/x-pack/plugins/fleet/server/constants/index.ts b/x-pack/plugins/fleet/server/constants/index.ts index 0ccbeb9f025e4..ec7a4a2664882 100644 --- a/x-pack/plugins/fleet/server/constants/index.ts +++ b/x-pack/plugins/fleet/server/constants/index.ts @@ -58,9 +58,15 @@ export { } from '../../common'; export { - FLEET_GLOBAL_COMPONENT_TEMPLATE_NAME, - FLEET_GLOBAL_COMPONENT_TEMPLATE_CONTENT, + FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME, + FLEET_GLOBALS_COMPONENT_TEMPLATE_CONTENT, + FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_NAME, + FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_CONTENT, + FLEET_COMPONENT_TEMPLATES, FLEET_FINAL_PIPELINE_ID, FLEET_FINAL_PIPELINE_CONTENT, FLEET_FINAL_PIPELINE_VERSION, + MAPPINGS_TEMPLATE_SUFFIX, + SETTINGS_TEMPLATE_SUFFIX, + USER_SETTINGS_TEMPLATE_SUFFIX, } from './fleet_es_assets'; diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/meta.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/meta.ts index a3ceaf44100d7..d691cd8c700e5 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/meta.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/meta.ts @@ -7,15 +7,9 @@ import { safeLoad, safeDump } from 'js-yaml'; -const MANAGED_BY_DEFAULT = 'fleet'; +import type { ESAssetMetadata } from '../../../../common/types'; -export interface ESAssetMetadata { - package?: { - name: string; - }; - managed_by: string; - managed: boolean; -} +const MANAGED_BY_DEFAULT = 'fleet'; /** * Build common metadata object for Elasticsearch assets installed by Fleet. Result should be diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap index e977c41cd69d8..efd51d5e0d997 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap @@ -2,613 +2,550 @@ exports[`EPM template tests loading base.yml: base.yml 1`] = ` { - "priority": 200, - "index_patterns": [ - "foo-*" - ], - "template": { - "settings": { - "index": {} + "properties": { + "user": { + "properties": { + "auid": { + "ignore_above": 1024, + "type": "keyword" + }, + "euid": { + "ignore_above": 1024, + "type": "keyword" + } + } }, - "mappings": { - "dynamic_templates": [ - { - "strings_as_keyword": { - "mapping": { - "ignore_above": 1024, - "type": "keyword" + "long": { + "properties": { + "nested": { + "properties": { + "foo": { + "type": "text" }, - "match_mapping_type": "string" + "bar": { + "type": "long" + } } } - ], - "date_detection": false, + } + }, + "nested": { + "properties": { + "bar": { + "ignore_above": 1024, + "type": "keyword" + }, + "baz": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "myalias": { + "type": "alias", + "path": "user.euid" + }, + "validarray": { + "type": "integer" + }, + "cycle_type": { + "type": "constant_keyword", + "value": "bicycle" + } + } +} +`; + +exports[`EPM template tests loading coredns.logs.yml: coredns.logs.yml 1`] = ` +{ + "properties": { + "coredns": { "properties": { - "user": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "query": { "properties": { - "auid": { + "size": { + "type": "long" + }, + "class": { "ignore_above": 1024, "type": "keyword" }, - "euid": { + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { "ignore_above": 1024, "type": "keyword" } } }, - "long": { - "properties": { - "nested": { - "properties": { - "foo": { - "type": "text" - }, - "bar": { - "type": "long" - } - } - } - } - }, - "nested": { + "response": { "properties": { - "bar": { + "code": { "ignore_above": 1024, "type": "keyword" }, - "baz": { + "flags": { "ignore_above": 1024, "type": "keyword" + }, + "size": { + "type": "long" } } }, - "myalias": { - "type": "alias", - "path": "user.euid" - }, - "validarray": { - "type": "integer" - }, - "cycle_type": { - "type": "constant_keyword", - "value": "bicycle" - } - }, - "_meta": { - "managed_by": "fleet", - "managed": true, - "package": { - "name": "nginx" + "dnssec_ok": { + "type": "boolean" } } } - }, - "data_stream": {}, - "composed_of": [ - ".fleet_component_template-1" - ], - "_meta": { - "managed_by": "fleet", - "managed": true, - "package": { - "name": "nginx" - } } } `; -exports[`EPM template tests loading coredns.logs.yml: coredns.logs.yml 1`] = ` +exports[`EPM template tests loading system.yml: system.yml 1`] = ` { - "priority": 200, - "index_patterns": [ - "foo-*" - ], - "template": { - "settings": { - "index": {} - }, - "mappings": { - "dynamic_templates": [ - { - "strings_as_keyword": { - "mapping": { - "ignore_above": 1024, - "type": "keyword" - }, - "match_mapping_type": "string" - } - } - ], - "date_detection": false, + "properties": { + "system": { "properties": { - "coredns": { + "core": { "properties": { "id": { - "ignore_above": 1024, - "type": "keyword" + "type": "long" }, - "query": { + "user": { "properties": { - "size": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + }, + "ticks": { "type": "long" + } + } + }, + "system": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 }, - "class": { - "ignore_above": 1024, - "type": "keyword" + "ticks": { + "type": "long" + } + } + }, + "nice": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 }, - "name": { - "ignore_above": 1024, - "type": "keyword" + "ticks": { + "type": "long" + } + } + }, + "idle": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 }, - "type": { - "ignore_above": 1024, - "type": "keyword" + "ticks": { + "type": "long" } } }, - "response": { + "iowait": { "properties": { - "code": { - "ignore_above": 1024, - "type": "keyword" + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 }, - "flags": { - "ignore_above": 1024, - "type": "keyword" + "ticks": { + "type": "long" + } + } + }, + "irq": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 }, - "size": { + "ticks": { "type": "long" } } }, - "dnssec_ok": { - "type": "boolean" - } - } - } - }, - "_meta": { - "managed_by": "fleet", - "managed": true, - "package": { - "name": "coredns" - } - } - } - }, - "data_stream": {}, - "composed_of": [ - ".fleet_component_template-1" - ], - "_meta": { - "managed_by": "fleet", - "managed": true, - "package": { - "name": "coredns" - } - } -} -`; - -exports[`EPM template tests loading system.yml: system.yml 1`] = ` -{ - "priority": 200, - "index_patterns": [ - "whatsthis-*" - ], - "template": { - "settings": { - "index": {} - }, - "mappings": { - "dynamic_templates": [ - { - "strings_as_keyword": { - "mapping": { - "ignore_above": 1024, - "type": "keyword" + "softirq": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + }, + "ticks": { + "type": "long" + } + } }, - "match_mapping_type": "string" + "steal": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + }, + "ticks": { + "type": "long" + } + } + } } - } - ], - "date_detection": false, - "properties": { - "system": { + }, + "cpu": { "properties": { - "core": { + "cores": { + "type": "long" + }, + "user": { "properties": { - "id": { - "type": "long" + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 }, - "user": { + "norm": { "properties": { "pct": { "type": "scaled_float", "scaling_factor": 1000 - }, - "ticks": { - "type": "long" } } }, - "system": { + "ticks": { + "type": "long" + } + } + }, + "system": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + }, + "norm": { "properties": { "pct": { "type": "scaled_float", "scaling_factor": 1000 - }, - "ticks": { - "type": "long" } } }, - "nice": { + "ticks": { + "type": "long" + } + } + }, + "nice": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + }, + "norm": { "properties": { "pct": { "type": "scaled_float", "scaling_factor": 1000 - }, - "ticks": { - "type": "long" } } }, - "idle": { + "ticks": { + "type": "long" + } + } + }, + "idle": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + }, + "norm": { "properties": { "pct": { "type": "scaled_float", "scaling_factor": 1000 - }, - "ticks": { - "type": "long" } } }, - "iowait": { + "ticks": { + "type": "long" + } + } + }, + "iowait": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + }, + "norm": { "properties": { "pct": { "type": "scaled_float", "scaling_factor": 1000 - }, - "ticks": { - "type": "long" } } }, - "irq": { + "ticks": { + "type": "long" + } + } + }, + "irq": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + }, + "norm": { "properties": { "pct": { "type": "scaled_float", "scaling_factor": 1000 - }, - "ticks": { - "type": "long" } } }, - "softirq": { + "ticks": { + "type": "long" + } + } + }, + "softirq": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + }, + "norm": { "properties": { "pct": { "type": "scaled_float", "scaling_factor": 1000 - }, - "ticks": { - "type": "long" } } }, - "steal": { + "ticks": { + "type": "long" + } + } + }, + "steal": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + }, + "norm": { "properties": { "pct": { "type": "scaled_float", "scaling_factor": 1000 - }, - "ticks": { - "type": "long" } } + }, + "ticks": { + "type": "long" } } }, - "cpu": { + "total": { "properties": { - "cores": { - "type": "long" - }, - "user": { - "properties": { - "pct": { - "type": "scaled_float", - "scaling_factor": 1000 - }, - "norm": { - "properties": { - "pct": { - "type": "scaled_float", - "scaling_factor": 1000 - } - } - }, - "ticks": { - "type": "long" - } - } - }, - "system": { - "properties": { - "pct": { - "type": "scaled_float", - "scaling_factor": 1000 - }, - "norm": { - "properties": { - "pct": { - "type": "scaled_float", - "scaling_factor": 1000 - } - } - }, - "ticks": { - "type": "long" - } - } - }, - "nice": { - "properties": { - "pct": { - "type": "scaled_float", - "scaling_factor": 1000 - }, - "norm": { - "properties": { - "pct": { - "type": "scaled_float", - "scaling_factor": 1000 - } - } - }, - "ticks": { - "type": "long" - } - } - }, - "idle": { - "properties": { - "pct": { - "type": "scaled_float", - "scaling_factor": 1000 - }, - "norm": { - "properties": { - "pct": { - "type": "scaled_float", - "scaling_factor": 1000 - } - } - }, - "ticks": { - "type": "long" - } - } - }, - "iowait": { - "properties": { - "pct": { - "type": "scaled_float", - "scaling_factor": 1000 - }, - "norm": { - "properties": { - "pct": { - "type": "scaled_float", - "scaling_factor": 1000 - } - } - }, - "ticks": { - "type": "long" - } - } + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 }, - "irq": { + "norm": { "properties": { "pct": { "type": "scaled_float", "scaling_factor": 1000 - }, - "norm": { - "properties": { - "pct": { - "type": "scaled_float", - "scaling_factor": 1000 - } - } - }, - "ticks": { - "type": "long" - } - } - }, - "softirq": { - "properties": { - "pct": { - "type": "scaled_float", - "scaling_factor": 1000 - }, - "norm": { - "properties": { - "pct": { - "type": "scaled_float", - "scaling_factor": 1000 - } - } - }, - "ticks": { - "type": "long" } } + } + } + } + } + }, + "diskio": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "serial_number": { + "ignore_above": 1024, + "type": "keyword" + }, + "read": { + "properties": { + "count": { + "type": "long" }, - "steal": { - "properties": { - "pct": { - "type": "scaled_float", - "scaling_factor": 1000 - }, - "norm": { - "properties": { - "pct": { - "type": "scaled_float", - "scaling_factor": 1000 - } - } - }, - "ticks": { - "type": "long" - } - } + "bytes": { + "type": "long" }, - "total": { - "properties": { - "pct": { - "type": "scaled_float", - "scaling_factor": 1000 - }, - "norm": { - "properties": { - "pct": { - "type": "scaled_float", - "scaling_factor": 1000 - } - } - } - } + "time": { + "type": "long" } } }, - "diskio": { + "write": { "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" + "count": { + "type": "long" }, - "serial_number": { - "ignore_above": 1024, - "type": "keyword" + "bytes": { + "type": "long" }, + "time": { + "type": "long" + } + } + }, + "io": { + "properties": { + "time": { + "type": "long" + } + } + }, + "iostat": { + "properties": { "read": { "properties": { - "count": { - "type": "long" - }, - "bytes": { - "type": "long" - }, - "time": { - "type": "long" - } - } - }, - "write": { - "properties": { - "count": { - "type": "long" - }, - "bytes": { - "type": "long" - }, - "time": { - "type": "long" - } - } - }, - "io": { - "properties": { - "time": { - "type": "long" - } - } - }, - "iostat": { - "properties": { - "read": { + "request": { "properties": { - "request": { - "properties": { - "merges_per_sec": { - "type": "float" - }, - "per_sec": { - "type": "float" - } - } + "merges_per_sec": { + "type": "float" }, "per_sec": { - "properties": { - "bytes": { - "type": "float" - } - } - }, - "await": { "type": "float" } } }, - "write": { + "per_sec": { "properties": { - "request": { - "properties": { - "merges_per_sec": { - "type": "float" - }, - "per_sec": { - "type": "float" - } - } - }, - "per_sec": { - "properties": { - "bytes": { - "type": "float" - } - } - }, - "await": { + "bytes": { "type": "float" } } }, + "await": { + "type": "float" + } + } + }, + "write": { + "properties": { "request": { "properties": { - "avg_size": { + "merges_per_sec": { + "type": "float" + }, + "per_sec": { "type": "float" } } }, - "queue": { + "per_sec": { "properties": { - "avg_size": { + "bytes": { "type": "float" } } }, "await": { "type": "float" - }, - "service_time": { + } + } + }, + "request": { + "properties": { + "avg_size": { "type": "float" - }, - "busy": { + } + } + }, + "queue": { + "properties": { + "avg_size": { "type": "float" } } + }, + "await": { + "type": "float" + }, + "service_time": { + "type": "float" + }, + "busy": { + "type": "float" } } + } + } + }, + "entropy": { + "properties": { + "available_bits": { + "type": "long" }, - "entropy": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + } + } + }, + "filesystem": { + "properties": { + "available": { + "type": "long" + }, + "device_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "mount_point": { + "ignore_above": 1024, + "type": "keyword" + }, + "files": { + "type": "long" + }, + "free": { + "type": "long" + }, + "free_files": { + "type": "long" + }, + "total": { + "type": "long" + }, + "used": { "properties": { - "available_bits": { + "bytes": { "type": "long" }, "pct": { @@ -616,36 +553,88 @@ exports[`EPM template tests loading system.yml: system.yml 1`] = ` "scaling_factor": 1000 } } + } + } + }, + "fsstat": { + "properties": { + "count": { + "type": "long" + }, + "total_files": { + "type": "long" }, - "filesystem": { + "total_size": { "properties": { - "available": { + "free": { "type": "long" }, - "device_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "mount_point": { - "ignore_above": 1024, - "type": "keyword" - }, - "files": { + "used": { "type": "long" }, - "free": { + "total": { "type": "long" + } + } + } + } + }, + "load": { + "properties": { + "1": { + "type": "scaled_float", + "scaling_factor": 100 + }, + "5": { + "type": "scaled_float", + "scaling_factor": 100 + }, + "15": { + "type": "scaled_float", + "scaling_factor": 100 + }, + "norm": { + "properties": { + "1": { + "type": "scaled_float", + "scaling_factor": 100 }, - "free_files": { - "type": "long" + "5": { + "type": "scaled_float", + "scaling_factor": 100 }, - "total": { + "15": { + "type": "scaled_float", + "scaling_factor": 100 + } + } + }, + "cores": { + "type": "long" + } + } + }, + "memory": { + "properties": { + "total": { + "type": "long" + }, + "used": { + "properties": { + "bytes": { "type": "long" }, + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + } + } + }, + "free": { + "type": "long" + }, + "actual": { + "properties": { "used": { "properties": { "bytes": { @@ -656,68 +645,58 @@ exports[`EPM template tests loading system.yml: system.yml 1`] = ` "scaling_factor": 1000 } } + }, + "free": { + "type": "long" } } }, - "fsstat": { + "swap": { "properties": { - "count": { - "type": "long" - }, - "total_files": { + "total": { "type": "long" }, - "total_size": { + "used": { "properties": { - "free": { - "type": "long" - }, - "used": { + "bytes": { "type": "long" }, - "total": { - "type": "long" + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 } } - } - } - }, - "load": { - "properties": { - "1": { - "type": "scaled_float", - "scaling_factor": 100 }, - "5": { - "type": "scaled_float", - "scaling_factor": 100 + "free": { + "type": "long" }, - "15": { - "type": "scaled_float", - "scaling_factor": 100 + "out": { + "properties": { + "pages": { + "type": "long" + } + } }, - "norm": { + "in": { "properties": { - "1": { - "type": "scaled_float", - "scaling_factor": 100 - }, - "5": { - "type": "scaled_float", - "scaling_factor": 100 - }, - "15": { - "type": "scaled_float", - "scaling_factor": 100 + "pages": { + "type": "long" } } }, - "cores": { - "type": "long" + "readahead": { + "properties": { + "pages": { + "type": "long" + }, + "cached": { + "type": "long" + } + } } } }, - "memory": { + "hugepages": { "properties": { "total": { "type": "long" @@ -728,296 +707,332 @@ exports[`EPM template tests loading system.yml: system.yml 1`] = ` "type": "long" }, "pct": { - "type": "scaled_float", - "scaling_factor": 1000 + "type": "long" } } }, "free": { "type": "long" }, - "actual": { + "reserved": { + "type": "long" + }, + "surplus": { + "type": "long" + }, + "default_size": { + "type": "long" + }, + "swap": { + "properties": { + "out": { + "properties": { + "pages": { + "type": "long" + }, + "fallback": { + "type": "long" + } + } + } + } + } + } + } + } + }, + "network": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "out": { + "properties": { + "bytes": { + "type": "long" + }, + "packets": { + "type": "long" + }, + "errors": { + "type": "long" + }, + "dropped": { + "type": "long" + } + } + }, + "in": { + "properties": { + "bytes": { + "type": "long" + }, + "packets": { + "type": "long" + }, + "errors": { + "type": "long" + }, + "dropped": { + "type": "long" + } + } + } + } + }, + "network_summary": { + "properties": { + "ip": { + "properties": { + "*": { + "type": "object" + } + } + }, + "tcp": { + "properties": { + "*": { + "type": "object" + } + } + }, + "udp": { + "properties": { + "*": { + "type": "object" + } + } + }, + "udp_lite": { + "properties": { + "*": { + "type": "object" + } + } + }, + "icmp": { + "properties": { + "*": { + "type": "object" + } + } + } + } + }, + "process": { + "properties": { + "state": { + "ignore_above": 1024, + "type": "keyword" + }, + "cmdline": { + "ignore_above": 2048, + "type": "keyword" + }, + "env": { + "type": "object" + }, + "cpu": { + "properties": { + "user": { "properties": { - "used": { - "properties": { - "bytes": { - "type": "long" - }, - "pct": { - "type": "scaled_float", - "scaling_factor": 1000 - } - } - }, - "free": { + "ticks": { "type": "long" } } }, - "swap": { + "total": { "properties": { - "total": { + "value": { "type": "long" }, - "used": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + }, + "norm": { "properties": { - "bytes": { - "type": "long" - }, "pct": { "type": "scaled_float", "scaling_factor": 1000 } } }, - "free": { + "ticks": { "type": "long" - }, - "out": { - "properties": { - "pages": { - "type": "long" - } - } - }, - "in": { - "properties": { - "pages": { - "type": "long" - } - } - }, - "readahead": { - "properties": { - "pages": { - "type": "long" - }, - "cached": { - "type": "long" - } - } } } }, - "hugepages": { + "system": { "properties": { - "total": { - "type": "long" - }, - "used": { - "properties": { - "bytes": { - "type": "long" - }, - "pct": { - "type": "long" - } - } - }, - "free": { - "type": "long" - }, - "reserved": { - "type": "long" - }, - "surplus": { - "type": "long" - }, - "default_size": { + "ticks": { "type": "long" - }, - "swap": { - "properties": { - "out": { - "properties": { - "pages": { - "type": "long" - }, - "fallback": { - "type": "long" - } - } - } - } } } + }, + "start_time": { + "type": "date" } } }, - "network": { + "memory": { "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" + "size": { + "type": "long" }, - "out": { + "rss": { "properties": { "bytes": { "type": "long" }, - "packets": { - "type": "long" - }, - "errors": { - "type": "long" - }, - "dropped": { - "type": "long" + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 } } }, - "in": { - "properties": { - "bytes": { - "type": "long" - }, - "packets": { - "type": "long" - }, - "errors": { - "type": "long" - }, - "dropped": { - "type": "long" - } - } + "share": { + "type": "long" } } }, - "network_summary": { + "fd": { "properties": { - "ip": { - "properties": { - "*": { - "type": "object" - } - } - }, - "tcp": { - "properties": { - "*": { - "type": "object" - } - } - }, - "udp": { - "properties": { - "*": { - "type": "object" - } - } - }, - "udp_lite": { - "properties": { - "*": { - "type": "object" - } - } + "open": { + "type": "long" }, - "icmp": { + "limit": { "properties": { - "*": { - "type": "object" + "soft": { + "type": "long" + }, + "hard": { + "type": "long" } } } } }, - "process": { + "cgroup": { "properties": { - "state": { + "id": { "ignore_above": 1024, "type": "keyword" }, - "cmdline": { - "ignore_above": 2048, + "path": { + "ignore_above": 1024, "type": "keyword" }, - "env": { - "type": "object" - }, "cpu": { "properties": { - "user": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "cfs": { "properties": { - "ticks": { + "period": { + "properties": { + "us": { + "type": "long" + } + } + }, + "quota": { + "properties": { + "us": { + "type": "long" + } + } + }, + "shares": { "type": "long" } } }, - "total": { + "rt": { "properties": { - "value": { - "type": "long" - }, - "pct": { - "type": "scaled_float", - "scaling_factor": 1000 - }, - "norm": { + "period": { "properties": { - "pct": { - "type": "scaled_float", - "scaling_factor": 1000 + "us": { + "type": "long" } } }, - "ticks": { - "type": "long" + "runtime": { + "properties": { + "us": { + "type": "long" + } + } } } }, - "system": { + "stats": { "properties": { - "ticks": { + "periods": { "type": "long" + }, + "throttled": { + "properties": { + "periods": { + "type": "long" + }, + "ns": { + "type": "long" + } + } } } - }, - "start_time": { - "type": "date" } } }, - "memory": { + "cpuacct": { "properties": { - "size": { - "type": "long" + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" }, - "rss": { + "total": { "properties": { - "bytes": { + "ns": { "type": "long" - }, - "pct": { - "type": "scaled_float", - "scaling_factor": 1000 } } }, - "share": { - "type": "long" - } - } - }, - "fd": { - "properties": { - "open": { - "type": "long" - }, - "limit": { + "stats": { "properties": { - "soft": { - "type": "long" + "user": { + "properties": { + "ns": { + "type": "long" + } + } }, - "hard": { - "type": "long" + "system": { + "properties": { + "ns": { + "type": "long" + } + } } } + }, + "percpu": { + "type": "object" } } }, - "cgroup": { + "memory": { "properties": { "id": { "ignore_above": 1024, @@ -1027,354 +1042,212 @@ exports[`EPM template tests loading system.yml: system.yml 1`] = ` "ignore_above": 1024, "type": "keyword" }, - "cpu": { + "mem": { "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "path": { - "ignore_above": 1024, - "type": "keyword" - }, - "cfs": { + "usage": { "properties": { - "period": { - "properties": { - "us": { - "type": "long" - } - } + "bytes": { + "type": "long" }, - "quota": { + "max": { "properties": { - "us": { + "bytes": { "type": "long" } } - }, - "shares": { + } + } + }, + "limit": { + "properties": { + "bytes": { "type": "long" } } }, - "rt": { + "failures": { + "type": "long" + } + } + }, + "memsw": { + "properties": { + "usage": { "properties": { - "period": { - "properties": { - "us": { - "type": "long" - } - } + "bytes": { + "type": "long" }, - "runtime": { + "max": { "properties": { - "us": { + "bytes": { "type": "long" } } } } }, - "stats": { + "limit": { "properties": { - "periods": { + "bytes": { + "type": "long" + } + } + }, + "failures": { + "type": "long" + } + } + }, + "kmem": { + "properties": { + "usage": { + "properties": { + "bytes": { "type": "long" }, - "throttled": { + "max": { "properties": { - "periods": { - "type": "long" - }, - "ns": { + "bytes": { "type": "long" } } } } - } - } - }, - "cpuacct": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "path": { - "ignore_above": 1024, - "type": "keyword" }, - "total": { + "limit": { "properties": { - "ns": { + "bytes": { "type": "long" } } }, - "stats": { + "failures": { + "type": "long" + } + } + }, + "kmem_tcp": { + "properties": { + "usage": { "properties": { - "user": { - "properties": { - "ns": { - "type": "long" - } - } + "bytes": { + "type": "long" }, - "system": { + "max": { "properties": { - "ns": { + "bytes": { "type": "long" } } } } }, - "percpu": { - "type": "object" + "limit": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "failures": { + "type": "long" } } }, - "memory": { + "stats": { "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "path": { - "ignore_above": 1024, - "type": "keyword" + "active_anon": { + "properties": { + "bytes": { + "type": "long" + } + } }, - "mem": { + "active_file": { "properties": { - "usage": { - "properties": { - "bytes": { - "type": "long" - }, - "max": { - "properties": { - "bytes": { - "type": "long" - } - } - } - } - }, - "limit": { - "properties": { - "bytes": { - "type": "long" - } - } - }, - "failures": { + "bytes": { "type": "long" } } }, - "memsw": { + "cache": { "properties": { - "usage": { - "properties": { - "bytes": { - "type": "long" - }, - "max": { - "properties": { - "bytes": { - "type": "long" - } - } - } - } - }, - "limit": { - "properties": { - "bytes": { - "type": "long" - } - } - }, - "failures": { + "bytes": { "type": "long" } } }, - "kmem": { + "hierarchical_memory_limit": { "properties": { - "usage": { - "properties": { - "bytes": { - "type": "long" - }, - "max": { - "properties": { - "bytes": { - "type": "long" - } - } - } - } - }, - "limit": { - "properties": { - "bytes": { - "type": "long" - } - } - }, - "failures": { + "bytes": { "type": "long" } } }, - "kmem_tcp": { + "hierarchical_memsw_limit": { "properties": { - "usage": { - "properties": { - "bytes": { - "type": "long" - }, - "max": { - "properties": { - "bytes": { - "type": "long" - } - } - } - } - }, - "limit": { - "properties": { - "bytes": { - "type": "long" - } - } - }, - "failures": { + "bytes": { "type": "long" } } }, - "stats": { + "inactive_anon": { "properties": { - "active_anon": { - "properties": { - "bytes": { - "type": "long" - } - } - }, - "active_file": { - "properties": { - "bytes": { - "type": "long" - } - } - }, - "cache": { - "properties": { - "bytes": { - "type": "long" - } - } - }, - "hierarchical_memory_limit": { - "properties": { - "bytes": { - "type": "long" - } - } - }, - "hierarchical_memsw_limit": { - "properties": { - "bytes": { - "type": "long" - } - } - }, - "inactive_anon": { - "properties": { - "bytes": { - "type": "long" - } - } - }, - "inactive_file": { - "properties": { - "bytes": { - "type": "long" - } - } - }, - "mapped_file": { - "properties": { - "bytes": { - "type": "long" - } - } - }, - "page_faults": { + "bytes": { "type": "long" - }, - "major_page_faults": { + } + } + }, + "inactive_file": { + "properties": { + "bytes": { "type": "long" - }, - "pages_in": { + } + } + }, + "mapped_file": { + "properties": { + "bytes": { "type": "long" - }, - "pages_out": { + } + } + }, + "page_faults": { + "type": "long" + }, + "major_page_faults": { + "type": "long" + }, + "pages_in": { + "type": "long" + }, + "pages_out": { + "type": "long" + }, + "rss": { + "properties": { + "bytes": { "type": "long" - }, - "rss": { - "properties": { - "bytes": { - "type": "long" - } - } - }, - "rss_huge": { - "properties": { - "bytes": { - "type": "long" - } - } - }, - "swap": { - "properties": { - "bytes": { - "type": "long" - } - } - }, - "unevictable": { - "properties": { - "bytes": { - "type": "long" - } - } } } - } - } - }, - "blkio": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" }, - "path": { - "ignore_above": 1024, - "type": "keyword" + "rss_huge": { + "properties": { + "bytes": { + "type": "long" + } + } }, - "total": { + "swap": { "properties": { "bytes": { "type": "long" - }, - "ios": { + } + } + }, + "unevictable": { + "properties": { + "bytes": { "type": "long" } } @@ -1383,286 +1256,290 @@ exports[`EPM template tests loading system.yml: system.yml 1`] = ` } } }, - "summary": { + "blkio": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "total": { + "properties": { + "bytes": { + "type": "long" + }, + "ios": { + "type": "long" + } + } + } + } + } + } + }, + "summary": { + "properties": { + "total": { + "type": "long" + }, + "running": { + "type": "long" + }, + "idle": { + "type": "long" + }, + "sleeping": { + "type": "long" + }, + "stopped": { + "type": "long" + }, + "zombie": { + "type": "long" + }, + "dead": { + "type": "long" + }, + "unknown": { + "type": "long" + } + } + } + } + }, + "raid": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + }, + "level": { + "ignore_above": 1024, + "type": "keyword" + }, + "sync_action": { + "ignore_above": 1024, + "type": "keyword" + }, + "disks": { + "properties": { + "active": { + "type": "long" + }, + "total": { + "type": "long" + }, + "spare": { + "type": "long" + }, + "failed": { + "type": "long" + }, + "states": { "properties": { - "total": { - "type": "long" - }, - "running": { - "type": "long" - }, - "idle": { - "type": "long" - }, - "sleeping": { - "type": "long" - }, - "stopped": { - "type": "long" - }, - "zombie": { - "type": "long" - }, - "dead": { - "type": "long" - }, - "unknown": { - "type": "long" + "*": { + "type": "object" } } } } }, - "raid": { + "blocks": { + "properties": { + "total": { + "type": "long" + }, + "synced": { + "type": "long" + } + } + } + } + }, + "socket": { + "properties": { + "local": { + "properties": { + "ip": { + "type": "ip" + }, + "port": { + "type": "long" + } + } + }, + "remote": { "properties": { - "name": { + "ip": { + "type": "ip" + }, + "port": { + "type": "long" + }, + "host": { "ignore_above": 1024, "type": "keyword" }, - "status": { + "etld_plus_one": { "ignore_above": 1024, "type": "keyword" }, - "level": { + "host_error": { "ignore_above": 1024, "type": "keyword" - }, - "sync_action": { + } + } + }, + "process": { + "properties": { + "cmdline": { "ignore_above": 1024, "type": "keyword" - }, - "disks": { - "properties": { - "active": { - "type": "long" - }, - "total": { - "type": "long" - }, - "spare": { - "type": "long" - }, - "failed": { - "type": "long" - }, - "states": { - "properties": { - "*": { - "type": "object" - } - } - } - } - }, - "blocks": { - "properties": { - "total": { - "type": "long" - }, - "synced": { - "type": "long" - } - } } } }, - "socket": { + "user": { + "properties": {} + }, + "summary": { "properties": { - "local": { + "all": { "properties": { - "ip": { - "type": "ip" + "count": { + "type": "long" }, - "port": { + "listening": { "type": "long" } } }, - "remote": { + "tcp": { "properties": { - "ip": { - "type": "ip" - }, - "port": { + "memory": { "type": "long" }, - "host": { - "ignore_above": 1024, - "type": "keyword" - }, - "etld_plus_one": { - "ignore_above": 1024, - "type": "keyword" - }, - "host_error": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "process": { - "properties": { - "cmdline": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "user": { - "properties": {} - }, - "summary": { - "properties": { "all": { "properties": { + "orphan": { + "type": "long" + }, "count": { "type": "long" }, "listening": { "type": "long" - } - } - }, - "tcp": { - "properties": { - "memory": { + }, + "established": { "type": "long" }, - "all": { - "properties": { - "orphan": { - "type": "long" - }, - "count": { - "type": "long" - }, - "listening": { - "type": "long" - }, - "established": { - "type": "long" - }, - "close_wait": { - "type": "long" - }, - "time_wait": { - "type": "long" - }, - "syn_sent": { - "type": "long" - }, - "syn_recv": { - "type": "long" - }, - "fin_wait1": { - "type": "long" - }, - "fin_wait2": { - "type": "long" - }, - "last_ack": { - "type": "long" - }, - "closing": { - "type": "long" - } - } - } - } - }, - "udp": { - "properties": { - "memory": { + "close_wait": { "type": "long" }, - "all": { - "properties": { - "count": { - "type": "long" - } - } + "time_wait": { + "type": "long" + }, + "syn_sent": { + "type": "long" + }, + "syn_recv": { + "type": "long" + }, + "fin_wait1": { + "type": "long" + }, + "fin_wait2": { + "type": "long" + }, + "last_ack": { + "type": "long" + }, + "closing": { + "type": "long" } } } } - } - } - }, - "uptime": { - "properties": { - "duration": { + }, + "udp": { "properties": { - "ms": { + "memory": { "type": "long" + }, + "all": { + "properties": { + "count": { + "type": "long" + } + } } } } } - }, - "users": { + } + } + }, + "uptime": { + "properties": { + "duration": { "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "seat": { - "ignore_above": 1024, - "type": "keyword" - }, - "path": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "service": { - "ignore_above": 1024, - "type": "keyword" - }, - "remote": { - "type": "boolean" - }, - "state": { - "ignore_above": 1024, - "type": "keyword" - }, - "scope": { - "ignore_above": 1024, - "type": "keyword" - }, - "leader": { + "ms": { "type": "long" - }, - "remote_host": { - "ignore_above": 1024, - "type": "keyword" } } } } - } - }, - "_meta": { - "managed_by": "fleet", - "managed": true, - "package": { - "name": "system" + }, + "users": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "seat": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "service": { + "ignore_above": 1024, + "type": "keyword" + }, + "remote": { + "type": "boolean" + }, + "state": { + "ignore_above": 1024, + "type": "keyword" + }, + "scope": { + "ignore_above": 1024, + "type": "keyword" + }, + "leader": { + "type": "long" + }, + "remote_host": { + "ignore_above": 1024, + "type": "keyword" + } + } } } } - }, - "data_stream": {}, - "composed_of": [ - ".fleet_component_template-1" - ], - "_meta": { - "managed_by": "fleet", - "managed": true, - "package": { - "name": "system" - } } } `; diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/default_settings.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/default_settings.test.ts index ee6d7086cdd3c..ea2d0868c6d17 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/default_settings.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/default_settings.test.ts @@ -46,11 +46,6 @@ describe('buildDefaultSettings', () => { "lifecycle": Object { "name": "logs", }, - "mapping": Object { - "total_fields": Object { - "limit": "10000", - }, - }, "query": Object { "default_field": Array [ "field1Keyword", diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/default_settings.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/default_settings.ts index 84ec75b9da065..7f8e8e8544109 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/default_settings.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/default_settings.ts @@ -67,12 +67,6 @@ export function buildDefaultSettings({ }, // What should be our default for the compression? codec: 'best_compression', - mapping: { - total_fields: { - limit: '10000', - }, - }, - // All the default fields which should be queried have to be added here. // So far we add all keyword and text fields here if there are any, otherwise // this setting is skipped. diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts index 1303db1a36c0e..894c2820fa2e1 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { merge } from 'lodash'; +import { merge, cloneDeep } from 'lodash'; import Boom from '@hapi/boom'; import type { ElasticsearchClient, Logger, SavedObjectsClientContract } from 'src/core/server'; @@ -17,18 +17,23 @@ import type { InstallablePackage, IndexTemplate, PackageInfo, + IndexTemplateMappings, + TemplateMapEntry, + TemplateMap, } from '../../../../types'; + import { loadFieldsFromYaml, processFields } from '../../fields/field'; import type { Field } from '../../fields/field'; import { getPipelineNameForInstallation } from '../ingest_pipeline/install'; import { getAsset, getPathParts } from '../../archive'; import { removeAssetTypesFromInstalledEs, saveInstalledEsRefs } from '../../packages/install'; import { - FLEET_GLOBAL_COMPONENT_TEMPLATE_NAME, - FLEET_GLOBAL_COMPONENT_TEMPLATE_CONTENT, + FLEET_COMPONENT_TEMPLATES, + MAPPINGS_TEMPLATE_SUFFIX, + SETTINGS_TEMPLATE_SUFFIX, + USER_SETTINGS_TEMPLATE_SUFFIX, } from '../../../../constants'; -import type { ESAssetMetadata } from '../meta'; import { getESAssetMetadata } from '../meta'; import { retryTransientEsErrors } from '../retry'; @@ -43,6 +48,8 @@ import { } from './template'; import { buildDefaultSettings } from './default_settings'; +const FLEET_COMPONENT_TEMPLATE_NAMES = FLEET_COMPONENT_TEMPLATES.map((tmpl) => tmpl.name); + export const installTemplates = async ( installablePackage: InstallablePackage, esClient: ElasticsearchClient, @@ -202,19 +209,6 @@ export async function installTemplateForDataStream({ }); } -interface TemplateMapEntry { - _meta: ESAssetMetadata; - template: - | { - mappings: NonNullable; - } - | { - settings: NonNullable | object; - }; -} - -type TemplateMap = Record; - function putComponentTemplate( esClient: ElasticsearchClient, logger: Logger, @@ -223,7 +217,10 @@ function putComponentTemplate( name: string; create?: boolean; } -): { clusterPromise: Promise; name: string } { +): { + clusterPromise: ReturnType; + name: string; +} { const { name, body, create = false } = params; return { clusterPromise: retryTransientEsErrors( @@ -234,41 +231,59 @@ function putComponentTemplate( }; } -const mappingsSuffix = '@mappings'; -const settingsSuffix = '@settings'; -const userSettingsSuffix = '@custom'; type TemplateBaseName = string; -type UserSettingsTemplateName = `${TemplateBaseName}${typeof userSettingsSuffix}`; +type UserSettingsTemplateName = `${TemplateBaseName}${typeof USER_SETTINGS_TEMPLATE_SUFFIX}`; const isUserSettingsTemplate = (name: string): name is UserSettingsTemplateName => - name.endsWith(userSettingsSuffix); + name.endsWith(USER_SETTINGS_TEMPLATE_SUFFIX); function buildComponentTemplates(params: { + mappings: IndexTemplateMappings; templateName: string; registryElasticsearch: RegistryElasticsearch | undefined; packageName: string; defaultSettings: IndexTemplate['template']['settings']; }) { - const { templateName, registryElasticsearch, packageName, defaultSettings } = params; - const mappingsTemplateName = `${templateName}${mappingsSuffix}`; - const settingsTemplateName = `${templateName}${settingsSuffix}`; - const userSettingsTemplateName = `${templateName}${userSettingsSuffix}`; + const { templateName, registryElasticsearch, packageName, defaultSettings, mappings } = params; + const mappingsTemplateName = `${templateName}${MAPPINGS_TEMPLATE_SUFFIX}`; + const settingsTemplateName = `${templateName}${SETTINGS_TEMPLATE_SUFFIX}`; + const userSettingsTemplateName = `${templateName}${USER_SETTINGS_TEMPLATE_SUFFIX}`; const templatesMap: TemplateMap = {}; const _meta = getESAssetMetadata({ packageName }); - if (registryElasticsearch && registryElasticsearch['index_template.mappings']) { - templatesMap[mappingsTemplateName] = { - template: { - mappings: registryElasticsearch['index_template.mappings'], - }, - _meta, - }; + const indexTemplateSettings = registryElasticsearch?.['index_template.settings'] ?? {}; + // @ts-expect-error no property .mapping (yes there is) + const indexTemplateMappingSettings = indexTemplateSettings?.index?.mapping; + const indexTemplateSettingsForTemplate = cloneDeep(indexTemplateSettings); + + // index.mapping settings must go on the mapping component template otherwise + // the template may be rejected e.g if nested_fields.limit has been increased + if (indexTemplateMappingSettings) { + // @ts-expect-error no property .mapping + delete indexTemplateSettingsForTemplate.index.mapping; } + templatesMap[mappingsTemplateName] = { + template: { + settings: { + index: { + mapping: { + total_fields: { + limit: '10000', + }, + ...indexTemplateMappingSettings, + }, + }, + }, + mappings: merge(mappings, registryElasticsearch?.['index_template.mappings'] ?? {}), + }, + _meta, + }; + templatesMap[settingsTemplateName] = { template: { - settings: merge(defaultSettings, registryElasticsearch?.['index_template.settings'] ?? {}), + settings: merge(defaultSettings, indexTemplateSettingsForTemplate), }, _meta, }; @@ -285,6 +300,7 @@ function buildComponentTemplates(params: { } async function installDataStreamComponentTemplates(params: { + mappings: IndexTemplateMappings; templateName: string; registryElasticsearch: RegistryElasticsearch | undefined; esClient: ElasticsearchClient; @@ -292,16 +308,23 @@ async function installDataStreamComponentTemplates(params: { packageName: string; defaultSettings: IndexTemplate['template']['settings']; }) { - const { templateName, registryElasticsearch, esClient, packageName, defaultSettings, logger } = - params; - const templates = buildComponentTemplates({ + const { + templateName, + registryElasticsearch, + esClient, + packageName, + defaultSettings, + logger, + mappings, + } = params; + const componentTemplates = buildComponentTemplates({ + mappings, templateName, registryElasticsearch, packageName, defaultSettings, }); - const templateNames = Object.keys(templates); - const templateEntries = Object.entries(templates); + const templateEntries = Object.entries(componentTemplates); // TODO: Check return values for errors await Promise.all( templateEntries.map(async ([name, body]) => { @@ -327,18 +350,31 @@ async function installDataStreamComponentTemplates(params: { }) ); - return templateNames; + return { componentTemplateNames: Object.keys(componentTemplates) }; } -export async function ensureDefaultComponentTemplate( +export async function ensureDefaultComponentTemplates( esClient: ElasticsearchClient, logger: Logger +) { + return Promise.all( + FLEET_COMPONENT_TEMPLATES.map(({ name, body }) => + ensureComponentTemplate(esClient, logger, name, body) + ) + ); +} + +export async function ensureComponentTemplate( + esClient: ElasticsearchClient, + logger: Logger, + name: string, + body: TemplateMapEntry ) { const getTemplateRes = await retryTransientEsErrors( () => esClient.cluster.getComponentTemplate( { - name: FLEET_GLOBAL_COMPONENT_TEMPLATE_NAME, + name, }, { ignore: [404], @@ -350,8 +386,8 @@ export async function ensureDefaultComponentTemplate( const existingTemplate = getTemplateRes?.component_templates?.[0]; if (!existingTemplate) { await putComponentTemplate(esClient, logger, { - name: FLEET_GLOBAL_COMPONENT_TEMPLATE_NAME, - body: FLEET_GLOBAL_COMPONENT_TEMPLATE_CONTENT, + name, + body, }).clusterPromise; } @@ -434,7 +470,8 @@ export async function installTemplate({ ilmPolicy: dataStream.ilm_policy, }); - const composedOfTemplates = await installDataStreamComponentTemplates({ + const { componentTemplateNames } = await installDataStreamComponentTemplates({ + mappings, templateName, registryElasticsearch: dataStream.elasticsearch, esClient, @@ -444,13 +481,10 @@ export async function installTemplate({ }); const template = getTemplate({ - type: dataStream.type, templateIndexPattern, - fields: validFields, - mappings, pipelineName, packageName, - composedOfTemplates, + composedOfTemplates: componentTemplateNames, templatePriority, hidden: dataStream.hidden, }); @@ -482,7 +516,9 @@ export function getAllTemplateRefs(installedTemplates: IndexTemplateEntry[]) { ]; const componentTemplates = installedTemplate.indexTemplate.composed_of // Filter global component template shared between integrations - .filter((componentTemplateId) => componentTemplateId !== FLEET_GLOBAL_COMPONENT_TEMPLATE_NAME) + .filter( + (componentTemplateId) => !FLEET_COMPONENT_TEMPLATE_NAMES.includes(componentTemplateId) + ) .map((componentTemplateId) => ({ id: componentTemplateId, type: ElasticsearchAssetType.componentTemplate, diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts index 5474d2c09cfc5..86edf1c5e4064 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts @@ -26,7 +26,7 @@ import { updateCurrentWriteIndices, } from './template'; -const FLEET_COMPONENT_TEMPLATE = '.fleet_component_template-1'; +const FLEET_COMPONENT_TEMPLATES = ['.fleet_globals-1', '.fleet_agent_id_verification-1']; // Add our own serialiser to just do JSON.stringify expect.addSnapshotSerializer({ @@ -48,11 +48,8 @@ describe('EPM template', () => { const templateIndexPattern = 'logs-nginx.access-abcd-*'; const template = getTemplate({ - type: 'logs', templateIndexPattern, packageName: 'nginx', - fields: [], - mappings: { properties: {} }, composedOfTemplates: [], templatePriority: 200, }); @@ -63,41 +60,35 @@ describe('EPM template', () => { const composedOfTemplates = ['component1', 'component2']; const template = getTemplate({ - type: 'logs', templateIndexPattern: 'name-*', packageName: 'nginx', - fields: [], - mappings: { properties: {} }, composedOfTemplates, templatePriority: 200, }); - expect(template.composed_of).toStrictEqual([...composedOfTemplates, FLEET_COMPONENT_TEMPLATE]); + expect(template.composed_of).toStrictEqual([ + ...composedOfTemplates, + ...FLEET_COMPONENT_TEMPLATES, + ]); }); it('adds empty composed_of correctly', () => { const composedOfTemplates: string[] = []; const template = getTemplate({ - type: 'logs', templateIndexPattern: 'name-*', packageName: 'nginx', - fields: [], - mappings: { properties: {} }, composedOfTemplates, templatePriority: 200, }); - expect(template.composed_of).toStrictEqual([FLEET_COMPONENT_TEMPLATE]); + expect(template.composed_of).toStrictEqual(FLEET_COMPONENT_TEMPLATES); }); it('adds hidden field correctly', () => { const templateIndexPattern = 'logs-nginx.access-abcd-*'; const templateWithHidden = getTemplate({ - type: 'logs', templateIndexPattern, packageName: 'nginx', - fields: [], - mappings: { properties: {} }, composedOfTemplates: [], templatePriority: 200, hidden: true, @@ -105,11 +96,8 @@ describe('EPM template', () => { expect(templateWithHidden.data_stream.hidden).toEqual(true); const templateWithoutHidden = getTemplate({ - type: 'logs', templateIndexPattern, packageName: 'nginx', - fields: [], - mappings: { properties: {} }, composedOfTemplates: [], templatePriority: 200, }); @@ -123,17 +111,8 @@ describe('EPM template', () => { const processedFields = processFields(fields); const mappings = generateMappings(processedFields); - const template = getTemplate({ - type: 'logs', - templateIndexPattern: 'foo-*', - packageName: 'nginx', - fields: processedFields, - mappings, - composedOfTemplates: [], - templatePriority: 200, - }); - expect(template).toMatchSnapshot(path.basename(ymlPath)); + expect(mappings).toMatchSnapshot(path.basename(ymlPath)); }); it('tests loading coredns.logs.yml', () => { @@ -143,37 +122,19 @@ describe('EPM template', () => { const processedFields = processFields(fields); const mappings = generateMappings(processedFields); - const template = getTemplate({ - type: 'logs', - templateIndexPattern: 'foo-*', - packageName: 'coredns', - fields: processedFields, - mappings, - composedOfTemplates: [], - templatePriority: 200, - }); - expect(template).toMatchSnapshot(path.basename(ymlPath)); + expect(mappings).toMatchSnapshot(path.basename(ymlPath)); }); it('tests loading system.yml', () => { const ymlPath = path.join(__dirname, '../../fields/tests/system.yml'); const fieldsYML = readFileSync(ymlPath, 'utf-8'); const fields: Field[] = safeLoad(fieldsYML); - const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); - const template = getTemplate({ - type: 'metrics', - templateIndexPattern: 'whatsthis-*', - packageName: 'system', - fields: processedFields, - mappings, - composedOfTemplates: [], - templatePriority: 200, - }); - expect(template).toMatchSnapshot(path.basename(ymlPath)); + expect(mappings).toMatchSnapshot(path.basename(ymlPath)); }); it('tests processing long field with index false', () => { @@ -874,6 +835,12 @@ describe('EPM template', () => { esClient.indices.getDataStream.mockResponse({ data_streams: [{ name: 'test.prefix1-default' }], } as any); + esClient.indices.simulateTemplate.mockResponse({ + template: { + settings: { index: {} }, + mappings: { properties: {} }, + }, + } as any); const logger = loggerMock.create(); await updateCurrentWriteIndices(esClient, logger, [ { @@ -902,6 +869,14 @@ describe('EPM template', () => { { name: 'test-replicated', replicated: true }, ], } as any); + + esClient.indices.simulateTemplate.mockResponse({ + template: { + settings: { index: {} }, + mappings: { properties: {} }, + }, + } as any); + const logger = loggerMock.create(); await updateCurrentWriteIndices(esClient, logger, [ { diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts index 4b3da8bae784d..21c7351b31384 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts @@ -6,6 +6,7 @@ */ import type { ElasticsearchClient, Logger } from 'kibana/server'; +import type { IndicesIndexSettings } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import type { Field, Fields } from '../../fields/field'; import type { @@ -16,7 +17,10 @@ import type { } from '../../../../types'; import { appContextService } from '../../../'; import { getRegistryDataStreamAssetBaseName } from '../index'; -import { FLEET_GLOBAL_COMPONENT_TEMPLATE_NAME } from '../../../../constants'; +import { + FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME, + FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_NAME, +} from '../../../../constants'; import { getESAssetMetadata } from '../meta'; import { retryTransientEsErrors } from '../retry'; @@ -51,20 +55,14 @@ const META_PROP_KEYS = ['metric_type', 'unit']; * @param indexPattern String with the index pattern */ export function getTemplate({ - type, templateIndexPattern, - fields, - mappings, pipelineName, packageName, composedOfTemplates, templatePriority, hidden, }: { - type: string; templateIndexPattern: string; - fields: Fields; - mappings: IndexTemplateMappings; pipelineName?: string | undefined; packageName: string; composedOfTemplates: string[]; @@ -72,10 +70,7 @@ export function getTemplate({ hidden?: boolean; }): IndexTemplate { const template = getBaseTemplate( - type, templateIndexPattern, - fields, - mappings, packageName, composedOfTemplates, templatePriority, @@ -88,10 +83,13 @@ export function getTemplate({ throw new Error(`Error template for ${templateIndexPattern} contains a final_pipeline`); } - if (appContextService.getConfig()?.agentIdVerificationEnabled) { - // Add fleet global assets - template.composed_of = [...(template.composed_of || []), FLEET_GLOBAL_COMPONENT_TEMPLATE_NAME]; - } + template.composed_of = [ + ...(template.composed_of || []), + FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME, + ...(appContextService.getConfig()?.agentIdVerificationEnabled + ? [FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_NAME] + : []), + ]; return template; } @@ -321,6 +319,14 @@ export function generateTemplateName(dataStream: RegistryDataStream): string { return getRegistryDataStreamAssetBaseName(dataStream); } +/** + * Given a data stream name, return the indexTemplate name + */ +function dataStreamNameToIndexTemplateName(dataStreamName: string): string { + const [type, dataset] = dataStreamName.split('-'); // ignore namespace at the end + return [type, dataset].join('-'); +} + export function generateTemplateIndexPattern(dataStream: RegistryDataStream): string { // undefined or explicitly set to false // See also https://github.com/elastic/package-spec/pull/102 @@ -387,45 +393,22 @@ const flattenFieldsToNameAndType = ( }; function getBaseTemplate( - type: string, templateIndexPattern: string, - fields: Fields, - mappings: IndexTemplateMappings, packageName: string, composedOfTemplates: string[], templatePriority: number, hidden?: boolean ): IndexTemplate { - // Meta information to identify Ingest Manager's managed templates and indices const _meta = getESAssetMetadata({ packageName }); return { priority: templatePriority, - // To be completed with the correct index patterns index_patterns: [templateIndexPattern], template: { settings: { index: {}, }, mappings: { - // All the dynamic field mappings - dynamic_templates: [ - // This makes sure all mappings are keywords by default - { - strings_as_keyword: { - mapping: { - ignore_above: 1024, - type: 'keyword', - }, - match_mapping_type: 'string', - }, - }, - ], - // As we define fields ahead, we don't need any automatic field detection - // This makes sure all the fields are mapped to keyword by default to prevent mapping conflicts - date_detection: false, - // All the properties we know from the fields.yml file - properties: mappings.properties, _meta, }, }, @@ -490,70 +473,81 @@ const getDataStreams = async ( })); }; +const rolloverDataStream = (dataStreamName: string, esClient: ElasticsearchClient) => { + try { + // Do no wrap rollovers in retryTransientEsErrors since it is not idempotent + return esClient.indices.rollover({ + alias: dataStreamName, + }); + } catch (error) { + throw new Error(`cannot rollover data stream [${dataStreamName}] due to error: ${error}`); + } +}; + const updateAllDataStreams = async ( indexNameWithTemplates: CurrentDataStream[], esClient: ElasticsearchClient, logger: Logger ): Promise => { - const updatedataStreamPromises = indexNameWithTemplates.map( - ({ dataStreamName, indexTemplate }) => { - return updateExistingDataStream({ dataStreamName, esClient, logger, indexTemplate }); - } - ); + const updatedataStreamPromises = indexNameWithTemplates.map((templateEntry) => { + return updateExistingDataStream({ + esClient, + logger, + dataStreamName: templateEntry.dataStreamName, + }); + }); await Promise.all(updatedataStreamPromises); }; const updateExistingDataStream = async ({ dataStreamName, esClient, logger, - indexTemplate, }: { dataStreamName: string; esClient: ElasticsearchClient; logger: Logger; - indexTemplate: IndexTemplate; }) => { - const { settings, mappings } = indexTemplate.template; - - // for now, remove from object so as not to update stream or data stream properties of the index until type and name - // are added in https://github.com/elastic/kibana/issues/66551. namespace value we will continue - // to skip updating and assume the value in the index mapping is correct - delete mappings.properties.stream; - delete mappings.properties.data_stream; - // try to update the mappings first + let settings: IndicesIndexSettings; try { + const simulateResult = await retryTransientEsErrors(() => + esClient.indices.simulateTemplate({ + name: dataStreamNameToIndexTemplateName(dataStreamName), + }) + ); + + settings = simulateResult.template.settings; + const mappings = simulateResult.template.mappings; + // for now, remove from object so as not to update stream or data stream properties of the index until type and name + // are added in https://github.com/elastic/kibana/issues/66551. namespace value we will continue + // to skip updating and assume the value in the index mapping is correct + if (mappings && mappings.properties) { + delete mappings.properties.stream; + delete mappings.properties.data_stream; + } await retryTransientEsErrors( () => esClient.indices.putMapping({ index: dataStreamName, - body: mappings, + body: mappings || {}, write_index_only: true, }), { logger } ); // if update fails, rollover data stream } catch (err) { - try { - // Do no wrap rollovers in retryTransientEsErrors since it is not idempotent - const path = `/${dataStreamName}/_rollover`; - await esClient.transport.request({ - method: 'POST', - path, - }); - } catch (error) { - throw new Error(`cannot rollover data stream ${error}`); - } + await rolloverDataStream(dataStreamName, esClient); + return; } // update settings after mappings was successful to ensure // pointing to the new pipeline is safe // for now, only update the pipeline - if (!settings.index.default_pipeline) return; + if (!settings?.index?.default_pipeline) return; try { await retryTransientEsErrors( () => esClient.indices.putSettings({ index: dataStreamName, - body: { default_pipeline: settings.index.default_pipeline }, + body: { default_pipeline: settings!.index!.default_pipeline }, }), { logger } ); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/index.ts b/x-pack/plugins/fleet/server/services/epm/packages/index.ts index fa2e5781a209e..d742103dccf12 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/index.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/index.ts @@ -25,6 +25,8 @@ export { getLimitedPackages, } from './get'; +export { getBundledPackages } from './bundled_packages'; + export type { BulkInstallResponse, IBulkInstallPackageError } from './install'; export { handleInstallPackageFailure, installPackage, ensureInstalledPackage } from './install'; export { removeInstallation } from './remove'; diff --git a/x-pack/plugins/fleet/server/services/setup.ts b/x-pack/plugins/fleet/server/services/setup.ts index ed6ba978251ff..6e3aed538fb07 100644 --- a/x-pack/plugins/fleet/server/services/setup.ts +++ b/x-pack/plugins/fleet/server/services/setup.ts @@ -10,7 +10,12 @@ import { compact } from 'lodash'; import type { ElasticsearchClient, SavedObjectsClientContract } from 'src/core/server'; import { AUTO_UPDATE_PACKAGES } from '../../common'; -import type { DefaultPackagesInstallationError, PreconfigurationError } from '../../common'; +import type { + DefaultPackagesInstallationError, + PreconfigurationError, + BundledPackage, + Installation, +} from '../../common'; import { SO_SEARCH_LIMIT } from '../constants'; import { DEFAULT_SPACE_ID } from '../../../spaces/common/constants'; @@ -25,13 +30,13 @@ import { generateEnrollmentAPIKey, hasEnrollementAPIKeysForPolicy } from './api_ import { settingsService } from '.'; import { awaitIfPending } from './setup_utils'; import { ensureFleetFinalPipelineIsInstalled } from './epm/elasticsearch/ingest_pipeline/install'; -import { ensureDefaultComponentTemplate } from './epm/elasticsearch/template/install'; +import { ensureDefaultComponentTemplates } from './epm/elasticsearch/template/install'; import { getInstallations, installPackage } from './epm/packages'; import { isPackageInstalled } from './epm/packages/install'; import { pkgToPkgKey } from './epm/registry'; import type { UpgradeManagedPackagePoliciesResult } from './managed_package_policies'; import { upgradeManagedPackagePolicies } from './managed_package_policies'; - +import { getBundledPackages } from './epm/packages'; export interface SetupStatus { isInitialized: boolean; nonFatalErrors: Array< @@ -139,21 +144,43 @@ export async function ensureFleetGlobalEsAssets( // Ensure Global Fleet ES assets are installed logger.debug('Creating Fleet component template and ingest pipeline'); const globalAssetsRes = await Promise.all([ - ensureDefaultComponentTemplate(esClient, logger), + ensureDefaultComponentTemplates(esClient, logger), // returns an array ensureFleetFinalPipelineIsInstalled(esClient, logger), ]); - - if (globalAssetsRes.some((asset) => asset.isCreated)) { + const assetResults = globalAssetsRes.flat(); + if (assetResults.some((asset) => asset.isCreated)) { // Update existing index template - const packages = await getInstallations(soClient); - + const installedPackages = await getInstallations(soClient); + const bundledPackages = await getBundledPackages(); + const findMatchingBundledPkg = (pkg: Installation) => + bundledPackages.find( + (bundledPkg: BundledPackage) => + bundledPkg.name === pkg.name && bundledPkg.version === pkg.version + ); await Promise.all( - packages.saved_objects.map(async ({ attributes: installation }) => { + installedPackages.saved_objects.map(async ({ attributes: installation }) => { if (installation.install_source !== 'registry') { - logger.error( - `Package needs to be manually reinstalled ${installation.name} after installing Fleet global assets` - ); - return; + const matchingBundledPackage = findMatchingBundledPkg(installation); + if (!matchingBundledPackage) { + logger.error( + `Package needs to be manually reinstalled ${installation.name} after installing Fleet global assets` + ); + return; + } else { + await installPackage({ + installSource: 'upload', + savedObjectsClient: soClient, + esClient, + spaceId: DEFAULT_SPACE_ID, + contentType: 'application/zip', + archiveBuffer: matchingBundledPackage.buffer, + }).catch((err) => { + logger.error( + `Bundled package needs to be manually reinstalled ${installation.name} after installing Fleet global assets: ${err.message}` + ); + }); + return; + } } await installPackage({ installSource: installation.install_source, diff --git a/x-pack/plugins/fleet/server/types/index.tsx b/x-pack/plugins/fleet/server/types/index.tsx index 91303046485d9..9024cd05e2dea 100644 --- a/x-pack/plugins/fleet/server/types/index.tsx +++ b/x-pack/plugins/fleet/server/types/index.tsx @@ -63,6 +63,8 @@ export type { RegistrySearchResult, IndexTemplateEntry, IndexTemplateMappings, + TemplateMap, + TemplateMapEntry, Settings, SettingsSOAttributes, InstallType, diff --git a/x-pack/test/fleet_api_integration/apis/epm/__snapshots__/install_by_upload.snap b/x-pack/test/fleet_api_integration/apis/epm/__snapshots__/install_by_upload.snap new file mode 100644 index 0000000000000..421a5fbdf1744 --- /dev/null +++ b/x-pack/test/fleet_api_integration/apis/epm/__snapshots__/install_by_upload.snap @@ -0,0 +1,735 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`Fleet Endpoints EPM Endpoints installs packages from direct upload should install a zip archive correctly and package info should return correctly after validation 1`] = ` +Object { + "assets": Object { + "elasticsearch": Object { + "ingest_pipeline": Array [ + Object { + "dataset": "access", + "file": "default.yml", + "path": "apache-0.1.4/data_stream/access/elasticsearch/ingest_pipeline/default.yml", + "pkgkey": "apache-0.1.4", + "service": "elasticsearch", + "type": "ingest_pipeline", + }, + Object { + "dataset": "error", + "file": "default.yml", + "path": "apache-0.1.4/data_stream/error/elasticsearch/ingest_pipeline/default.yml", + "pkgkey": "apache-0.1.4", + "service": "elasticsearch", + "type": "ingest_pipeline", + }, + ], + }, + "kibana": Object { + "dashboard": Array [ + Object { + "file": "apache-Logs-Apache-Dashboard-ecs.json", + "path": "apache-0.1.4/kibana/dashboard/apache-Logs-Apache-Dashboard-ecs.json", + "pkgkey": "apache-0.1.4", + "service": "kibana", + "type": "dashboard", + }, + Object { + "file": "apache-Metrics-Apache-HTTPD-server-status-ecs.json", + "path": "apache-0.1.4/kibana/dashboard/apache-Metrics-Apache-HTTPD-server-status-ecs.json", + "pkgkey": "apache-0.1.4", + "service": "kibana", + "type": "dashboard", + }, + ], + "search": Array [ + Object { + "file": "Apache-HTTPD-ecs.json", + "path": "apache-0.1.4/kibana/search/Apache-HTTPD-ecs.json", + "pkgkey": "apache-0.1.4", + "service": "kibana", + "type": "search", + }, + Object { + "file": "Apache-access-logs-ecs.json", + "path": "apache-0.1.4/kibana/search/Apache-access-logs-ecs.json", + "pkgkey": "apache-0.1.4", + "service": "kibana", + "type": "search", + }, + Object { + "file": "Apache-errors-log-ecs.json", + "path": "apache-0.1.4/kibana/search/Apache-errors-log-ecs.json", + "pkgkey": "apache-0.1.4", + "service": "kibana", + "type": "search", + }, + ], + "visualization": Array [ + Object { + "file": "Apache-HTTPD-CPU-ecs.json", + "path": "apache-0.1.4/kibana/visualization/Apache-HTTPD-CPU-ecs.json", + "pkgkey": "apache-0.1.4", + "service": "kibana", + "type": "visualization", + }, + Object { + "file": "Apache-HTTPD-Hostname-list-ecs.json", + "path": "apache-0.1.4/kibana/visualization/Apache-HTTPD-Hostname-list-ecs.json", + "pkgkey": "apache-0.1.4", + "service": "kibana", + "type": "visualization", + }, + Object { + "file": "Apache-HTTPD-Load1-slash-5-slash-15-ecs.json", + "path": "apache-0.1.4/kibana/visualization/Apache-HTTPD-Load1-slash-5-slash-15-ecs.json", + "pkgkey": "apache-0.1.4", + "service": "kibana", + "type": "visualization", + }, + Object { + "file": "Apache-HTTPD-Scoreboard-ecs.json", + "path": "apache-0.1.4/kibana/visualization/Apache-HTTPD-Scoreboard-ecs.json", + "pkgkey": "apache-0.1.4", + "service": "kibana", + "type": "visualization", + }, + Object { + "file": "Apache-HTTPD-Total-accesses-and-kbytes-ecs.json", + "path": "apache-0.1.4/kibana/visualization/Apache-HTTPD-Total-accesses-and-kbytes-ecs.json", + "pkgkey": "apache-0.1.4", + "service": "kibana", + "type": "visualization", + }, + Object { + "file": "Apache-HTTPD-Uptime-ecs.json", + "path": "apache-0.1.4/kibana/visualization/Apache-HTTPD-Uptime-ecs.json", + "pkgkey": "apache-0.1.4", + "service": "kibana", + "type": "visualization", + }, + Object { + "file": "Apache-HTTPD-Workers-ecs.json", + "path": "apache-0.1.4/kibana/visualization/Apache-HTTPD-Workers-ecs.json", + "pkgkey": "apache-0.1.4", + "service": "kibana", + "type": "visualization", + }, + Object { + "file": "Apache-access-unique-IPs-map-ecs.json", + "path": "apache-0.1.4/kibana/visualization/Apache-access-unique-IPs-map-ecs.json", + "pkgkey": "apache-0.1.4", + "service": "kibana", + "type": "visualization", + }, + Object { + "file": "Apache-browsers-ecs.json", + "path": "apache-0.1.4/kibana/visualization/Apache-browsers-ecs.json", + "pkgkey": "apache-0.1.4", + "service": "kibana", + "type": "visualization", + }, + Object { + "file": "Apache-error-logs-over-time-ecs.json", + "path": "apache-0.1.4/kibana/visualization/Apache-error-logs-over-time-ecs.json", + "pkgkey": "apache-0.1.4", + "service": "kibana", + "type": "visualization", + }, + Object { + "file": "Apache-operating-systems-ecs.json", + "path": "apache-0.1.4/kibana/visualization/Apache-operating-systems-ecs.json", + "pkgkey": "apache-0.1.4", + "service": "kibana", + "type": "visualization", + }, + Object { + "file": "Apache-response-codes-of-top-URLs-ecs.json", + "path": "apache-0.1.4/kibana/visualization/Apache-response-codes-of-top-URLs-ecs.json", + "pkgkey": "apache-0.1.4", + "service": "kibana", + "type": "visualization", + }, + Object { + "file": "Apache-response-codes-over-time-ecs.json", + "path": "apache-0.1.4/kibana/visualization/Apache-response-codes-over-time-ecs.json", + "pkgkey": "apache-0.1.4", + "service": "kibana", + "type": "visualization", + }, + ], + }, + }, + "categories": Array [ + "web", + ], + "conditions": Object { + "kibana": Object { + "version": "^7.9.0", + }, + }, + "data_streams": Array [ + Object { + "dataset": "apache.access", + "ingest_pipeline": "default", + "package": "apache", + "path": "access", + "release": "experimental", + "streams": Array [ + Object { + "description": "Collect Apache access logs", + "enabled": true, + "input": "logfile", + "template_path": "log.yml.hbs", + "title": "Apache access logs", + "vars": Array [ + Object { + "default": Array [ + "/var/log/apache2/access.log*", + "/var/log/apache2/other_vhosts_access.log*", + "/var/log/httpd/access_log*", + ], + "multi": true, + "name": "paths", + "required": true, + "show_user": true, + "title": "Paths", + "type": "text", + }, + ], + }, + ], + "title": "Apache access logs", + "type": "logs", + }, + Object { + "dataset": "apache.error", + "ingest_pipeline": "default", + "package": "apache", + "path": "error", + "release": "experimental", + "streams": Array [ + Object { + "description": "Collect Apache error logs", + "enabled": true, + "input": "logfile", + "template_path": "log.yml.hbs", + "title": "Apache error logs", + "vars": Array [ + Object { + "default": Array [ + "/var/log/apache2/error.log*", + "/var/log/httpd/error_log*", + ], + "multi": true, + "name": "paths", + "required": true, + "show_user": true, + "title": "Paths", + "type": "text", + }, + ], + }, + ], + "title": "Apache error logs", + "type": "logs", + }, + Object { + "dataset": "apache.status", + "package": "apache", + "path": "status", + "release": "experimental", + "streams": Array [ + Object { + "description": "Collect Apache status metrics", + "enabled": true, + "input": "apache/metrics", + "template_path": "stream.yml.hbs", + "title": "Apache status metrics", + "vars": Array [ + Object { + "default": "10s", + "multi": false, + "name": "period", + "required": true, + "show_user": true, + "title": "Period", + "type": "text", + }, + Object { + "default": "/server-status", + "multi": false, + "name": "server_status_path", + "required": true, + "show_user": false, + "title": "Server Status Path", + "type": "text", + }, + ], + }, + ], + "title": "Apache status metrics", + "type": "metrics", + }, + ], + "description": "Apache Integration", + "download": "/epr/apache/apache-0.1.4.zip", + "format_version": "1.0.0", + "icons": Array [ + Object { + "path": "/package/apache/0.1.4/img/logo_apache.svg", + "size": "32x32", + "src": "/img/logo_apache.svg", + "title": "Apache Logo", + "type": "image/svg+xml", + }, + ], + "keepPoliciesUpToDate": false, + "license": "basic", + "name": "apache", + "owner": Object { + "github": "elastic/integrations-services", + }, + "path": "/package/apache/0.1.4", + "policy_templates": Array [ + Object { + "description": "Collect logs and metrics from Apache instances", + "inputs": Array [ + Object { + "description": "Collecting Apache access and error logs", + "title": "Collect logs from Apache instances", + "type": "logfile", + }, + Object { + "description": "Collecting Apache status metrics", + "title": "Collect metrics from Apache instances", + "type": "apache/metrics", + "vars": Array [ + Object { + "default": Array [ + "http://127.0.0.1", + ], + "multi": true, + "name": "hosts", + "required": true, + "show_user": true, + "title": "Hosts", + "type": "text", + }, + ], + }, + ], + "multiple": true, + "name": "apache", + "title": "Apache logs and metrics", + }, + ], + "readme": "/package/apache/0.1.4/docs/README.md", + "release": "experimental", + "removable": true, + "savedObject": Object { + "attributes": Object { + "es_index_patterns": Object { + "access": "logs-apache.access-*", + "error": "logs-apache.error-*", + "status": "metrics-apache.status-*", + }, + "install_source": "upload", + "install_status": "installed", + "install_version": "0.1.4", + "installed_es": Array [ + Object { + "id": "logs-apache.access-0.1.4-default", + "type": "ingest_pipeline", + }, + Object { + "id": "logs-apache.error-0.1.4-default", + "type": "ingest_pipeline", + }, + Object { + "id": "logs-apache.access", + "type": "index_template", + }, + Object { + "id": "logs-apache.access@mappings", + "type": "component_template", + }, + Object { + "id": "logs-apache.access@settings", + "type": "component_template", + }, + Object { + "id": "logs-apache.access@custom", + "type": "component_template", + }, + Object { + "id": "metrics-apache.status", + "type": "index_template", + }, + Object { + "id": "metrics-apache.status@mappings", + "type": "component_template", + }, + Object { + "id": "metrics-apache.status@settings", + "type": "component_template", + }, + Object { + "id": "metrics-apache.status@custom", + "type": "component_template", + }, + Object { + "id": "logs-apache.error", + "type": "index_template", + }, + Object { + "id": "logs-apache.error@mappings", + "type": "component_template", + }, + Object { + "id": "logs-apache.error@settings", + "type": "component_template", + }, + Object { + "id": "logs-apache.error@custom", + "type": "component_template", + }, + ], + "installed_kibana": Array [ + Object { + "id": "apache-Logs-Apache-Dashboard-ecs", + "type": "dashboard", + }, + Object { + "id": "apache-Metrics-Apache-HTTPD-server-status-ecs", + "type": "dashboard", + }, + Object { + "id": "Apache-access-unique-IPs-map-ecs", + "type": "visualization", + }, + Object { + "id": "Apache-HTTPD-CPU-ecs", + "type": "visualization", + }, + Object { + "id": "Apache-HTTPD-Load1-slash-5-slash-15-ecs", + "type": "visualization", + }, + Object { + "id": "Apache-response-codes-over-time-ecs", + "type": "visualization", + }, + Object { + "id": "Apache-HTTPD-Workers-ecs", + "type": "visualization", + }, + Object { + "id": "Apache-HTTPD-Hostname-list-ecs", + "type": "visualization", + }, + Object { + "id": "Apache-error-logs-over-time-ecs", + "type": "visualization", + }, + Object { + "id": "Apache-HTTPD-Scoreboard-ecs", + "type": "visualization", + }, + Object { + "id": "Apache-HTTPD-Uptime-ecs", + "type": "visualization", + }, + Object { + "id": "Apache-operating-systems-ecs", + "type": "visualization", + }, + Object { + "id": "Apache-HTTPD-Total-accesses-and-kbytes-ecs", + "type": "visualization", + }, + Object { + "id": "Apache-browsers-ecs", + "type": "visualization", + }, + Object { + "id": "Apache-response-codes-of-top-URLs-ecs", + "type": "visualization", + }, + Object { + "id": "Apache-access-logs-ecs", + "type": "search", + }, + Object { + "id": "Apache-errors-log-ecs", + "type": "search", + }, + Object { + "id": "Apache-HTTPD-ecs", + "type": "search", + }, + ], + "installed_kibana_space_id": "default", + "name": "apache", + "package_assets": Array [ + Object { + "id": "2f1ab9c0-8cf6-5e83-afcd-0d12851c8108", + "type": "epm-packages-assets", + }, + Object { + "id": "841166f1-6db0-5f7a-a8d9-768e88ddf984", + "type": "epm-packages-assets", + }, + Object { + "id": "b12ae5e1-daf2-51a7-99d8-0888d1f13b5b", + "type": "epm-packages-assets", + }, + Object { + "id": "2f263b24-c36a-5ea8-a707-76d1f274c888", + "type": "epm-packages-assets", + }, + Object { + "id": "bd5ff9ad-ba4a-5215-b5af-cef58a3aa886", + "type": "epm-packages-assets", + }, + Object { + "id": "5fc59aa9-1d7e-50ae-8ce5-b875ab44cfc5", + "type": "epm-packages-assets", + }, + Object { + "id": "7c850453-346b-5010-a946-28b83fc69e48", + "type": "epm-packages-assets", + }, + Object { + "id": "f02f8adb-3e0c-5f2f-b4f2-a04dc645b713", + "type": "epm-packages-assets", + }, + Object { + "id": "889d88db-6214-5836-aeff-1a87f8513b27", + "type": "epm-packages-assets", + }, + Object { + "id": "06a6b940-a745-563c-abf4-83eb3335926b", + "type": "epm-packages-assets", + }, + Object { + "id": "e68fd7ac-302e-5b75-bbbb-d69b441c8848", + "type": "epm-packages-assets", + }, + Object { + "id": "2c57fe0f-3b1a-57da-a63b-28f9b9e82bce", + "type": "epm-packages-assets", + }, + Object { + "id": "13db43e8-f8f9-57f0-b131-a171c2f2070f", + "type": "epm-packages-assets", + }, + Object { + "id": "e8750081-1c0b-5c55-bcab-fa6d47f01a85", + "type": "epm-packages-assets", + }, + Object { + "id": "71af57fe-25c4-5935-9879-ca4a2fba730e", + "type": "epm-packages-assets", + }, + Object { + "id": "cc287718-9573-5c56-a9ed-6dfef6589506", + "type": "epm-packages-assets", + }, + Object { + "id": "8badd8ba-289a-5e60-a1c0-f3d39e15cda3", + "type": "epm-packages-assets", + }, + Object { + "id": "20300efc-10eb-5fac-ba90-f6aa9b467e84", + "type": "epm-packages-assets", + }, + Object { + "id": "047c89df-33c2-5d74-b0a4-8b441879761c", + "type": "epm-packages-assets", + }, + Object { + "id": "9838a13f-1b89-5c54-844e-978620d66a1d", + "type": "epm-packages-assets", + }, + Object { + "id": "e105414b-221d-5433-8b24-452625f59b7c", + "type": "epm-packages-assets", + }, + Object { + "id": "eb166c25-843b-5271-8d43-6fb005d2df5a", + "type": "epm-packages-assets", + }, + Object { + "id": "342dbf4d-d88d-53e8-b365-d3639ebbbb14", + "type": "epm-packages-assets", + }, + Object { + "id": "f98c44a3-eaea-505f-8598-3b7f1097ef59", + "type": "epm-packages-assets", + }, + Object { + "id": "12da8c6c-d0e3-589c-9244-88d857ea76b6", + "type": "epm-packages-assets", + }, + Object { + "id": "e2d151ed-709c-542d-b797-cb95f353b9b3", + "type": "epm-packages-assets", + }, + Object { + "id": "f434cffe-0b00-59de-a17f-c1e71bd4ab0f", + "type": "epm-packages-assets", + }, + Object { + "id": "5bd0c25f-04a5-5fd0-8298-ba9aa2f6fe5e", + "type": "epm-packages-assets", + }, + Object { + "id": "279da3a3-8e9b-589b-86e0-bd7364821bab", + "type": "epm-packages-assets", + }, + Object { + "id": "b8758fcb-08bf-50fa-89bd-24398955298a", + "type": "epm-packages-assets", + }, + Object { + "id": "96e4eb36-03c3-5856-af44-559fd5133f2b", + "type": "epm-packages-assets", + }, + Object { + "id": "a59a79c3-66bd-5cfc-91f5-ee84f7227855", + "type": "epm-packages-assets", + }, + Object { + "id": "395143f9-54bf-5b46-b1be-a7b2a6142ad9", + "type": "epm-packages-assets", + }, + Object { + "id": "3449b8d2-ffd5-5aec-bb32-4245f2fbcde4", + "type": "epm-packages-assets", + }, + Object { + "id": "ab44094e-6c9d-50b8-b5c4-2e518d89912e", + "type": "epm-packages-assets", + }, + Object { + "id": "b093bfc0-6e98-5a1b-a502-e838a36f6568", + "type": "epm-packages-assets", + }, + Object { + "id": "03d86823-b756-5b91-850d-7ad231d33546", + "type": "epm-packages-assets", + }, + Object { + "id": "a76af2f0-049b-5be1-8d20-e87c9d1c2709", + "type": "epm-packages-assets", + }, + Object { + "id": "bc2f0c1e-992e-5407-9435-fedb39ff74ea", + "type": "epm-packages-assets", + }, + Object { + "id": "84668ac1-d5ef-545b-88f3-1e49f8f1c8ad", + "type": "epm-packages-assets", + }, + Object { + "id": "69b41271-91a0-5a2e-a62c-60364d5a9c8f", + "type": "epm-packages-assets", + }, + Object { + "id": "8e4ec555-5fbf-55d3-bea3-3af12c9aca3f", + "type": "epm-packages-assets", + }, + Object { + "id": "aa18f3f9-f62a-5ab8-9b34-75696efa5c48", + "type": "epm-packages-assets", + }, + Object { + "id": "71c8c6b1-2116-5817-b65f-7a87ef5ef2b7", + "type": "epm-packages-assets", + }, + Object { + "id": "8f6d7a1f-1e7f-5a60-8fe7-ce19115ed460", + "type": "epm-packages-assets", + }, + Object { + "id": "c115dbbf-edad-59f2-b046-c65a0373a81c", + "type": "epm-packages-assets", + }, + Object { + "id": "b7d696c3-8106-585c-9ecc-94a75cf1e3da", + "type": "epm-packages-assets", + }, + Object { + "id": "639e6a78-59d8-5ce8-9687-64e8f9af7e71", + "type": "epm-packages-assets", + }, + Object { + "id": "ae60c853-7a90-58d2-ab6c-04d3be5f1847", + "type": "epm-packages-assets", + }, + Object { + "id": "0cd33163-2ae4-57eb-96f6-c50af6685cab", + "type": "epm-packages-assets", + }, + Object { + "id": "39e0f78f-1172-5e61-9446-65ef3c0cb46c", + "type": "epm-packages-assets", + }, + Object { + "id": "b08f10ee-6afd-5e89-b9b4-569064fbdd9f", + "type": "epm-packages-assets", + }, + Object { + "id": "efcbe1c6-b2d5-521c-b27a-2146f08a604d", + "type": "epm-packages-assets", + }, + Object { + "id": "f9422c02-d43f-5ebb-b7c5-9e32f9b77c21", + "type": "epm-packages-assets", + }, + Object { + "id": "c276e880-3ba8-58e7-a5d5-c07707dba6b7", + "type": "epm-packages-assets", + }, + Object { + "id": "561a3711-c386-541c-9a77-2d0fa256caf6", + "type": "epm-packages-assets", + }, + Object { + "id": "1378350d-2e2b-52dd-ab3a-d8b9a09df92f", + "type": "epm-packages-assets", + }, + Object { + "id": "94e40729-4aea-59c8-86ba-075137c000dc", + "type": "epm-packages-assets", + }, + ], + "removable": true, + "version": "0.1.4", + }, + "id": "apache", + "namespaces": Array [], + "references": Array [], + "type": "epm-packages", + }, + "screenshots": Array [ + Object { + "path": "/package/apache/0.1.4/img/kibana-apache.png", + "size": "1215x1199", + "src": "/img/kibana-apache.png", + "title": "Apache Integration", + "type": "image/png", + }, + Object { + "path": "/package/apache/0.1.4/img/apache_httpd_server_status.png", + "size": "1919x1079", + "src": "/img/apache_httpd_server_status.png", + "title": "Apache HTTPD Server Status", + "type": "image/png", + }, + ], + "status": "installed", + "title": "Apache", + "type": "integration", + "version": "0.1.4", +} +`; diff --git a/x-pack/test/fleet_api_integration/apis/epm/final_pipeline.ts b/x-pack/test/fleet_api_integration/apis/epm/final_pipeline.ts index 3b01b1f861aef..1c8e605cedd72 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/final_pipeline.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/final_pipeline.ts @@ -109,8 +109,9 @@ export default function (providerContext: FtrProviderContext) { expect(pipelineRes).to.have.property(FINAL_PIPELINE_ID); const res = await es.indices.getIndexTemplate({ name: 'logs-log.log' }); expect(res.index_templates.length).to.be(FINAL_PIPELINE_VERSION); + expect(res.index_templates[0]?.index_template?.composed_of).to.contain('.fleet_globals-1'); expect(res.index_templates[0]?.index_template?.composed_of).to.contain( - '.fleet_component_template-1' + '.fleet_agent_id_verification-1' ); }); diff --git a/x-pack/test/fleet_api_integration/apis/epm/install_by_upload.ts b/x-pack/test/fleet_api_integration/apis/epm/install_by_upload.ts index 28b68609ce15e..68cac70e8fed8 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/install_by_upload.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/install_by_upload.ts @@ -75,7 +75,7 @@ export default function (providerContext: FtrProviderContext) { .type('application/gzip') .send(buf) .expect(200); - expect(res.body.items.length).to.be(29); + expect(res.body.items.length).to.be(32); }); it('should install a zip archive correctly and package info should return correctly after validation', async function () { @@ -86,7 +86,7 @@ export default function (providerContext: FtrProviderContext) { .type('application/zip') .send(buf) .expect(200); - expect(res.body.items.length).to.be(29); + expect(res.body.items.length).to.be(32); }); it('should throw an error if the archive is zip but content type is gzip', async function () { diff --git a/x-pack/test/fleet_api_integration/apis/epm/install_overrides.ts b/x-pack/test/fleet_api_integration/apis/epm/install_overrides.ts index 603e18931c227..eee9525a4f062 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/install_overrides.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/install_overrides.ts @@ -55,7 +55,8 @@ export default function (providerContext: FtrProviderContext) { `${templateName}@mappings`, `${templateName}@settings`, `${templateName}@custom`, - '.fleet_component_template-1', + '.fleet_globals-1', + '.fleet_agent_id_verification-1', ]); ({ body } = await es.transport.request( @@ -151,6 +152,24 @@ export default function (providerContext: FtrProviderContext) { }, mappings: { dynamic: 'false', + properties: { + '@timestamp': { + type: 'date', + }, + data_stream: { + properties: { + dataset: { + type: 'constant_keyword', + }, + namespace: { + type: 'constant_keyword', + }, + type: { + type: 'constant_keyword', + }, + }, + }, + }, }, aliases: {}, }, diff --git a/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts b/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts index af807d4393daf..a44b8be478874 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts @@ -548,6 +548,10 @@ const expectAssetsInstalled = ({ id: 'logs-all_assets.test_logs@custom', type: 'component_template', }, + { + id: 'metrics-all_assets.test_metrics@mappings', + type: 'component_template', + }, { id: 'metrics-all_assets.test_metrics@settings', type: 'component_template', diff --git a/x-pack/test/fleet_api_integration/apis/epm/template.ts b/x-pack/test/fleet_api_integration/apis/epm/template.ts index d8856ec392218..9cef0d3f28415 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/template.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/template.ts @@ -14,19 +14,6 @@ export default function ({ getService }: FtrProviderContext) { const templateName = 'bar'; const templateIndexPattern = 'bar-*'; const es = getService('es'); - const mappings = { - properties: { - foo: { - type: 'keyword', - }, - }, - }; - const fields = [ - { - name: 'foo', - type: 'keyword', - }, - ]; // This test was inspired by https://github.com/elastic/kibana/blob/main/x-pack/test/api_integration/apis/monitoring/common/mappings_exist.js describe('EPM - template', async () => { @@ -43,10 +30,7 @@ export default function ({ getService }: FtrProviderContext) { it('can be loaded', async () => { const template = getTemplate({ - type: 'logs', templateIndexPattern, - fields, - mappings, packageName: 'system', composedOfTemplates: [], templatePriority: 200, diff --git a/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts b/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts index 1947396b8a2bd..7d28b04c28a53 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts @@ -81,32 +81,8 @@ export default function (providerContext: FtrProviderContext) { { meta: true } ); expect(resLogsTemplate.statusCode).equal(200); - expect( - resLogsTemplate.body.index_templates[0].index_template.template.mappings.properties - ).eql({ - '@timestamp': { - type: 'date', - }, - logs_test_name: { - type: 'text', - }, - new_field_name: { - ignore_above: 1024, - type: 'keyword', - }, - data_stream: { - properties: { - dataset: { - type: 'constant_keyword', - }, - namespace: { - type: 'constant_keyword', - }, - type: { - type: 'constant_keyword', - }, - }, - }, + expect(resLogsTemplate.body.index_templates[0].index_template.template.mappings).eql({ + _meta: { package: { name: 'all_assets' }, managed_by: 'fleet', managed: true }, }); const resMetricsTemplate = await es.transport.request( { @@ -116,27 +92,12 @@ export default function (providerContext: FtrProviderContext) { { meta: true } ); expect(resMetricsTemplate.statusCode).equal(200); - expect( - resMetricsTemplate.body.index_templates[0].index_template.template.mappings.properties - ).eql({ - '@timestamp': { - type: 'date', - }, - metrics_test_name2: { - ignore_above: 1024, - type: 'keyword', - }, - data_stream: { - properties: { - dataset: { - type: 'constant_keyword', - }, - namespace: { - type: 'constant_keyword', - }, - type: { - type: 'constant_keyword', - }, + expect(resMetricsTemplate.body.index_templates[0].index_template.template.mappings).eql({ + _meta: { + managed: true, + managed_by: 'fleet', + package: { + name: 'all_assets', }, }, }); @@ -150,8 +111,27 @@ export default function (providerContext: FtrProviderContext) { { meta: true } ); expect(resLogsTemplate.statusCode).equal(200); + expect(resLogsTemplate.body.index_templates[0].index_template.template.mappings).eql({ + _meta: { + managed: true, + managed_by: 'fleet', + package: { + name: 'all_assets', + }, + }, + }); + }); + it('should have populated the new component template with the correct mapping', async () => { + const resMappings = await es.transport.request( + { + method: 'GET', + path: `/_component_template/${logsTemplateName2}@mappings`, + }, + { meta: true } + ); + expect(resMappings.statusCode).equal(200); expect( - resLogsTemplate.body.index_templates[0].index_template.template.mappings.properties + resMappings.body.component_templates[0].component_template.template.mappings.properties ).eql({ '@timestamp': { type: 'date', @@ -223,7 +203,7 @@ export default function (providerContext: FtrProviderContext) { ); expect(resPipeline2.statusCode).equal(404); }); - it('should have updated the component templates', async function () { + it('should have updated the logs component templates', async function () { const resMappings = await es.transport.request( { method: 'GET', @@ -232,8 +212,42 @@ export default function (providerContext: FtrProviderContext) { { meta: true } ); expect(resMappings.statusCode).equal(200); + expect(resMappings.body.component_templates[0].component_template.template.settings).eql({ + index: { + mapping: { + total_fields: { + limit: '10000', + }, + }, + }, + }); expect(resMappings.body.component_templates[0].component_template.template.mappings).eql({ dynamic: true, + properties: { + '@timestamp': { + type: 'date', + }, + data_stream: { + properties: { + dataset: { + type: 'constant_keyword', + }, + namespace: { + type: 'constant_keyword', + }, + type: { + type: 'constant_keyword', + }, + }, + }, + logs_test_name: { + type: 'text', + }, + new_field_name: { + ignore_above: 1024, + type: 'keyword', + }, + }, }); const resSettings = await es.transport.request( { @@ -247,11 +261,6 @@ export default function (providerContext: FtrProviderContext) { index: { lifecycle: { name: 'reference2' }, codec: 'best_compression', - mapping: { - total_fields: { - limit: '10000', - }, - }, query: { default_field: ['logs_test_name', 'new_field_name'], }, @@ -285,6 +294,40 @@ export default function (providerContext: FtrProviderContext) { ], }); }); + it('should have updated the metrics mapping component template', async function () { + const resMappings = await es.transport.request( + { + method: 'GET', + path: `/_component_template/${metricsTemplateName}@mappings`, + }, + { meta: true } + ); + expect(resMappings.statusCode).equal(200); + expect( + resMappings.body.component_templates[0].component_template.template.mappings.properties + ).eql({ + '@timestamp': { + type: 'date', + }, + metrics_test_name2: { + ignore_above: 1024, + type: 'keyword', + }, + data_stream: { + properties: { + dataset: { + type: 'constant_keyword', + }, + namespace: { + type: 'constant_keyword', + }, + type: { + type: 'constant_keyword', + }, + }, + }, + }); + }); it('should have updated the kibana assets', async function () { const resDashboard = await kibanaServer.savedObjects.get({ type: 'dashboard', @@ -396,6 +439,10 @@ export default function (providerContext: FtrProviderContext) { id: 'logs-all_assets.test_logs2', type: 'index_template', }, + { + id: 'logs-all_assets.test_logs2@mappings', + type: 'component_template', + }, { id: 'logs-all_assets.test_logs2@settings', type: 'component_template', @@ -408,6 +455,10 @@ export default function (providerContext: FtrProviderContext) { id: 'metrics-all_assets.test_metrics', type: 'index_template', }, + { + id: 'metrics-all_assets.test_metrics@mappings', + type: 'component_template', + }, { id: 'metrics-all_assets.test_metrics@settings', type: 'component_template',