From c28b89af3c0ac7c5d86cb0e56bef2a07f6c62b0e Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Fri, 28 Jan 2022 10:24:52 +0000 Subject: [PATCH 01/29] cherry pick from old branch --- .../__snapshots__/template.test.ts.snap | 2249 ++++++++--------- .../epm/elasticsearch/template/install.ts | 108 +- .../elasticsearch/template/template.test.ts | 50 +- .../epm/elasticsearch/template/template.ts | 34 +- .../apis/epm/template.ts | 16 - 5 files changed, 1162 insertions(+), 1295 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap index e977c41cd69d8..efd51d5e0d997 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap @@ -2,613 +2,550 @@ exports[`EPM template tests loading base.yml: base.yml 1`] = ` { - "priority": 200, - "index_patterns": [ - "foo-*" - ], - "template": { - "settings": { - "index": {} + "properties": { + "user": { + "properties": { + "auid": { + "ignore_above": 1024, + "type": "keyword" + }, + "euid": { + "ignore_above": 1024, + "type": "keyword" + } + } }, - "mappings": { - "dynamic_templates": [ - { - "strings_as_keyword": { - "mapping": { - "ignore_above": 1024, - "type": "keyword" + "long": { + "properties": { + "nested": { + "properties": { + "foo": { + "type": "text" }, - "match_mapping_type": "string" + "bar": { + "type": "long" + } } } - ], - "date_detection": false, + } + }, + "nested": { + "properties": { + "bar": { + "ignore_above": 1024, + "type": "keyword" + }, + "baz": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "myalias": { + "type": "alias", + "path": "user.euid" + }, + "validarray": { + "type": "integer" + }, + "cycle_type": { + "type": "constant_keyword", + "value": "bicycle" + } + } +} +`; + +exports[`EPM template tests loading coredns.logs.yml: coredns.logs.yml 1`] = ` +{ + "properties": { + "coredns": { "properties": { - "user": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "query": { "properties": { - "auid": { + "size": { + "type": "long" + }, + "class": { "ignore_above": 1024, "type": "keyword" }, - "euid": { + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { "ignore_above": 1024, "type": "keyword" } } }, - "long": { - "properties": { - "nested": { - "properties": { - "foo": { - "type": "text" - }, - "bar": { - "type": "long" - } - } - } - } - }, - "nested": { + "response": { "properties": { - "bar": { + "code": { "ignore_above": 1024, "type": "keyword" }, - "baz": { + "flags": { "ignore_above": 1024, "type": "keyword" + }, + "size": { + "type": "long" } } }, - "myalias": { - "type": "alias", - "path": "user.euid" - }, - "validarray": { - "type": "integer" - }, - "cycle_type": { - "type": "constant_keyword", - "value": "bicycle" - } - }, - "_meta": { - "managed_by": "fleet", - "managed": true, - "package": { - "name": "nginx" + "dnssec_ok": { + "type": "boolean" } } } - }, - "data_stream": {}, - "composed_of": [ - ".fleet_component_template-1" - ], - "_meta": { - "managed_by": "fleet", - "managed": true, - "package": { - "name": "nginx" - } } } `; -exports[`EPM template tests loading coredns.logs.yml: coredns.logs.yml 1`] = ` +exports[`EPM template tests loading system.yml: system.yml 1`] = ` { - "priority": 200, - "index_patterns": [ - "foo-*" - ], - "template": { - "settings": { - "index": {} - }, - "mappings": { - "dynamic_templates": [ - { - "strings_as_keyword": { - "mapping": { - "ignore_above": 1024, - "type": "keyword" - }, - "match_mapping_type": "string" - } - } - ], - "date_detection": false, + "properties": { + "system": { "properties": { - "coredns": { + "core": { "properties": { "id": { - "ignore_above": 1024, - "type": "keyword" + "type": "long" }, - "query": { + "user": { "properties": { - "size": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + }, + "ticks": { "type": "long" + } + } + }, + "system": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 }, - "class": { - "ignore_above": 1024, - "type": "keyword" + "ticks": { + "type": "long" + } + } + }, + "nice": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 }, - "name": { - "ignore_above": 1024, - "type": "keyword" + "ticks": { + "type": "long" + } + } + }, + "idle": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 }, - "type": { - "ignore_above": 1024, - "type": "keyword" + "ticks": { + "type": "long" } } }, - "response": { + "iowait": { "properties": { - "code": { - "ignore_above": 1024, - "type": "keyword" + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 }, - "flags": { - "ignore_above": 1024, - "type": "keyword" + "ticks": { + "type": "long" + } + } + }, + "irq": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 }, - "size": { + "ticks": { "type": "long" } } }, - "dnssec_ok": { - "type": "boolean" - } - } - } - }, - "_meta": { - "managed_by": "fleet", - "managed": true, - "package": { - "name": "coredns" - } - } - } - }, - "data_stream": {}, - "composed_of": [ - ".fleet_component_template-1" - ], - "_meta": { - "managed_by": "fleet", - "managed": true, - "package": { - "name": "coredns" - } - } -} -`; - -exports[`EPM template tests loading system.yml: system.yml 1`] = ` -{ - "priority": 200, - "index_patterns": [ - "whatsthis-*" - ], - "template": { - "settings": { - "index": {} - }, - "mappings": { - "dynamic_templates": [ - { - "strings_as_keyword": { - "mapping": { - "ignore_above": 1024, - "type": "keyword" + "softirq": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + }, + "ticks": { + "type": "long" + } + } }, - "match_mapping_type": "string" + "steal": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + }, + "ticks": { + "type": "long" + } + } + } } - } - ], - "date_detection": false, - "properties": { - "system": { + }, + "cpu": { "properties": { - "core": { + "cores": { + "type": "long" + }, + "user": { "properties": { - "id": { - "type": "long" + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 }, - "user": { + "norm": { "properties": { "pct": { "type": "scaled_float", "scaling_factor": 1000 - }, - "ticks": { - "type": "long" } } }, - "system": { + "ticks": { + "type": "long" + } + } + }, + "system": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + }, + "norm": { "properties": { "pct": { "type": "scaled_float", "scaling_factor": 1000 - }, - "ticks": { - "type": "long" } } }, - "nice": { + "ticks": { + "type": "long" + } + } + }, + "nice": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + }, + "norm": { "properties": { "pct": { "type": "scaled_float", "scaling_factor": 1000 - }, - "ticks": { - "type": "long" } } }, - "idle": { + "ticks": { + "type": "long" + } + } + }, + "idle": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + }, + "norm": { "properties": { "pct": { "type": "scaled_float", "scaling_factor": 1000 - }, - "ticks": { - "type": "long" } } }, - "iowait": { + "ticks": { + "type": "long" + } + } + }, + "iowait": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + }, + "norm": { "properties": { "pct": { "type": "scaled_float", "scaling_factor": 1000 - }, - "ticks": { - "type": "long" } } }, - "irq": { + "ticks": { + "type": "long" + } + } + }, + "irq": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + }, + "norm": { "properties": { "pct": { "type": "scaled_float", "scaling_factor": 1000 - }, - "ticks": { - "type": "long" } } }, - "softirq": { + "ticks": { + "type": "long" + } + } + }, + "softirq": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + }, + "norm": { "properties": { "pct": { "type": "scaled_float", "scaling_factor": 1000 - }, - "ticks": { - "type": "long" } } }, - "steal": { + "ticks": { + "type": "long" + } + } + }, + "steal": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + }, + "norm": { "properties": { "pct": { "type": "scaled_float", "scaling_factor": 1000 - }, - "ticks": { - "type": "long" } } + }, + "ticks": { + "type": "long" } } }, - "cpu": { + "total": { "properties": { - "cores": { - "type": "long" - }, - "user": { - "properties": { - "pct": { - "type": "scaled_float", - "scaling_factor": 1000 - }, - "norm": { - "properties": { - "pct": { - "type": "scaled_float", - "scaling_factor": 1000 - } - } - }, - "ticks": { - "type": "long" - } - } - }, - "system": { - "properties": { - "pct": { - "type": "scaled_float", - "scaling_factor": 1000 - }, - "norm": { - "properties": { - "pct": { - "type": "scaled_float", - "scaling_factor": 1000 - } - } - }, - "ticks": { - "type": "long" - } - } - }, - "nice": { - "properties": { - "pct": { - "type": "scaled_float", - "scaling_factor": 1000 - }, - "norm": { - "properties": { - "pct": { - "type": "scaled_float", - "scaling_factor": 1000 - } - } - }, - "ticks": { - "type": "long" - } - } - }, - "idle": { - "properties": { - "pct": { - "type": "scaled_float", - "scaling_factor": 1000 - }, - "norm": { - "properties": { - "pct": { - "type": "scaled_float", - "scaling_factor": 1000 - } - } - }, - "ticks": { - "type": "long" - } - } - }, - "iowait": { - "properties": { - "pct": { - "type": "scaled_float", - "scaling_factor": 1000 - }, - "norm": { - "properties": { - "pct": { - "type": "scaled_float", - "scaling_factor": 1000 - } - } - }, - "ticks": { - "type": "long" - } - } + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 }, - "irq": { + "norm": { "properties": { "pct": { "type": "scaled_float", "scaling_factor": 1000 - }, - "norm": { - "properties": { - "pct": { - "type": "scaled_float", - "scaling_factor": 1000 - } - } - }, - "ticks": { - "type": "long" - } - } - }, - "softirq": { - "properties": { - "pct": { - "type": "scaled_float", - "scaling_factor": 1000 - }, - "norm": { - "properties": { - "pct": { - "type": "scaled_float", - "scaling_factor": 1000 - } - } - }, - "ticks": { - "type": "long" } } + } + } + } + } + }, + "diskio": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "serial_number": { + "ignore_above": 1024, + "type": "keyword" + }, + "read": { + "properties": { + "count": { + "type": "long" }, - "steal": { - "properties": { - "pct": { - "type": "scaled_float", - "scaling_factor": 1000 - }, - "norm": { - "properties": { - "pct": { - "type": "scaled_float", - "scaling_factor": 1000 - } - } - }, - "ticks": { - "type": "long" - } - } + "bytes": { + "type": "long" }, - "total": { - "properties": { - "pct": { - "type": "scaled_float", - "scaling_factor": 1000 - }, - "norm": { - "properties": { - "pct": { - "type": "scaled_float", - "scaling_factor": 1000 - } - } - } - } + "time": { + "type": "long" } } }, - "diskio": { + "write": { "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" + "count": { + "type": "long" }, - "serial_number": { - "ignore_above": 1024, - "type": "keyword" + "bytes": { + "type": "long" }, + "time": { + "type": "long" + } + } + }, + "io": { + "properties": { + "time": { + "type": "long" + } + } + }, + "iostat": { + "properties": { "read": { "properties": { - "count": { - "type": "long" - }, - "bytes": { - "type": "long" - }, - "time": { - "type": "long" - } - } - }, - "write": { - "properties": { - "count": { - "type": "long" - }, - "bytes": { - "type": "long" - }, - "time": { - "type": "long" - } - } - }, - "io": { - "properties": { - "time": { - "type": "long" - } - } - }, - "iostat": { - "properties": { - "read": { + "request": { "properties": { - "request": { - "properties": { - "merges_per_sec": { - "type": "float" - }, - "per_sec": { - "type": "float" - } - } + "merges_per_sec": { + "type": "float" }, "per_sec": { - "properties": { - "bytes": { - "type": "float" - } - } - }, - "await": { "type": "float" } } }, - "write": { + "per_sec": { "properties": { - "request": { - "properties": { - "merges_per_sec": { - "type": "float" - }, - "per_sec": { - "type": "float" - } - } - }, - "per_sec": { - "properties": { - "bytes": { - "type": "float" - } - } - }, - "await": { + "bytes": { "type": "float" } } }, + "await": { + "type": "float" + } + } + }, + "write": { + "properties": { "request": { "properties": { - "avg_size": { + "merges_per_sec": { + "type": "float" + }, + "per_sec": { "type": "float" } } }, - "queue": { + "per_sec": { "properties": { - "avg_size": { + "bytes": { "type": "float" } } }, "await": { "type": "float" - }, - "service_time": { + } + } + }, + "request": { + "properties": { + "avg_size": { "type": "float" - }, - "busy": { + } + } + }, + "queue": { + "properties": { + "avg_size": { "type": "float" } } + }, + "await": { + "type": "float" + }, + "service_time": { + "type": "float" + }, + "busy": { + "type": "float" } } + } + } + }, + "entropy": { + "properties": { + "available_bits": { + "type": "long" }, - "entropy": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + } + } + }, + "filesystem": { + "properties": { + "available": { + "type": "long" + }, + "device_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "mount_point": { + "ignore_above": 1024, + "type": "keyword" + }, + "files": { + "type": "long" + }, + "free": { + "type": "long" + }, + "free_files": { + "type": "long" + }, + "total": { + "type": "long" + }, + "used": { "properties": { - "available_bits": { + "bytes": { "type": "long" }, "pct": { @@ -616,36 +553,88 @@ exports[`EPM template tests loading system.yml: system.yml 1`] = ` "scaling_factor": 1000 } } + } + } + }, + "fsstat": { + "properties": { + "count": { + "type": "long" + }, + "total_files": { + "type": "long" }, - "filesystem": { + "total_size": { "properties": { - "available": { + "free": { "type": "long" }, - "device_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "mount_point": { - "ignore_above": 1024, - "type": "keyword" - }, - "files": { + "used": { "type": "long" }, - "free": { + "total": { "type": "long" + } + } + } + } + }, + "load": { + "properties": { + "1": { + "type": "scaled_float", + "scaling_factor": 100 + }, + "5": { + "type": "scaled_float", + "scaling_factor": 100 + }, + "15": { + "type": "scaled_float", + "scaling_factor": 100 + }, + "norm": { + "properties": { + "1": { + "type": "scaled_float", + "scaling_factor": 100 }, - "free_files": { - "type": "long" + "5": { + "type": "scaled_float", + "scaling_factor": 100 }, - "total": { + "15": { + "type": "scaled_float", + "scaling_factor": 100 + } + } + }, + "cores": { + "type": "long" + } + } + }, + "memory": { + "properties": { + "total": { + "type": "long" + }, + "used": { + "properties": { + "bytes": { "type": "long" }, + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + } + } + }, + "free": { + "type": "long" + }, + "actual": { + "properties": { "used": { "properties": { "bytes": { @@ -656,68 +645,58 @@ exports[`EPM template tests loading system.yml: system.yml 1`] = ` "scaling_factor": 1000 } } + }, + "free": { + "type": "long" } } }, - "fsstat": { + "swap": { "properties": { - "count": { - "type": "long" - }, - "total_files": { + "total": { "type": "long" }, - "total_size": { + "used": { "properties": { - "free": { - "type": "long" - }, - "used": { + "bytes": { "type": "long" }, - "total": { - "type": "long" + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 } } - } - } - }, - "load": { - "properties": { - "1": { - "type": "scaled_float", - "scaling_factor": 100 }, - "5": { - "type": "scaled_float", - "scaling_factor": 100 + "free": { + "type": "long" }, - "15": { - "type": "scaled_float", - "scaling_factor": 100 + "out": { + "properties": { + "pages": { + "type": "long" + } + } }, - "norm": { + "in": { "properties": { - "1": { - "type": "scaled_float", - "scaling_factor": 100 - }, - "5": { - "type": "scaled_float", - "scaling_factor": 100 - }, - "15": { - "type": "scaled_float", - "scaling_factor": 100 + "pages": { + "type": "long" } } }, - "cores": { - "type": "long" + "readahead": { + "properties": { + "pages": { + "type": "long" + }, + "cached": { + "type": "long" + } + } } } }, - "memory": { + "hugepages": { "properties": { "total": { "type": "long" @@ -728,296 +707,332 @@ exports[`EPM template tests loading system.yml: system.yml 1`] = ` "type": "long" }, "pct": { - "type": "scaled_float", - "scaling_factor": 1000 + "type": "long" } } }, "free": { "type": "long" }, - "actual": { + "reserved": { + "type": "long" + }, + "surplus": { + "type": "long" + }, + "default_size": { + "type": "long" + }, + "swap": { + "properties": { + "out": { + "properties": { + "pages": { + "type": "long" + }, + "fallback": { + "type": "long" + } + } + } + } + } + } + } + } + }, + "network": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "out": { + "properties": { + "bytes": { + "type": "long" + }, + "packets": { + "type": "long" + }, + "errors": { + "type": "long" + }, + "dropped": { + "type": "long" + } + } + }, + "in": { + "properties": { + "bytes": { + "type": "long" + }, + "packets": { + "type": "long" + }, + "errors": { + "type": "long" + }, + "dropped": { + "type": "long" + } + } + } + } + }, + "network_summary": { + "properties": { + "ip": { + "properties": { + "*": { + "type": "object" + } + } + }, + "tcp": { + "properties": { + "*": { + "type": "object" + } + } + }, + "udp": { + "properties": { + "*": { + "type": "object" + } + } + }, + "udp_lite": { + "properties": { + "*": { + "type": "object" + } + } + }, + "icmp": { + "properties": { + "*": { + "type": "object" + } + } + } + } + }, + "process": { + "properties": { + "state": { + "ignore_above": 1024, + "type": "keyword" + }, + "cmdline": { + "ignore_above": 2048, + "type": "keyword" + }, + "env": { + "type": "object" + }, + "cpu": { + "properties": { + "user": { "properties": { - "used": { - "properties": { - "bytes": { - "type": "long" - }, - "pct": { - "type": "scaled_float", - "scaling_factor": 1000 - } - } - }, - "free": { + "ticks": { "type": "long" } } }, - "swap": { + "total": { "properties": { - "total": { + "value": { "type": "long" }, - "used": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + }, + "norm": { "properties": { - "bytes": { - "type": "long" - }, "pct": { "type": "scaled_float", "scaling_factor": 1000 } } }, - "free": { + "ticks": { "type": "long" - }, - "out": { - "properties": { - "pages": { - "type": "long" - } - } - }, - "in": { - "properties": { - "pages": { - "type": "long" - } - } - }, - "readahead": { - "properties": { - "pages": { - "type": "long" - }, - "cached": { - "type": "long" - } - } } } }, - "hugepages": { + "system": { "properties": { - "total": { - "type": "long" - }, - "used": { - "properties": { - "bytes": { - "type": "long" - }, - "pct": { - "type": "long" - } - } - }, - "free": { - "type": "long" - }, - "reserved": { - "type": "long" - }, - "surplus": { - "type": "long" - }, - "default_size": { + "ticks": { "type": "long" - }, - "swap": { - "properties": { - "out": { - "properties": { - "pages": { - "type": "long" - }, - "fallback": { - "type": "long" - } - } - } - } } } + }, + "start_time": { + "type": "date" } } }, - "network": { + "memory": { "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" + "size": { + "type": "long" }, - "out": { + "rss": { "properties": { "bytes": { "type": "long" }, - "packets": { - "type": "long" - }, - "errors": { - "type": "long" - }, - "dropped": { - "type": "long" + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 } } }, - "in": { - "properties": { - "bytes": { - "type": "long" - }, - "packets": { - "type": "long" - }, - "errors": { - "type": "long" - }, - "dropped": { - "type": "long" - } - } + "share": { + "type": "long" } } }, - "network_summary": { + "fd": { "properties": { - "ip": { - "properties": { - "*": { - "type": "object" - } - } - }, - "tcp": { - "properties": { - "*": { - "type": "object" - } - } - }, - "udp": { - "properties": { - "*": { - "type": "object" - } - } - }, - "udp_lite": { - "properties": { - "*": { - "type": "object" - } - } + "open": { + "type": "long" }, - "icmp": { + "limit": { "properties": { - "*": { - "type": "object" + "soft": { + "type": "long" + }, + "hard": { + "type": "long" } } } } }, - "process": { + "cgroup": { "properties": { - "state": { + "id": { "ignore_above": 1024, "type": "keyword" }, - "cmdline": { - "ignore_above": 2048, + "path": { + "ignore_above": 1024, "type": "keyword" }, - "env": { - "type": "object" - }, "cpu": { "properties": { - "user": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "cfs": { "properties": { - "ticks": { + "period": { + "properties": { + "us": { + "type": "long" + } + } + }, + "quota": { + "properties": { + "us": { + "type": "long" + } + } + }, + "shares": { "type": "long" } } }, - "total": { + "rt": { "properties": { - "value": { - "type": "long" - }, - "pct": { - "type": "scaled_float", - "scaling_factor": 1000 - }, - "norm": { + "period": { "properties": { - "pct": { - "type": "scaled_float", - "scaling_factor": 1000 + "us": { + "type": "long" } } }, - "ticks": { - "type": "long" + "runtime": { + "properties": { + "us": { + "type": "long" + } + } } } }, - "system": { + "stats": { "properties": { - "ticks": { + "periods": { "type": "long" + }, + "throttled": { + "properties": { + "periods": { + "type": "long" + }, + "ns": { + "type": "long" + } + } } } - }, - "start_time": { - "type": "date" } } }, - "memory": { + "cpuacct": { "properties": { - "size": { - "type": "long" + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" }, - "rss": { + "total": { "properties": { - "bytes": { + "ns": { "type": "long" - }, - "pct": { - "type": "scaled_float", - "scaling_factor": 1000 } } }, - "share": { - "type": "long" - } - } - }, - "fd": { - "properties": { - "open": { - "type": "long" - }, - "limit": { + "stats": { "properties": { - "soft": { - "type": "long" + "user": { + "properties": { + "ns": { + "type": "long" + } + } }, - "hard": { - "type": "long" + "system": { + "properties": { + "ns": { + "type": "long" + } + } } } + }, + "percpu": { + "type": "object" } } }, - "cgroup": { + "memory": { "properties": { "id": { "ignore_above": 1024, @@ -1027,354 +1042,212 @@ exports[`EPM template tests loading system.yml: system.yml 1`] = ` "ignore_above": 1024, "type": "keyword" }, - "cpu": { + "mem": { "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "path": { - "ignore_above": 1024, - "type": "keyword" - }, - "cfs": { + "usage": { "properties": { - "period": { - "properties": { - "us": { - "type": "long" - } - } + "bytes": { + "type": "long" }, - "quota": { + "max": { "properties": { - "us": { + "bytes": { "type": "long" } } - }, - "shares": { + } + } + }, + "limit": { + "properties": { + "bytes": { "type": "long" } } }, - "rt": { + "failures": { + "type": "long" + } + } + }, + "memsw": { + "properties": { + "usage": { "properties": { - "period": { - "properties": { - "us": { - "type": "long" - } - } + "bytes": { + "type": "long" }, - "runtime": { + "max": { "properties": { - "us": { + "bytes": { "type": "long" } } } } }, - "stats": { + "limit": { "properties": { - "periods": { + "bytes": { + "type": "long" + } + } + }, + "failures": { + "type": "long" + } + } + }, + "kmem": { + "properties": { + "usage": { + "properties": { + "bytes": { "type": "long" }, - "throttled": { + "max": { "properties": { - "periods": { - "type": "long" - }, - "ns": { + "bytes": { "type": "long" } } } } - } - } - }, - "cpuacct": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "path": { - "ignore_above": 1024, - "type": "keyword" }, - "total": { + "limit": { "properties": { - "ns": { + "bytes": { "type": "long" } } }, - "stats": { + "failures": { + "type": "long" + } + } + }, + "kmem_tcp": { + "properties": { + "usage": { "properties": { - "user": { - "properties": { - "ns": { - "type": "long" - } - } + "bytes": { + "type": "long" }, - "system": { + "max": { "properties": { - "ns": { + "bytes": { "type": "long" } } } } }, - "percpu": { - "type": "object" + "limit": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "failures": { + "type": "long" } } }, - "memory": { + "stats": { "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "path": { - "ignore_above": 1024, - "type": "keyword" + "active_anon": { + "properties": { + "bytes": { + "type": "long" + } + } }, - "mem": { + "active_file": { "properties": { - "usage": { - "properties": { - "bytes": { - "type": "long" - }, - "max": { - "properties": { - "bytes": { - "type": "long" - } - } - } - } - }, - "limit": { - "properties": { - "bytes": { - "type": "long" - } - } - }, - "failures": { + "bytes": { "type": "long" } } }, - "memsw": { + "cache": { "properties": { - "usage": { - "properties": { - "bytes": { - "type": "long" - }, - "max": { - "properties": { - "bytes": { - "type": "long" - } - } - } - } - }, - "limit": { - "properties": { - "bytes": { - "type": "long" - } - } - }, - "failures": { + "bytes": { "type": "long" } } }, - "kmem": { + "hierarchical_memory_limit": { "properties": { - "usage": { - "properties": { - "bytes": { - "type": "long" - }, - "max": { - "properties": { - "bytes": { - "type": "long" - } - } - } - } - }, - "limit": { - "properties": { - "bytes": { - "type": "long" - } - } - }, - "failures": { + "bytes": { "type": "long" } } }, - "kmem_tcp": { + "hierarchical_memsw_limit": { "properties": { - "usage": { - "properties": { - "bytes": { - "type": "long" - }, - "max": { - "properties": { - "bytes": { - "type": "long" - } - } - } - } - }, - "limit": { - "properties": { - "bytes": { - "type": "long" - } - } - }, - "failures": { + "bytes": { "type": "long" } } }, - "stats": { + "inactive_anon": { "properties": { - "active_anon": { - "properties": { - "bytes": { - "type": "long" - } - } - }, - "active_file": { - "properties": { - "bytes": { - "type": "long" - } - } - }, - "cache": { - "properties": { - "bytes": { - "type": "long" - } - } - }, - "hierarchical_memory_limit": { - "properties": { - "bytes": { - "type": "long" - } - } - }, - "hierarchical_memsw_limit": { - "properties": { - "bytes": { - "type": "long" - } - } - }, - "inactive_anon": { - "properties": { - "bytes": { - "type": "long" - } - } - }, - "inactive_file": { - "properties": { - "bytes": { - "type": "long" - } - } - }, - "mapped_file": { - "properties": { - "bytes": { - "type": "long" - } - } - }, - "page_faults": { + "bytes": { "type": "long" - }, - "major_page_faults": { + } + } + }, + "inactive_file": { + "properties": { + "bytes": { "type": "long" - }, - "pages_in": { + } + } + }, + "mapped_file": { + "properties": { + "bytes": { "type": "long" - }, - "pages_out": { + } + } + }, + "page_faults": { + "type": "long" + }, + "major_page_faults": { + "type": "long" + }, + "pages_in": { + "type": "long" + }, + "pages_out": { + "type": "long" + }, + "rss": { + "properties": { + "bytes": { "type": "long" - }, - "rss": { - "properties": { - "bytes": { - "type": "long" - } - } - }, - "rss_huge": { - "properties": { - "bytes": { - "type": "long" - } - } - }, - "swap": { - "properties": { - "bytes": { - "type": "long" - } - } - }, - "unevictable": { - "properties": { - "bytes": { - "type": "long" - } - } } } - } - } - }, - "blkio": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" }, - "path": { - "ignore_above": 1024, - "type": "keyword" + "rss_huge": { + "properties": { + "bytes": { + "type": "long" + } + } }, - "total": { + "swap": { "properties": { "bytes": { "type": "long" - }, - "ios": { + } + } + }, + "unevictable": { + "properties": { + "bytes": { "type": "long" } } @@ -1383,286 +1256,290 @@ exports[`EPM template tests loading system.yml: system.yml 1`] = ` } } }, - "summary": { + "blkio": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "total": { + "properties": { + "bytes": { + "type": "long" + }, + "ios": { + "type": "long" + } + } + } + } + } + } + }, + "summary": { + "properties": { + "total": { + "type": "long" + }, + "running": { + "type": "long" + }, + "idle": { + "type": "long" + }, + "sleeping": { + "type": "long" + }, + "stopped": { + "type": "long" + }, + "zombie": { + "type": "long" + }, + "dead": { + "type": "long" + }, + "unknown": { + "type": "long" + } + } + } + } + }, + "raid": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + }, + "level": { + "ignore_above": 1024, + "type": "keyword" + }, + "sync_action": { + "ignore_above": 1024, + "type": "keyword" + }, + "disks": { + "properties": { + "active": { + "type": "long" + }, + "total": { + "type": "long" + }, + "spare": { + "type": "long" + }, + "failed": { + "type": "long" + }, + "states": { "properties": { - "total": { - "type": "long" - }, - "running": { - "type": "long" - }, - "idle": { - "type": "long" - }, - "sleeping": { - "type": "long" - }, - "stopped": { - "type": "long" - }, - "zombie": { - "type": "long" - }, - "dead": { - "type": "long" - }, - "unknown": { - "type": "long" + "*": { + "type": "object" } } } } }, - "raid": { + "blocks": { + "properties": { + "total": { + "type": "long" + }, + "synced": { + "type": "long" + } + } + } + } + }, + "socket": { + "properties": { + "local": { + "properties": { + "ip": { + "type": "ip" + }, + "port": { + "type": "long" + } + } + }, + "remote": { "properties": { - "name": { + "ip": { + "type": "ip" + }, + "port": { + "type": "long" + }, + "host": { "ignore_above": 1024, "type": "keyword" }, - "status": { + "etld_plus_one": { "ignore_above": 1024, "type": "keyword" }, - "level": { + "host_error": { "ignore_above": 1024, "type": "keyword" - }, - "sync_action": { + } + } + }, + "process": { + "properties": { + "cmdline": { "ignore_above": 1024, "type": "keyword" - }, - "disks": { - "properties": { - "active": { - "type": "long" - }, - "total": { - "type": "long" - }, - "spare": { - "type": "long" - }, - "failed": { - "type": "long" - }, - "states": { - "properties": { - "*": { - "type": "object" - } - } - } - } - }, - "blocks": { - "properties": { - "total": { - "type": "long" - }, - "synced": { - "type": "long" - } - } } } }, - "socket": { + "user": { + "properties": {} + }, + "summary": { "properties": { - "local": { + "all": { "properties": { - "ip": { - "type": "ip" + "count": { + "type": "long" }, - "port": { + "listening": { "type": "long" } } }, - "remote": { + "tcp": { "properties": { - "ip": { - "type": "ip" - }, - "port": { + "memory": { "type": "long" }, - "host": { - "ignore_above": 1024, - "type": "keyword" - }, - "etld_plus_one": { - "ignore_above": 1024, - "type": "keyword" - }, - "host_error": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "process": { - "properties": { - "cmdline": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "user": { - "properties": {} - }, - "summary": { - "properties": { "all": { "properties": { + "orphan": { + "type": "long" + }, "count": { "type": "long" }, "listening": { "type": "long" - } - } - }, - "tcp": { - "properties": { - "memory": { + }, + "established": { "type": "long" }, - "all": { - "properties": { - "orphan": { - "type": "long" - }, - "count": { - "type": "long" - }, - "listening": { - "type": "long" - }, - "established": { - "type": "long" - }, - "close_wait": { - "type": "long" - }, - "time_wait": { - "type": "long" - }, - "syn_sent": { - "type": "long" - }, - "syn_recv": { - "type": "long" - }, - "fin_wait1": { - "type": "long" - }, - "fin_wait2": { - "type": "long" - }, - "last_ack": { - "type": "long" - }, - "closing": { - "type": "long" - } - } - } - } - }, - "udp": { - "properties": { - "memory": { + "close_wait": { "type": "long" }, - "all": { - "properties": { - "count": { - "type": "long" - } - } + "time_wait": { + "type": "long" + }, + "syn_sent": { + "type": "long" + }, + "syn_recv": { + "type": "long" + }, + "fin_wait1": { + "type": "long" + }, + "fin_wait2": { + "type": "long" + }, + "last_ack": { + "type": "long" + }, + "closing": { + "type": "long" } } } } - } - } - }, - "uptime": { - "properties": { - "duration": { + }, + "udp": { "properties": { - "ms": { + "memory": { "type": "long" + }, + "all": { + "properties": { + "count": { + "type": "long" + } + } } } } } - }, - "users": { + } + } + }, + "uptime": { + "properties": { + "duration": { "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "seat": { - "ignore_above": 1024, - "type": "keyword" - }, - "path": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "service": { - "ignore_above": 1024, - "type": "keyword" - }, - "remote": { - "type": "boolean" - }, - "state": { - "ignore_above": 1024, - "type": "keyword" - }, - "scope": { - "ignore_above": 1024, - "type": "keyword" - }, - "leader": { + "ms": { "type": "long" - }, - "remote_host": { - "ignore_above": 1024, - "type": "keyword" } } } } - } - }, - "_meta": { - "managed_by": "fleet", - "managed": true, - "package": { - "name": "system" + }, + "users": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "seat": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "service": { + "ignore_above": 1024, + "type": "keyword" + }, + "remote": { + "type": "boolean" + }, + "state": { + "ignore_above": 1024, + "type": "keyword" + }, + "scope": { + "ignore_above": 1024, + "type": "keyword" + }, + "leader": { + "type": "long" + }, + "remote_host": { + "ignore_above": 1024, + "type": "keyword" + } + } } } } - }, - "data_stream": {}, - "composed_of": [ - ".fleet_component_template-1" - ], - "_meta": { - "managed_by": "fleet", - "managed": true, - "package": { - "name": "system" - } } } `; diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts index 1303db1a36c0e..1ba71cbcb0be6 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts @@ -5,10 +5,12 @@ * 2.0. */ -import { merge } from 'lodash'; +import { merge, mergeWith, isArray } from 'lodash'; import Boom from '@hapi/boom'; import type { ElasticsearchClient, Logger, SavedObjectsClientContract } from 'src/core/server'; +import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; + import { ElasticsearchAssetType } from '../../../../types'; import type { RegistryDataStream, @@ -17,7 +19,9 @@ import type { InstallablePackage, IndexTemplate, PackageInfo, + IndexTemplateMappings, } from '../../../../types'; + import { loadFieldsFromYaml, processFields } from '../../fields/field'; import type { Field } from '../../fields/field'; import { getPipelineNameForInstallation } from '../ingest_pipeline/install'; @@ -43,6 +47,14 @@ import { } from './template'; import { buildDefaultSettings } from './default_settings'; +// lodash mergeWith but with a customizer function that +// concatenates arrays together +const mergeWithArrayConcat = (a: any, b: any) => + mergeWith(a, b, (val1, val2) => { + if (isArray(val1)) { + return val1.concat(val2); + } + }); export const installTemplates = async ( installablePackage: InstallablePackage, esClient: ElasticsearchClient, @@ -244,12 +256,13 @@ const isUserSettingsTemplate = (name: string): name is UserSettingsTemplateName name.endsWith(userSettingsSuffix); function buildComponentTemplates(params: { + mappings: IndexTemplateMappings; templateName: string; registryElasticsearch: RegistryElasticsearch | undefined; packageName: string; defaultSettings: IndexTemplate['template']['settings']; }) { - const { templateName, registryElasticsearch, packageName, defaultSettings } = params; + const { templateName, registryElasticsearch, packageName, defaultSettings, mappings } = params; const mappingsTemplateName = `${templateName}${mappingsSuffix}`; const settingsTemplateName = `${templateName}${settingsSuffix}`; const userSettingsTemplateName = `${templateName}${userSettingsSuffix}`; @@ -257,15 +270,36 @@ function buildComponentTemplates(params: { const templatesMap: TemplateMap = {}; const _meta = getESAssetMetadata({ packageName }); - if (registryElasticsearch && registryElasticsearch['index_template.mappings']) { - templatesMap[mappingsTemplateName] = { - template: { - mappings: registryElasticsearch['index_template.mappings'], + const baseMapping = { + // All the dynamic field mappings + dynamic_templates: [ + // This makes sure all mappings are keywords by default + { + strings_as_keyword: { + mapping: { + ignore_above: 1024, + type: 'keyword', + }, + match_mapping_type: 'string', + }, }, - _meta, - }; - } + ], + // As we define fields ahead, we don't need any automatic field detection + // This makes sure all the fields are mapped to keyword by default to prevent mapping conflicts + date_detection: false, + // All the properties we know from the fields.yml file + properties: mappings.properties, + _meta, + }; + const packageMappings = registryElasticsearch && registryElasticsearch['index_template.mappings']; + templatesMap[mappingsTemplateName] = { + template: { + // dynamic_templates arrays should be concatenated together not overlayed + mappings: mergeWithArrayConcat(baseMapping, packageMappings || {}), + }, + _meta, + }; templatesMap[settingsTemplateName] = { template: { settings: merge(defaultSettings, registryElasticsearch?.['index_template.settings'] ?? {}), @@ -284,7 +318,47 @@ function buildComponentTemplates(params: { return templatesMap; } +function buildMappingComponentTemplate({ + mappings, + packageMappings = {}, + _meta, +}: { + mappings: IndexTemplateMappings; + packageMappings?: MappingTypeMapping; + _meta: ESAssetMetadata; +}) { + const baseMapping = { + // All the dynamic field mappings + dynamic_templates: [ + // This makes sure all mappings are keywords by default + { + strings_as_keyword: { + mapping: { + ignore_above: 1024, + type: 'keyword', + }, + match_mapping_type: 'string', + }, + }, + ], + // As we define fields ahead, we don't need any automatic field detection + // This makes sure all the fields are mapped to keyword by default to prevent mapping conflicts + date_detection: false, + // All the properties we know from the fields.yml file + properties: mappings.properties, + _meta, + }; + + return { + template: { + // dynamic_templates arrays should be concatenated together not overwritten + mappings: mergeWithArrayConcat(baseMapping, packageMappings), + }, + _meta, + }; +} async function installDataStreamComponentTemplates(params: { + mappings: IndexTemplateMappings; templateName: string; registryElasticsearch: RegistryElasticsearch | undefined; esClient: ElasticsearchClient; @@ -292,9 +366,17 @@ async function installDataStreamComponentTemplates(params: { packageName: string; defaultSettings: IndexTemplate['template']['settings']; }) { - const { templateName, registryElasticsearch, esClient, packageName, defaultSettings, logger } = - params; + const { + templateName, + registryElasticsearch, + esClient, + packageName, + defaultSettings, + logger, + mappings, + } = params; const templates = buildComponentTemplates({ + mappings, templateName, registryElasticsearch, packageName, @@ -435,6 +517,7 @@ export async function installTemplate({ }); const composedOfTemplates = await installDataStreamComponentTemplates({ + mappings, templateName, registryElasticsearch: dataStream.elasticsearch, esClient, @@ -444,10 +527,7 @@ export async function installTemplate({ }); const template = getTemplate({ - type: dataStream.type, templateIndexPattern, - fields: validFields, - mappings, pipelineName, packageName, composedOfTemplates, diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts index 7650caf73d714..1e617c749ba3d 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts @@ -48,11 +48,8 @@ describe('EPM template', () => { const templateIndexPattern = 'logs-nginx.access-abcd-*'; const template = getTemplate({ - type: 'logs', templateIndexPattern, packageName: 'nginx', - fields: [], - mappings: { properties: {} }, composedOfTemplates: [], templatePriority: 200, }); @@ -63,11 +60,8 @@ describe('EPM template', () => { const composedOfTemplates = ['component1', 'component2']; const template = getTemplate({ - type: 'logs', templateIndexPattern: 'name-*', packageName: 'nginx', - fields: [], - mappings: { properties: {} }, composedOfTemplates, templatePriority: 200, }); @@ -78,11 +72,8 @@ describe('EPM template', () => { const composedOfTemplates: string[] = []; const template = getTemplate({ - type: 'logs', templateIndexPattern: 'name-*', packageName: 'nginx', - fields: [], - mappings: { properties: {} }, composedOfTemplates, templatePriority: 200, }); @@ -93,11 +84,8 @@ describe('EPM template', () => { const templateIndexPattern = 'logs-nginx.access-abcd-*'; const templateWithHidden = getTemplate({ - type: 'logs', templateIndexPattern, packageName: 'nginx', - fields: [], - mappings: { properties: {} }, composedOfTemplates: [], templatePriority: 200, hidden: true, @@ -105,11 +93,8 @@ describe('EPM template', () => { expect(templateWithHidden.data_stream.hidden).toEqual(true); const templateWithoutHidden = getTemplate({ - type: 'logs', templateIndexPattern, packageName: 'nginx', - fields: [], - mappings: { properties: {} }, composedOfTemplates: [], templatePriority: 200, }); @@ -123,17 +108,8 @@ describe('EPM template', () => { const processedFields = processFields(fields); const mappings = generateMappings(processedFields); - const template = getTemplate({ - type: 'logs', - templateIndexPattern: 'foo-*', - packageName: 'nginx', - fields: processedFields, - mappings, - composedOfTemplates: [], - templatePriority: 200, - }); - expect(template).toMatchSnapshot(path.basename(ymlPath)); + expect(mappings).toMatchSnapshot(path.basename(ymlPath)); }); it('tests loading coredns.logs.yml', () => { @@ -143,37 +119,19 @@ describe('EPM template', () => { const processedFields = processFields(fields); const mappings = generateMappings(processedFields); - const template = getTemplate({ - type: 'logs', - templateIndexPattern: 'foo-*', - packageName: 'coredns', - fields: processedFields, - mappings, - composedOfTemplates: [], - templatePriority: 200, - }); - expect(template).toMatchSnapshot(path.basename(ymlPath)); + expect(mappings).toMatchSnapshot(path.basename(ymlPath)); }); it('tests loading system.yml', () => { const ymlPath = path.join(__dirname, '../../fields/tests/system.yml'); const fieldsYML = readFileSync(ymlPath, 'utf-8'); const fields: Field[] = safeLoad(fieldsYML); - const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); - const template = getTemplate({ - type: 'metrics', - templateIndexPattern: 'whatsthis-*', - packageName: 'system', - fields: processedFields, - mappings, - composedOfTemplates: [], - templatePriority: 200, - }); - expect(template).toMatchSnapshot(path.basename(ymlPath)); + expect(mappings).toMatchSnapshot(path.basename(ymlPath)); }); it('tests processing long field with index false', () => { diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts index 73ad218d1a9fd..b0c21c18b59fb 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts @@ -51,20 +51,14 @@ const META_PROP_KEYS = ['metric_type', 'unit']; * @param indexPattern String with the index pattern */ export function getTemplate({ - type, templateIndexPattern, - fields, - mappings, pipelineName, packageName, composedOfTemplates, templatePriority, hidden, }: { - type: string; templateIndexPattern: string; - fields: Fields; - mappings: IndexTemplateMappings; pipelineName?: string | undefined; packageName: string; composedOfTemplates: string[]; @@ -72,10 +66,7 @@ export function getTemplate({ hidden?: boolean; }): IndexTemplate { const template = getBaseTemplate( - type, templateIndexPattern, - fields, - mappings, packageName, composedOfTemplates, templatePriority, @@ -386,10 +377,7 @@ const flattenFieldsToNameAndType = ( }; function getBaseTemplate( - type: string, templateIndexPattern: string, - fields: Fields, - mappings: IndexTemplateMappings, packageName: string, composedOfTemplates: string[], templatePriority: number, @@ -406,27 +394,7 @@ function getBaseTemplate( settings: { index: {}, }, - mappings: { - // All the dynamic field mappings - dynamic_templates: [ - // This makes sure all mappings are keywords by default - { - strings_as_keyword: { - mapping: { - ignore_above: 1024, - type: 'keyword', - }, - match_mapping_type: 'string', - }, - }, - ], - // As we define fields ahead, we don't need any automatic field detection - // This makes sure all the fields are mapped to keyword by default to prevent mapping conflicts - date_detection: false, - // All the properties we know from the fields.yml file - properties: mappings.properties, - _meta, - }, + mappings: {}, }, data_stream: { hidden }, composed_of: composedOfTemplates, diff --git a/x-pack/test/fleet_api_integration/apis/epm/template.ts b/x-pack/test/fleet_api_integration/apis/epm/template.ts index d8856ec392218..9cef0d3f28415 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/template.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/template.ts @@ -14,19 +14,6 @@ export default function ({ getService }: FtrProviderContext) { const templateName = 'bar'; const templateIndexPattern = 'bar-*'; const es = getService('es'); - const mappings = { - properties: { - foo: { - type: 'keyword', - }, - }, - }; - const fields = [ - { - name: 'foo', - type: 'keyword', - }, - ]; // This test was inspired by https://github.com/elastic/kibana/blob/main/x-pack/test/api_integration/apis/monitoring/common/mappings_exist.js describe('EPM - template', async () => { @@ -43,10 +30,7 @@ export default function ({ getService }: FtrProviderContext) { it('can be loaded', async () => { const template = getTemplate({ - type: 'logs', templateIndexPattern, - fields, - mappings, packageName: 'system', composedOfTemplates: [], templatePriority: 200, From f826fd65a6aced797c3817444740d440c5ef53dc Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Fri, 28 Jan 2022 11:15:01 +0000 Subject: [PATCH 02/29] use global template --- .../fleet/server/constants/fleet_es_assets.ts | 18 ++++- .../epm/elasticsearch/template/install.ts | 81 ++----------------- 2 files changed, 23 insertions(+), 76 deletions(-) diff --git a/x-pack/plugins/fleet/server/constants/fleet_es_assets.ts b/x-pack/plugins/fleet/server/constants/fleet_es_assets.ts index 8cdb93be28148..f8d6628417eed 100644 --- a/x-pack/plugins/fleet/server/constants/fleet_es_assets.ts +++ b/x-pack/plugins/fleet/server/constants/fleet_es_assets.ts @@ -11,7 +11,7 @@ const meta = getESAssetMetadata(); export const FLEET_FINAL_PIPELINE_ID = '.fleet_final_pipeline-1'; -export const FLEET_GLOBAL_COMPONENT_TEMPLATE_NAME = '.fleet_component_template-1'; +export const FLEET_GLOBAL_COMPONENT_TEMPLATE_NAME = '.fleet_component_template-2'; export const FLEET_GLOBAL_COMPONENT_TEMPLATE_CONTENT = { _meta: meta, @@ -22,6 +22,22 @@ export const FLEET_GLOBAL_COMPONENT_TEMPLATE_CONTENT = { }, }, mappings: { + // All the dynamic field mappings + dynamic_templates: [ + // This makes sure all mappings are keywords by default + { + strings_as_keyword: { + mapping: { + ignore_above: 1024, + type: 'keyword', + }, + match_mapping_type: 'string', + }, + }, + ], + // As we define fields ahead, we don't need any automatic field detection + // This makes sure all the fields are mapped to keyword by default to prevent mapping conflicts + date_detection: false, properties: { event: { properties: { diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts index 1ba71cbcb0be6..0758b8e3662c8 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts @@ -5,12 +5,10 @@ * 2.0. */ -import { merge, mergeWith, isArray } from 'lodash'; +import { merge } from 'lodash'; import Boom from '@hapi/boom'; import type { ElasticsearchClient, Logger, SavedObjectsClientContract } from 'src/core/server'; -import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; - import { ElasticsearchAssetType } from '../../../../types'; import type { RegistryDataStream, @@ -47,14 +45,6 @@ import { } from './template'; import { buildDefaultSettings } from './default_settings'; -// lodash mergeWith but with a customizer function that -// concatenates arrays together -const mergeWithArrayConcat = (a: any, b: any) => - mergeWith(a, b, (val1, val2) => { - if (isArray(val1)) { - return val1.concat(val2); - } - }); export const installTemplates = async ( installablePackage: InstallablePackage, esClient: ElasticsearchClient, @@ -270,36 +260,16 @@ function buildComponentTemplates(params: { const templatesMap: TemplateMap = {}; const _meta = getESAssetMetadata({ packageName }); - const baseMapping = { - // All the dynamic field mappings - dynamic_templates: [ - // This makes sure all mappings are keywords by default - { - strings_as_keyword: { - mapping: { - ignore_above: 1024, - type: 'keyword', - }, - match_mapping_type: 'string', - }, - }, - ], - // As we define fields ahead, we don't need any automatic field detection - // This makes sure all the fields are mapped to keyword by default to prevent mapping conflicts - date_detection: false, - // All the properties we know from the fields.yml file - properties: mappings.properties, - _meta, - }; - - const packageMappings = registryElasticsearch && registryElasticsearch['index_template.mappings']; templatesMap[mappingsTemplateName] = { template: { - // dynamic_templates arrays should be concatenated together not overlayed - mappings: mergeWithArrayConcat(baseMapping, packageMappings || {}), + mappings: merge( + { properties: mappings.properties }, + registryElasticsearch?.['index_template.mappings'] ?? {} + ), }, _meta, }; + templatesMap[settingsTemplateName] = { template: { settings: merge(defaultSettings, registryElasticsearch?.['index_template.settings'] ?? {}), @@ -318,45 +288,6 @@ function buildComponentTemplates(params: { return templatesMap; } -function buildMappingComponentTemplate({ - mappings, - packageMappings = {}, - _meta, -}: { - mappings: IndexTemplateMappings; - packageMappings?: MappingTypeMapping; - _meta: ESAssetMetadata; -}) { - const baseMapping = { - // All the dynamic field mappings - dynamic_templates: [ - // This makes sure all mappings are keywords by default - { - strings_as_keyword: { - mapping: { - ignore_above: 1024, - type: 'keyword', - }, - match_mapping_type: 'string', - }, - }, - ], - // As we define fields ahead, we don't need any automatic field detection - // This makes sure all the fields are mapped to keyword by default to prevent mapping conflicts - date_detection: false, - // All the properties we know from the fields.yml file - properties: mappings.properties, - _meta, - }; - - return { - template: { - // dynamic_templates arrays should be concatenated together not overwritten - mappings: mergeWithArrayConcat(baseMapping, packageMappings), - }, - _meta, - }; -} async function installDataStreamComponentTemplates(params: { mappings: IndexTemplateMappings; templateName: string; From cfbf18d03b49c9c38e5b4c82ec7d53b9af0c9de5 Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Fri, 28 Jan 2022 14:55:46 +0000 Subject: [PATCH 03/29] re-add _meta to index template --- .../server/services/epm/elasticsearch/template/template.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts index b0c21c18b59fb..aee7443da8608 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts @@ -394,7 +394,9 @@ function getBaseTemplate( settings: { index: {}, }, - mappings: {}, + mappings: { + _meta, + }, }, data_stream: { hidden }, composed_of: composedOfTemplates, From 197cb06712e9dd5802426a9fb77f9589ea92005d Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Fri, 28 Jan 2022 14:56:26 +0000 Subject: [PATCH 04/29] simplify merge --- .../server/services/epm/elasticsearch/template/install.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts index 0758b8e3662c8..eb3d1ef01a1a4 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts @@ -262,10 +262,7 @@ function buildComponentTemplates(params: { templatesMap[mappingsTemplateName] = { template: { - mappings: merge( - { properties: mappings.properties }, - registryElasticsearch?.['index_template.mappings'] ?? {} - ), + mappings: merge(mappings, registryElasticsearch?.['index_template.mappings'] ?? {}), }, _meta, }; From fee70b8f8da291789e70491f9a281f3bba931a76 Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Fri, 28 Jan 2022 16:27:21 +0000 Subject: [PATCH 05/29] fix component_template test --- .../server/services/epm/elasticsearch/template/template.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts index 1e617c749ba3d..aa73686f91e5d 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts @@ -26,7 +26,7 @@ import { updateCurrentWriteIndices, } from './template'; -const FLEET_COMPONENT_TEMPLATE = '.fleet_component_template-1'; +const FLEET_COMPONENT_TEMPLATE = '.fleet_component_template-2'; // Add our own serialiser to just do JSON.stringify expect.addSnapshotSerializer({ From 5a7a5167f2a34e7e41738cd62553c9ad457ba143 Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Fri, 28 Jan 2022 17:56:33 +0000 Subject: [PATCH 06/29] add settings to mapping component template --- .../services/epm/elasticsearch/template/install.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts index eb3d1ef01a1a4..f68c134bdfdcd 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts @@ -262,6 +262,15 @@ function buildComponentTemplates(params: { templatesMap[mappingsTemplateName] = { template: { + settings: { + index: { + mapping: { + total_fields: { + limit: '10000', + }, + }, + }, + }, mappings: merge(mappings, registryElasticsearch?.['index_template.mappings'] ?? {}), }, _meta, From 2f5ff81e115b30ddbf2e6ebbd5474409f6f372ae Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Mon, 31 Jan 2022 13:40:36 +0000 Subject: [PATCH 07/29] put mapping settings on @mapping component template --- .../template/default_settings.ts | 6 ---- .../epm/elasticsearch/template/install.ts | 29 ++++++++++++++----- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/default_settings.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/default_settings.ts index 84ec75b9da065..7f8e8e8544109 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/default_settings.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/default_settings.ts @@ -67,12 +67,6 @@ export function buildDefaultSettings({ }, // What should be our default for the compression? codec: 'best_compression', - mapping: { - total_fields: { - limit: '10000', - }, - }, - // All the default fields which should be queried have to be added here. // So far we add all keyword and text fields here if there are any, otherwise // this setting is skipped. diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts index f68c134bdfdcd..49c63106a09c4 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { merge } from 'lodash'; +import { merge, cloneDeep } from 'lodash'; import Boom from '@hapi/boom'; import type { ElasticsearchClient, Logger, SavedObjectsClientContract } from 'src/core/server'; @@ -260,6 +260,25 @@ function buildComponentTemplates(params: { const templatesMap: TemplateMap = {}; const _meta = getESAssetMetadata({ packageName }); + const registrySettings = registryElasticsearch?.['index_template.settings'] ?? {}; + // @ts-expect-error 2339 + const mappingSettings = registrySettings?.index?.mapping; + + const registrySettingsForTemplate = cloneDeep(registrySettings); + + // index.mapping settings must go on the mapping component template otherwise + // the template may be rejected e.g if nested_fields.limit has been increased + if (mappingSettings) { + // @ts-expect-error 2339 + delete registrySettingsForTemplate.index.mapping; + } + templatesMap[settingsTemplateName] = { + template: { + settings: merge(defaultSettings, registrySettingsForTemplate), + }, + _meta, + }; + templatesMap[mappingsTemplateName] = { template: { settings: { @@ -268,6 +287,7 @@ function buildComponentTemplates(params: { total_fields: { limit: '10000', }, + ...mappingSettings, }, }, }, @@ -276,13 +296,6 @@ function buildComponentTemplates(params: { _meta, }; - templatesMap[settingsTemplateName] = { - template: { - settings: merge(defaultSettings, registryElasticsearch?.['index_template.settings'] ?? {}), - }, - _meta, - }; - // return empty/stub template templatesMap[userSettingsTemplateName] = { template: { From 7c99da4546993e5af80fbdee02282b3c689ed2dd Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Mon, 31 Jan 2022 15:05:32 +0000 Subject: [PATCH 08/29] fix snapshot --- .../epm/elasticsearch/template/default_settings.test.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/default_settings.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/default_settings.test.ts index ee6d7086cdd3c..ea2d0868c6d17 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/default_settings.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/default_settings.test.ts @@ -46,11 +46,6 @@ describe('buildDefaultSettings', () => { "lifecycle": Object { "name": "logs", }, - "mapping": Object { - "total_fields": Object { - "limit": "10000", - }, - }, "query": Object { "default_field": Array [ "field1Keyword", From 9b011890525150d6bead3443a98e4e8c36591388 Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Mon, 31 Jan 2022 15:14:39 +0000 Subject: [PATCH 09/29] fix integration tests --- .../apis/epm/install_by_upload.ts | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/x-pack/test/fleet_api_integration/apis/epm/install_by_upload.ts b/x-pack/test/fleet_api_integration/apis/epm/install_by_upload.ts index 97f0bce458791..8f84bb00f8c9d 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/install_by_upload.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/install_by_upload.ts @@ -75,7 +75,7 @@ export default function (providerContext: FtrProviderContext) { .type('application/gzip') .send(buf) .expect(200); - expect(res.body.items.length).to.be(27); + expect(res.body.items.length).to.be(30); }); it('should install a zip archive correctly and package info should return correctly after validation', async function () { @@ -86,7 +86,21 @@ export default function (providerContext: FtrProviderContext) { .type('application/zip') .send(buf) .expect(200); - expect(res.body.items.length).to.be(27); + expect(res.body.items.length).to.be(30); + + const packageInfoRes = await supertest + .get(`/api/fleet/epm/packages/${testPkgName}/${testPkgVersion}`) + .set('kbn-xsrf', 'xxxx') + .expect(200); + + delete packageInfoRes.body.item.latestVersion; + delete packageInfoRes.body.item.savedObject.attributes.install_started_at; + delete packageInfoRes.body.item.savedObject.version; + delete packageInfoRes.body.item.savedObject.updated_at; + delete packageInfoRes.body.item.savedObject.coreMigrationVersion; + delete packageInfoRes.body.item.savedObject.migrationVersion; + + expectSnapshot(packageInfoRes.body.item).toMatch(); }); it('should throw an error if the archive is zip but content type is gzip', async function () { From aa119222e14a4134b6693f27d7a6a30fcb49eb5c Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Mon, 31 Jan 2022 17:19:02 +0000 Subject: [PATCH 10/29] fix component template order and test --- .../epm/elasticsearch/template/install.ts | 13 ++++++------ .../apis/epm/install_overrides.ts | 20 ++++++++++++++++++- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts index 49c63106a09c4..9a4b17fa37cc2 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts @@ -272,12 +272,6 @@ function buildComponentTemplates(params: { // @ts-expect-error 2339 delete registrySettingsForTemplate.index.mapping; } - templatesMap[settingsTemplateName] = { - template: { - settings: merge(defaultSettings, registrySettingsForTemplate), - }, - _meta, - }; templatesMap[mappingsTemplateName] = { template: { @@ -296,6 +290,13 @@ function buildComponentTemplates(params: { _meta, }; + templatesMap[settingsTemplateName] = { + template: { + settings: merge(defaultSettings, registrySettingsForTemplate), + }, + _meta, + }; + // return empty/stub template templatesMap[userSettingsTemplateName] = { template: { diff --git a/x-pack/test/fleet_api_integration/apis/epm/install_overrides.ts b/x-pack/test/fleet_api_integration/apis/epm/install_overrides.ts index 603e18931c227..2d2e9246e66f9 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/install_overrides.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/install_overrides.ts @@ -55,7 +55,7 @@ export default function (providerContext: FtrProviderContext) { `${templateName}@mappings`, `${templateName}@settings`, `${templateName}@custom`, - '.fleet_component_template-1', + '.fleet_component_template-2', ]); ({ body } = await es.transport.request( @@ -151,6 +151,24 @@ export default function (providerContext: FtrProviderContext) { }, mappings: { dynamic: 'false', + properties: { + '@timestamp': { + type: 'date', + }, + data_stream: { + properties: { + dataset: { + type: 'constant_keyword', + }, + namespace: { + type: 'constant_keyword', + }, + type: { + type: 'constant_keyword', + }, + }, + }, + }, }, aliases: {}, }, From d00189fa3726ffa38d860aadd14f3ed095cc31da Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Fri, 4 Feb 2022 09:54:24 +0000 Subject: [PATCH 11/29] use mapping component template for updating backing indices --- .../plugins/fleet/common/types/models/epm.ts | 20 ++++++++ .../fleet/server/constants/fleet_es_assets.ts | 6 +++ .../plugins/fleet/server/constants/index.ts | 3 ++ .../server/services/epm/elasticsearch/meta.ts | 10 +--- .../epm/elasticsearch/template/install.ts | 49 ++++++++----------- .../elasticsearch/template/template.test.ts | 12 +++++ .../epm/elasticsearch/template/template.ts | 44 ++++++++++++----- x-pack/plugins/fleet/server/types/index.tsx | 2 + 8 files changed, 96 insertions(+), 50 deletions(-) diff --git a/x-pack/plugins/fleet/common/types/models/epm.ts b/x-pack/plugins/fleet/common/types/models/epm.ts index 1c7e09a51c5da..4e9e84912ef78 100644 --- a/x-pack/plugins/fleet/common/types/models/epm.ts +++ b/x-pack/plugins/fleet/common/types/models/epm.ts @@ -481,7 +481,27 @@ export interface IndexTemplate { _meta: object; } +export interface ESAssetMetadata { + package?: { + name: string; + }; + managed_by: string; + managed: boolean; +} +export interface TemplateMapEntry { + _meta: ESAssetMetadata; + template: + | { + mappings: NonNullable; + } + | { + settings: NonNullable | object; + }; +} + +export type TemplateMap = Record; export interface IndexTemplateEntry { templateName: string; indexTemplate: IndexTemplate; + composedOfTemplates: TemplateMap; } diff --git a/x-pack/plugins/fleet/server/constants/fleet_es_assets.ts b/x-pack/plugins/fleet/server/constants/fleet_es_assets.ts index f8d6628417eed..0789e6a9af2f5 100644 --- a/x-pack/plugins/fleet/server/constants/fleet_es_assets.ts +++ b/x-pack/plugins/fleet/server/constants/fleet_es_assets.ts @@ -9,6 +9,12 @@ import { getESAssetMetadata } from '../services/epm/elasticsearch/meta'; const meta = getESAssetMetadata(); +export const MAPPINGS_TEMPLATE_SUFFIX = '@mappings'; + +export const SETTINGS_TEMPLATE_SUFFIX = '@settings'; + +export const USER_SETTINGS_TEMPLATE_SUFFIX = '@custom'; + export const FLEET_FINAL_PIPELINE_ID = '.fleet_final_pipeline-1'; export const FLEET_GLOBAL_COMPONENT_TEMPLATE_NAME = '.fleet_component_template-2'; diff --git a/x-pack/plugins/fleet/server/constants/index.ts b/x-pack/plugins/fleet/server/constants/index.ts index 0ccbeb9f025e4..ee2877df68fec 100644 --- a/x-pack/plugins/fleet/server/constants/index.ts +++ b/x-pack/plugins/fleet/server/constants/index.ts @@ -63,4 +63,7 @@ export { FLEET_FINAL_PIPELINE_ID, FLEET_FINAL_PIPELINE_CONTENT, FLEET_FINAL_PIPELINE_VERSION, + MAPPINGS_TEMPLATE_SUFFIX, + SETTINGS_TEMPLATE_SUFFIX, + USER_SETTINGS_TEMPLATE_SUFFIX, } from './fleet_es_assets'; diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/meta.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/meta.ts index a3ceaf44100d7..d691cd8c700e5 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/meta.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/meta.ts @@ -7,15 +7,9 @@ import { safeLoad, safeDump } from 'js-yaml'; -const MANAGED_BY_DEFAULT = 'fleet'; +import type { ESAssetMetadata } from '../../../../common/types'; -export interface ESAssetMetadata { - package?: { - name: string; - }; - managed_by: string; - managed: boolean; -} +const MANAGED_BY_DEFAULT = 'fleet'; /** * Build common metadata object for Elasticsearch assets installed by Fleet. Result should be diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts index 9a4b17fa37cc2..4b6daf39a03b6 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts @@ -18,6 +18,8 @@ import type { IndexTemplate, PackageInfo, IndexTemplateMappings, + TemplateMapEntry, + TemplateMap, } from '../../../../types'; import { loadFieldsFromYaml, processFields } from '../../fields/field'; @@ -28,9 +30,11 @@ import { removeAssetTypesFromInstalledEs, saveInstalledEsRefs } from '../../pack import { FLEET_GLOBAL_COMPONENT_TEMPLATE_NAME, FLEET_GLOBAL_COMPONENT_TEMPLATE_CONTENT, + MAPPINGS_TEMPLATE_SUFFIX, + SETTINGS_TEMPLATE_SUFFIX, + USER_SETTINGS_TEMPLATE_SUFFIX, } from '../../../../constants'; -import type { ESAssetMetadata } from '../meta'; import { getESAssetMetadata } from '../meta'; import { retryTransientEsErrors } from '../retry'; @@ -204,19 +208,6 @@ export async function installTemplateForDataStream({ }); } -interface TemplateMapEntry { - _meta: ESAssetMetadata; - template: - | { - mappings: NonNullable; - } - | { - settings: NonNullable | object; - }; -} - -type TemplateMap = Record; - function putComponentTemplate( esClient: ElasticsearchClient, logger: Logger, @@ -225,7 +216,10 @@ function putComponentTemplate( name: string; create?: boolean; } -): { clusterPromise: Promise; name: string } { +): { + clusterPromise: ReturnType; + name: string; +} { const { name, body, create = false } = params; return { clusterPromise: retryTransientEsErrors( @@ -236,14 +230,11 @@ function putComponentTemplate( }; } -const mappingsSuffix = '@mappings'; -const settingsSuffix = '@settings'; -const userSettingsSuffix = '@custom'; type TemplateBaseName = string; -type UserSettingsTemplateName = `${TemplateBaseName}${typeof userSettingsSuffix}`; +type UserSettingsTemplateName = `${TemplateBaseName}${typeof USER_SETTINGS_TEMPLATE_SUFFIX}`; const isUserSettingsTemplate = (name: string): name is UserSettingsTemplateName => - name.endsWith(userSettingsSuffix); + name.endsWith(USER_SETTINGS_TEMPLATE_SUFFIX); function buildComponentTemplates(params: { mappings: IndexTemplateMappings; @@ -253,9 +244,9 @@ function buildComponentTemplates(params: { defaultSettings: IndexTemplate['template']['settings']; }) { const { templateName, registryElasticsearch, packageName, defaultSettings, mappings } = params; - const mappingsTemplateName = `${templateName}${mappingsSuffix}`; - const settingsTemplateName = `${templateName}${settingsSuffix}`; - const userSettingsTemplateName = `${templateName}${userSettingsSuffix}`; + const mappingsTemplateName = `${templateName}${MAPPINGS_TEMPLATE_SUFFIX}`; + const settingsTemplateName = `${templateName}${SETTINGS_TEMPLATE_SUFFIX}`; + const userSettingsTemplateName = `${templateName}${USER_SETTINGS_TEMPLATE_SUFFIX}`; const templatesMap: TemplateMap = {}; const _meta = getESAssetMetadata({ packageName }); @@ -326,15 +317,14 @@ async function installDataStreamComponentTemplates(params: { logger, mappings, } = params; - const templates = buildComponentTemplates({ + const componentTemplates = buildComponentTemplates({ mappings, templateName, registryElasticsearch, packageName, defaultSettings, }); - const templateNames = Object.keys(templates); - const templateEntries = Object.entries(templates); + const templateEntries = Object.entries(componentTemplates); // TODO: Check return values for errors await Promise.all( templateEntries.map(async ([name, body]) => { @@ -360,7 +350,7 @@ async function installDataStreamComponentTemplates(params: { }) ); - return templateNames; + return { componentTemplateNames: Object.keys(componentTemplates), componentTemplates }; } export async function ensureDefaultComponentTemplate( @@ -467,7 +457,7 @@ export async function installTemplate({ ilmPolicy: dataStream.ilm_policy, }); - const composedOfTemplates = await installDataStreamComponentTemplates({ + const { componentTemplateNames, componentTemplates } = await installDataStreamComponentTemplates({ mappings, templateName, registryElasticsearch: dataStream.elasticsearch, @@ -481,7 +471,7 @@ export async function installTemplate({ templateIndexPattern, pipelineName, packageName, - composedOfTemplates, + composedOfTemplates: componentTemplateNames, templatePriority, hidden: dataStream.hidden, }); @@ -500,6 +490,7 @@ export async function installTemplate({ return { templateName, indexTemplate: template, + composedOfTemplates: componentTemplates, }; } diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts index aa73686f91e5d..d0ad411b81dd7 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts @@ -837,6 +837,12 @@ describe('EPM template', () => { await updateCurrentWriteIndices(esClient, logger, [ { templateName: 'test', + composedOfTemplates: { + 'test@mappings': { + _meta: {} as any, + template: { mappings: { properties: { hello: { type: 'text' } } } }, + }, + }, indexTemplate: { index_patterns: ['test.*-*'], template: { @@ -865,6 +871,12 @@ describe('EPM template', () => { await updateCurrentWriteIndices(esClient, logger, [ { templateName: 'test', + composedOfTemplates: { + 'test@mappings': { + _meta: {} as any, + template: { mappings: { properties: { hello: { type: 'text' } } } }, + }, + }, indexTemplate: { index_patterns: ['test-*'], template: { diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts index aee7443da8608..1189edca8613a 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts @@ -13,10 +13,14 @@ import type { IndexTemplateEntry, IndexTemplate, IndexTemplateMappings, + TemplateMap, } from '../../../../types'; import { appContextService } from '../../../'; import { getRegistryDataStreamAssetBaseName } from '../index'; -import { FLEET_GLOBAL_COMPONENT_TEMPLATE_NAME } from '../../../../constants'; +import { + FLEET_GLOBAL_COMPONENT_TEMPLATE_NAME, + MAPPINGS_TEMPLATE_SUFFIX, +} from '../../../../constants'; import { getESAssetMetadata } from '../meta'; import { retryTransientEsErrors } from '../retry'; @@ -35,6 +39,7 @@ export interface CurrentDataStream { dataStreamName: string; replicated: boolean; indexTemplate: IndexTemplate; + composedOfTemplates: TemplateMap; } const DEFAULT_SCALING_FACTOR = 1000; const DEFAULT_IGNORE_ABOVE = 1024; @@ -444,7 +449,7 @@ const getDataStreams = async ( esClient: ElasticsearchClient, template: IndexTemplateEntry ): Promise => { - const { indexTemplate } = template; + const { indexTemplate, composedOfTemplates } = template; const body = await esClient.indices.getDataStream({ name: indexTemplate.index_patterns.join(','), @@ -456,6 +461,7 @@ const getDataStreams = async ( dataStreamName: dataStream.name, replicated: dataStream.replicated, indexTemplate, + composedOfTemplates, })); }; @@ -464,11 +470,9 @@ const updateAllDataStreams = async ( esClient: ElasticsearchClient, logger: Logger ): Promise => { - const updatedataStreamPromises = indexNameWithTemplates.map( - ({ dataStreamName, indexTemplate }) => { - return updateExistingDataStream({ dataStreamName, esClient, logger, indexTemplate }); - } - ); + const updatedataStreamPromises = indexNameWithTemplates.map((templateEntry) => { + return updateExistingDataStream({ esClient, logger, ...templateEntry }); + }); await Promise.all(updatedataStreamPromises); }; const updateExistingDataStream = async ({ @@ -476,21 +480,35 @@ const updateExistingDataStream = async ({ esClient, logger, indexTemplate, + composedOfTemplates, }: { dataStreamName: string; esClient: ElasticsearchClient; logger: Logger; indexTemplate: IndexTemplate; + composedOfTemplates: TemplateMap; }) => { - const { settings, mappings } = indexTemplate.template; + const { settings } = indexTemplate.template; - // for now, remove from object so as not to update stream or data stream properties of the index until type and name - // are added in https://github.com/elastic/kibana/issues/66551. namespace value we will continue - // to skip updating and assume the value in the index mapping is correct - delete mappings.properties.stream; - delete mappings.properties.data_stream; // try to update the mappings first try { + const mappingComponentTemplateName = + Object.keys(composedOfTemplates).find((templateName) => + templateName.endsWith(MAPPINGS_TEMPLATE_SUFFIX) + ) || ''; + const mappingComponentTemplate = composedOfTemplates[mappingComponentTemplateName]; + // @ts-expect-error 2339 + const mappings = mappingComponentTemplate?.template?.mappings; + + if (!mappingComponentTemplate || !mappings?.properties) { + throw new Error('mapping not found'); + } + + // for now, remove from object so as not to update stream or data stream properties of the index until type and name + // are added in https://github.com/elastic/kibana/issues/66551. namespace value we will continue + // to skip updating and assume the value in the index mapping is correct + delete mappings.properties.stream; + delete mappings.properties.data_stream; await retryTransientEsErrors( () => esClient.indices.putMapping({ diff --git a/x-pack/plugins/fleet/server/types/index.tsx b/x-pack/plugins/fleet/server/types/index.tsx index 91303046485d9..9024cd05e2dea 100644 --- a/x-pack/plugins/fleet/server/types/index.tsx +++ b/x-pack/plugins/fleet/server/types/index.tsx @@ -63,6 +63,8 @@ export type { RegistrySearchResult, IndexTemplateEntry, IndexTemplateMappings, + TemplateMap, + TemplateMapEntry, Settings, SettingsSOAttributes, InstallType, From a9950a3f39d6fe30ef185775f636e1bba81eb7cd Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Fri, 4 Feb 2022 09:56:19 +0000 Subject: [PATCH 12/29] fix tests --- .../apis/epm/final_pipeline.ts | 2 +- .../apis/epm/install_remove_assets.ts | 4 + .../apis/epm/update_assets.ts | 129 ++++++++---------- 3 files changed, 61 insertions(+), 74 deletions(-) diff --git a/x-pack/test/fleet_api_integration/apis/epm/final_pipeline.ts b/x-pack/test/fleet_api_integration/apis/epm/final_pipeline.ts index 3b01b1f861aef..cd8a6e92aaefb 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/final_pipeline.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/final_pipeline.ts @@ -110,7 +110,7 @@ export default function (providerContext: FtrProviderContext) { const res = await es.indices.getIndexTemplate({ name: 'logs-log.log' }); expect(res.index_templates.length).to.be(FINAL_PIPELINE_VERSION); expect(res.index_templates[0]?.index_template?.composed_of).to.contain( - '.fleet_component_template-1' + '.fleet_component_template-2' ); }); diff --git a/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts b/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts index af807d4393daf..a44b8be478874 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts @@ -548,6 +548,10 @@ const expectAssetsInstalled = ({ id: 'logs-all_assets.test_logs@custom', type: 'component_template', }, + { + id: 'metrics-all_assets.test_metrics@mappings', + type: 'component_template', + }, { id: 'metrics-all_assets.test_metrics@settings', type: 'component_template', diff --git a/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts b/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts index 1947396b8a2bd..7b8c3bc09db1a 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts @@ -81,32 +81,8 @@ export default function (providerContext: FtrProviderContext) { { meta: true } ); expect(resLogsTemplate.statusCode).equal(200); - expect( - resLogsTemplate.body.index_templates[0].index_template.template.mappings.properties - ).eql({ - '@timestamp': { - type: 'date', - }, - logs_test_name: { - type: 'text', - }, - new_field_name: { - ignore_above: 1024, - type: 'keyword', - }, - data_stream: { - properties: { - dataset: { - type: 'constant_keyword', - }, - namespace: { - type: 'constant_keyword', - }, - type: { - type: 'constant_keyword', - }, - }, - }, + expect(resLogsTemplate.body.index_templates[0].index_template.template.mappings).eql({ + _meta: { package: { name: 'all_assets' }, managed_by: 'fleet', managed: true }, }); const resMetricsTemplate = await es.transport.request( { @@ -116,27 +92,12 @@ export default function (providerContext: FtrProviderContext) { { meta: true } ); expect(resMetricsTemplate.statusCode).equal(200); - expect( - resMetricsTemplate.body.index_templates[0].index_template.template.mappings.properties - ).eql({ - '@timestamp': { - type: 'date', - }, - metrics_test_name2: { - ignore_above: 1024, - type: 'keyword', - }, - data_stream: { - properties: { - dataset: { - type: 'constant_keyword', - }, - namespace: { - type: 'constant_keyword', - }, - type: { - type: 'constant_keyword', - }, + expect(resMetricsTemplate.body.index_templates[0].index_template.template.mappings).eql({ + _meta: { + managed: true, + managed_by: 'fleet', + package: { + name: 'all_assets', }, }, }); @@ -150,27 +111,12 @@ export default function (providerContext: FtrProviderContext) { { meta: true } ); expect(resLogsTemplate.statusCode).equal(200); - expect( - resLogsTemplate.body.index_templates[0].index_template.template.mappings.properties - ).eql({ - '@timestamp': { - type: 'date', - }, - test_logs2: { - ignore_above: 1024, - type: 'keyword', - }, - data_stream: { - properties: { - dataset: { - type: 'constant_keyword', - }, - namespace: { - type: 'constant_keyword', - }, - type: { - type: 'constant_keyword', - }, + expect(resLogsTemplate.body.index_templates[0].index_template.template.mappings).eql({ + _meta: { + managed: true, + managed_by: 'fleet', + package: { + name: 'all_assets', }, }, }); @@ -232,8 +178,42 @@ export default function (providerContext: FtrProviderContext) { { meta: true } ); expect(resMappings.statusCode).equal(200); + expect(resMappings.body.component_templates[0].component_template.template.settings).eql({ + index: { + mapping: { + total_fields: { + limit: '10000', + }, + }, + }, + }); expect(resMappings.body.component_templates[0].component_template.template.mappings).eql({ dynamic: true, + properties: { + '@timestamp': { + type: 'date', + }, + data_stream: { + properties: { + dataset: { + type: 'constant_keyword', + }, + namespace: { + type: 'constant_keyword', + }, + type: { + type: 'constant_keyword', + }, + }, + }, + logs_test_name: { + type: 'text', + }, + new_field_name: { + ignore_above: 1024, + type: 'keyword', + }, + }, }); const resSettings = await es.transport.request( { @@ -247,11 +227,6 @@ export default function (providerContext: FtrProviderContext) { index: { lifecycle: { name: 'reference2' }, codec: 'best_compression', - mapping: { - total_fields: { - limit: '10000', - }, - }, query: { default_field: ['logs_test_name', 'new_field_name'], }, @@ -396,6 +371,10 @@ export default function (providerContext: FtrProviderContext) { id: 'logs-all_assets.test_logs2', type: 'index_template', }, + { + id: 'logs-all_assets.test_logs2@mappings', + type: 'component_template', + }, { id: 'logs-all_assets.test_logs2@settings', type: 'component_template', @@ -408,6 +387,10 @@ export default function (providerContext: FtrProviderContext) { id: 'metrics-all_assets.test_metrics', type: 'index_template', }, + { + id: 'metrics-all_assets.test_metrics@mappings', + type: 'component_template', + }, { id: 'metrics-all_assets.test_metrics@settings', type: 'component_template', From bbd973f3d9a5e947467a9dfe9a95e5dbd5d26dc3 Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Fri, 4 Feb 2022 10:11:13 +0000 Subject: [PATCH 13/29] move code to util functions --- .../epm/elasticsearch/template/template.ts | 52 +++++++++++-------- 1 file changed, 31 insertions(+), 21 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts index 1189edca8613a..5e49aeb15fc20 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts @@ -465,6 +465,26 @@ const getDataStreams = async ( })); }; +const getTemplateWithNameEndingIn = (composedOfTemplates: TemplateMap, suffix: string) => { + const mappingComponentTemplateName = + Object.keys(composedOfTemplates).find((templateName) => templateName.endsWith(suffix)) || ''; + + return composedOfTemplates[mappingComponentTemplateName]; +}; + +const rolloverDataStream = (dataStreamName: string, esClient: ElasticsearchClient) => { + try { + // Do no wrap rollovers in retryTransientEsErrors since it is not idempotent + const path = `/${dataStreamName}/_rollover`; + return esClient.transport.request({ + method: 'POST', + path, + }); + } catch (error) { + throw new Error(`cannot rollover data stream ${dataStreamName} - ${error}`); + } +}; + const updateAllDataStreams = async ( indexNameWithTemplates: CurrentDataStream[], esClient: ElasticsearchClient, @@ -488,20 +508,17 @@ const updateExistingDataStream = async ({ indexTemplate: IndexTemplate; composedOfTemplates: TemplateMap; }) => { - const { settings } = indexTemplate.template; + const mappingComponentTemplate = getTemplateWithNameEndingIn( + composedOfTemplates, + MAPPINGS_TEMPLATE_SUFFIX + ); + // @ts-expect-error 2339 + const mappings = mappingComponentTemplate?.template?.mappings; - // try to update the mappings first try { - const mappingComponentTemplateName = - Object.keys(composedOfTemplates).find((templateName) => - templateName.endsWith(MAPPINGS_TEMPLATE_SUFFIX) - ) || ''; - const mappingComponentTemplate = composedOfTemplates[mappingComponentTemplateName]; - // @ts-expect-error 2339 - const mappings = mappingComponentTemplate?.template?.mappings; - if (!mappingComponentTemplate || !mappings?.properties) { - throw new Error('mapping not found'); + await rolloverDataStream(dataStreamName, esClient); + return; } // for now, remove from object so as not to update stream or data stream properties of the index until type and name @@ -520,20 +537,13 @@ const updateExistingDataStream = async ({ ); // if update fails, rollover data stream } catch (err) { - try { - // Do no wrap rollovers in retryTransientEsErrors since it is not idempotent - const path = `/${dataStreamName}/_rollover`; - await esClient.transport.request({ - method: 'POST', - path, - }); - } catch (error) { - throw new Error(`cannot rollover data stream ${error}`); - } + await rolloverDataStream(dataStreamName, esClient); + return; } // update settings after mappings was successful to ensure // pointing to the new pipeline is safe // for now, only update the pipeline + const { settings } = indexTemplate.template; if (!settings.index.default_pipeline) return; try { await retryTransientEsErrors( From e10054e794f27ca09312751492fc48725bb5f94b Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Fri, 4 Feb 2022 11:58:17 +0000 Subject: [PATCH 14/29] add comment --- .../server/services/epm/elasticsearch/template/template.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts index 5e49aeb15fc20..a3852146a73c5 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts @@ -517,6 +517,8 @@ const updateExistingDataStream = async ({ try { if (!mappingComponentTemplate || !mappings?.properties) { + // if there is no mapping component template then something has + // gone wrong, so rollover the stream to get the new mapping. await rolloverDataStream(dataStreamName, esClient); return; } From 5ded64210d691b064528a4882548d7b6022f4057 Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Wed, 23 Feb 2022 13:45:01 +0000 Subject: [PATCH 15/29] split global templates --- .../fleet/server/constants/fleet_es_assets.ts | 40 +++++++++++++++++-- .../plugins/fleet/server/constants/index.ts | 7 +++- .../epm/elasticsearch/template/install.ts | 30 ++++++++++---- .../elasticsearch/template/template.test.ts | 9 +++-- .../epm/elasticsearch/template/template.ts | 11 ++--- x-pack/plugins/fleet/server/services/setup.ts | 8 ++-- 6 files changed, 81 insertions(+), 24 deletions(-) diff --git a/x-pack/plugins/fleet/server/constants/fleet_es_assets.ts b/x-pack/plugins/fleet/server/constants/fleet_es_assets.ts index 0789e6a9af2f5..0b685b7c4f69e 100644 --- a/x-pack/plugins/fleet/server/constants/fleet_es_assets.ts +++ b/x-pack/plugins/fleet/server/constants/fleet_es_assets.ts @@ -8,7 +8,6 @@ import { getESAssetMetadata } from '../services/epm/elasticsearch/meta'; const meta = getESAssetMetadata(); - export const MAPPINGS_TEMPLATE_SUFFIX = '@mappings'; export const SETTINGS_TEMPLATE_SUFFIX = '@settings'; @@ -17,9 +16,36 @@ export const USER_SETTINGS_TEMPLATE_SUFFIX = '@custom'; export const FLEET_FINAL_PIPELINE_ID = '.fleet_final_pipeline-1'; -export const FLEET_GLOBAL_COMPONENT_TEMPLATE_NAME = '.fleet_component_template-2'; +export const FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME = '.fleet_globals-1'; + +export const FLEET_GLOBALS_COMPONENT_TEMPLATE_CONTENT = { + _meta: meta, + template: { + settings: {}, + mappings: { + _meta: meta, + // All the dynamic field mappings + dynamic_templates: [ + // This makes sure all mappings are keywords by default + { + strings_as_keyword: { + mapping: { + ignore_above: 1024, + type: 'keyword', + }, + match_mapping_type: 'string', + }, + }, + ], + // As we define fields ahead, we don't need any automatic field detection + // This makes sure all the fields are mapped to keyword by default to prevent mapping conflicts + date_detection: false, + }, + }, +}; +export const FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_NAME = '.fleet_agent_id_verification-1'; -export const FLEET_GLOBAL_COMPONENT_TEMPLATE_CONTENT = { +export const FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_CONTENT = { _meta: meta, template: { settings: { @@ -62,6 +88,14 @@ export const FLEET_GLOBAL_COMPONENT_TEMPLATE_CONTENT = { }, }; +export const FLEET_COMPONENT_TEMPLATES = [ + { name: FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME, body: FLEET_GLOBALS_COMPONENT_TEMPLATE_CONTENT }, + { + name: FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_NAME, + body: FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_CONTENT, + }, +]; + export const FLEET_FINAL_PIPELINE_VERSION = 2; // If the content is updated you probably need to update the FLEET_FINAL_PIPELINE_VERSION too to allow upgrade of the pipeline diff --git a/x-pack/plugins/fleet/server/constants/index.ts b/x-pack/plugins/fleet/server/constants/index.ts index ee2877df68fec..ec7a4a2664882 100644 --- a/x-pack/plugins/fleet/server/constants/index.ts +++ b/x-pack/plugins/fleet/server/constants/index.ts @@ -58,8 +58,11 @@ export { } from '../../common'; export { - FLEET_GLOBAL_COMPONENT_TEMPLATE_NAME, - FLEET_GLOBAL_COMPONENT_TEMPLATE_CONTENT, + FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME, + FLEET_GLOBALS_COMPONENT_TEMPLATE_CONTENT, + FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_NAME, + FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_CONTENT, + FLEET_COMPONENT_TEMPLATES, FLEET_FINAL_PIPELINE_ID, FLEET_FINAL_PIPELINE_CONTENT, FLEET_FINAL_PIPELINE_VERSION, diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts index 4b6daf39a03b6..be78164f0b976 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts @@ -28,8 +28,7 @@ import { getPipelineNameForInstallation } from '../ingest_pipeline/install'; import { getAsset, getPathParts } from '../../archive'; import { removeAssetTypesFromInstalledEs, saveInstalledEsRefs } from '../../packages/install'; import { - FLEET_GLOBAL_COMPONENT_TEMPLATE_NAME, - FLEET_GLOBAL_COMPONENT_TEMPLATE_CONTENT, + FLEET_COMPONENT_TEMPLATES, MAPPINGS_TEMPLATE_SUFFIX, SETTINGS_TEMPLATE_SUFFIX, USER_SETTINGS_TEMPLATE_SUFFIX, @@ -49,6 +48,8 @@ import { } from './template'; import { buildDefaultSettings } from './default_settings'; +const FLEET_COMPONENT_TEMPLATE_NAMES = FLEET_COMPONENT_TEMPLATES.map((tmpl) => tmpl.name); + export const installTemplates = async ( installablePackage: InstallablePackage, esClient: ElasticsearchClient, @@ -353,15 +354,28 @@ async function installDataStreamComponentTemplates(params: { return { componentTemplateNames: Object.keys(componentTemplates), componentTemplates }; } -export async function ensureDefaultComponentTemplate( +export async function ensureDefaultComponentTemplates( esClient: ElasticsearchClient, logger: Logger +) { + return Promise.all( + FLEET_COMPONENT_TEMPLATES.map(({ name, body }) => + ensureComponentTemplate(esClient, logger, name, body) + ) + ); +} + +export async function ensureComponentTemplate( + esClient: ElasticsearchClient, + logger: Logger, + name: string, + body: TemplateMapEntry ) { const getTemplateRes = await retryTransientEsErrors( () => esClient.cluster.getComponentTemplate( { - name: FLEET_GLOBAL_COMPONENT_TEMPLATE_NAME, + name, }, { ignore: [404], @@ -373,8 +387,8 @@ export async function ensureDefaultComponentTemplate( const existingTemplate = getTemplateRes?.component_templates?.[0]; if (!existingTemplate) { await putComponentTemplate(esClient, logger, { - name: FLEET_GLOBAL_COMPONENT_TEMPLATE_NAME, - body: FLEET_GLOBAL_COMPONENT_TEMPLATE_CONTENT, + name, + body, }).clusterPromise; } @@ -504,7 +518,9 @@ export function getAllTemplateRefs(installedTemplates: IndexTemplateEntry[]) { ]; const componentTemplates = installedTemplate.indexTemplate.composed_of // Filter global component template shared between integrations - .filter((componentTemplateId) => componentTemplateId !== FLEET_GLOBAL_COMPONENT_TEMPLATE_NAME) + .filter( + (componentTemplateId) => !FLEET_COMPONENT_TEMPLATE_NAMES.includes(componentTemplateId) + ) .map((componentTemplateId) => ({ id: componentTemplateId, type: ElasticsearchAssetType.componentTemplate, diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts index d0ad411b81dd7..343c0e22eb3c5 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts @@ -26,7 +26,7 @@ import { updateCurrentWriteIndices, } from './template'; -const FLEET_COMPONENT_TEMPLATE = '.fleet_component_template-2'; +const FLEET_COMPONENT_TEMPLATES = ['.fleet_globals-1', '.fleet_agent_id_verification-1']; // Add our own serialiser to just do JSON.stringify expect.addSnapshotSerializer({ @@ -65,7 +65,10 @@ describe('EPM template', () => { composedOfTemplates, templatePriority: 200, }); - expect(template.composed_of).toStrictEqual([...composedOfTemplates, FLEET_COMPONENT_TEMPLATE]); + expect(template.composed_of).toStrictEqual([ + ...composedOfTemplates, + ...FLEET_COMPONENT_TEMPLATES, + ]); }); it('adds empty composed_of correctly', () => { @@ -77,7 +80,7 @@ describe('EPM template', () => { composedOfTemplates, templatePriority: 200, }); - expect(template.composed_of).toStrictEqual([FLEET_COMPONENT_TEMPLATE]); + expect(template.composed_of).toStrictEqual(FLEET_COMPONENT_TEMPLATES); }); it('adds hidden field correctly', () => { diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts index a3852146a73c5..32465964619e1 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts @@ -18,7 +18,8 @@ import type { import { appContextService } from '../../../'; import { getRegistryDataStreamAssetBaseName } from '../index'; import { - FLEET_GLOBAL_COMPONENT_TEMPLATE_NAME, + FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME, + FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_NAME, MAPPINGS_TEMPLATE_SUFFIX, } from '../../../../constants'; import { getESAssetMetadata } from '../meta'; @@ -84,10 +85,12 @@ export function getTemplate({ throw new Error(`Error template for ${templateIndexPattern} contains a final_pipeline`); } + const fleetTemplatesToAdd = [FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME]; + if (appContextService.getConfig()?.agentIdVerificationEnabled) { - // Add fleet global assets - template.composed_of = [...(template.composed_of || []), FLEET_GLOBAL_COMPONENT_TEMPLATE_NAME]; + fleetTemplatesToAdd.push(FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_NAME); } + template.composed_of = [...(template.composed_of || []), ...fleetTemplatesToAdd]; return template; } @@ -388,12 +391,10 @@ function getBaseTemplate( templatePriority: number, hidden?: boolean ): IndexTemplate { - // Meta information to identify Ingest Manager's managed templates and indices const _meta = getESAssetMetadata({ packageName }); return { priority: templatePriority, - // To be completed with the correct index patterns index_patterns: [templateIndexPattern], template: { settings: { diff --git a/x-pack/plugins/fleet/server/services/setup.ts b/x-pack/plugins/fleet/server/services/setup.ts index e7ba627f5cbdf..2502ab9cb2221 100644 --- a/x-pack/plugins/fleet/server/services/setup.ts +++ b/x-pack/plugins/fleet/server/services/setup.ts @@ -28,7 +28,7 @@ import { generateEnrollmentAPIKey, hasEnrollementAPIKeysForPolicy } from './api_ import { settingsService } from '.'; import { awaitIfPending } from './setup_utils'; import { ensureFleetFinalPipelineIsInstalled } from './epm/elasticsearch/ingest_pipeline/install'; -import { ensureDefaultComponentTemplate } from './epm/elasticsearch/template/install'; +import { ensureDefaultComponentTemplates } from './epm/elasticsearch/template/install'; import { getInstallations, installPackage } from './epm/packages'; import { isPackageInstalled } from './epm/packages/install'; import { pkgToPkgKey } from './epm/registry'; @@ -145,11 +145,11 @@ export async function ensureFleetGlobalEsAssets( // Ensure Global Fleet ES assets are installed logger.debug('Creating Fleet component template and ingest pipeline'); const globalAssetsRes = await Promise.all([ - ensureDefaultComponentTemplate(esClient, logger), + ensureDefaultComponentTemplates(esClient, logger), // returns an array ensureFleetFinalPipelineIsInstalled(esClient, logger), ]); - - if (globalAssetsRes.some((asset) => asset.isCreated)) { + const assetResults = globalAssetsRes.flat(); + if (assetResults.some((asset) => asset.isCreated)) { // Update existing index template const packages = await getInstallations(soClient); From d9d25dc862b5635905c7b0bae34687ee40828a6e Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Thu, 24 Feb 2022 10:47:58 +0000 Subject: [PATCH 16/29] update snaphsot --- .../epm/__snapshots__/install_by_upload.snap | 735 ++++++++++++++++++ 1 file changed, 735 insertions(+) create mode 100644 x-pack/test/fleet_api_integration/apis/epm/__snapshots__/install_by_upload.snap diff --git a/x-pack/test/fleet_api_integration/apis/epm/__snapshots__/install_by_upload.snap b/x-pack/test/fleet_api_integration/apis/epm/__snapshots__/install_by_upload.snap new file mode 100644 index 0000000000000..421a5fbdf1744 --- /dev/null +++ b/x-pack/test/fleet_api_integration/apis/epm/__snapshots__/install_by_upload.snap @@ -0,0 +1,735 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`Fleet Endpoints EPM Endpoints installs packages from direct upload should install a zip archive correctly and package info should return correctly after validation 1`] = ` +Object { + "assets": Object { + "elasticsearch": Object { + "ingest_pipeline": Array [ + Object { + "dataset": "access", + "file": "default.yml", + "path": "apache-0.1.4/data_stream/access/elasticsearch/ingest_pipeline/default.yml", + "pkgkey": "apache-0.1.4", + "service": "elasticsearch", + "type": "ingest_pipeline", + }, + Object { + "dataset": "error", + "file": "default.yml", + "path": "apache-0.1.4/data_stream/error/elasticsearch/ingest_pipeline/default.yml", + "pkgkey": "apache-0.1.4", + "service": "elasticsearch", + "type": "ingest_pipeline", + }, + ], + }, + "kibana": Object { + "dashboard": Array [ + Object { + "file": "apache-Logs-Apache-Dashboard-ecs.json", + "path": "apache-0.1.4/kibana/dashboard/apache-Logs-Apache-Dashboard-ecs.json", + "pkgkey": "apache-0.1.4", + "service": "kibana", + "type": "dashboard", + }, + Object { + "file": "apache-Metrics-Apache-HTTPD-server-status-ecs.json", + "path": "apache-0.1.4/kibana/dashboard/apache-Metrics-Apache-HTTPD-server-status-ecs.json", + "pkgkey": "apache-0.1.4", + "service": "kibana", + "type": "dashboard", + }, + ], + "search": Array [ + Object { + "file": "Apache-HTTPD-ecs.json", + "path": "apache-0.1.4/kibana/search/Apache-HTTPD-ecs.json", + "pkgkey": "apache-0.1.4", + "service": "kibana", + "type": "search", + }, + Object { + "file": "Apache-access-logs-ecs.json", + "path": "apache-0.1.4/kibana/search/Apache-access-logs-ecs.json", + "pkgkey": "apache-0.1.4", + "service": "kibana", + "type": "search", + }, + Object { + "file": "Apache-errors-log-ecs.json", + "path": "apache-0.1.4/kibana/search/Apache-errors-log-ecs.json", + "pkgkey": "apache-0.1.4", + "service": "kibana", + "type": "search", + }, + ], + "visualization": Array [ + Object { + "file": "Apache-HTTPD-CPU-ecs.json", + "path": "apache-0.1.4/kibana/visualization/Apache-HTTPD-CPU-ecs.json", + "pkgkey": "apache-0.1.4", + "service": "kibana", + "type": "visualization", + }, + Object { + "file": "Apache-HTTPD-Hostname-list-ecs.json", + "path": "apache-0.1.4/kibana/visualization/Apache-HTTPD-Hostname-list-ecs.json", + "pkgkey": "apache-0.1.4", + "service": "kibana", + "type": "visualization", + }, + Object { + "file": "Apache-HTTPD-Load1-slash-5-slash-15-ecs.json", + "path": "apache-0.1.4/kibana/visualization/Apache-HTTPD-Load1-slash-5-slash-15-ecs.json", + "pkgkey": "apache-0.1.4", + "service": "kibana", + "type": "visualization", + }, + Object { + "file": "Apache-HTTPD-Scoreboard-ecs.json", + "path": "apache-0.1.4/kibana/visualization/Apache-HTTPD-Scoreboard-ecs.json", + "pkgkey": "apache-0.1.4", + "service": "kibana", + "type": "visualization", + }, + Object { + "file": "Apache-HTTPD-Total-accesses-and-kbytes-ecs.json", + "path": "apache-0.1.4/kibana/visualization/Apache-HTTPD-Total-accesses-and-kbytes-ecs.json", + "pkgkey": "apache-0.1.4", + "service": "kibana", + "type": "visualization", + }, + Object { + "file": "Apache-HTTPD-Uptime-ecs.json", + "path": "apache-0.1.4/kibana/visualization/Apache-HTTPD-Uptime-ecs.json", + "pkgkey": "apache-0.1.4", + "service": "kibana", + "type": "visualization", + }, + Object { + "file": "Apache-HTTPD-Workers-ecs.json", + "path": "apache-0.1.4/kibana/visualization/Apache-HTTPD-Workers-ecs.json", + "pkgkey": "apache-0.1.4", + "service": "kibana", + "type": "visualization", + }, + Object { + "file": "Apache-access-unique-IPs-map-ecs.json", + "path": "apache-0.1.4/kibana/visualization/Apache-access-unique-IPs-map-ecs.json", + "pkgkey": "apache-0.1.4", + "service": "kibana", + "type": "visualization", + }, + Object { + "file": "Apache-browsers-ecs.json", + "path": "apache-0.1.4/kibana/visualization/Apache-browsers-ecs.json", + "pkgkey": "apache-0.1.4", + "service": "kibana", + "type": "visualization", + }, + Object { + "file": "Apache-error-logs-over-time-ecs.json", + "path": "apache-0.1.4/kibana/visualization/Apache-error-logs-over-time-ecs.json", + "pkgkey": "apache-0.1.4", + "service": "kibana", + "type": "visualization", + }, + Object { + "file": "Apache-operating-systems-ecs.json", + "path": "apache-0.1.4/kibana/visualization/Apache-operating-systems-ecs.json", + "pkgkey": "apache-0.1.4", + "service": "kibana", + "type": "visualization", + }, + Object { + "file": "Apache-response-codes-of-top-URLs-ecs.json", + "path": "apache-0.1.4/kibana/visualization/Apache-response-codes-of-top-URLs-ecs.json", + "pkgkey": "apache-0.1.4", + "service": "kibana", + "type": "visualization", + }, + Object { + "file": "Apache-response-codes-over-time-ecs.json", + "path": "apache-0.1.4/kibana/visualization/Apache-response-codes-over-time-ecs.json", + "pkgkey": "apache-0.1.4", + "service": "kibana", + "type": "visualization", + }, + ], + }, + }, + "categories": Array [ + "web", + ], + "conditions": Object { + "kibana": Object { + "version": "^7.9.0", + }, + }, + "data_streams": Array [ + Object { + "dataset": "apache.access", + "ingest_pipeline": "default", + "package": "apache", + "path": "access", + "release": "experimental", + "streams": Array [ + Object { + "description": "Collect Apache access logs", + "enabled": true, + "input": "logfile", + "template_path": "log.yml.hbs", + "title": "Apache access logs", + "vars": Array [ + Object { + "default": Array [ + "/var/log/apache2/access.log*", + "/var/log/apache2/other_vhosts_access.log*", + "/var/log/httpd/access_log*", + ], + "multi": true, + "name": "paths", + "required": true, + "show_user": true, + "title": "Paths", + "type": "text", + }, + ], + }, + ], + "title": "Apache access logs", + "type": "logs", + }, + Object { + "dataset": "apache.error", + "ingest_pipeline": "default", + "package": "apache", + "path": "error", + "release": "experimental", + "streams": Array [ + Object { + "description": "Collect Apache error logs", + "enabled": true, + "input": "logfile", + "template_path": "log.yml.hbs", + "title": "Apache error logs", + "vars": Array [ + Object { + "default": Array [ + "/var/log/apache2/error.log*", + "/var/log/httpd/error_log*", + ], + "multi": true, + "name": "paths", + "required": true, + "show_user": true, + "title": "Paths", + "type": "text", + }, + ], + }, + ], + "title": "Apache error logs", + "type": "logs", + }, + Object { + "dataset": "apache.status", + "package": "apache", + "path": "status", + "release": "experimental", + "streams": Array [ + Object { + "description": "Collect Apache status metrics", + "enabled": true, + "input": "apache/metrics", + "template_path": "stream.yml.hbs", + "title": "Apache status metrics", + "vars": Array [ + Object { + "default": "10s", + "multi": false, + "name": "period", + "required": true, + "show_user": true, + "title": "Period", + "type": "text", + }, + Object { + "default": "/server-status", + "multi": false, + "name": "server_status_path", + "required": true, + "show_user": false, + "title": "Server Status Path", + "type": "text", + }, + ], + }, + ], + "title": "Apache status metrics", + "type": "metrics", + }, + ], + "description": "Apache Integration", + "download": "/epr/apache/apache-0.1.4.zip", + "format_version": "1.0.0", + "icons": Array [ + Object { + "path": "/package/apache/0.1.4/img/logo_apache.svg", + "size": "32x32", + "src": "/img/logo_apache.svg", + "title": "Apache Logo", + "type": "image/svg+xml", + }, + ], + "keepPoliciesUpToDate": false, + "license": "basic", + "name": "apache", + "owner": Object { + "github": "elastic/integrations-services", + }, + "path": "/package/apache/0.1.4", + "policy_templates": Array [ + Object { + "description": "Collect logs and metrics from Apache instances", + "inputs": Array [ + Object { + "description": "Collecting Apache access and error logs", + "title": "Collect logs from Apache instances", + "type": "logfile", + }, + Object { + "description": "Collecting Apache status metrics", + "title": "Collect metrics from Apache instances", + "type": "apache/metrics", + "vars": Array [ + Object { + "default": Array [ + "http://127.0.0.1", + ], + "multi": true, + "name": "hosts", + "required": true, + "show_user": true, + "title": "Hosts", + "type": "text", + }, + ], + }, + ], + "multiple": true, + "name": "apache", + "title": "Apache logs and metrics", + }, + ], + "readme": "/package/apache/0.1.4/docs/README.md", + "release": "experimental", + "removable": true, + "savedObject": Object { + "attributes": Object { + "es_index_patterns": Object { + "access": "logs-apache.access-*", + "error": "logs-apache.error-*", + "status": "metrics-apache.status-*", + }, + "install_source": "upload", + "install_status": "installed", + "install_version": "0.1.4", + "installed_es": Array [ + Object { + "id": "logs-apache.access-0.1.4-default", + "type": "ingest_pipeline", + }, + Object { + "id": "logs-apache.error-0.1.4-default", + "type": "ingest_pipeline", + }, + Object { + "id": "logs-apache.access", + "type": "index_template", + }, + Object { + "id": "logs-apache.access@mappings", + "type": "component_template", + }, + Object { + "id": "logs-apache.access@settings", + "type": "component_template", + }, + Object { + "id": "logs-apache.access@custom", + "type": "component_template", + }, + Object { + "id": "metrics-apache.status", + "type": "index_template", + }, + Object { + "id": "metrics-apache.status@mappings", + "type": "component_template", + }, + Object { + "id": "metrics-apache.status@settings", + "type": "component_template", + }, + Object { + "id": "metrics-apache.status@custom", + "type": "component_template", + }, + Object { + "id": "logs-apache.error", + "type": "index_template", + }, + Object { + "id": "logs-apache.error@mappings", + "type": "component_template", + }, + Object { + "id": "logs-apache.error@settings", + "type": "component_template", + }, + Object { + "id": "logs-apache.error@custom", + "type": "component_template", + }, + ], + "installed_kibana": Array [ + Object { + "id": "apache-Logs-Apache-Dashboard-ecs", + "type": "dashboard", + }, + Object { + "id": "apache-Metrics-Apache-HTTPD-server-status-ecs", + "type": "dashboard", + }, + Object { + "id": "Apache-access-unique-IPs-map-ecs", + "type": "visualization", + }, + Object { + "id": "Apache-HTTPD-CPU-ecs", + "type": "visualization", + }, + Object { + "id": "Apache-HTTPD-Load1-slash-5-slash-15-ecs", + "type": "visualization", + }, + Object { + "id": "Apache-response-codes-over-time-ecs", + "type": "visualization", + }, + Object { + "id": "Apache-HTTPD-Workers-ecs", + "type": "visualization", + }, + Object { + "id": "Apache-HTTPD-Hostname-list-ecs", + "type": "visualization", + }, + Object { + "id": "Apache-error-logs-over-time-ecs", + "type": "visualization", + }, + Object { + "id": "Apache-HTTPD-Scoreboard-ecs", + "type": "visualization", + }, + Object { + "id": "Apache-HTTPD-Uptime-ecs", + "type": "visualization", + }, + Object { + "id": "Apache-operating-systems-ecs", + "type": "visualization", + }, + Object { + "id": "Apache-HTTPD-Total-accesses-and-kbytes-ecs", + "type": "visualization", + }, + Object { + "id": "Apache-browsers-ecs", + "type": "visualization", + }, + Object { + "id": "Apache-response-codes-of-top-URLs-ecs", + "type": "visualization", + }, + Object { + "id": "Apache-access-logs-ecs", + "type": "search", + }, + Object { + "id": "Apache-errors-log-ecs", + "type": "search", + }, + Object { + "id": "Apache-HTTPD-ecs", + "type": "search", + }, + ], + "installed_kibana_space_id": "default", + "name": "apache", + "package_assets": Array [ + Object { + "id": "2f1ab9c0-8cf6-5e83-afcd-0d12851c8108", + "type": "epm-packages-assets", + }, + Object { + "id": "841166f1-6db0-5f7a-a8d9-768e88ddf984", + "type": "epm-packages-assets", + }, + Object { + "id": "b12ae5e1-daf2-51a7-99d8-0888d1f13b5b", + "type": "epm-packages-assets", + }, + Object { + "id": "2f263b24-c36a-5ea8-a707-76d1f274c888", + "type": "epm-packages-assets", + }, + Object { + "id": "bd5ff9ad-ba4a-5215-b5af-cef58a3aa886", + "type": "epm-packages-assets", + }, + Object { + "id": "5fc59aa9-1d7e-50ae-8ce5-b875ab44cfc5", + "type": "epm-packages-assets", + }, + Object { + "id": "7c850453-346b-5010-a946-28b83fc69e48", + "type": "epm-packages-assets", + }, + Object { + "id": "f02f8adb-3e0c-5f2f-b4f2-a04dc645b713", + "type": "epm-packages-assets", + }, + Object { + "id": "889d88db-6214-5836-aeff-1a87f8513b27", + "type": "epm-packages-assets", + }, + Object { + "id": "06a6b940-a745-563c-abf4-83eb3335926b", + "type": "epm-packages-assets", + }, + Object { + "id": "e68fd7ac-302e-5b75-bbbb-d69b441c8848", + "type": "epm-packages-assets", + }, + Object { + "id": "2c57fe0f-3b1a-57da-a63b-28f9b9e82bce", + "type": "epm-packages-assets", + }, + Object { + "id": "13db43e8-f8f9-57f0-b131-a171c2f2070f", + "type": "epm-packages-assets", + }, + Object { + "id": "e8750081-1c0b-5c55-bcab-fa6d47f01a85", + "type": "epm-packages-assets", + }, + Object { + "id": "71af57fe-25c4-5935-9879-ca4a2fba730e", + "type": "epm-packages-assets", + }, + Object { + "id": "cc287718-9573-5c56-a9ed-6dfef6589506", + "type": "epm-packages-assets", + }, + Object { + "id": "8badd8ba-289a-5e60-a1c0-f3d39e15cda3", + "type": "epm-packages-assets", + }, + Object { + "id": "20300efc-10eb-5fac-ba90-f6aa9b467e84", + "type": "epm-packages-assets", + }, + Object { + "id": "047c89df-33c2-5d74-b0a4-8b441879761c", + "type": "epm-packages-assets", + }, + Object { + "id": "9838a13f-1b89-5c54-844e-978620d66a1d", + "type": "epm-packages-assets", + }, + Object { + "id": "e105414b-221d-5433-8b24-452625f59b7c", + "type": "epm-packages-assets", + }, + Object { + "id": "eb166c25-843b-5271-8d43-6fb005d2df5a", + "type": "epm-packages-assets", + }, + Object { + "id": "342dbf4d-d88d-53e8-b365-d3639ebbbb14", + "type": "epm-packages-assets", + }, + Object { + "id": "f98c44a3-eaea-505f-8598-3b7f1097ef59", + "type": "epm-packages-assets", + }, + Object { + "id": "12da8c6c-d0e3-589c-9244-88d857ea76b6", + "type": "epm-packages-assets", + }, + Object { + "id": "e2d151ed-709c-542d-b797-cb95f353b9b3", + "type": "epm-packages-assets", + }, + Object { + "id": "f434cffe-0b00-59de-a17f-c1e71bd4ab0f", + "type": "epm-packages-assets", + }, + Object { + "id": "5bd0c25f-04a5-5fd0-8298-ba9aa2f6fe5e", + "type": "epm-packages-assets", + }, + Object { + "id": "279da3a3-8e9b-589b-86e0-bd7364821bab", + "type": "epm-packages-assets", + }, + Object { + "id": "b8758fcb-08bf-50fa-89bd-24398955298a", + "type": "epm-packages-assets", + }, + Object { + "id": "96e4eb36-03c3-5856-af44-559fd5133f2b", + "type": "epm-packages-assets", + }, + Object { + "id": "a59a79c3-66bd-5cfc-91f5-ee84f7227855", + "type": "epm-packages-assets", + }, + Object { + "id": "395143f9-54bf-5b46-b1be-a7b2a6142ad9", + "type": "epm-packages-assets", + }, + Object { + "id": "3449b8d2-ffd5-5aec-bb32-4245f2fbcde4", + "type": "epm-packages-assets", + }, + Object { + "id": "ab44094e-6c9d-50b8-b5c4-2e518d89912e", + "type": "epm-packages-assets", + }, + Object { + "id": "b093bfc0-6e98-5a1b-a502-e838a36f6568", + "type": "epm-packages-assets", + }, + Object { + "id": "03d86823-b756-5b91-850d-7ad231d33546", + "type": "epm-packages-assets", + }, + Object { + "id": "a76af2f0-049b-5be1-8d20-e87c9d1c2709", + "type": "epm-packages-assets", + }, + Object { + "id": "bc2f0c1e-992e-5407-9435-fedb39ff74ea", + "type": "epm-packages-assets", + }, + Object { + "id": "84668ac1-d5ef-545b-88f3-1e49f8f1c8ad", + "type": "epm-packages-assets", + }, + Object { + "id": "69b41271-91a0-5a2e-a62c-60364d5a9c8f", + "type": "epm-packages-assets", + }, + Object { + "id": "8e4ec555-5fbf-55d3-bea3-3af12c9aca3f", + "type": "epm-packages-assets", + }, + Object { + "id": "aa18f3f9-f62a-5ab8-9b34-75696efa5c48", + "type": "epm-packages-assets", + }, + Object { + "id": "71c8c6b1-2116-5817-b65f-7a87ef5ef2b7", + "type": "epm-packages-assets", + }, + Object { + "id": "8f6d7a1f-1e7f-5a60-8fe7-ce19115ed460", + "type": "epm-packages-assets", + }, + Object { + "id": "c115dbbf-edad-59f2-b046-c65a0373a81c", + "type": "epm-packages-assets", + }, + Object { + "id": "b7d696c3-8106-585c-9ecc-94a75cf1e3da", + "type": "epm-packages-assets", + }, + Object { + "id": "639e6a78-59d8-5ce8-9687-64e8f9af7e71", + "type": "epm-packages-assets", + }, + Object { + "id": "ae60c853-7a90-58d2-ab6c-04d3be5f1847", + "type": "epm-packages-assets", + }, + Object { + "id": "0cd33163-2ae4-57eb-96f6-c50af6685cab", + "type": "epm-packages-assets", + }, + Object { + "id": "39e0f78f-1172-5e61-9446-65ef3c0cb46c", + "type": "epm-packages-assets", + }, + Object { + "id": "b08f10ee-6afd-5e89-b9b4-569064fbdd9f", + "type": "epm-packages-assets", + }, + Object { + "id": "efcbe1c6-b2d5-521c-b27a-2146f08a604d", + "type": "epm-packages-assets", + }, + Object { + "id": "f9422c02-d43f-5ebb-b7c5-9e32f9b77c21", + "type": "epm-packages-assets", + }, + Object { + "id": "c276e880-3ba8-58e7-a5d5-c07707dba6b7", + "type": "epm-packages-assets", + }, + Object { + "id": "561a3711-c386-541c-9a77-2d0fa256caf6", + "type": "epm-packages-assets", + }, + Object { + "id": "1378350d-2e2b-52dd-ab3a-d8b9a09df92f", + "type": "epm-packages-assets", + }, + Object { + "id": "94e40729-4aea-59c8-86ba-075137c000dc", + "type": "epm-packages-assets", + }, + ], + "removable": true, + "version": "0.1.4", + }, + "id": "apache", + "namespaces": Array [], + "references": Array [], + "type": "epm-packages", + }, + "screenshots": Array [ + Object { + "path": "/package/apache/0.1.4/img/kibana-apache.png", + "size": "1215x1199", + "src": "/img/kibana-apache.png", + "title": "Apache Integration", + "type": "image/png", + }, + Object { + "path": "/package/apache/0.1.4/img/apache_httpd_server_status.png", + "size": "1919x1079", + "src": "/img/apache_httpd_server_status.png", + "title": "Apache HTTPD Server Status", + "type": "image/png", + }, + ], + "status": "installed", + "title": "Apache", + "type": "integration", + "version": "0.1.4", +} +`; From e52e1681e36f041f5d4d566db1f47e289986bf19 Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Thu, 24 Feb 2022 14:21:16 +0000 Subject: [PATCH 17/29] rename variables --- .../epm/elasticsearch/template/install.ts | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts index be78164f0b976..0d52827aa13a9 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts @@ -252,17 +252,16 @@ function buildComponentTemplates(params: { const templatesMap: TemplateMap = {}; const _meta = getESAssetMetadata({ packageName }); - const registrySettings = registryElasticsearch?.['index_template.settings'] ?? {}; - // @ts-expect-error 2339 - const mappingSettings = registrySettings?.index?.mapping; - - const registrySettingsForTemplate = cloneDeep(registrySettings); + const indexTemplateSettings = registryElasticsearch?.['index_template.settings'] ?? {}; + // @ts-expect-error no property .mapping (yes there is) + const indexTemplateMappingSettings = indexTemplateSettings?.index?.mapping; + const indexTemplateSettingsForTemplate = cloneDeep(indexTemplateSettings); // index.mapping settings must go on the mapping component template otherwise // the template may be rejected e.g if nested_fields.limit has been increased - if (mappingSettings) { - // @ts-expect-error 2339 - delete registrySettingsForTemplate.index.mapping; + if (indexTemplateMappingSettings) { + // @ts-expect-error no property .mapping + delete indexTemplateSettingsForTemplate.index.mapping; } templatesMap[mappingsTemplateName] = { @@ -273,7 +272,7 @@ function buildComponentTemplates(params: { total_fields: { limit: '10000', }, - ...mappingSettings, + ...indexTemplateMappingSettings, }, }, }, @@ -284,7 +283,7 @@ function buildComponentTemplates(params: { templatesMap[settingsTemplateName] = { template: { - settings: merge(defaultSettings, registrySettingsForTemplate), + settings: merge(defaultSettings, indexTemplateSettingsForTemplate), }, _meta, }; From 9f91941a34873e7a8199fde699eb122ba4bb8efe Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Thu, 24 Feb 2022 14:31:24 +0000 Subject: [PATCH 18/29] fix tests again --- x-pack/test/fleet_api_integration/apis/epm/final_pipeline.ts | 3 ++- .../test/fleet_api_integration/apis/epm/install_overrides.ts | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/x-pack/test/fleet_api_integration/apis/epm/final_pipeline.ts b/x-pack/test/fleet_api_integration/apis/epm/final_pipeline.ts index cd8a6e92aaefb..1c8e605cedd72 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/final_pipeline.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/final_pipeline.ts @@ -109,8 +109,9 @@ export default function (providerContext: FtrProviderContext) { expect(pipelineRes).to.have.property(FINAL_PIPELINE_ID); const res = await es.indices.getIndexTemplate({ name: 'logs-log.log' }); expect(res.index_templates.length).to.be(FINAL_PIPELINE_VERSION); + expect(res.index_templates[0]?.index_template?.composed_of).to.contain('.fleet_globals-1'); expect(res.index_templates[0]?.index_template?.composed_of).to.contain( - '.fleet_component_template-2' + '.fleet_agent_id_verification-1' ); }); diff --git a/x-pack/test/fleet_api_integration/apis/epm/install_overrides.ts b/x-pack/test/fleet_api_integration/apis/epm/install_overrides.ts index 2d2e9246e66f9..eee9525a4f062 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/install_overrides.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/install_overrides.ts @@ -55,7 +55,8 @@ export default function (providerContext: FtrProviderContext) { `${templateName}@mappings`, `${templateName}@settings`, `${templateName}@custom`, - '.fleet_component_template-2', + '.fleet_globals-1', + '.fleet_agent_id_verification-1', ]); ({ body } = await es.transport.request( From 627f1018d73926c215d9276f1d8c02ebc4874ada Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Tue, 1 Mar 2022 14:35:32 +0000 Subject: [PATCH 19/29] reinstall bundled packages when component templates change --- .../server/services/epm/packages/index.ts | 2 + x-pack/plugins/fleet/server/services/setup.ts | 45 +++++++++++++++---- 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/packages/index.ts b/x-pack/plugins/fleet/server/services/epm/packages/index.ts index fa2e5781a209e..d742103dccf12 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/index.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/index.ts @@ -25,6 +25,8 @@ export { getLimitedPackages, } from './get'; +export { getBundledPackages } from './bundled_packages'; + export type { BulkInstallResponse, IBulkInstallPackageError } from './install'; export { handleInstallPackageFailure, installPackage, ensureInstalledPackage } from './install'; export { removeInstallation } from './remove'; diff --git a/x-pack/plugins/fleet/server/services/setup.ts b/x-pack/plugins/fleet/server/services/setup.ts index 2502ab9cb2221..29ceb157477b0 100644 --- a/x-pack/plugins/fleet/server/services/setup.ts +++ b/x-pack/plugins/fleet/server/services/setup.ts @@ -10,7 +10,12 @@ import { compact } from 'lodash'; import type { ElasticsearchClient, SavedObjectsClientContract } from 'src/core/server'; import { AUTO_UPDATE_PACKAGES } from '../../common'; -import type { DefaultPackagesInstallationError, PreconfigurationError } from '../../common'; +import type { + DefaultPackagesInstallationError, + PreconfigurationError, + BundledPackage, + Installation, +} from '../../common'; import { SO_SEARCH_LIMIT } from '../constants'; import { DEFAULT_SPACE_ID } from '../../../spaces/common/constants'; @@ -34,7 +39,7 @@ import { isPackageInstalled } from './epm/packages/install'; import { pkgToPkgKey } from './epm/registry'; import type { UpgradeManagedPackagePoliciesResult } from './managed_package_policies'; import { upgradeManagedPackagePolicies } from './managed_package_policies'; - +import { getBundledPackages } from './epm/packages'; export interface SetupStatus { isInitialized: boolean; nonFatalErrors: Array< @@ -151,15 +156,37 @@ export async function ensureFleetGlobalEsAssets( const assetResults = globalAssetsRes.flat(); if (assetResults.some((asset) => asset.isCreated)) { // Update existing index template - const packages = await getInstallations(soClient); - + const installedPackages = await getInstallations(soClient); + const bundledPackages = await getBundledPackages(); + const findMatchingBundledPkg = (pkg: Installation) => + bundledPackages.find( + (bundledPkg: BundledPackage) => + bundledPkg.name === pkg.name && bundledPkg.version === pkg.version + ); await Promise.all( - packages.saved_objects.map(async ({ attributes: installation }) => { + installedPackages.saved_objects.map(async ({ attributes: installation }) => { if (installation.install_source !== 'registry') { - logger.error( - `Package needs to be manually reinstalled ${installation.name} after installing Fleet global assets` - ); - return; + const matchingBundledPackage = findMatchingBundledPkg(installation); + if (!matchingBundledPackage) { + logger.error( + `Package needs to be manually reinstalled ${installation.name} after installing Fleet global assets` + ); + return; + } else { + await installPackage({ + installSource: 'upload', + savedObjectsClient: soClient, + esClient, + spaceId: DEFAULT_SPACE_ID, + contentType: 'application/zip', + archiveBuffer: matchingBundledPackage.buffer, + }).catch((err) => { + logger.error( + `Bundled package needs to be manually reinstalled ${installation.name} after installing Fleet global assets: ${err.message}` + ); + }); + return; + } } await installPackage({ installSource: installation.install_source, From 6b7c00ca8d5f5c5b129bd00b4ea1cda8de20656b Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Wed, 2 Mar 2022 16:29:32 +0000 Subject: [PATCH 20/29] combine compoised_of generation statements --- .../services/epm/elasticsearch/template/template.ts | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts index 32465964619e1..3d50a2e66e930 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts @@ -85,12 +85,13 @@ export function getTemplate({ throw new Error(`Error template for ${templateIndexPattern} contains a final_pipeline`); } - const fleetTemplatesToAdd = [FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME]; - - if (appContextService.getConfig()?.agentIdVerificationEnabled) { - fleetTemplatesToAdd.push(FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_NAME); - } - template.composed_of = [...(template.composed_of || []), ...fleetTemplatesToAdd]; + template.composed_of = [ + ...(template.composed_of || []), + FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME, + ...(appContextService.getConfig()?.agentIdVerificationEnabled + ? [FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_NAME] + : []), + ]; return template; } From 064a9d6888242882ad22dfbb70ed28db67e34148 Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Wed, 2 Mar 2022 16:34:42 +0000 Subject: [PATCH 21/29] remove duplicated global fields --- .../fleet/server/constants/fleet_es_assets.ts | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/x-pack/plugins/fleet/server/constants/fleet_es_assets.ts b/x-pack/plugins/fleet/server/constants/fleet_es_assets.ts index 0b685b7c4f69e..859a25a0ec7c7 100644 --- a/x-pack/plugins/fleet/server/constants/fleet_es_assets.ts +++ b/x-pack/plugins/fleet/server/constants/fleet_es_assets.ts @@ -54,22 +54,6 @@ export const FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_CONTENT = { }, }, mappings: { - // All the dynamic field mappings - dynamic_templates: [ - // This makes sure all mappings are keywords by default - { - strings_as_keyword: { - mapping: { - ignore_above: 1024, - type: 'keyword', - }, - match_mapping_type: 'string', - }, - }, - ], - // As we define fields ahead, we don't need any automatic field detection - // This makes sure all the fields are mapped to keyword by default to prevent mapping conflicts - date_detection: false, properties: { event: { properties: { From bbd8730cf801f886f2991bb01e98bba0a4319782 Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Wed, 2 Mar 2022 17:05:20 +0000 Subject: [PATCH 22/29] remove unnecessary rollover call --- .../epm/elasticsearch/template/template.ts | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts index 3d50a2e66e930..00334f3b06a20 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts @@ -518,23 +518,18 @@ const updateExistingDataStream = async ({ const mappings = mappingComponentTemplate?.template?.mappings; try { - if (!mappingComponentTemplate || !mappings?.properties) { - // if there is no mapping component template then something has - // gone wrong, so rollover the stream to get the new mapping. - await rolloverDataStream(dataStreamName, esClient); - return; - } - // for now, remove from object so as not to update stream or data stream properties of the index until type and name // are added in https://github.com/elastic/kibana/issues/66551. namespace value we will continue // to skip updating and assume the value in the index mapping is correct - delete mappings.properties.stream; - delete mappings.properties.data_stream; + if (mappings && mappings.properties) { + delete mappings.properties.stream; + delete mappings.properties.data_stream; + } await retryTransientEsErrors( () => esClient.indices.putMapping({ index: dataStreamName, - body: mappings, + body: mappings || {}, write_index_only: true, }), { logger } From 65cec5501851da6753d0781c45cc1a59238d03d7 Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Wed, 2 Mar 2022 17:06:55 +0000 Subject: [PATCH 23/29] use rollover API --- .../server/services/epm/elasticsearch/template/template.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts index 00334f3b06a20..786b460215f40 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts @@ -477,10 +477,8 @@ const getTemplateWithNameEndingIn = (composedOfTemplates: TemplateMap, suffix: s const rolloverDataStream = (dataStreamName: string, esClient: ElasticsearchClient) => { try { // Do no wrap rollovers in retryTransientEsErrors since it is not idempotent - const path = `/${dataStreamName}/_rollover`; - return esClient.transport.request({ - method: 'POST', - path, + return esClient.indices.rollover({ + alias: dataStreamName, }); } catch (error) { throw new Error(`cannot rollover data stream ${dataStreamName} - ${error}`); From c067e5e5815538e3b0a6013d4102a41e289dad8e Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Mon, 7 Mar 2022 15:07:54 +0000 Subject: [PATCH 24/29] use simulate API to get template content --- .../epm/elasticsearch/template/template.ts | 34 +++++++++++-------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts index 786b460215f40..a5440cca56add 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts @@ -320,6 +320,14 @@ export function generateTemplateName(dataStream: RegistryDataStream): string { return getRegistryDataStreamAssetBaseName(dataStream); } +/** + * Given a data stream name, return the indexTemplate name + */ +function dataStreamNameToIndexTemplateName(dataStreamName: string): string { + const [type, dataset] = dataStreamName.split('-'); // ignore namespace at the end + return [type, dataset].join('-'); +} + export function generateTemplateIndexPattern(dataStream: RegistryDataStream): string { // undefined or explicitly set to false // See also https://github.com/elastic/package-spec/pull/102 @@ -491,7 +499,11 @@ const updateAllDataStreams = async ( logger: Logger ): Promise => { const updatedataStreamPromises = indexNameWithTemplates.map((templateEntry) => { - return updateExistingDataStream({ esClient, logger, ...templateEntry }); + return updateExistingDataStream({ + esClient, + logger, + dataStreamName: templateEntry.dataStreamName, + }); }); await Promise.all(updatedataStreamPromises); }; @@ -499,21 +511,16 @@ const updateExistingDataStream = async ({ dataStreamName, esClient, logger, - indexTemplate, - composedOfTemplates, }: { dataStreamName: string; esClient: ElasticsearchClient; logger: Logger; - indexTemplate: IndexTemplate; - composedOfTemplates: TemplateMap; }) => { - const mappingComponentTemplate = getTemplateWithNameEndingIn( - composedOfTemplates, - MAPPINGS_TEMPLATE_SUFFIX - ); - // @ts-expect-error 2339 - const mappings = mappingComponentTemplate?.template?.mappings; + const { + template: { mappings, settings }, + } = await esClient.indices.simulateTemplate({ + name: dataStreamNameToIndexTemplateName(dataStreamName), + }); try { // for now, remove from object so as not to update stream or data stream properties of the index until type and name @@ -540,14 +547,13 @@ const updateExistingDataStream = async ({ // update settings after mappings was successful to ensure // pointing to the new pipeline is safe // for now, only update the pipeline - const { settings } = indexTemplate.template; - if (!settings.index.default_pipeline) return; + if (!settings?.index?.default_pipeline) return; try { await retryTransientEsErrors( () => esClient.indices.putSettings({ index: dataStreamName, - body: { default_pipeline: settings.index.default_pipeline }, + body: { default_pipeline: settings!.index!.default_pipeline }, }), { logger } ); From 74ccc144e3ca3b1664a1ae02a6fb48efd0b1cd4c Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Mon, 7 Mar 2022 15:33:48 +0000 Subject: [PATCH 25/29] remove unused parameters --- x-pack/plugins/fleet/common/types/models/epm.ts | 1 - .../services/epm/elasticsearch/template/install.ts | 5 ++--- .../epm/elasticsearch/template/template.test.ts | 12 ------------ .../services/epm/elasticsearch/template/template.ts | 13 +------------ 4 files changed, 3 insertions(+), 28 deletions(-) diff --git a/x-pack/plugins/fleet/common/types/models/epm.ts b/x-pack/plugins/fleet/common/types/models/epm.ts index 4e9e84912ef78..920f5e4f34084 100644 --- a/x-pack/plugins/fleet/common/types/models/epm.ts +++ b/x-pack/plugins/fleet/common/types/models/epm.ts @@ -503,5 +503,4 @@ export type TemplateMap = Record; export interface IndexTemplateEntry { templateName: string; indexTemplate: IndexTemplate; - composedOfTemplates: TemplateMap; } diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts index 0d52827aa13a9..894c2820fa2e1 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts @@ -350,7 +350,7 @@ async function installDataStreamComponentTemplates(params: { }) ); - return { componentTemplateNames: Object.keys(componentTemplates), componentTemplates }; + return { componentTemplateNames: Object.keys(componentTemplates) }; } export async function ensureDefaultComponentTemplates( @@ -470,7 +470,7 @@ export async function installTemplate({ ilmPolicy: dataStream.ilm_policy, }); - const { componentTemplateNames, componentTemplates } = await installDataStreamComponentTemplates({ + const { componentTemplateNames } = await installDataStreamComponentTemplates({ mappings, templateName, registryElasticsearch: dataStream.elasticsearch, @@ -503,7 +503,6 @@ export async function installTemplate({ return { templateName, indexTemplate: template, - composedOfTemplates: componentTemplates, }; } diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts index 343c0e22eb3c5..d36bf9fae603c 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts @@ -840,12 +840,6 @@ describe('EPM template', () => { await updateCurrentWriteIndices(esClient, logger, [ { templateName: 'test', - composedOfTemplates: { - 'test@mappings': { - _meta: {} as any, - template: { mappings: { properties: { hello: { type: 'text' } } } }, - }, - }, indexTemplate: { index_patterns: ['test.*-*'], template: { @@ -874,12 +868,6 @@ describe('EPM template', () => { await updateCurrentWriteIndices(esClient, logger, [ { templateName: 'test', - composedOfTemplates: { - 'test@mappings': { - _meta: {} as any, - template: { mappings: { properties: { hello: { type: 'text' } } } }, - }, - }, indexTemplate: { index_patterns: ['test-*'], template: { diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts index a5440cca56add..d304f0a36af9b 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts @@ -13,14 +13,12 @@ import type { IndexTemplateEntry, IndexTemplate, IndexTemplateMappings, - TemplateMap, } from '../../../../types'; import { appContextService } from '../../../'; import { getRegistryDataStreamAssetBaseName } from '../index'; import { FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME, FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_NAME, - MAPPINGS_TEMPLATE_SUFFIX, } from '../../../../constants'; import { getESAssetMetadata } from '../meta'; import { retryTransientEsErrors } from '../retry'; @@ -40,7 +38,6 @@ export interface CurrentDataStream { dataStreamName: string; replicated: boolean; indexTemplate: IndexTemplate; - composedOfTemplates: TemplateMap; } const DEFAULT_SCALING_FACTOR = 1000; const DEFAULT_IGNORE_ABOVE = 1024; @@ -459,7 +456,7 @@ const getDataStreams = async ( esClient: ElasticsearchClient, template: IndexTemplateEntry ): Promise => { - const { indexTemplate, composedOfTemplates } = template; + const { indexTemplate } = template; const body = await esClient.indices.getDataStream({ name: indexTemplate.index_patterns.join(','), @@ -471,17 +468,9 @@ const getDataStreams = async ( dataStreamName: dataStream.name, replicated: dataStream.replicated, indexTemplate, - composedOfTemplates, })); }; -const getTemplateWithNameEndingIn = (composedOfTemplates: TemplateMap, suffix: string) => { - const mappingComponentTemplateName = - Object.keys(composedOfTemplates).find((templateName) => templateName.endsWith(suffix)) || ''; - - return composedOfTemplates[mappingComponentTemplateName]; -}; - const rolloverDataStream = (dataStreamName: string, esClient: ElasticsearchClient) => { try { // Do no wrap rollovers in retryTransientEsErrors since it is not idempotent From 8ddebf099927e841e21963cad76e75031d07aa99 Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Mon, 7 Mar 2022 20:05:01 +0000 Subject: [PATCH 26/29] fix unit test --- .../epm/elasticsearch/template/template.test.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts index d36bf9fae603c..057c0cfb72ecf 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts @@ -836,6 +836,12 @@ describe('EPM template', () => { esClient.indices.getDataStream.mockResponse({ data_streams: [{ name: 'test.prefix1-default' }], } as any); + esClient.indices.simulateTemplate.mockResponse({ + template: { + settings: { index: {} }, + mappings: { properties: {} }, + }, + } as any); const logger = loggerMock.create(); await updateCurrentWriteIndices(esClient, logger, [ { @@ -864,6 +870,14 @@ describe('EPM template', () => { { name: 'test-replicated', replicated: true }, ], } as any); + + esClient.indices.simulateTemplate.mockResponse({ + template: { + settings: { index: {} }, + mappings: { properties: {} }, + }, + } as any); + const logger = loggerMock.create(); await updateCurrentWriteIndices(esClient, logger, [ { From 8270184dab4c789c3b51a178badddb90addb100a Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Thu, 10 Mar 2022 16:14:20 +0000 Subject: [PATCH 27/29] improve simulate error handling --- .../epm/elasticsearch/template/template.ts | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts index f4ff484844d6e..21c7351b31384 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts @@ -6,6 +6,7 @@ */ import type { ElasticsearchClient, Logger } from 'kibana/server'; +import type { IndicesIndexSettings } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import type { Field, Fields } from '../../fields/field'; import type { @@ -479,7 +480,7 @@ const rolloverDataStream = (dataStreamName: string, esClient: ElasticsearchClien alias: dataStreamName, }); } catch (error) { - throw new Error(`cannot rollover data stream ${dataStreamName} - ${error}`); + throw new Error(`cannot rollover data stream [${dataStreamName}] due to error: ${error}`); } }; @@ -506,13 +507,16 @@ const updateExistingDataStream = async ({ esClient: ElasticsearchClient; logger: Logger; }) => { - const { - template: { mappings, settings }, - } = await esClient.indices.simulateTemplate({ - name: dataStreamNameToIndexTemplateName(dataStreamName), - }); - + let settings: IndicesIndexSettings; try { + const simulateResult = await retryTransientEsErrors(() => + esClient.indices.simulateTemplate({ + name: dataStreamNameToIndexTemplateName(dataStreamName), + }) + ); + + settings = simulateResult.template.settings; + const mappings = simulateResult.template.mappings; // for now, remove from object so as not to update stream or data stream properties of the index until type and name // are added in https://github.com/elastic/kibana/issues/66551. namespace value we will continue // to skip updating and assume the value in the index mapping is correct From b98aa0c71e929f644b9f7af4b2b47536514557ee Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Thu, 10 Mar 2022 16:14:48 +0000 Subject: [PATCH 28/29] re-add removed mapping assertions --- .../apis/epm/update_assets.ts | 70 ++++++++++++++++++- 1 file changed, 69 insertions(+), 1 deletion(-) diff --git a/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts b/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts index 7b8c3bc09db1a..7d28b04c28a53 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts @@ -121,6 +121,40 @@ export default function (providerContext: FtrProviderContext) { }, }); }); + it('should have populated the new component template with the correct mapping', async () => { + const resMappings = await es.transport.request( + { + method: 'GET', + path: `/_component_template/${logsTemplateName2}@mappings`, + }, + { meta: true } + ); + expect(resMappings.statusCode).equal(200); + expect( + resMappings.body.component_templates[0].component_template.template.mappings.properties + ).eql({ + '@timestamp': { + type: 'date', + }, + test_logs2: { + ignore_above: 1024, + type: 'keyword', + }, + data_stream: { + properties: { + dataset: { + type: 'constant_keyword', + }, + namespace: { + type: 'constant_keyword', + }, + type: { + type: 'constant_keyword', + }, + }, + }, + }); + }); it('should have installed the new versionized pipelines', async function () { const res = await es.ingest.getPipeline( { @@ -169,7 +203,7 @@ export default function (providerContext: FtrProviderContext) { ); expect(resPipeline2.statusCode).equal(404); }); - it('should have updated the component templates', async function () { + it('should have updated the logs component templates', async function () { const resMappings = await es.transport.request( { method: 'GET', @@ -260,6 +294,40 @@ export default function (providerContext: FtrProviderContext) { ], }); }); + it('should have updated the metrics mapping component template', async function () { + const resMappings = await es.transport.request( + { + method: 'GET', + path: `/_component_template/${metricsTemplateName}@mappings`, + }, + { meta: true } + ); + expect(resMappings.statusCode).equal(200); + expect( + resMappings.body.component_templates[0].component_template.template.mappings.properties + ).eql({ + '@timestamp': { + type: 'date', + }, + metrics_test_name2: { + ignore_above: 1024, + type: 'keyword', + }, + data_stream: { + properties: { + dataset: { + type: 'constant_keyword', + }, + namespace: { + type: 'constant_keyword', + }, + type: { + type: 'constant_keyword', + }, + }, + }, + }); + }); it('should have updated the kibana assets', async function () { const resDashboard = await kibanaServer.savedObjects.get({ type: 'dashboard', From 43568ceb8fccfc1b9990e26499a15dccace235d1 Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Mon, 14 Mar 2022 11:54:45 +0000 Subject: [PATCH 29/29] fix test --- .../test/fleet_api_integration/apis/epm/install_by_upload.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/test/fleet_api_integration/apis/epm/install_by_upload.ts b/x-pack/test/fleet_api_integration/apis/epm/install_by_upload.ts index e693beed2e988..68cac70e8fed8 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/install_by_upload.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/install_by_upload.ts @@ -75,7 +75,7 @@ export default function (providerContext: FtrProviderContext) { .type('application/gzip') .send(buf) .expect(200); - expect(res.body.items.length).to.be(30); + expect(res.body.items.length).to.be(32); }); it('should install a zip archive correctly and package info should return correctly after validation', async function () { @@ -86,7 +86,7 @@ export default function (providerContext: FtrProviderContext) { .type('application/zip') .send(buf) .expect(200); - expect(res.body.items.length).to.be(30); + expect(res.body.items.length).to.be(32); }); it('should throw an error if the archive is zip but content type is gzip', async function () {