From 26a50f71562ad713903e8543b7793f482f717935 Mon Sep 17 00:00:00 2001 From: Milton Hultgren Date: Wed, 18 Sep 2024 16:03:17 +0200 Subject: [PATCH] [EEM] Add built in definitions for hosts and containers (#193157) ### Summary This PR adds built in definitions for `hosts` and `containers` based on ECS data. ### How to test 1. Check out this branch of Kibana 2. Start ES and Kibana, verify that start up works and that the two definitions are installed (by calling `GET /internal/entities/definition` or checking that the transforms are installed). 3. Ingest some data for each definition to work with, verify that you get data in `entities-host-*` and `entities-container-*` and that the data matches the expected shape (metadata and metrics[1]) [[1]](https://github.com/elastic/kibana/pull/193157#issuecomment-2355609821) --- .../built_in/containers_from_ecs_data.ts | 158 ++++++++++++++++ .../entities/built_in/hosts_from_ecs_data.ts | 179 ++++++++++++++++++ .../server/lib/entities/built_in/index.ts | 10 +- ...{services.ts => services_from_ecs_data.ts} | 2 +- 4 files changed, 346 insertions(+), 3 deletions(-) create mode 100644 x-pack/plugins/entity_manager/server/lib/entities/built_in/containers_from_ecs_data.ts create mode 100644 x-pack/plugins/entity_manager/server/lib/entities/built_in/hosts_from_ecs_data.ts rename x-pack/plugins/entity_manager/server/lib/entities/built_in/{services.ts => services_from_ecs_data.ts} (98%) diff --git a/x-pack/plugins/entity_manager/server/lib/entities/built_in/containers_from_ecs_data.ts b/x-pack/plugins/entity_manager/server/lib/entities/built_in/containers_from_ecs_data.ts new file mode 100644 index 0000000000000..b9119143ec37e --- /dev/null +++ b/x-pack/plugins/entity_manager/server/lib/entities/built_in/containers_from_ecs_data.ts @@ -0,0 +1,158 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EntityDefinition, entityDefinitionSchema } from '@kbn/entities-schema'; +import { BUILT_IN_ID_PREFIX } from './constants'; + +export const builtInContainersFromEcsEntityDefinition: EntityDefinition = + entityDefinitionSchema.parse({ + id: `${BUILT_IN_ID_PREFIX}containers_from_ecs_data`, + managed: true, + version: '1.0.0', + name: 'Containers from ECS data', + description: + 'This definition extracts container entities from common data streams by looking for the ECS field container.id', + type: 'container', + indexPatterns: ['filebeat-*', 'logs-*', 'metrics-*', 'metricbeat-*'], + identityFields: ['container.id'], + displayNameTemplate: '{{container.id}}', + history: { + timestampField: '@timestamp', + interval: '5m', + settings: { + frequency: '5m', + }, + }, + metadata: [ + { + source: '_index', + destination: 'source_index', + }, + { + source: 'data_stream.type', + destination: 'source_data_stream.type', + }, + { + source: 'data_stream.dataset', + destination: 'source_data_stream.dataset', + }, + 'container.name', + 'container.image.name', + 'container.image.tag', + 'container.runtime', + 'host.name', + 'host.ip', + 'host.mac', + 'host.architecture', + 'host.os.family', + 'host.os.kernel', + 'host.os.name', + 'host.os.platform', + 'host.os.type', + 'host.os.version', + 'cloud.provider', + 'cloud.region', + 'cloud.availability_zone', + 'cloud.instance.id', + 'cloud.instance.name', + 'cloud.machine.type', + 'cloud.service.name', + 'agent.name', + 'agent.type', + 'agent.ephemeral_id', + ], + metrics: [ + { + name: 'log_rate', + equation: 'A', + metrics: [ + { + name: 'A', + aggregation: 'doc_count', + filter: 'log.level: * OR error.log.level: *', + }, + ], + }, + { + name: 'error_log_rate', + equation: 'A', + metrics: [ + { + name: 'A', + aggregation: 'doc_count', + filter: '(log.level: "error" OR "ERROR") OR (error.log.level: "error" OR "ERROR")', + }, + ], + }, + { + name: 'cpu_usage_avg', + equation: 'A', + metrics: [ + { + name: 'A', + aggregation: 'avg', + field: 'docker.cpu.total.pct', + }, + ], + }, + { + name: 'memory_usage_avg', + equation: 'A', + metrics: [ + { + name: 'A', + aggregation: 'avg', + field: 'docker.memory.usage.pct', + }, + ], + }, + { + name: 'network_in_avg', + equation: 'A', + metrics: [ + { + name: 'A', + aggregation: 'avg', + field: 'docker.network.in.bytes', + }, + ], + }, + { + name: 'network_out_avg', + equation: 'A', + metrics: [ + { + name: 'A', + aggregation: 'avg', + field: 'docker.network.out.bytes', + }, + ], + }, + { + name: 'disk_read_avg', + equation: 'A', + metrics: [ + { + name: 'A', + aggregation: 'avg', + field: 'docker.diskio.read.ops', + }, + ], + }, + { + name: 'disk_write_avg', + equation: 'A', + metrics: [ + { + name: 'A', + aggregation: 'avg', + field: 'docker.diskio.write.ops', + }, + ], + }, + ], + }); diff --git a/x-pack/plugins/entity_manager/server/lib/entities/built_in/hosts_from_ecs_data.ts b/x-pack/plugins/entity_manager/server/lib/entities/built_in/hosts_from_ecs_data.ts new file mode 100644 index 0000000000000..5fead32f5c0e8 --- /dev/null +++ b/x-pack/plugins/entity_manager/server/lib/entities/built_in/hosts_from_ecs_data.ts @@ -0,0 +1,179 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EntityDefinition, entityDefinitionSchema } from '@kbn/entities-schema'; +import { BUILT_IN_ID_PREFIX } from './constants'; + +export const builtInHostsFromEcsEntityDefinition: EntityDefinition = entityDefinitionSchema.parse({ + id: `${BUILT_IN_ID_PREFIX}hosts_from_ecs_data`, + managed: true, + version: '1.0.0', + name: 'Hosts from ECS data', + description: + 'This definition extracts host entities from common data streams by looking for the ECS field host.name', + type: 'host', + indexPatterns: ['filebeat-*', 'logs-*', 'metrics-*', 'metricbeat-*'], + identityFields: ['host.name'], + displayNameTemplate: '{{host.name}}', + history: { + timestampField: '@timestamp', + interval: '5m', + settings: { + frequency: '5m', + }, + }, + metadata: [ + { + source: '_index', + destination: 'source_index', + }, + { + source: 'data_stream.type', + destination: 'source_data_stream.type', + }, + { + source: 'data_stream.dataset', + destination: 'source_data_stream.dataset', + }, + 'host.hostname', + 'host.ip', + 'host.mac', + 'host.architecture', + 'host.containerized', + 'host.os.platform', + 'host.os.name', + 'host.os.type', + 'host.os.codename', + 'host.os.family', + 'host.os.kernel', + 'host.os.version', + 'cloud.provider', + 'cloud.region', + 'cloud.availability_zone', + 'cloud.instance.id', + 'cloud.instance.name', + 'cloud.service.name', + 'cloud.machine.type', + 'cloud.account.id', + 'cloud.project.id', + 'agent.id', + 'agent.name', + 'agent.type', + 'agent.version', + ], + metrics: [ + { + name: 'log_rate', + equation: 'A', + metrics: [ + { + name: 'A', + aggregation: 'doc_count', + filter: 'log.level: * OR error.log.level: *', + }, + ], + }, + { + name: 'error_log_rate', + equation: 'A', + metrics: [ + { + name: 'A', + aggregation: 'doc_count', + filter: '(log.level: "error" OR "ERROR") OR (error.log.level: "error" OR "ERROR")', + }, + ], + }, + { + name: 'cpu_usage_avg', + equation: 'A', + metrics: [ + { + name: 'A', + aggregation: 'avg', + field: 'system.cpu.total.norm.pct', + }, + ], + }, + { + name: 'normalized_load_avg', + equation: 'A / B', + metrics: [ + { + name: 'A', + aggregation: 'avg', + field: 'system.load.1', + }, + { + name: 'B', + aggregation: 'max', + field: 'system.load.cores', + }, + ], + }, + { + name: 'memory_usage_avg', + equation: 'A', + metrics: [ + { + name: 'A', + aggregation: 'avg', + field: 'system.memory.actual.used.pct', + }, + ], + }, + { + name: 'memory_free_avg', + equation: 'A - B', + metrics: [ + { + name: 'A', + aggregation: 'max', + field: 'system.memory.total', + }, + { + name: 'B', + aggregation: 'avg', + field: 'system.memory.actual.used.bytes', + }, + ], + }, + { + name: 'disk_usage_max', + equation: 'A', + metrics: [ + { + name: 'A', + aggregation: 'max', + field: 'system.filesystem.used.pct', + }, + ], + }, + { + name: 'rx_avg', + equation: 'A * 8', + metrics: [ + { + name: 'A', + aggregation: 'sum', + field: 'host.network.ingress.bytes', + }, + ], + }, + { + name: 'tx_avg', + equation: 'A * 8', + metrics: [ + { + name: 'A', + aggregation: 'sum', + field: 'host.network.egress.bytes', + }, + ], + }, + ], +}); diff --git a/x-pack/plugins/entity_manager/server/lib/entities/built_in/index.ts b/x-pack/plugins/entity_manager/server/lib/entities/built_in/index.ts index d091e21f446d2..6c0d4c5995c63 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/built_in/index.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/built_in/index.ts @@ -6,8 +6,14 @@ */ import { EntityDefinition } from '@kbn/entities-schema'; -import { builtInServicesFromLogsEntityDefinition } from './services'; +import { builtInServicesFromEcsEntityDefinition } from './services_from_ecs_data'; +import { builtInHostsFromEcsEntityDefinition } from './hosts_from_ecs_data'; +import { builtInContainersFromEcsEntityDefinition } from './containers_from_ecs_data'; export { BUILT_IN_ID_PREFIX } from './constants'; -export const builtInDefinitions: EntityDefinition[] = [builtInServicesFromLogsEntityDefinition]; +export const builtInDefinitions: EntityDefinition[] = [ + builtInServicesFromEcsEntityDefinition, + builtInHostsFromEcsEntityDefinition, + builtInContainersFromEcsEntityDefinition, +]; diff --git a/x-pack/plugins/entity_manager/server/lib/entities/built_in/services.ts b/x-pack/plugins/entity_manager/server/lib/entities/built_in/services_from_ecs_data.ts similarity index 98% rename from x-pack/plugins/entity_manager/server/lib/entities/built_in/services.ts rename to x-pack/plugins/entity_manager/server/lib/entities/built_in/services_from_ecs_data.ts index aa1d86ee25adf..96667fb4d0af4 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/built_in/services.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/built_in/services_from_ecs_data.ts @@ -18,7 +18,7 @@ const serviceTransactionFilter = (additionalFilters: string[] = []) => { return [...baseFilters, ...additionalFilters].join(' AND '); }; -export const builtInServicesFromLogsEntityDefinition: EntityDefinition = +export const builtInServicesFromEcsEntityDefinition: EntityDefinition = entityDefinitionSchema.parse({ version: '1.0.3', id: `${BUILT_IN_ID_PREFIX}services_from_ecs_data`,