From 3fad1e9d93de5c445eb43031aefc8eeeda8277cf Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Mon, 1 Aug 2022 15:01:06 -0400 Subject: [PATCH] [Fleet] Support dynamic_template mappings from object field --- .../plugins/fleet/common/types/models/epm.ts | 1 + .../__snapshots__/template.test.ts.snap | 111 ++++++- .../epm/elasticsearch/template/mappings.ts | 48 +++ .../elasticsearch/template/template.test.ts | 11 + .../epm/elasticsearch/template/template.ts | 290 ++++++++++++------ .../fleet/server/services/epm/fields/field.ts | 1 + .../tests/cockroachdb_dynamic_templates.yml | 37 +++ 7 files changed, 388 insertions(+), 111 deletions(-) create mode 100644 x-pack/plugins/fleet/server/services/epm/elasticsearch/template/mappings.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/fields/tests/cockroachdb_dynamic_templates.yml diff --git a/x-pack/plugins/fleet/common/types/models/epm.ts b/x-pack/plugins/fleet/common/types/models/epm.ts index 2a97719c811c1..8c8e6288d474e 100644 --- a/x-pack/plugins/fleet/common/types/models/epm.ts +++ b/x-pack/plugins/fleet/common/types/models/epm.ts @@ -483,6 +483,7 @@ export type PackageAssetReference = Pick & { export interface IndexTemplateMappings { properties: any; + dynamic_templates?: any; } // This is an index template v2, see https://github.com/elastic/elasticsearch/issues/53101 diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap index efd51d5e0d997..3e4c9659892b2 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap @@ -56,6 +56,74 @@ exports[`EPM template tests loading base.yml: base.yml 1`] = ` } `; +exports[`EPM template tests loading cockroachdb_dynamic_templates.yml: cockroachdb_dynamic_templates.yml 1`] = ` +{ + "properties": { + "cockroachdb": { + "properties": { + "status": { + "properties": { + "labels": { + "properties": {} + }, + "*": { + "properties": {} + } + } + } + } + } + }, + "dynamic_templates": [ + { + "cockroachdb.status.labels.*": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "cockroachdb.status.labels.*" + } + }, + { + "cockroachdb.status.*.value": { + "mapping": { + "type": "double" + }, + "match_mapping_type": "*", + "path_match": "cockroachdb.status.*.value" + } + }, + { + "cockroachdb.status.*.counter": { + "mapping": { + "type": "double" + }, + "match_mapping_type": "*", + "path_match": "cockroachdb.status.*.counter" + } + }, + { + "cockroachdb.status.*.rate": { + "mapping": { + "type": "double" + }, + "match_mapping_type": "*", + "path_match": "cockroachdb.status.*.rate" + } + }, + { + "cockroachdb.status.*.histogram": { + "mapping": { + "type": "histogram" + }, + "match_mapping_type": "*", + "path_match": "cockroachdb.status.*.histogram" + } + } + ] +} +`; + exports[`EPM template tests loading coredns.logs.yml: coredns.logs.yml 1`] = ` { "properties": { @@ -830,9 +898,6 @@ exports[`EPM template tests loading system.yml: system.yml 1`] = ` "ignore_above": 2048, "type": "keyword" }, - "env": { - "type": "object" - }, "cpu": { "properties": { "user": { @@ -1026,9 +1091,6 @@ exports[`EPM template tests loading system.yml: system.yml 1`] = ` } } } - }, - "percpu": { - "type": "object" } } }, @@ -1343,11 +1405,7 @@ exports[`EPM template tests loading system.yml: system.yml 1`] = ` "type": "long" }, "states": { - "properties": { - "*": { - "type": "object" - } - } + "properties": {} } } }, @@ -1540,6 +1598,35 @@ exports[`EPM template tests loading system.yml: system.yml 1`] = ` } } } - } + }, + "dynamic_templates": [ + { + "system.process.env": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "system.process.env.*" + } + }, + { + "system.process.cgroup.cpuacct.percpu": { + "mapping": { + "type": "long" + }, + "match_mapping_type": "long", + "path_match": "system.process.cgroup.cpuacct.percpu.*" + } + }, + { + "system.raid.disks.states.*": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "system.raid.disks.states.*" + } + } + ] } `; diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/mappings.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/mappings.ts new file mode 100644 index 0000000000000..a398f4fde99d9 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/mappings.ts @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Field } from '../../fields/field'; + +const DEFAULT_SCALING_FACTOR = 1000; + +interface Properties { + [key: string]: any; +} + +export function getDefaultProperties(field: Field): Properties { + const properties: Properties = {}; + + if (field.index !== undefined) { + properties.index = field.index; + } + if (field.doc_values !== undefined) { + properties.doc_values = field.doc_values; + } + if (field.copy_to) { + properties.copy_to = field.copy_to; + } + + return properties; +} + +export function scaledFloat(field: Field): Properties { + const fieldProps = getDefaultProperties(field); + fieldProps.type = 'scaled_float'; + fieldProps.scaling_factor = field.scaling_factor || DEFAULT_SCALING_FACTOR; + if (field.metric_type) { + fieldProps.time_series_metric = field.metric_type; + } + + return fieldProps; +} + +export function histogram(field: Field): Properties { + const fieldProps = getDefaultProperties(field); + fieldProps.type = 'histogram'; + + return fieldProps; +} diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts index d565a22347b39..5cd0081d20cdf 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts @@ -137,6 +137,17 @@ describe('EPM template', () => { expect(mappings).toMatchSnapshot(path.basename(ymlPath)); }); + it('tests loading cockroachdb_dynamic_templates.yml', () => { + const ymlPath = path.join(__dirname, '../../fields/tests/cockroachdb_dynamic_templates.yml'); + const fieldsYML = readFileSync(ymlPath, 'utf-8'); + const fields: Field[] = safeLoad(fieldsYML); + const processedFields = processFields(fields); + + const mappings = generateMappings(processedFields); + + expect(mappings).toMatchSnapshot(path.basename(ymlPath)); + }); + it('tests processing long field with index false', () => { const longWithIndexFalseYml = ` - name: longIndexFalse diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts index d6f203f29c2d1..de31cb51bb7a1 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts @@ -24,6 +24,8 @@ import { import { getESAssetMetadata } from '../meta'; import { retryTransientEsErrors } from '../retry'; +import { getDefaultProperties, histogram, scaledFloat } from './mappings'; + interface Properties { [key: string]: any; } @@ -40,7 +42,7 @@ export interface CurrentDataStream { replicated: boolean; indexTemplate: IndexTemplate; } -const DEFAULT_SCALING_FACTOR = 1000; + const DEFAULT_IGNORE_ABOVE = 1024; // see discussion in https://github.com/elastic/kibana/issues/88307 @@ -103,6 +105,60 @@ export function getTemplate({ * @param fields */ export function generateMappings(fields: Field[]): IndexTemplateMappings { + const dynamicTemplates: Array<{ [k: string]: Properties }> = []; + const dynamicTemplateNames = new Set(); + + const { properties } = _generateMappings(fields, { + addDynamicMapping: (dynamicMapping: { + path: string; + matchingType: string; + pathMatch: string; + properties: string; + }) => { + const name = dynamicMapping.path; + if (dynamicTemplateNames.has(name)) { + return; + } + + const dynamicTemplate: Properties = { + mapping: dynamicMapping.properties, + }; + + if (dynamicMapping.matchingType) { + dynamicTemplate.match_mapping_type = dynamicMapping.matchingType; + } + + if (dynamicMapping.pathMatch) { + dynamicTemplate.path_match = dynamicMapping.pathMatch; + } + dynamicTemplateNames.add(name); + dynamicTemplates.push({ [dynamicMapping.path]: dynamicTemplate }); + }, + }); + + return dynamicTemplates.length + ? { + properties, + dynamic_templates: dynamicTemplates, + } + : { properties }; +} + +/** + * Generate mapping takes the given nested fields array and creates the Elasticsearch + * mapping properties out of it. + * + * This assumes that all fields with dotted.names have been expanded in a previous step. + * + * @param fields + */ +function _generateMappings( + fields: Field[], + ctx: { + addDynamicMapping: any; + groupFieldName?: string; + } +): IndexTemplateMappings { const props: Properties = {}; // TODO: this can happen when the fields property in fields.yml is present but empty // Maybe validation should be moved to fields/field.ts @@ -111,97 +167,149 @@ export function generateMappings(fields: Field[]): IndexTemplateMappings { // If type is not defined, assume keyword const type = field.type || 'keyword'; - let fieldProps = getDefaultProperties(field); + if (type === 'object' && field.object_type) { + const path = ctx.groupFieldName ? `${ctx.groupFieldName}.${field.name}` : field.name; + const pathMatch = path.includes('*') ? path : `${path}.*`; - switch (type) { - case 'group': - fieldProps = { ...generateMappings(field.fields!), ...generateDynamicAndEnabled(field) }; - break; - case 'group-nested': - fieldProps = { - ...generateMappings(field.fields!), - ...generateNestedProps(field), - type: 'nested', - }; - break; - case 'integer': - fieldProps.type = 'long'; - break; - case 'scaled_float': - fieldProps.type = 'scaled_float'; - fieldProps.scaling_factor = field.scaling_factor || DEFAULT_SCALING_FACTOR; - if (field.metric_type) { - fieldProps.time_series_metric = field.metric_type; - } - break; - case 'text': - const textMapping = generateTextMapping(field); - fieldProps = { ...fieldProps, ...textMapping, type: 'text' }; - if (field.multi_fields) { - fieldProps.fields = generateMultiFields(field.multi_fields); - } - break; - case 'keyword': - const keywordMapping = generateKeywordMapping(field); - fieldProps = { ...fieldProps, ...keywordMapping, type: 'keyword' }; - if (field.multi_fields) { - fieldProps.fields = generateMultiFields(field.multi_fields); - } - break; - case 'wildcard': - const wildcardMapping = generateWildcardMapping(field); - fieldProps = { ...fieldProps, ...wildcardMapping, type: 'wildcard' }; - if (field.multi_fields) { - fieldProps.fields = generateMultiFields(field.multi_fields); - } - break; - case 'constant_keyword': - fieldProps.type = field.type; - if (field.value) { - fieldProps.value = field.value; - } - break; - case 'object': - fieldProps = { ...fieldProps, ...generateDynamicAndEnabled(field), type: 'object' }; - break; - case 'nested': - fieldProps = { ...fieldProps, ...generateNestedProps(field), type: 'nested' }; - break; - case 'array': - // this assumes array fields were validated in an earlier step - // adding an array field with no object_type would result in an error - // when the template is added to ES - if (field.object_type) { - fieldProps.type = field.object_type; - } - break; - case 'alias': - // this assumes alias fields were validated in an earlier step - // adding a path to a field that doesn't exist would result in an error - // when the template is added to ES. - fieldProps.type = 'alias'; - fieldProps.path = field.path; - break; - default: - fieldProps.type = type; - } + let dynProperties: Properties = getDefaultProperties(field); + let matchingType: string | undefined; + switch (field.object_type) { + case 'histogram': + dynProperties = histogram(field); + matchingType = field.object_type_mapping_type ?? '*'; + break; + case 'text': + dynProperties.type = field.object_type; + matchingType = field.object_type_mapping_type ?? 'string'; + break; + case 'keyword': + dynProperties.type = field.object_type; + matchingType = field.object_type_mapping_type ?? 'string'; + break; + case 'byte': + case 'double': + case 'float': + case 'long': + case 'short': + case 'boolean': + dynProperties = { + type: field.object_type, + }; + matchingType = field.object_type_mapping_type ?? field.object_type; + default: + break; + } + + if (dynProperties && matchingType) { + ctx.addDynamicMapping({ + path, + pathMatch, + matchingType, + properties: dynProperties, + }); + } + } else { + let fieldProps = getDefaultProperties(field); - const fieldHasMetaProps = META_PROP_KEYS.some((key) => key in field); - if (fieldHasMetaProps) { switch (type) { case 'group': + fieldProps = { + ..._generateMappings(field.fields!, { + ...ctx, + groupFieldName: ctx.groupFieldName + ? `${ctx.groupFieldName}.${field.name}` + : field.name, + }), + ...generateDynamicAndEnabled(field), + }; + break; case 'group-nested': + fieldProps = { + ..._generateMappings(field.fields!, { + ...ctx, + groupFieldName: ctx.groupFieldName + ? `${ctx.groupFieldName}.${field.name}` + : field.name, + }), + ...generateNestedProps(field), + type: 'nested', + }; + break; + case 'integer': + fieldProps.type = 'long'; + break; + case 'scaled_float': + fieldProps = scaledFloat(field); + break; + case 'text': + const textMapping = generateTextMapping(field); + fieldProps = { ...fieldProps, ...textMapping, type: 'text' }; + if (field.multi_fields) { + fieldProps.fields = generateMultiFields(field.multi_fields); + } break; - default: { - const meta = {}; - if ('metric_type' in field) Reflect.set(meta, 'metric_type', field.metric_type); - if ('unit' in field) Reflect.set(meta, 'unit', field.unit); - fieldProps.meta = meta; + case 'object': + fieldProps = { ...fieldProps, ...generateDynamicAndEnabled(field), type: 'object' }; + break; + case 'keyword': + const keywordMapping = generateKeywordMapping(field); + fieldProps = { ...fieldProps, ...keywordMapping, type: 'keyword' }; + if (field.multi_fields) { + fieldProps.fields = generateMultiFields(field.multi_fields); + } + break; + case 'wildcard': + const wildcardMapping = generateWildcardMapping(field); + fieldProps = { ...fieldProps, ...wildcardMapping, type: 'wildcard' }; + if (field.multi_fields) { + fieldProps.fields = generateMultiFields(field.multi_fields); + } + break; + case 'constant_keyword': + fieldProps.type = field.type; + if (field.value) { + fieldProps.value = field.value; + } + break; + case 'nested': + fieldProps = { ...fieldProps, ...generateNestedProps(field), type: 'nested' }; + break; + case 'array': + // this assumes array fields were validated in an earlier step + // adding an array field with no object_type would result in an error + // when the template is added to ES + if (field.object_type) { + fieldProps.type = field.object_type; + } + break; + case 'alias': + // this assumes alias fields were validated in an earlier step + // adding a path to a field that doesn't exist would result in an error + // when the template is added to ES. + fieldProps.type = 'alias'; + fieldProps.path = field.path; + break; + default: + fieldProps.type = type; + } + + const fieldHasMetaProps = META_PROP_KEYS.some((key) => key in field); + if (fieldHasMetaProps) { + switch (type) { + case 'group': + case 'group-nested': + break; + default: { + const meta = {}; + if ('metric_type' in field) Reflect.set(meta, 'metric_type', field.metric_type); + if ('unit' in field) Reflect.set(meta, 'unit', field.unit); + fieldProps.meta = meta; + } } } - } - props[field.name] = fieldProps; + props[field.name] = fieldProps; + } }); } @@ -295,22 +403,6 @@ function generateWildcardMapping(field: Field): IndexTemplateMapping { return mapping; } -function getDefaultProperties(field: Field): Properties { - const properties: Properties = {}; - - if (field.index !== undefined) { - properties.index = field.index; - } - if (field.doc_values !== undefined) { - properties.doc_values = field.doc_values; - } - if (field.copy_to) { - properties.copy_to = field.copy_to; - } - - return properties; -} - /** * Generates the template name out of the given information */ diff --git a/x-pack/plugins/fleet/server/services/epm/fields/field.ts b/x-pack/plugins/fleet/server/services/epm/fields/field.ts index 0e00840b0c74e..8d784e0cffc22 100644 --- a/x-pack/plugins/fleet/server/services/epm/fields/field.ts +++ b/x-pack/plugins/fleet/server/services/epm/fields/field.ts @@ -30,6 +30,7 @@ export interface Field { search_analyzer?: string; ignore_above?: number; object_type?: string; + object_type_mapping_type?: string; scaling_factor?: number; dynamic?: 'strict' | boolean; include_in_parent?: boolean; diff --git a/x-pack/plugins/fleet/server/services/epm/fields/tests/cockroachdb_dynamic_templates.yml b/x-pack/plugins/fleet/server/services/epm/fields/tests/cockroachdb_dynamic_templates.yml new file mode 100644 index 0000000000000..5fa4ff937bd14 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/fields/tests/cockroachdb_dynamic_templates.yml @@ -0,0 +1,37 @@ +- name: cockroachdb.status + type: group + release: beta + fields: + - name: labels.* + type: object + object_type: keyword + description: > + Prometheus metric labels + +- name: cockroachdb.status.*.value + type: object + object_type: double + object_type_mapping_type: '*' + description: > + Prometheus gauge metric + +- name: cockroachdb.status.*.counter + type: object + object_type: double + object_type_mapping_type: '*' + description: > + Prometheus counter metric + +- name: cockroachdb.status.*.rate + type: object + object_type: double + object_type_mapping_type: '*' + description: > + Prometheus rated counter metric + +- name: cockroachdb.status.*.histogram + type: object + object_type: histogram + object_type_mapping_type: '*' + description: >- + Prometheus histogram metric