diff --git a/libbeat/common/field.go b/libbeat/common/field.go new file mode 100644 index 000000000000..0cbafb37161e --- /dev/null +++ b/libbeat/common/field.go @@ -0,0 +1,116 @@ +package common + +import ( + "fmt" + "strings" + + "github.com/elastic/go-ucfg/yaml" +) + +//This reflects allowed attributes for field definitions in the fields.yml. +//No logic is put into this data structure. +//The purpose is to enable using different kinds of transformation, on top of the same data structure. +//Current transformation: +// -ElasticSearch Template +//Future: +// -Kibana Index Pattern + +type Field struct { + Name string `config:"name"` + Type string `config:"type"` + Description string `config:"description"` + Format string `config:"format"` + ScalingFactor int `config:"scaling_factor"` + Fields Fields `config:"fields"` + MultiFields Fields `config:"multi_fields"` + ObjectType string `config:"object_type"` + Enabled *bool `config:"enabled"` + Analyzer string `config:"analyzer"` + SearchAnalyzer string `config:"search_analyzer"` + Norms bool `config:"norms"` + Dynamic DynamicType `config:"dynamic"` + Index *bool `config:"index"` + DocValues *bool `config:"doc_values"` + + Path string +} + +type Fields []Field + +func LoadFieldsYaml(path string) (Fields, error) { + keys := []Field{} + + cfg, err := yaml.NewConfigWithFile(path) + if err != nil { + return nil, err + } + cfg.Unpack(&keys) + + fields := Fields{} + + for _, key := range keys { + fields = append(fields, key.Fields...) + } + return fields, nil +} + +// HasKey checks if inside fields the given key exists +// The key can be in the form of a.b.c and it will check if the nested field exist +// In case the key is `a` and there is a value `a.b` false is return as it only +// returns true if it's a leave node +func (f Fields) HasKey(key string) bool { + keys := strings.Split(key, ".") + return f.hasKey(keys) +} + +// Recursively generates the correct key based on the dots +// The mapping requires "properties" between each layer. This is added here. +func GenerateKey(key string) string { + if strings.Contains(key, ".") { + keys := strings.SplitN(key, ".", 2) + key = keys[0] + ".properties." + GenerateKey(keys[1]) + } + return key +} + +type DynamicType struct{ Value interface{} } + +func (d *DynamicType) Unpack(s string) error { + switch s { + case "true": + d.Value = true + case "false": + d.Value = false + case "strict": + d.Value = s + default: + return fmt.Errorf("'%v' is invalid dynamic setting", s) + } + return nil +} + +func (f Fields) hasKey(keys []string) bool { + // Nothing to compare anymore + if len(keys) == 0 { + return false + } + + key := keys[0] + keys = keys[1:] + + for _, field := range f { + if field.Name == key { + + if len(field.Fields) > 0 { + return field.Fields.hasKey(keys) + } + // Last entry in the tree but still more keys + if len(keys) > 0 { + return false + } + + return true + } + } + return false +} diff --git a/libbeat/common/field_test.go b/libbeat/common/field_test.go new file mode 100644 index 000000000000..2bab75a67d6c --- /dev/null +++ b/libbeat/common/field_test.go @@ -0,0 +1,117 @@ +package common + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/go-ucfg/yaml" +) + +func TestFieldsHasKey(t *testing.T) { + tests := []struct { + key string + fields Fields + result bool + }{ + { + key: "test.find", + fields: Fields{}, + result: false, + }, + { + key: "test.find", + fields: Fields{ + Field{Name: "test"}, + Field{Name: "find"}, + }, + result: false, + }, + { + key: "test.find", + fields: Fields{ + Field{ + Name: "test", Fields: Fields{ + Field{ + Name: "find", + }, + }, + }, + }, + result: true, + }, + { + key: "test", + fields: Fields{ + Field{ + Name: "test", Fields: Fields{ + Field{ + Name: "find", + }, + }, + }, + }, + result: false, + }, + } + + for _, test := range tests { + assert.Equal(t, test.result, test.fields.HasKey(test.key)) + } +} + +func TestDynamicYaml(t *testing.T) { + tests := []struct { + input []byte + output Field + error bool + }{ + { + input: []byte(` +name: test +dynamic: true`), + output: Field{ + Name: "test", + Dynamic: DynamicType{true}, + }, + }, + { + input: []byte(` +name: test +dynamic: "true"`), + output: Field{ + Name: "test", + Dynamic: DynamicType{true}, + }, + }, + { + input: []byte(` +name: test +dynamic: "blue"`), + error: true, + }, + { + input: []byte(` +name: test +dynamic: "strict"`), + output: Field{ + Name: "test", + Dynamic: DynamicType{"strict"}, + }, + }, + } + + for _, test := range tests { + keys := Field{} + + cfg, err := yaml.NewConfig(test.input) + assert.NoError(t, err) + err = cfg.Unpack(&keys) + + if err != nil { + assert.True(t, test.error) + } else { + assert.Equal(t, test.output.Dynamic, keys.Dynamic) + } + } +} diff --git a/libbeat/template/field.go b/libbeat/template/field.go deleted file mode 100644 index a2a401ffe216..000000000000 --- a/libbeat/template/field.go +++ /dev/null @@ -1,238 +0,0 @@ -package template - -import ( - "fmt" - "strings" - - "github.com/elastic/beats/libbeat/common" -) - -var ( - defaultScalingFactor = 1000 -) - -type Field struct { - Name string `config:"name"` - Type string `config:"type"` - Description string `config:"description"` - Format string `config:"format"` - ScalingFactor int `config:"scaling_factor"` - Fields Fields `config:"fields"` - MultiFields Fields `config:"multi_fields"` - ObjectType string `config:"object_type"` - Enabled *bool `config:"enabled"` - Analyzer string `config:"analyzer"` - SearchAnalyzer string `config:"search_analyzer"` - Norms bool `config:"norms"` - Dynamic dynamicType `config:"dynamic"` - Index *bool `config:"index"` - DocValues *bool `config:"doc_values"` - - path string - esVersion common.Version -} - -// This includes all entries without special handling for different versions. -// Currently this is: -// long, geo_point, date, short, byte, float, double, boolean -func (f *Field) other() common.MapStr { - property := f.getDefaultProperties() - if f.Type != "" { - property["type"] = f.Type - } - - return property -} - -func (f *Field) integer() common.MapStr { - property := f.getDefaultProperties() - property["type"] = "long" - return property -} - -func (f *Field) scaledFloat() common.MapStr { - property := f.getDefaultProperties() - property["type"] = "scaled_float" - - if f.esVersion.IsMajor(2) { - property["type"] = "float" - } else { - // Set default scaling factor - if f.ScalingFactor == 0 { - f.ScalingFactor = defaultScalingFactor - } - property["scaling_factor"] = f.ScalingFactor - } - return property -} - -func (f *Field) halfFloat() common.MapStr { - property := f.getDefaultProperties() - property["type"] = "half_float" - - if f.esVersion.IsMajor(2) { - property["type"] = "float" - } - return property -} - -func (f *Field) ip() common.MapStr { - property := f.getDefaultProperties() - - property["type"] = "ip" - - if f.esVersion.IsMajor(2) { - property["type"] = "string" - property["ignore_above"] = 1024 - property["index"] = "not_analyzed" - } - return property -} - -func (f *Field) keyword() common.MapStr { - property := f.getDefaultProperties() - - property["type"] = "keyword" - property["ignore_above"] = 1024 - - if f.esVersion.IsMajor(2) { - property["type"] = "string" - property["ignore_above"] = 1024 - property["index"] = "not_analyzed" - } - return property -} - -func (f *Field) text() common.MapStr { - properties := f.getDefaultProperties() - - properties["type"] = "text" - - if f.esVersion.IsMajor(2) { - properties["type"] = "string" - properties["index"] = "analyzed" - if !f.Norms { - properties["norms"] = common.MapStr{ - "enabled": false, - } - } - } else { - if !f.Norms { - properties["norms"] = false - } - } - - if f.Analyzer != "" { - properties["analyzer"] = f.Analyzer - } - - if f.SearchAnalyzer != "" { - properties["search_analyzer"] = f.SearchAnalyzer - } - - if len(f.MultiFields) > 0 { - fields := common.MapStr{} - f.MultiFields.process("", f.esVersion, fields) - properties["fields"] = fields - } - - return properties -} - -func (f *Field) array() common.MapStr { - properties := f.getDefaultProperties() - if f.ObjectType != "" { - properties["type"] = f.ObjectType - } - return properties -} - -func (f *Field) object() common.MapStr { - dynProperties := f.getDefaultProperties() - - switch f.ObjectType { - case "text": - dynProperties["type"] = "text" - - if f.esVersion.IsMajor(2) { - dynProperties["type"] = "string" - dynProperties["index"] = "analyzed" - } - f.addDynamicTemplate(dynProperties, "string") - case "long": - dynProperties["type"] = f.ObjectType - f.addDynamicTemplate(dynProperties, "long") - case "keyword": - dynProperties["type"] = f.ObjectType - f.addDynamicTemplate(dynProperties, "string") - } - - properties := f.getDefaultProperties() - properties["type"] = "object" - if f.Enabled != nil { - properties["enabled"] = *f.Enabled - } - - if f.Dynamic.value != nil { - properties["dynamic"] = f.Dynamic.value - } - - return properties -} - -func (f *Field) addDynamicTemplate(properties common.MapStr, matchType string) { - path := "" - if len(f.path) > 0 { - path = f.path + "." - } - template := common.MapStr{ - // Set the path of the field as name - path + f.Name: common.MapStr{ - "mapping": properties, - "match_mapping_type": matchType, - "path_match": path + f.Name + ".*", - }, - } - - dynamicTemplates = append(dynamicTemplates, template) -} - -func (f *Field) getDefaultProperties() common.MapStr { - // Currently no defaults exist - properties := common.MapStr{} - - if f.Index != nil { - properties["index"] = *f.Index - } - - if f.DocValues != nil { - properties["doc_values"] = *f.DocValues - } - return properties -} - -// Recursively generates the correct key based on the dots -// The mapping requires "properties" between each layer. This is added here. -func generateKey(key string) string { - if strings.Contains(key, ".") { - keys := strings.SplitN(key, ".", 2) - key = keys[0] + ".properties." + generateKey(keys[1]) - } - return key -} - -type dynamicType struct{ value interface{} } - -func (d *dynamicType) Unpack(s string) error { - switch s { - case "true": - d.value = true - case "false": - d.value = false - case "strict": - d.value = s - default: - return fmt.Errorf("'%v' is invalid dynamic setting", s) - } - return nil -} diff --git a/libbeat/template/field_test.go b/libbeat/template/field_test.go deleted file mode 100644 index 6a5e56e6fd8d..000000000000 --- a/libbeat/template/field_test.go +++ /dev/null @@ -1,325 +0,0 @@ -package template - -import ( - "testing" - - "github.com/elastic/beats/libbeat/common" - - "github.com/stretchr/testify/assert" - - "github.com/elastic/go-ucfg/yaml" -) - -func TestField(t *testing.T) { - esVersion2, err := common.NewVersion("2.0.0") - assert.NoError(t, err) - - falseVar := false - trueVar := true - - tests := []struct { - field Field - method func(f Field) common.MapStr - output common.MapStr - }{ - { - field: Field{Type: "long"}, - method: func(f Field) common.MapStr { return f.other() }, - output: common.MapStr{ - "type": "long", - }, - }, - { - field: Field{Type: "scaled_float"}, - method: func(f Field) common.MapStr { return f.scaledFloat() }, - output: common.MapStr{ - "type": "scaled_float", - "scaling_factor": 1000, - }, - }, - { - field: Field{Type: "scaled_float", ScalingFactor: 100}, - method: func(f Field) common.MapStr { return f.scaledFloat() }, - output: common.MapStr{ - "type": "scaled_float", - "scaling_factor": 100, - }, - }, - { - field: Field{Type: "scaled_float", esVersion: *esVersion2}, - method: func(f Field) common.MapStr { return f.scaledFloat() }, - output: common.MapStr{ - "type": "float", - }, - }, - { - field: Field{Type: "object", Enabled: &falseVar}, - method: func(f Field) common.MapStr { return f.object() }, - output: common.MapStr{ - "type": "object", - "enabled": false, - }, - }, - { - field: Field{Type: "array"}, - method: func(f Field) common.MapStr { return f.array() }, - output: common.MapStr{}, - }, - { - field: Field{Type: "array", ObjectType: "text"}, - method: func(f Field) common.MapStr { return f.array() }, - output: common.MapStr{"type": "text"}, - }, - { - field: Field{Type: "array", Index: &falseVar, ObjectType: "keyword"}, - method: func(f Field) common.MapStr { return f.array() }, - output: common.MapStr{"index": false, "type": "keyword"}, - }, - { - field: Field{Type: "object", Enabled: &falseVar}, - method: func(f Field) common.MapStr { return f.object() }, - output: common.MapStr{ - "type": "object", - "enabled": false, - }, - }, - { - field: Field{Type: "text", Analyzer: "autocomplete"}, - method: func(f Field) common.MapStr { return f.text() }, - output: common.MapStr{ - "type": "text", - "analyzer": "autocomplete", - "norms": false, - }, - }, - { - field: Field{Type: "text", Analyzer: "autocomplete", Norms: true}, - method: func(f Field) common.MapStr { return f.text() }, - output: common.MapStr{ - "type": "text", - "analyzer": "autocomplete", - }, - }, - { - field: Field{Type: "text", SearchAnalyzer: "standard", Norms: true}, - method: func(f Field) common.MapStr { return f.text() }, - output: common.MapStr{ - "type": "text", - "search_analyzer": "standard", - }, - }, - { - field: Field{Type: "text", Analyzer: "autocomplete", SearchAnalyzer: "standard", Norms: true}, - method: func(f Field) common.MapStr { return f.text() }, - output: common.MapStr{ - "type": "text", - "analyzer": "autocomplete", - "search_analyzer": "standard", - }, - }, - { - field: Field{Type: "text", MultiFields: Fields{Field{Name: "raw", Type: "keyword"}}, Norms: true}, - method: func(f Field) common.MapStr { return f.text() }, - output: common.MapStr{ - "type": "text", - "fields": common.MapStr{ - "raw": common.MapStr{ - "type": "keyword", - "ignore_above": 1024, - }, - }, - }, - }, - { - field: Field{Type: "text", MultiFields: Fields{ - Field{Name: "raw", Type: "keyword"}, - Field{Name: "indexed", Type: "text"}, - }, Norms: true}, - method: func(f Field) common.MapStr { return f.text() }, - output: common.MapStr{ - "type": "text", - "fields": common.MapStr{ - "raw": common.MapStr{ - "type": "keyword", - "ignore_above": 1024, - }, - "indexed": common.MapStr{ - "type": "text", - "norms": false, - }, - }, - }, - }, - { - field: Field{Dynamic: dynamicType{false}}, - method: func(f Field) common.MapStr { return f.object() }, - output: common.MapStr{ - "dynamic": false, "type": "object", - }, - }, - { - field: Field{Dynamic: dynamicType{true}}, - method: func(f Field) common.MapStr { return f.object() }, - output: common.MapStr{ - "dynamic": true, "type": "object", - }, - }, - { - field: Field{Dynamic: dynamicType{"strict"}}, - method: func(f Field) common.MapStr { return f.object() }, - output: common.MapStr{ - "dynamic": "strict", "type": "object", - }, - }, - { - field: Field{Type: "long", Index: &falseVar}, - method: func(f Field) common.MapStr { return f.other() }, - output: common.MapStr{ - "type": "long", "index": false, - }, - }, - { - field: Field{Type: "text", Index: &trueVar}, - method: func(f Field) common.MapStr { return f.other() }, - output: common.MapStr{ - "type": "text", "index": true, - }, - }, - { - field: Field{Type: "long", DocValues: &falseVar}, - method: func(f Field) common.MapStr { return f.other() }, - output: common.MapStr{ - "type": "long", "doc_values": false, - }, - }, - { - field: Field{Type: "text", DocValues: &trueVar}, - method: func(f Field) common.MapStr { return f.other() }, - output: common.MapStr{ - "type": "text", "doc_values": true, - }, - }, - } - - for _, test := range tests { - output := test.method(test.field) - assert.Equal(t, test.output, output) - } -} - -func TestDynamicTemplate(t *testing.T) { - tests := []struct { - field Field - method func(f Field) common.MapStr - output common.MapStr - }{ - { - field: Field{ - Type: "object", ObjectType: "keyword", - Name: "context", - }, - method: func(f Field) common.MapStr { return f.object() }, - output: common.MapStr{ - "context": common.MapStr{ - "mapping": common.MapStr{"type": "keyword"}, - "match_mapping_type": "string", - "path_match": "context.*", - }, - }, - }, - { - field: Field{ - Type: "object", ObjectType: "long", - path: "language", Name: "english", - }, - method: func(f Field) common.MapStr { return f.object() }, - output: common.MapStr{ - "language.english": common.MapStr{ - "mapping": common.MapStr{"type": "long"}, - "match_mapping_type": "long", - "path_match": "language.english.*", - }, - }, - }, - { - field: Field{ - Type: "object", ObjectType: "text", - path: "language", Name: "english", - }, - method: func(f Field) common.MapStr { return f.object() }, - output: common.MapStr{ - "language.english": common.MapStr{ - "mapping": common.MapStr{"type": "text"}, - "match_mapping_type": "string", - "path_match": "language.english.*", - }, - }, - }, - } - - for _, test := range tests { - dynamicTemplates = nil - test.method(test.field) - assert.Equal(t, test.output, dynamicTemplates[0]) - } -} - -func TestDynamicYaml(t *testing.T) { - tests := []struct { - input []byte - output Field - error bool - }{ - { - input: []byte(` -name: test -dynamic: true -`), - output: Field{ - Name: "test", - Dynamic: dynamicType{true}, - }, - }, - { - input: []byte(` -name: test -dynamic: "true" -`), - output: Field{ - Name: "test", - Dynamic: dynamicType{true}, - }, - }, - { - input: []byte(` -name: test -dynamic: "blue" -`), - error: true, - }, - { - input: []byte(` -name: test -dynamic: "strict" -`), - output: Field{ - Name: "test", - Dynamic: dynamicType{"strict"}, - }, - }, - } - - for _, test := range tests { - keys := Field{} - - cfg, err := yaml.NewConfig(test.input) - assert.NoError(t, err) - err = cfg.Unpack(&keys) - - if err != nil { - assert.True(t, test.error) - } else { - assert.Equal(t, test.output.Dynamic, keys.Dynamic) - } - } -} diff --git a/libbeat/template/fields.go b/libbeat/template/fields.go deleted file mode 100644 index 937c0f6a398d..000000000000 --- a/libbeat/template/fields.go +++ /dev/null @@ -1,120 +0,0 @@ -package template - -import ( - "errors" - "strings" - - "github.com/elastic/beats/libbeat/common" -) - -var ( - defaultType = "keyword" -) - -type Fields []Field - -func (f Fields) process(path string, esVersion common.Version, output common.MapStr) error { - for _, field := range f { - - var mapping common.MapStr - field.path = path - field.esVersion = esVersion - - // If not type is defined, it assumes keyword - if field.Type == "" { - field.Type = defaultType - } - - switch field.Type { - case "ip": - mapping = field.ip() - case "scaled_float": - mapping = field.scaledFloat() - case "half_float": - mapping = field.halfFloat() - case "integer": - mapping = field.integer() - case "text": - mapping = field.text() - case "keyword": - mapping = field.keyword() - case "object": - mapping = field.object() - case "array": - mapping = field.array() - case "group": - var newPath string - if path == "" { - newPath = field.Name - } else { - newPath = path + "." + field.Name - } - mapping = common.MapStr{} - if field.Dynamic.value != nil { - mapping["dynamic"] = field.Dynamic.value - } - - // Combine properties with previous field definitions (if any) - properties := common.MapStr{} - key := generateKey(field.Name) + ".properties" - currentProperties, err := output.GetValue(key) - if err == nil { - var ok bool - properties, ok = currentProperties.(common.MapStr) - if !ok { - // This should never happen - return errors.New(key + " is expected to be a MapStr") - } - } - - if err := field.Fields.process(newPath, esVersion, properties); err != nil { - return err - } - mapping["properties"] = properties - - default: - mapping = field.other() - } - - if len(mapping) > 0 { - output.Put(generateKey(field.Name), mapping) - } - } - - return nil -} - -// HasKey checks if inside fields the given key exists -// The key can be in the form of a.b.c and it will check if the nested field exist -// In case the key is `a` and there is a value `a.b` false is return as it only -// returns true if it's a leave node -func (f Fields) HasKey(key string) bool { - keys := strings.Split(key, ".") - return f.hasKey(keys) -} - -func (f Fields) hasKey(keys []string) bool { - // Nothing to compare anymore - if len(keys) == 0 { - return false - } - - key := keys[0] - keys = keys[1:] - - for _, field := range f { - if field.Name == key { - - if len(field.Fields) > 0 { - return field.Fields.hasKey(keys) - } - // Last entry in the tree but still more keys - if len(keys) > 0 { - return false - } - - return true - } - } - return false -} diff --git a/libbeat/template/fields_test.go b/libbeat/template/fields_test.go deleted file mode 100644 index d39fa131afcb..000000000000 --- a/libbeat/template/fields_test.go +++ /dev/null @@ -1,110 +0,0 @@ -package template - -import ( - "testing" - - "github.com/stretchr/testify/assert" - - "github.com/elastic/beats/libbeat/common" -) - -func TestHasKey(t *testing.T) { - tests := []struct { - key string - fields Fields - result bool - }{ - { - key: "test.find", - fields: Fields{}, - result: false, - }, - { - key: "test.find", - fields: Fields{ - Field{Name: "test"}, - Field{Name: "find"}, - }, - result: false, - }, - { - key: "test.find", - fields: Fields{ - Field{ - Name: "test", Fields: Fields{ - Field{ - Name: "find", - }, - }, - }, - }, - result: true, - }, - { - key: "test", - fields: Fields{ - Field{ - Name: "test", Fields: Fields{ - Field{ - Name: "find", - }, - }, - }, - }, - result: false, - }, - } - - for _, test := range tests { - assert.Equal(t, test.result, test.fields.HasKey(test.key)) - } -} - -func TestPropertiesCombine(t *testing.T) { - // Test common fields are combined even if they come from different objects - fields := Fields{ - Field{ - Name: "test", - Type: "group", - Fields: Fields{ - Field{ - Name: "one", - Type: "text", - }, - }, - }, - Field{ - Name: "test", - Type: "group", - Fields: Fields{ - Field{ - Name: "two", - Type: "text", - }, - }, - }, - } - - output := common.MapStr{} - version, err := common.NewVersion("6.0.0") - if err != nil { - t.Fatal(err) - } - - err = fields.process("", *version, output) - if err != nil { - t.Fatal(err) - } - - v1, err := output.GetValue("test.properties.one") - if err != nil { - t.Fatal(err) - } - v2, err := output.GetValue("test.properties.two") - if err != nil { - t.Fatal(err) - } - - assert.Equal(t, v1, common.MapStr{"type": "text", "norms": false}) - assert.Equal(t, v2, common.MapStr{"type": "text", "norms": false}) -} diff --git a/libbeat/template/processor.go b/libbeat/template/processor.go new file mode 100644 index 000000000000..af9a6f73d5c5 --- /dev/null +++ b/libbeat/template/processor.go @@ -0,0 +1,259 @@ +package template + +import ( + "errors" + + "github.com/elastic/beats/libbeat/common" +) + +type Processor struct { + EsVersion common.Version +} + +var ( + defaultScalingFactor = 1000 +) + +// This includes all entries without special handling for different versions. +// Currently this is: +// long, geo_point, date, short, byte, float, double, boolean +func (p *Processor) process(fields common.Fields, path string, output common.MapStr) error { + for _, field := range fields { + + field.Path = path + var mapping common.MapStr + + switch field.Type { + case "ip": + mapping = p.ip(&field) + case "scaled_float": + mapping = p.scaledFloat(&field) + case "half_float": + mapping = p.halfFloat(&field) + case "integer": + mapping = p.integer(&field) + case "text": + mapping = p.text(&field) + case "", "keyword": + mapping = p.keyword(&field) + case "object": + mapping = p.object(&field) + case "array": + mapping = p.array(&field) + case "group": + var newPath string + if path == "" { + newPath = field.Name + } else { + newPath = path + "." + field.Name + } + mapping = common.MapStr{} + if field.Dynamic.Value != nil { + mapping["dynamic"] = field.Dynamic.Value + } + + // Combine properties with previous field definitions (if any) + properties := common.MapStr{} + key := common.GenerateKey(field.Name) + ".properties" + currentProperties, err := output.GetValue(key) + if err == nil { + var ok bool + properties, ok = currentProperties.(common.MapStr) + if !ok { + // This should never happen + return errors.New(key + " is expected to be a MapStr") + } + } + + if err := p.process(field.Fields, newPath, properties); err != nil { + return err + } + mapping["properties"] = properties + + default: + mapping = p.other(&field) + } + + if len(mapping) > 0 { + output.Put(common.GenerateKey(field.Name), mapping) + } + } + return nil +} + +func (p *Processor) other(f *common.Field) common.MapStr { + property := getDefaultProperties(f) + if f.Type != "" { + property["type"] = f.Type + } + + return property +} + +func (p *Processor) integer(f *common.Field) common.MapStr { + property := getDefaultProperties(f) + property["type"] = "long" + return property +} + +func (p *Processor) scaledFloat(f *common.Field) common.MapStr { + property := getDefaultProperties(f) + property["type"] = "scaled_float" + + if p.EsVersion.IsMajor(2) { + property["type"] = "float" + } else { + scalingFactor := f.ScalingFactor + // Set default scaling factor + if scalingFactor == 0 { + scalingFactor = defaultScalingFactor + } + property["scaling_factor"] = scalingFactor + } + return property +} + +func (p *Processor) halfFloat(f *common.Field) common.MapStr { + property := getDefaultProperties(f) + property["type"] = "half_float" + + if p.EsVersion.IsMajor(2) { + property["type"] = "float" + } + return property +} + +func (p *Processor) ip(f *common.Field) common.MapStr { + property := getDefaultProperties(f) + + property["type"] = "ip" + + if p.EsVersion.IsMajor(2) { + property["type"] = "string" + property["ignore_above"] = 1024 + property["index"] = "not_analyzed" + } + return property +} + +func (p *Processor) keyword(f *common.Field) common.MapStr { + property := getDefaultProperties(f) + + property["type"] = "keyword" + property["ignore_above"] = 1024 + + if p.EsVersion.IsMajor(2) { + property["type"] = "string" + property["ignore_above"] = 1024 + property["index"] = "not_analyzed" + } + return property +} + +func (p *Processor) text(f *common.Field) common.MapStr { + properties := getDefaultProperties(f) + + properties["type"] = "text" + + if p.EsVersion.IsMajor(2) { + properties["type"] = "string" + properties["index"] = "analyzed" + if !f.Norms { + properties["norms"] = common.MapStr{ + "enabled": false, + } + } + } else { + if !f.Norms { + properties["norms"] = false + } + } + + if f.Analyzer != "" { + properties["analyzer"] = f.Analyzer + } + + if f.SearchAnalyzer != "" { + properties["search_analyzer"] = f.SearchAnalyzer + } + + if len(f.MultiFields) > 0 { + fields := common.MapStr{} + p.process(f.MultiFields, "", fields) + properties["fields"] = fields + } + + return properties +} + +func (p *Processor) array(f *common.Field) common.MapStr { + properties := getDefaultProperties(f) + if f.ObjectType != "" { + properties["type"] = f.ObjectType + } + return properties +} + +func (p *Processor) object(f *common.Field) common.MapStr { + dynProperties := getDefaultProperties(f) + + switch f.ObjectType { + case "text": + dynProperties["type"] = "text" + + if p.EsVersion.IsMajor(2) { + dynProperties["type"] = "string" + dynProperties["index"] = "analyzed" + } + addDynamicTemplate(f, dynProperties, "string") + case "long": + dynProperties["type"] = f.ObjectType + addDynamicTemplate(f, dynProperties, "long") + case "keyword": + dynProperties["type"] = f.ObjectType + addDynamicTemplate(f, dynProperties, "string") + } + + properties := getDefaultProperties(f) + properties["type"] = "object" + if f.Enabled != nil { + properties["enabled"] = *f.Enabled + } + + if f.Dynamic.Value != nil { + properties["dynamic"] = f.Dynamic.Value + } + + return properties +} + +func addDynamicTemplate(f *common.Field, properties common.MapStr, matchType string) { + path := "" + if len(f.Path) > 0 { + path = f.Path + "." + } + template := common.MapStr{ + // Set the path of the field as name + path + f.Name: common.MapStr{ + "mapping": properties, + "match_mapping_type": matchType, + "path_match": path + f.Name + ".*", + }, + } + + dynamicTemplates = append(dynamicTemplates, template) +} + +func getDefaultProperties(f *common.Field) common.MapStr { + // Currently no defaults exist + properties := common.MapStr{} + + if f.Index != nil { + properties["index"] = *f.Index + } + + if f.DocValues != nil { + properties["doc_values"] = *f.DocValues + } + return properties +} diff --git a/libbeat/template/processor_test.go b/libbeat/template/processor_test.go new file mode 100644 index 000000000000..45133e3dce5b --- /dev/null +++ b/libbeat/template/processor_test.go @@ -0,0 +1,284 @@ +package template + +import ( + "testing" + + "github.com/elastic/beats/libbeat/common" + + "github.com/stretchr/testify/assert" +) + +func TestProcessor(t *testing.T) { + esVersion2, err := common.NewVersion("2.0.0") + assert.NoError(t, err) + + falseVar := false + trueVar := true + p := &Processor{} + pEsVersion2 := &Processor{EsVersion: *esVersion2} + + tests := []struct { + output common.MapStr + expected common.MapStr + }{ + { + output: p.other(&common.Field{Type: "long"}), + expected: common.MapStr{"type": "long"}, + }, + { + output: p.scaledFloat(&common.Field{Type: "scaled_float"}), + expected: common.MapStr{ + "type": "scaled_float", + "scaling_factor": 1000, + }, + }, + { + output: p.scaledFloat(&common.Field{Type: "scaled_float", ScalingFactor: 100}), + expected: common.MapStr{ + "type": "scaled_float", + "scaling_factor": 100, + }, + }, + { + output: pEsVersion2.scaledFloat(&common.Field{Type: "scaled_float"}), + expected: common.MapStr{"type": "float"}, + }, + { + output: p.object(&common.Field{Type: "object", Enabled: &falseVar}), + expected: common.MapStr{ + "type": "object", + "enabled": false, + }, + }, + { + output: p.array(&common.Field{Type: "array"}), + expected: common.MapStr{}, + }, + { + output: p.array(&common.Field{Type: "array", ObjectType: "text"}), + expected: common.MapStr{"type": "text"}, + }, + { + output: p.array(&common.Field{Type: "array", Index: &falseVar, ObjectType: "keyword"}), + expected: common.MapStr{"index": false, "type": "keyword"}, + }, + { + output: p.object(&common.Field{Type: "object", Enabled: &falseVar}), + expected: common.MapStr{ + "type": "object", + "enabled": false, + }, + }, + { + output: p.text(&common.Field{Type: "text", Analyzer: "autocomplete"}), + expected: common.MapStr{ + "type": "text", + "analyzer": "autocomplete", + "norms": false, + }, + }, + { + output: p.text(&common.Field{Type: "text", Analyzer: "autocomplete", Norms: true}), + expected: common.MapStr{ + "type": "text", + "analyzer": "autocomplete", + }, + }, + { + output: p.text(&common.Field{Type: "text", SearchAnalyzer: "standard", Norms: true}), + expected: common.MapStr{ + "type": "text", + "search_analyzer": "standard", + }, + }, + { + output: p.text(&common.Field{Type: "text", Analyzer: "autocomplete", SearchAnalyzer: "standard", Norms: true}), + expected: common.MapStr{ + "type": "text", + "analyzer": "autocomplete", + "search_analyzer": "standard", + }, + }, + { + output: p.text(&common.Field{Type: "text", MultiFields: common.Fields{common.Field{Name: "raw", Type: "keyword"}}, Norms: true}), + expected: common.MapStr{ + "type": "text", + "fields": common.MapStr{ + "raw": common.MapStr{ + "type": "keyword", + "ignore_above": 1024, + }, + }, + }, + }, + { + output: p.text(&common.Field{Type: "text", MultiFields: common.Fields{ + common.Field{Name: "raw", Type: "keyword"}, + common.Field{Name: "indexed", Type: "text"}, + }, Norms: true}), + expected: common.MapStr{ + "type": "text", + "fields": common.MapStr{ + "raw": common.MapStr{ + "type": "keyword", + "ignore_above": 1024, + }, + "indexed": common.MapStr{ + "type": "text", + "norms": false, + }, + }, + }, + }, + { + output: p.object(&common.Field{Dynamic: common.DynamicType{false}}), + expected: common.MapStr{ + "dynamic": false, "type": "object", + }, + }, + { + output: p.object(&common.Field{Dynamic: common.DynamicType{true}}), + expected: common.MapStr{ + "dynamic": true, "type": "object", + }, + }, + { + output: p.object(&common.Field{Dynamic: common.DynamicType{"strict"}}), + expected: common.MapStr{ + "dynamic": "strict", "type": "object", + }, + }, + { + output: p.other(&common.Field{Type: "long", Index: &falseVar}), + expected: common.MapStr{ + "type": "long", "index": false, + }, + }, + { + output: p.other(&common.Field{Type: "text", Index: &trueVar}), + expected: common.MapStr{ + "type": "text", "index": true, + }, + }, + { + output: p.other(&common.Field{Type: "long", DocValues: &falseVar}), + expected: common.MapStr{ + "type": "long", "doc_values": false, + }, + }, + { + output: p.other(&common.Field{Type: "text", DocValues: &trueVar}), + expected: common.MapStr{ + "type": "text", "doc_values": true, + }, + }, + } + + for _, test := range tests { + assert.Equal(t, test.expected, test.output) + } +} + +func TestDynamicTemplate(t *testing.T) { + p := &Processor{} + tests := []struct { + field common.Field + expected common.MapStr + }{ + { + field: common.Field{ + Type: "object", ObjectType: "keyword", + Name: "context", + }, + expected: common.MapStr{ + "context": common.MapStr{ + "mapping": common.MapStr{"type": "keyword"}, + "match_mapping_type": "string", + "path_match": "context.*", + }, + }, + }, + { + field: common.Field{ + Type: "object", ObjectType: "long", + Path: "language", Name: "english", + }, + expected: common.MapStr{ + "language.english": common.MapStr{ + "mapping": common.MapStr{"type": "long"}, + "match_mapping_type": "long", + "path_match": "language.english.*", + }, + }, + }, + { + field: common.Field{ + Type: "object", ObjectType: "text", + Path: "language", Name: "english", + }, + expected: common.MapStr{ + "language.english": common.MapStr{ + "mapping": common.MapStr{"type": "text"}, + "match_mapping_type": "string", + "path_match": "language.english.*", + }, + }, + }, + } + + for _, test := range tests { + dynamicTemplates = nil + p.object(&test.field) + assert.Equal(t, test.expected, dynamicTemplates[0]) + } +} + +func TestPropertiesCombine(t *testing.T) { + // Test common fields are combined even if they come from different objects + fields := common.Fields{ + common.Field{ + Name: "test", + Type: "group", + Fields: common.Fields{ + common.Field{ + Name: "one", + Type: "text", + }, + }, + }, + common.Field{ + Name: "test", + Type: "group", + Fields: common.Fields{ + common.Field{ + Name: "two", + Type: "text", + }, + }, + }, + } + + output := common.MapStr{} + version, err := common.NewVersion("6.0.0") + if err != nil { + t.Fatal(err) + } + + p := Processor{EsVersion: *version} + err = p.process(fields, "", output) + if err != nil { + t.Fatal(err) + } + + v1, err := output.GetValue("test.properties.one") + if err != nil { + t.Fatal(err) + } + v2, err := output.GetValue("test.properties.two") + if err != nil { + t.Fatal(err) + } + + assert.Equal(t, v1, common.MapStr{"type": "text", "norms": false}) + assert.Equal(t, v2, common.MapStr{"type": "text", "norms": false}) +} diff --git a/libbeat/template/template.go b/libbeat/template/template.go index 646614f6fc30..81b4148ef4db 100644 --- a/libbeat/template/template.go +++ b/libbeat/template/template.go @@ -7,7 +7,6 @@ import ( "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/common/fmtstr" - "github.com/elastic/go-ucfg/yaml" ) var ( @@ -93,14 +92,15 @@ func New(beatVersion string, beatName string, esVersion string, config TemplateC // Load the given input and generates the input based on it func (t *Template) Load(file string) (common.MapStr, error) { - fields, err := loadYaml(file) + fields, err := common.LoadFieldsYaml(file) if err != nil { return nil, err } // Start processing at the root properties := common.MapStr{} - if err := fields.process("", t.esVersion, properties); err != nil { + processor := Processor{EsVersion: t.esVersion} + if err := processor.process(fields, "", properties); err != nil { return nil, err } output := t.generate(properties, dynamicTemplates) @@ -192,20 +192,3 @@ func (t *Template) generate(properties common.MapStr, dynamicTemplates []common. return basicStructure } - -func loadYaml(path string) (Fields, error) { - keys := []Field{} - - cfg, err := yaml.NewConfigWithFile(path) - if err != nil { - return nil, err - } - cfg.Unpack(&keys) - - fields := Fields{} - - for _, key := range keys { - fields = append(fields, key.Fields...) - } - return fields, nil -}