diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc index e8ab8e7f82d..c3e92e3558a 100644 --- a/CHANGELOG-developer.next.asciidoc +++ b/CHANGELOG-developer.next.asciidoc @@ -22,6 +22,7 @@ The list below covers the major changes between 7.0.0-beta1 and master only. - Remove support for deprecated `GenRootCmd` methods. {pull}10721[10721] - Remove SkipNormalization, SkipAgentMetadata, SkipAddHostName. {pull}10801[10801] {pull}10769[10769] +- Move Fields from package libbeat/common to libbeat/mapping. {pull}11198[11198] ==== Bugfixes diff --git a/auditbeat/module/auditd/audit_linux_test.go b/auditbeat/module/auditd/audit_linux_test.go index 5c5cc10f979..1d69d375ea3 100644 --- a/auditbeat/module/auditd/audit_linux_test.go +++ b/auditbeat/module/auditd/audit_linux_test.go @@ -33,8 +33,8 @@ import ( "github.com/prometheus/procfs" "github.com/elastic/beats/auditbeat/core" - "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/libbeat/mapping" "github.com/elastic/beats/metricbeat/mb" mbtest "github.com/elastic/beats/metricbeat/mb/testing" "github.com/elastic/go-libaudit" @@ -102,7 +102,7 @@ func TestData(t *testing.T) { // assertFieldsAreDocumented mimics assert_fields_are_documented in Python system tests. func assertFieldsAreDocumented(t *testing.T, events []mb.Event) { - fieldsYml, err := common.LoadFieldsYaml("../../fields.yml") + fieldsYml, err := mapping.LoadFieldsYaml("../../fields.yml") if err != nil { t.Fatal(err) } diff --git a/libbeat/idxmgmt/idxmgmt_test.go b/libbeat/idxmgmt/idxmgmt_test.go index 3d7a841b797..339e8510be9 100644 --- a/libbeat/idxmgmt/idxmgmt_test.go +++ b/libbeat/idxmgmt/idxmgmt_test.go @@ -28,6 +28,7 @@ import ( "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/idxmgmt/ilm" + "github.com/elastic/beats/libbeat/mapping" "github.com/elastic/beats/libbeat/template" ) @@ -96,7 +97,7 @@ func TestDefaultSupport_TemplateConfig(t *testing.T) { cloneCfg := func(c template.TemplateConfig) template.TemplateConfig { if c.AppendFields != nil { - tmp := make(common.Fields, len(c.AppendFields)) + tmp := make(mapping.Fields, len(c.AppendFields)) copy(tmp, c.AppendFields) c.AppendFields = tmp } diff --git a/libbeat/kibana/fields_transformer.go b/libbeat/kibana/fields_transformer.go index 3e385144c62..b5ffd82fa04 100644 --- a/libbeat/kibana/fields_transformer.go +++ b/libbeat/kibana/fields_transformer.go @@ -22,12 +22,13 @@ import ( "fmt" "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/mapping" ) var v640 = common.MustNewVersion("6.4.0") type fieldsTransformer struct { - fields common.Fields + fields mapping.Fields transformedFields []common.MapStr transformedFieldFormatMap common.MapStr version *common.Version @@ -35,7 +36,7 @@ type fieldsTransformer struct { migration bool } -func newFieldsTransformer(version *common.Version, fields common.Fields, migration bool) (*fieldsTransformer, error) { +func newFieldsTransformer(version *common.Version, fields mapping.Fields, migration bool) (*fieldsTransformer, error) { if version == nil { return nil, errors.New("Version must be given") } @@ -64,10 +65,10 @@ func (t *fieldsTransformer) transform() (transformed common.MapStr, err error) { // add some meta fields truthy := true falsy := false - t.add(common.Field{Path: "_id", Type: "keyword", Index: &falsy, Analyzed: &falsy, DocValues: &falsy, Searchable: &falsy, Aggregatable: &falsy}) - t.add(common.Field{Path: "_type", Type: "keyword", Index: &falsy, Analyzed: &falsy, DocValues: &falsy, Searchable: &truthy, Aggregatable: &truthy}) - t.add(common.Field{Path: "_index", Type: "keyword", Index: &falsy, Analyzed: &falsy, DocValues: &falsy, Searchable: &falsy, Aggregatable: &falsy}) - t.add(common.Field{Path: "_score", Type: "integer", Index: &falsy, Analyzed: &falsy, DocValues: &falsy, Searchable: &falsy, Aggregatable: &falsy}) + t.add(mapping.Field{Path: "_id", Type: "keyword", Index: &falsy, Analyzed: &falsy, DocValues: &falsy, Searchable: &falsy, Aggregatable: &falsy}) + t.add(mapping.Field{Path: "_type", Type: "keyword", Index: &falsy, Analyzed: &falsy, DocValues: &falsy, Searchable: &truthy, Aggregatable: &truthy}) + t.add(mapping.Field{Path: "_index", Type: "keyword", Index: &falsy, Analyzed: &falsy, DocValues: &falsy, Searchable: &falsy, Aggregatable: &falsy}) + t.add(mapping.Field{Path: "_score", Type: "integer", Index: &falsy, Analyzed: &falsy, DocValues: &falsy, Searchable: &falsy, Aggregatable: &falsy}) transformed = common.MapStr{ "fields": t.transformedFields, @@ -76,7 +77,7 @@ func (t *fieldsTransformer) transform() (transformed common.MapStr, err error) { return } -func (t *fieldsTransformer) transformFields(commonFields common.Fields, path string) { +func (t *fieldsTransformer) transformFields(commonFields mapping.Fields, path string) { for _, f := range commonFields { f.Path = f.Name if path != "" { @@ -119,7 +120,7 @@ func (t *fieldsTransformer) transformFields(commonFields common.Fields, path str } } -func (t *fieldsTransformer) update(target *common.MapStr, override common.Field) error { +func (t *fieldsTransformer) update(target *common.MapStr, override mapping.Field) error { field, _ := transformField(t.version, override) if override.Type == "" || (*target)["type"] == field["type"] { target.Update(field) @@ -133,7 +134,7 @@ func (t *fieldsTransformer) update(target *common.MapStr, override common.Field) return fmt.Errorf("field <%s> is duplicated", override.Path) } -func (t *fieldsTransformer) add(f common.Field) { +func (t *fieldsTransformer) add(f mapping.Field) { if idx := t.keys[f.Path]; idx > 0 { target := &t.transformedFields[idx-1] // 1-indexed if err := t.update(target, f); err != nil { @@ -150,7 +151,7 @@ func (t *fieldsTransformer) add(f common.Field) { } } -func transformField(version *common.Version, f common.Field) (common.MapStr, common.MapStr) { +func transformField(version *common.Version, f mapping.Field) (common.MapStr, common.MapStr) { field := common.MapStr{ "name": f.Path, "count": f.Count, @@ -217,7 +218,7 @@ func getVal(valP *bool, def bool) bool { return def } -func addParams(format *common.MapStr, version *common.Version, f common.Field) { +func addParams(format *common.MapStr, version *common.Version, f mapping.Field) { addFormatParam(format, "pattern", f.Pattern) addFormatParam(format, "inputFormat", f.InputFormat) addFormatParam(format, "outputFormat", f.OutputFormat) @@ -248,7 +249,7 @@ func addFormatParam(f *common.MapStr, key string, val interface{}) { } // takes the highest version where major version <= given version -func addVersionedFormatParam(f *common.MapStr, version *common.Version, key string, val []common.VersionizedString) { +func addVersionedFormatParam(f *common.MapStr, version *common.Version, key string, val []mapping.VersionizedString) { if len(val) == 0 { return } diff --git a/libbeat/kibana/fields_transformer_test.go b/libbeat/kibana/fields_transformer_test.go index 1fb1e8b3a0b..be2fd6864fc 100644 --- a/libbeat/kibana/fields_transformer_test.go +++ b/libbeat/kibana/fields_transformer_test.go @@ -25,6 +25,7 @@ import ( "github.com/stretchr/testify/require" "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/mapping" ) var ( @@ -35,7 +36,7 @@ var ( ) func TestEmpty(t *testing.T) { - trans, err := newFieldsTransformer(version, common.Fields{}, true) + trans, err := newFieldsTransformer(version, mapping.Fields{}, true) assert.NoError(t, err) out, err := trans.transform() assert.NoError(t, err) @@ -93,26 +94,26 @@ func TestEmpty(t *testing.T) { func TestMissingVersion(t *testing.T) { var c *common.Version - _, err := newFieldsTransformer(c, common.Fields{}, true) + _, err := newFieldsTransformer(c, mapping.Fields{}, true) assert.Error(t, err) } func TestDuplicateField(t *testing.T) { testCases := []struct { - commonFields []common.Field + commonFields []mapping.Field }{ // type change - {commonFields: []common.Field{ + {commonFields: []mapping.Field{ {Name: "context", Path: "something"}, {Name: "context", Path: "something", Type: "date"}, }}, // missing overwrite - {commonFields: []common.Field{ + {commonFields: []mapping.Field{ {Name: "context", Path: "something"}, {Name: "context", Path: "something"}, }}, // missing overwrite in source - {commonFields: []common.Field{ + {commonFields: []mapping.Field{ {Name: "context", Path: "something", Overwrite: true}, {Name: "context", Path: "something"}, }}, @@ -127,9 +128,9 @@ func TestDuplicateField(t *testing.T) { } func TestValidDuplicateField(t *testing.T) { - commonFields := common.Fields{ - common.Field{Name: "context", Path: "something", Type: "keyword", Description: "original description"}, - common.Field{Name: "context", Path: "something", Overwrite: true, Description: "updated description", + commonFields := mapping.Fields{ + mapping.Field{Name: "context", Path: "something", Type: "keyword", Description: "original description"}, + mapping.Field{Name: "context", Path: "something", Overwrite: true, Description: "updated description", Aggregatable: &falsy, Analyzed: &truthy, Count: 2, @@ -137,18 +138,18 @@ func TestValidDuplicateField(t *testing.T) { Index: &falsy, Searchable: &falsy, }, - common.Field{ + mapping.Field{ Name: "context", Type: "group", - Fields: common.Fields{ - common.Field{Name: "another", Type: "date"}, + Fields: mapping.Fields{ + mapping.Field{Name: "another", Type: "date"}, }, }, - common.Field{ + mapping.Field{ Name: "context", Type: "group", - Fields: common.Fields{ - common.Field{Name: "another", Overwrite: true}, + Fields: mapping.Fields{ + mapping.Field{Name: "another", Overwrite: true}, }, }, } @@ -171,11 +172,11 @@ func TestValidDuplicateField(t *testing.T) { } func TestInvalidVersion(t *testing.T) { - commonFields := common.Fields{ - common.Field{ + commonFields := mapping.Fields{ + mapping.Field{ Name: "versionTest", Format: "url", - UrlTemplate: []common.VersionizedString{ + UrlTemplate: []mapping.VersionizedString{ {MinVersion: "3", Value: ""}, }, }, @@ -188,26 +189,26 @@ func TestInvalidVersion(t *testing.T) { func TestTransformTypes(t *testing.T) { tests := []struct { - commonField common.Field + commonField mapping.Field expected interface{} }{ - {commonField: common.Field{}, expected: "string"}, - {commonField: common.Field{Type: "half_float"}, expected: "number"}, - {commonField: common.Field{Type: "scaled_float"}, expected: "number"}, - {commonField: common.Field{Type: "float"}, expected: "number"}, - {commonField: common.Field{Type: "integer"}, expected: "number"}, - {commonField: common.Field{Type: "long"}, expected: "number"}, - {commonField: common.Field{Type: "short"}, expected: "number"}, - {commonField: common.Field{Type: "byte"}, expected: "number"}, - {commonField: common.Field{Type: "keyword"}, expected: "string"}, - {commonField: common.Field{Type: "text"}, expected: "string"}, - {commonField: common.Field{Type: "string"}, expected: nil}, - {commonField: common.Field{Type: "date"}, expected: "date"}, - {commonField: common.Field{Type: "geo_point"}, expected: "geo_point"}, - {commonField: common.Field{Type: "invalid"}, expected: nil}, + {commonField: mapping.Field{}, expected: "string"}, + {commonField: mapping.Field{Type: "half_float"}, expected: "number"}, + {commonField: mapping.Field{Type: "scaled_float"}, expected: "number"}, + {commonField: mapping.Field{Type: "float"}, expected: "number"}, + {commonField: mapping.Field{Type: "integer"}, expected: "number"}, + {commonField: mapping.Field{Type: "long"}, expected: "number"}, + {commonField: mapping.Field{Type: "short"}, expected: "number"}, + {commonField: mapping.Field{Type: "byte"}, expected: "number"}, + {commonField: mapping.Field{Type: "keyword"}, expected: "string"}, + {commonField: mapping.Field{Type: "text"}, expected: "string"}, + {commonField: mapping.Field{Type: "string"}, expected: nil}, + {commonField: mapping.Field{Type: "date"}, expected: "date"}, + {commonField: mapping.Field{Type: "geo_point"}, expected: "geo_point"}, + {commonField: mapping.Field{Type: "invalid"}, expected: nil}, } for idx, test := range tests { - trans, _ := newFieldsTransformer(version, common.Fields{test.commonField}, true) + trans, _ := newFieldsTransformer(version, mapping.Fields{test.commonField}, true) transformed, err := trans.transform() assert.NoError(t, err) out := transformed["fields"].([]common.MapStr)[0] @@ -217,32 +218,32 @@ func TestTransformTypes(t *testing.T) { func TestTransformGroup(t *testing.T) { tests := []struct { - commonFields common.Fields + commonFields mapping.Fields expected []string }{ { - commonFields: common.Fields{common.Field{Name: "context", Path: "something"}}, + commonFields: mapping.Fields{mapping.Field{Name: "context", Path: "something"}}, expected: []string{"context"}, }, { - commonFields: common.Fields{ - common.Field{ + commonFields: mapping.Fields{ + mapping.Field{ Name: "context", Type: "group", - Fields: common.Fields{ - common.Field{Name: "another", Type: ""}, + Fields: mapping.Fields{ + mapping.Field{Name: "another", Type: ""}, }, }, - common.Field{ + mapping.Field{ Name: "context", Type: "group", - Fields: common.Fields{ - common.Field{Name: "type", Type: ""}, - common.Field{ + Fields: mapping.Fields{ + mapping.Field{Name: "type", Type: ""}, + mapping.Field{ Name: "metric", Type: "group", - Fields: common.Fields{ - common.Field{Name: "object"}, + Fields: mapping.Fields{ + mapping.Field{Name: "object"}, }, }, }, @@ -265,70 +266,70 @@ func TestTransformGroup(t *testing.T) { func TestTransformMisc(t *testing.T) { tests := []struct { - commonField common.Field + commonField mapping.Field expected interface{} attr string }{ - {commonField: common.Field{}, expected: 0, attr: "count"}, - {commonField: common.Field{Count: 4}, expected: 4, attr: "count"}, + {commonField: mapping.Field{}, expected: 0, attr: "count"}, + {commonField: mapping.Field{Count: 4}, expected: 4, attr: "count"}, // searchable - {commonField: common.Field{}, expected: true, attr: "searchable"}, - {commonField: common.Field{Searchable: &truthy}, expected: true, attr: "searchable"}, - {commonField: common.Field{Searchable: &falsy}, expected: false, attr: "searchable"}, - {commonField: common.Field{Type: "binary"}, expected: false, attr: "searchable"}, - {commonField: common.Field{Searchable: &truthy, Type: "binary"}, expected: false, attr: "searchable"}, + {commonField: mapping.Field{}, expected: true, attr: "searchable"}, + {commonField: mapping.Field{Searchable: &truthy}, expected: true, attr: "searchable"}, + {commonField: mapping.Field{Searchable: &falsy}, expected: false, attr: "searchable"}, + {commonField: mapping.Field{Type: "binary"}, expected: false, attr: "searchable"}, + {commonField: mapping.Field{Searchable: &truthy, Type: "binary"}, expected: false, attr: "searchable"}, // aggregatable - {commonField: common.Field{}, expected: true, attr: "aggregatable"}, - {commonField: common.Field{Aggregatable: &truthy}, expected: true, attr: "aggregatable"}, - {commonField: common.Field{Aggregatable: &falsy}, expected: false, attr: "aggregatable"}, - {commonField: common.Field{Type: "binary"}, expected: false, attr: "aggregatable"}, - {commonField: common.Field{Aggregatable: &truthy, Type: "binary"}, expected: false, attr: "aggregatable"}, - {commonField: common.Field{Type: "keyword"}, expected: true, attr: "aggregatable"}, - {commonField: common.Field{Aggregatable: &truthy, Type: "text"}, expected: false, attr: "aggregatable"}, - {commonField: common.Field{Type: "text"}, expected: false, attr: "aggregatable"}, + {commonField: mapping.Field{}, expected: true, attr: "aggregatable"}, + {commonField: mapping.Field{Aggregatable: &truthy}, expected: true, attr: "aggregatable"}, + {commonField: mapping.Field{Aggregatable: &falsy}, expected: false, attr: "aggregatable"}, + {commonField: mapping.Field{Type: "binary"}, expected: false, attr: "aggregatable"}, + {commonField: mapping.Field{Aggregatable: &truthy, Type: "binary"}, expected: false, attr: "aggregatable"}, + {commonField: mapping.Field{Type: "keyword"}, expected: true, attr: "aggregatable"}, + {commonField: mapping.Field{Aggregatable: &truthy, Type: "text"}, expected: false, attr: "aggregatable"}, + {commonField: mapping.Field{Type: "text"}, expected: false, attr: "aggregatable"}, // analyzed - {commonField: common.Field{}, expected: false, attr: "analyzed"}, - {commonField: common.Field{Analyzed: &truthy}, expected: true, attr: "analyzed"}, - {commonField: common.Field{Analyzed: &falsy}, expected: false, attr: "analyzed"}, - {commonField: common.Field{Type: "binary"}, expected: false, attr: "analyzed"}, - {commonField: common.Field{Analyzed: &truthy, Type: "binary"}, expected: false, attr: "analyzed"}, + {commonField: mapping.Field{}, expected: false, attr: "analyzed"}, + {commonField: mapping.Field{Analyzed: &truthy}, expected: true, attr: "analyzed"}, + {commonField: mapping.Field{Analyzed: &falsy}, expected: false, attr: "analyzed"}, + {commonField: mapping.Field{Type: "binary"}, expected: false, attr: "analyzed"}, + {commonField: mapping.Field{Analyzed: &truthy, Type: "binary"}, expected: false, attr: "analyzed"}, // doc_values always set to true except for meta fields - {commonField: common.Field{}, expected: true, attr: "doc_values"}, - {commonField: common.Field{DocValues: &truthy}, expected: true, attr: "doc_values"}, - {commonField: common.Field{DocValues: &falsy}, expected: false, attr: "doc_values"}, - {commonField: common.Field{Script: "doc[]"}, expected: false, attr: "doc_values"}, - {commonField: common.Field{DocValues: &truthy, Script: "doc[]"}, expected: false, attr: "doc_values"}, - {commonField: common.Field{Type: "binary"}, expected: false, attr: "doc_values"}, - {commonField: common.Field{DocValues: &truthy, Type: "binary"}, expected: true, attr: "doc_values"}, + {commonField: mapping.Field{}, expected: true, attr: "doc_values"}, + {commonField: mapping.Field{DocValues: &truthy}, expected: true, attr: "doc_values"}, + {commonField: mapping.Field{DocValues: &falsy}, expected: false, attr: "doc_values"}, + {commonField: mapping.Field{Script: "doc[]"}, expected: false, attr: "doc_values"}, + {commonField: mapping.Field{DocValues: &truthy, Script: "doc[]"}, expected: false, attr: "doc_values"}, + {commonField: mapping.Field{Type: "binary"}, expected: false, attr: "doc_values"}, + {commonField: mapping.Field{DocValues: &truthy, Type: "binary"}, expected: true, attr: "doc_values"}, // enabled - only applies to objects (and only if set) - {commonField: common.Field{Type: "binary", Enabled: &falsy}, expected: nil, attr: "enabled"}, - {commonField: common.Field{Type: "binary", Enabled: &truthy}, expected: nil, attr: "enabled"}, - {commonField: common.Field{Type: "object", Enabled: &truthy}, expected: true, attr: "enabled"}, - {commonField: common.Field{Type: "object", Enabled: &falsy}, expected: false, attr: "enabled"}, - {commonField: common.Field{Type: "object", Enabled: &falsy}, expected: false, attr: "doc_values"}, + {commonField: mapping.Field{Type: "binary", Enabled: &falsy}, expected: nil, attr: "enabled"}, + {commonField: mapping.Field{Type: "binary", Enabled: &truthy}, expected: nil, attr: "enabled"}, + {commonField: mapping.Field{Type: "object", Enabled: &truthy}, expected: true, attr: "enabled"}, + {commonField: mapping.Field{Type: "object", Enabled: &falsy}, expected: false, attr: "enabled"}, + {commonField: mapping.Field{Type: "object", Enabled: &falsy}, expected: false, attr: "doc_values"}, // indexed - {commonField: common.Field{Type: "binary"}, expected: false, attr: "indexed"}, - {commonField: common.Field{Index: &truthy, Type: "binary"}, expected: false, attr: "indexed"}, + {commonField: mapping.Field{Type: "binary"}, expected: false, attr: "indexed"}, + {commonField: mapping.Field{Index: &truthy, Type: "binary"}, expected: false, attr: "indexed"}, // script, scripted - {commonField: common.Field{}, expected: false, attr: "scripted"}, - {commonField: common.Field{}, expected: nil, attr: "script"}, - {commonField: common.Field{Script: "doc[]"}, expected: true, attr: "scripted"}, - {commonField: common.Field{Script: "doc[]"}, expected: "doc[]", attr: "script"}, - {commonField: common.Field{Type: "binary"}, expected: false, attr: "scripted"}, + {commonField: mapping.Field{}, expected: false, attr: "scripted"}, + {commonField: mapping.Field{}, expected: nil, attr: "script"}, + {commonField: mapping.Field{Script: "doc[]"}, expected: true, attr: "scripted"}, + {commonField: mapping.Field{Script: "doc[]"}, expected: "doc[]", attr: "script"}, + {commonField: mapping.Field{Type: "binary"}, expected: false, attr: "scripted"}, // language - {commonField: common.Field{}, expected: nil, attr: "lang"}, - {commonField: common.Field{Script: "doc[]"}, expected: "painless", attr: "lang"}, + {commonField: mapping.Field{}, expected: nil, attr: "lang"}, + {commonField: mapping.Field{Script: "doc[]"}, expected: "painless", attr: "lang"}, } for idx, test := range tests { - trans, _ := newFieldsTransformer(version, common.Fields{test.commonField}, true) + trans, _ := newFieldsTransformer(version, mapping.Fields{test.commonField}, true) transformed, err := trans.transform() assert.NoError(t, err) out := transformed["fields"].([]common.MapStr)[0] @@ -344,27 +345,27 @@ func TestTransformFieldFormatMap(t *testing.T) { falsy := false tests := []struct { - commonField common.Field + commonField mapping.Field version *common.Version expected common.MapStr }{ { - commonField: common.Field{Name: "c"}, + commonField: mapping.Field{Name: "c"}, expected: common.MapStr{}, version: version, }, { - commonField: common.Field{Name: "c", Format: "url"}, + commonField: mapping.Field{Name: "c", Format: "url"}, expected: common.MapStr{"c": common.MapStr{"id": "url"}}, version: version, }, { - commonField: common.Field{Name: "c", Pattern: "p"}, + commonField: mapping.Field{Name: "c", Pattern: "p"}, expected: common.MapStr{"c": common.MapStr{"params": common.MapStr{"pattern": "p"}}}, version: version, }, { - commonField: common.Field{ + commonField: mapping.Field{ Name: "c", Format: "url", Pattern: "p", @@ -378,7 +379,7 @@ func TestTransformFieldFormatMap(t *testing.T) { version: version, }, { - commonField: common.Field{ + commonField: mapping.Field{ Name: "c", Format: "url", InputFormat: "string", @@ -394,7 +395,7 @@ func TestTransformFieldFormatMap(t *testing.T) { version: version, }, { - commonField: common.Field{ + commonField: mapping.Field{ Name: "c", Format: "url", Pattern: "[^-]", @@ -414,7 +415,7 @@ func TestTransformFieldFormatMap(t *testing.T) { version: version, }, { - commonField: common.Field{ + commonField: mapping.Field{ Name: "c", InputFormat: "string", }, @@ -423,7 +424,7 @@ func TestTransformFieldFormatMap(t *testing.T) { }, { version: version620, - commonField: common.Field{ + commonField: mapping.Field{ Name: "c", Format: "url", Pattern: "[^-]", @@ -432,7 +433,7 @@ func TestTransformFieldFormatMap(t *testing.T) { OutputFormat: "float", OutputPrecision: &precision, LabelTemplate: "lblT", - UrlTemplate: []common.VersionizedString{ + UrlTemplate: []mapping.VersionizedString{ {MinVersion: "5.0.0", Value: "5x.urlTemplate"}, {MinVersion: "6.0.0", Value: "6x.urlTemplate"}, }, @@ -454,10 +455,10 @@ func TestTransformFieldFormatMap(t *testing.T) { }, { version: version620, - commonField: common.Field{ + commonField: mapping.Field{ Name: "c", Format: "url", - UrlTemplate: []common.VersionizedString{ + UrlTemplate: []mapping.VersionizedString{ {MinVersion: "6.4.0", Value: "6x.urlTemplate"}, }, }, @@ -467,10 +468,10 @@ func TestTransformFieldFormatMap(t *testing.T) { }, { version: version620, - commonField: common.Field{ + commonField: mapping.Field{ Name: "c", Format: "url", - UrlTemplate: []common.VersionizedString{ + UrlTemplate: []mapping.VersionizedString{ {MinVersion: "4.7.2", Value: "4x.urlTemplate"}, {MinVersion: "6.5.1", Value: "6x.urlTemplate"}, }, @@ -486,10 +487,10 @@ func TestTransformFieldFormatMap(t *testing.T) { }, { version: version620, - commonField: common.Field{ + commonField: mapping.Field{ Name: "c", Format: "url", - UrlTemplate: []common.VersionizedString{ + UrlTemplate: []mapping.VersionizedString{ {MinVersion: "6.2.0", Value: "6.2.0.urlTemplate"}, {MinVersion: "6.2.0-alpha", Value: "6.2.0-alpha.urlTemplate"}, {MinVersion: "6.2.7", Value: "6.2.7.urlTemplate"}, @@ -506,10 +507,10 @@ func TestTransformFieldFormatMap(t *testing.T) { }, { version: version620, - commonField: common.Field{ + commonField: mapping.Field{ Name: "c", Format: "url", - UrlTemplate: []common.VersionizedString{ + UrlTemplate: []mapping.VersionizedString{ {MinVersion: "4.1.0", Value: "4x.urlTemplate"}, {MinVersion: "5.2.0-rc2", Value: "5.2.0-rc2.urlTemplate"}, {MinVersion: "5.2.0-rc3", Value: "5.2.0-rc3.urlTemplate"}, @@ -527,7 +528,7 @@ func TestTransformFieldFormatMap(t *testing.T) { }, } for idx, test := range tests { - trans, _ := newFieldsTransformer(test.version, common.Fields{test.commonField}, true) + trans, _ := newFieldsTransformer(test.version, mapping.Fields{test.commonField}, true) transformed, err := trans.transform() assert.NoError(t, err) out := transformed["fieldFormatMap"] @@ -537,25 +538,25 @@ func TestTransformFieldFormatMap(t *testing.T) { func TestTransformGroupAndEnabled(t *testing.T) { tests := []struct { - commonFields common.Fields + commonFields mapping.Fields expected []string }{ { - commonFields: common.Fields{common.Field{Name: "context", Path: "something"}}, + commonFields: mapping.Fields{mapping.Field{Name: "context", Path: "something"}}, expected: []string{"context"}, }, { - commonFields: common.Fields{ - common.Field{ + commonFields: mapping.Fields{ + mapping.Field{ Name: "context", Type: "group", - Fields: common.Fields{ - common.Field{Name: "type", Type: ""}, - common.Field{ + Fields: mapping.Fields{ + mapping.Field{Name: "type", Type: ""}, + mapping.Field{ Name: "metric", Type: "group", - Fields: common.Fields{ - common.Field{Name: "object"}, + Fields: mapping.Fields{ + mapping.Field{Name: "object"}, }, }, }, @@ -564,28 +565,28 @@ func TestTransformGroupAndEnabled(t *testing.T) { expected: []string{"context.type", "context.metric.object"}, }, { - commonFields: common.Fields{ - common.Field{Name: "enabledField"}, - common.Field{Name: "disabledField", Enabled: &falsy}, //enabled is ignored for Type!=group - common.Field{ + commonFields: mapping.Fields{ + mapping.Field{Name: "enabledField"}, + mapping.Field{Name: "disabledField", Enabled: &falsy}, //enabled is ignored for Type!=group + mapping.Field{ Name: "enabledGroup", Type: "group", Enabled: &truthy, - Fields: common.Fields{ - common.Field{Name: "type", Type: ""}, + Fields: mapping.Fields{ + mapping.Field{Name: "type", Type: ""}, }, }, - common.Field{ + mapping.Field{ Name: "context", Type: "group", Enabled: &falsy, - Fields: common.Fields{ - common.Field{Name: "type", Type: ""}, - common.Field{ + Fields: mapping.Fields{ + mapping.Field{Name: "type", Type: ""}, + mapping.Field{ Name: "metric", Type: "group", - Fields: common.Fields{ - common.Field{Name: "object"}, + Fields: mapping.Fields{ + mapping.Field{Name: "object"}, }, }, }, @@ -607,15 +608,15 @@ func TestTransformGroupAndEnabled(t *testing.T) { } func TestTransformMultiField(t *testing.T) { - f := common.Field{ + f := mapping.Field{ Name: "context", Type: "", - MultiFields: common.Fields{ - common.Field{Name: "keyword", Type: "keyword"}, - common.Field{Name: "text", Type: "text"}, + MultiFields: mapping.Fields{ + mapping.Field{Name: "keyword", Type: "keyword"}, + mapping.Field{Name: "text", Type: "text"}, }, } - trans, _ := newFieldsTransformer(version, common.Fields{f}, true) + trans, _ := newFieldsTransformer(version, mapping.Fields{f}, true) transformed, err := trans.transform() assert.NoError(t, err) out := transformed["fields"].([]common.MapStr) diff --git a/libbeat/kibana/index_pattern_generator.go b/libbeat/kibana/index_pattern_generator.go index fe426d0a033..93b5406c2de 100644 --- a/libbeat/kibana/index_pattern_generator.go +++ b/libbeat/kibana/index_pattern_generator.go @@ -25,6 +25,7 @@ import ( "regexp" "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/mapping" ) type IndexPatternGenerator struct { @@ -110,7 +111,7 @@ func (i *IndexPatternGenerator) addGeneral(indexPattern *common.MapStr) error { } func (i *IndexPatternGenerator) addFieldsSpecific(indexPattern *common.MapStr) error { - fields, err := common.LoadFields(i.fields) + fields, err := mapping.LoadFields(i.fields) if err != nil { return err } diff --git a/libbeat/common/field.go b/libbeat/mapping/field.go similarity index 99% rename from libbeat/common/field.go rename to libbeat/mapping/field.go index e6f14a50ba5..fc3a7d3073f 100644 --- a/libbeat/common/field.go +++ b/libbeat/mapping/field.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package common +package mapping import ( "fmt" diff --git a/libbeat/common/field_test.go b/libbeat/mapping/field_test.go similarity index 93% rename from libbeat/common/field_test.go rename to libbeat/mapping/field_test.go index 31d792cc98b..6d2bfe49551 100644 --- a/libbeat/common/field_test.go +++ b/libbeat/mapping/field_test.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package common +package mapping import ( "strings" @@ -25,6 +25,7 @@ import ( "github.com/stretchr/testify/assert" + "github.com/elastic/beats/libbeat/common" "github.com/elastic/go-ucfg/yaml" ) @@ -298,44 +299,44 @@ func TestGetField(t *testing.T) { func TestFieldValidate(t *testing.T) { tests := map[string]struct { - cfg MapStr + cfg common.MapStr field Field err bool }{ "top level object type config": { - cfg: MapStr{"object_type": "scaled_float", "object_type_mapping_type": "float", "scaling_factor": 10}, + cfg: common.MapStr{"object_type": "scaled_float", "object_type_mapping_type": "float", "scaling_factor": 10}, field: Field{ObjectType: "scaled_float", ObjectTypeMappingType: "float", ScalingFactor: 10}, err: false, }, "multiple object type configs": { - cfg: MapStr{"object_type_params": []MapStr{ + cfg: common.MapStr{"object_type_params": []common.MapStr{ {"object_type": "scaled_float", "object_type_mapping_type": "float", "scaling_factor": 100}}}, field: Field{ObjectTypeParams: []ObjectTypeCfg{{ObjectType: "scaled_float", ObjectTypeMappingType: "float", ScalingFactor: 100}}}, err: false, }, "invalid config mixing object_type and object_type_params": { - cfg: MapStr{ + cfg: common.MapStr{ "object_type": "scaled_float", - "object_type_params": []MapStr{{"object_type": "scaled_float", "object_type_mapping_type": "float"}}}, + "object_type_params": []common.MapStr{{"object_type": "scaled_float", "object_type_mapping_type": "float"}}}, err: true, }, "invalid config mixing object_type_mapping_type and object_type_params": { - cfg: MapStr{ + cfg: common.MapStr{ "object_type_mapping_type": "float", - "object_type_params": []MapStr{{"object_type": "scaled_float", "object_type_mapping_type": "float"}}}, + "object_type_params": []common.MapStr{{"object_type": "scaled_float", "object_type_mapping_type": "float"}}}, err: true, }, "invalid config mixing scaling_factor and object_type_params": { - cfg: MapStr{ + cfg: common.MapStr{ "scaling_factor": 100, - "object_type_params": []MapStr{{"object_type": "scaled_float", "object_type_mapping_type": "float"}}}, + "object_type_params": []common.MapStr{{"object_type": "scaled_float", "object_type_mapping_type": "float"}}}, err: true, }, } for name, test := range tests { t.Run(name, func(t *testing.T) { - cfg, err := NewConfigFrom(test.cfg) + cfg, err := common.NewConfigFrom(test.cfg) require.NoError(t, err) var f Field err = cfg.Unpack(&f) diff --git a/libbeat/template/config.go b/libbeat/template/config.go index ed939fcc9a0..53d9ac5eacb 100644 --- a/libbeat/template/config.go +++ b/libbeat/template/config.go @@ -17,7 +17,7 @@ package template -import "github.com/elastic/beats/libbeat/common" +import "github.com/elastic/beats/libbeat/mapping" type TemplateConfig struct { Enabled bool `config:"enabled"` @@ -29,7 +29,7 @@ type TemplateConfig struct { Path string `config:"path"` Name string `config:"name"` } `config:"json"` - AppendFields common.Fields `config:"append_fields"` + AppendFields mapping.Fields `config:"append_fields"` Overwrite bool `config:"overwrite"` Settings TemplateSettings `config:"settings"` } diff --git a/libbeat/template/processor.go b/libbeat/template/processor.go index 4e25bcec03f..4d316c52892 100644 --- a/libbeat/template/processor.go +++ b/libbeat/template/processor.go @@ -22,6 +22,7 @@ import ( "strings" "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/mapping" ) // Processor struct to process fields to template @@ -38,7 +39,7 @@ var ( const scalingFactorKey = "scalingFactor" // Process recursively processes the given fields and writes the template in the given output -func (p *Processor) Process(fields common.Fields, path string, output common.MapStr) error { +func (p *Processor) Process(fields mapping.Fields, path string, output common.MapStr) error { for _, field := range fields { if field.Name == "" { @@ -46,27 +47,27 @@ func (p *Processor) Process(fields common.Fields, path string, output common.Map } field.Path = path - var mapping common.MapStr + var indexMapping common.MapStr switch field.Type { case "ip": - mapping = p.ip(&field) + indexMapping = p.ip(&field) case "scaled_float": - mapping = p.scaledFloat(&field) + indexMapping = p.scaledFloat(&field) case "half_float": - mapping = p.halfFloat(&field) + indexMapping = p.halfFloat(&field) case "integer": - mapping = p.integer(&field) + indexMapping = p.integer(&field) case "text": - mapping = p.text(&field) + indexMapping = p.text(&field) case "", "keyword": - mapping = p.keyword(&field) + indexMapping = p.keyword(&field) case "object": - mapping = p.object(&field) + indexMapping = p.object(&field) case "array": - mapping = p.array(&field) + indexMapping = p.array(&field) case "alias": - mapping = p.alias(&field) + indexMapping = p.alias(&field) case "group": var newPath string if path == "" { @@ -74,14 +75,14 @@ func (p *Processor) Process(fields common.Fields, path string, output common.Map } else { newPath = path + "." + field.Name } - mapping = common.MapStr{} + indexMapping = common.MapStr{} if field.Dynamic.Value != nil { - mapping["dynamic"] = field.Dynamic.Value + indexMapping["dynamic"] = field.Dynamic.Value } // Combine properties with previous field definitions (if any) properties := common.MapStr{} - key := common.GenerateKey(field.Name) + ".properties" + key := mapping.GenerateKey(field.Name) + ".properties" currentProperties, err := output.GetValue(key) if err == nil { var ok bool @@ -95,9 +96,9 @@ func (p *Processor) Process(fields common.Fields, path string, output common.Map if err := p.Process(field.Fields, newPath, properties); err != nil { return err } - mapping["properties"] = properties + indexMapping["properties"] = properties default: - mapping = p.other(&field) + indexMapping = p.other(&field) } switch field.Type { @@ -105,14 +106,14 @@ func (p *Processor) Process(fields common.Fields, path string, output common.Map addToDefaultFields(&field) } - if len(mapping) > 0 { - output.Put(common.GenerateKey(field.Name), mapping) + if len(indexMapping) > 0 { + output.Put(mapping.GenerateKey(field.Name), indexMapping) } } return nil } -func addToDefaultFields(f *common.Field) { +func addToDefaultFields(f *mapping.Field) { fullName := f.Name if f.Path != "" { fullName = f.Path + "." + f.Name @@ -123,7 +124,7 @@ func addToDefaultFields(f *common.Field) { } } -func (p *Processor) other(f *common.Field) common.MapStr { +func (p *Processor) other(f *mapping.Field) common.MapStr { property := getDefaultProperties(f) if f.Type != "" { property["type"] = f.Type @@ -132,13 +133,13 @@ func (p *Processor) other(f *common.Field) common.MapStr { return property } -func (p *Processor) integer(f *common.Field) common.MapStr { +func (p *Processor) integer(f *mapping.Field) common.MapStr { property := getDefaultProperties(f) property["type"] = "long" return property } -func (p *Processor) scaledFloat(f *common.Field, params ...common.MapStr) common.MapStr { +func (p *Processor) scaledFloat(f *mapping.Field, params ...common.MapStr) common.MapStr { property := getDefaultProperties(f) property["type"] = "scaled_float" @@ -163,7 +164,7 @@ func (p *Processor) scaledFloat(f *common.Field, params ...common.MapStr) common return property } -func (p *Processor) halfFloat(f *common.Field) common.MapStr { +func (p *Processor) halfFloat(f *mapping.Field) common.MapStr { property := getDefaultProperties(f) property["type"] = "half_float" @@ -173,7 +174,7 @@ func (p *Processor) halfFloat(f *common.Field) common.MapStr { return property } -func (p *Processor) ip(f *common.Field) common.MapStr { +func (p *Processor) ip(f *mapping.Field) common.MapStr { property := getDefaultProperties(f) property["type"] = "ip" @@ -186,7 +187,7 @@ func (p *Processor) ip(f *common.Field) common.MapStr { return property } -func (p *Processor) keyword(f *common.Field) common.MapStr { +func (p *Processor) keyword(f *mapping.Field) common.MapStr { property := getDefaultProperties(f) property["type"] = "keyword" @@ -213,7 +214,7 @@ func (p *Processor) keyword(f *common.Field) common.MapStr { return property } -func (p *Processor) text(f *common.Field) common.MapStr { +func (p *Processor) text(f *mapping.Field) common.MapStr { properties := getDefaultProperties(f) properties["type"] = "text" @@ -249,7 +250,7 @@ func (p *Processor) text(f *common.Field) common.MapStr { return properties } -func (p *Processor) array(f *common.Field) common.MapStr { +func (p *Processor) array(f *mapping.Field) common.MapStr { properties := getDefaultProperties(f) if f.ObjectType != "" { properties["type"] = f.ObjectType @@ -257,7 +258,7 @@ func (p *Processor) array(f *common.Field) common.MapStr { return properties } -func (p *Processor) alias(f *common.Field) common.MapStr { +func (p *Processor) alias(f *mapping.Field) common.MapStr { // Aliases were introduced in Elasticsearch 6.4, ignore if unsupported if p.EsVersion.LessThan(common.MustNewVersion("6.4.0")) { return nil @@ -274,7 +275,7 @@ func (p *Processor) alias(f *common.Field) common.MapStr { return properties } -func (p *Processor) object(f *common.Field) common.MapStr { +func (p *Processor) object(f *mapping.Field) common.MapStr { matchType := func(onlyType string, mt string) string { if mt != "" { return mt @@ -282,11 +283,11 @@ func (p *Processor) object(f *common.Field) common.MapStr { return onlyType } - var otParams []common.ObjectTypeCfg + var otParams []mapping.ObjectTypeCfg if len(f.ObjectTypeParams) != 0 { otParams = f.ObjectTypeParams } else { - otParams = []common.ObjectTypeCfg{common.ObjectTypeCfg{ + otParams = []mapping.ObjectTypeCfg{mapping.ObjectTypeCfg{ ObjectType: f.ObjectType, ObjectTypeMappingType: f.ObjectTypeMappingType, ScalingFactor: f.ScalingFactor}} } @@ -327,7 +328,7 @@ func (p *Processor) object(f *common.Field) common.MapStr { return properties } -func addDynamicTemplate(f *common.Field, properties common.MapStr, matchType string) { +func addDynamicTemplate(f *mapping.Field, properties common.MapStr, matchType string) { path := "" if len(f.Path) > 0 { path = f.Path + "." @@ -348,7 +349,7 @@ func addDynamicTemplate(f *common.Field, properties common.MapStr, matchType str dynamicTemplates = append(dynamicTemplates, template) } -func getDefaultProperties(f *common.Field) common.MapStr { +func getDefaultProperties(f *mapping.Field) common.MapStr { // Currently no defaults exist properties := common.MapStr{} diff --git a/libbeat/template/processor_test.go b/libbeat/template/processor_test.go index ff8688107aa..e97bb35219e 100644 --- a/libbeat/template/processor_test.go +++ b/libbeat/template/processor_test.go @@ -23,6 +23,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/mapping" ) func TestProcessor(t *testing.T) { @@ -39,92 +40,92 @@ func TestProcessor(t *testing.T) { expected common.MapStr }{ { - output: p.other(&common.Field{Type: "long"}), + output: p.other(&mapping.Field{Type: "long"}), expected: common.MapStr{"type": "long"}, }, { - output: p.scaledFloat(&common.Field{Type: "scaled_float"}), + output: p.scaledFloat(&mapping.Field{Type: "scaled_float"}), expected: common.MapStr{ "type": "scaled_float", "scaling_factor": 1000, }, }, { - output: p.scaledFloat(&common.Field{Type: "scaled_float", ScalingFactor: 100}), + output: p.scaledFloat(&mapping.Field{Type: "scaled_float", ScalingFactor: 100}), expected: common.MapStr{ "type": "scaled_float", "scaling_factor": 100, }, }, { - output: p.scaledFloat(&common.Field{Type: "scaled_float"}, common.MapStr{scalingFactorKey: 0}), + output: p.scaledFloat(&mapping.Field{Type: "scaled_float"}, common.MapStr{scalingFactorKey: 0}), expected: common.MapStr{ "type": "scaled_float", "scaling_factor": 1000, }, }, { - output: p.scaledFloat(&common.Field{Type: "scaled_float"}, common.MapStr{"someKey": 10}), + output: p.scaledFloat(&mapping.Field{Type: "scaled_float"}, common.MapStr{"someKey": 10}), expected: common.MapStr{ "type": "scaled_float", "scaling_factor": 1000, }, }, { - output: p.scaledFloat(&common.Field{Type: "scaled_float"}, common.MapStr{scalingFactorKey: 10}), + output: p.scaledFloat(&mapping.Field{Type: "scaled_float"}, common.MapStr{scalingFactorKey: 10}), expected: common.MapStr{ "type": "scaled_float", "scaling_factor": 10, }, }, { - output: pEsVersion2.scaledFloat(&common.Field{Type: "scaled_float"}), + output: pEsVersion2.scaledFloat(&mapping.Field{Type: "scaled_float"}), expected: common.MapStr{"type": "float"}, }, { - output: p.object(&common.Field{Type: "object", Enabled: &falseVar}), + output: p.object(&mapping.Field{Type: "object", Enabled: &falseVar}), expected: common.MapStr{ "type": "object", "enabled": false, }, }, { - output: p.integer(&common.Field{Type: "long", CopyTo: "hello.world"}), + output: p.integer(&mapping.Field{Type: "long", CopyTo: "hello.world"}), expected: common.MapStr{ "type": "long", "copy_to": "hello.world", }, }, { - output: p.array(&common.Field{Type: "array"}), + output: p.array(&mapping.Field{Type: "array"}), expected: common.MapStr{}, }, { - output: p.array(&common.Field{Type: "array", ObjectType: "text"}), + output: p.array(&mapping.Field{Type: "array", ObjectType: "text"}), expected: common.MapStr{"type": "text"}, }, { - output: p.array(&common.Field{Type: "array", Index: &falseVar, ObjectType: "keyword"}), + output: p.array(&mapping.Field{Type: "array", Index: &falseVar, ObjectType: "keyword"}), expected: common.MapStr{"index": false, "type": "keyword"}, }, { - output: pEsVersion64.alias(&common.Field{Type: "alias", AliasPath: "a.b"}), + output: pEsVersion64.alias(&mapping.Field{Type: "alias", AliasPath: "a.b"}), expected: common.MapStr{"path": "a.b", "type": "alias"}, }, { // alias unsupported in ES < 6.4 - output: pEsVersion63.alias(&common.Field{Type: "alias", AliasPath: "a.b"}), + output: pEsVersion63.alias(&mapping.Field{Type: "alias", AliasPath: "a.b"}), expected: nil, }, { - output: p.object(&common.Field{Type: "object", Enabled: &falseVar}), + output: p.object(&mapping.Field{Type: "object", Enabled: &falseVar}), expected: common.MapStr{ "type": "object", "enabled": false, }, }, { - output: p.text(&common.Field{Type: "text", Analyzer: "autocomplete"}), + output: p.text(&mapping.Field{Type: "text", Analyzer: "autocomplete"}), expected: common.MapStr{ "type": "text", "analyzer": "autocomplete", @@ -132,21 +133,21 @@ func TestProcessor(t *testing.T) { }, }, { - output: p.text(&common.Field{Type: "text", Analyzer: "autocomplete", Norms: true}), + output: p.text(&mapping.Field{Type: "text", Analyzer: "autocomplete", Norms: true}), expected: common.MapStr{ "type": "text", "analyzer": "autocomplete", }, }, { - output: p.text(&common.Field{Type: "text", SearchAnalyzer: "standard", Norms: true}), + output: p.text(&mapping.Field{Type: "text", SearchAnalyzer: "standard", Norms: true}), expected: common.MapStr{ "type": "text", "search_analyzer": "standard", }, }, { - output: p.text(&common.Field{Type: "text", Analyzer: "autocomplete", SearchAnalyzer: "standard", Norms: true}), + output: p.text(&mapping.Field{Type: "text", Analyzer: "autocomplete", SearchAnalyzer: "standard", Norms: true}), expected: common.MapStr{ "type": "text", "analyzer": "autocomplete", @@ -154,7 +155,7 @@ func TestProcessor(t *testing.T) { }, }, { - output: p.text(&common.Field{Type: "text", MultiFields: common.Fields{common.Field{Name: "raw", Type: "keyword"}}, Norms: true}), + output: p.text(&mapping.Field{Type: "text", MultiFields: mapping.Fields{mapping.Field{Name: "raw", Type: "keyword"}}, Norms: true}), expected: common.MapStr{ "type": "text", "fields": common.MapStr{ @@ -166,7 +167,7 @@ func TestProcessor(t *testing.T) { }, }, { - output: p.keyword(&common.Field{Type: "keyword", MultiFields: common.Fields{common.Field{Name: "analyzed", Type: "text", Norms: true}}}), + output: p.keyword(&mapping.Field{Type: "keyword", MultiFields: mapping.Fields{mapping.Field{Name: "analyzed", Type: "text", Norms: true}}}), expected: common.MapStr{ "type": "keyword", "ignore_above": 1024, @@ -178,29 +179,29 @@ func TestProcessor(t *testing.T) { }, }, { - output: p.keyword(&common.Field{Type: "keyword", IgnoreAbove: 256}), + output: p.keyword(&mapping.Field{Type: "keyword", IgnoreAbove: 256}), expected: common.MapStr{ "type": "keyword", "ignore_above": 256, }, }, { - output: p.keyword(&common.Field{Type: "keyword", IgnoreAbove: -1}), + output: p.keyword(&mapping.Field{Type: "keyword", IgnoreAbove: -1}), expected: common.MapStr{ "type": "keyword", }, }, { - output: p.keyword(&common.Field{Type: "keyword"}), + output: p.keyword(&mapping.Field{Type: "keyword"}), expected: common.MapStr{ "type": "keyword", "ignore_above": 1024, }, }, { - output: p.text(&common.Field{Type: "text", MultiFields: common.Fields{ - common.Field{Name: "raw", Type: "keyword"}, - common.Field{Name: "indexed", Type: "text"}, + output: p.text(&mapping.Field{Type: "text", MultiFields: mapping.Fields{ + mapping.Field{Name: "raw", Type: "keyword"}, + mapping.Field{Name: "indexed", Type: "text"}, }, Norms: true}), expected: common.MapStr{ "type": "text", @@ -217,9 +218,9 @@ func TestProcessor(t *testing.T) { }, }, { - output: p.text(&common.Field{Type: "text", MultiFields: common.Fields{ - common.Field{Name: "raw", Type: "keyword"}, - common.Field{Name: "indexed", Type: "text"}, + output: p.text(&mapping.Field{Type: "text", MultiFields: mapping.Fields{ + mapping.Field{Name: "raw", Type: "keyword"}, + mapping.Field{Name: "indexed", Type: "text"}, }, Norms: true}), expected: common.MapStr{ "type": "text", @@ -236,67 +237,67 @@ func TestProcessor(t *testing.T) { }, }, { - output: p.object(&common.Field{Dynamic: common.DynamicType{Value: false}}), + output: p.object(&mapping.Field{Dynamic: mapping.DynamicType{Value: false}}), expected: common.MapStr{ "dynamic": false, "type": "object", }, }, { - output: p.object(&common.Field{Dynamic: common.DynamicType{Value: true}}), + output: p.object(&mapping.Field{Dynamic: mapping.DynamicType{Value: true}}), expected: common.MapStr{ "dynamic": true, "type": "object", }, }, { - output: p.object(&common.Field{Dynamic: common.DynamicType{Value: "strict"}}), + output: p.object(&mapping.Field{Dynamic: mapping.DynamicType{Value: "strict"}}), expected: common.MapStr{ "dynamic": "strict", "type": "object", }, }, { - output: p.other(&common.Field{Type: "long", Index: &falseVar}), + output: p.other(&mapping.Field{Type: "long", Index: &falseVar}), expected: common.MapStr{ "type": "long", "index": false, }, }, { - output: p.other(&common.Field{Type: "text", Index: &trueVar}), + output: p.other(&mapping.Field{Type: "text", Index: &trueVar}), expected: common.MapStr{ "type": "text", "index": true, }, }, { - output: p.other(&common.Field{Type: "long", DocValues: &falseVar}), + output: p.other(&mapping.Field{Type: "long", DocValues: &falseVar}), expected: common.MapStr{ "type": "long", "doc_values": false, }, }, { - output: p.other(&common.Field{Type: "double", DocValues: &falseVar}), + output: p.other(&mapping.Field{Type: "double", DocValues: &falseVar}), expected: common.MapStr{ "type": "double", "doc_values": false, }, }, { - output: p.other(&common.Field{Type: "text", DocValues: &trueVar}), + output: p.other(&mapping.Field{Type: "text", DocValues: &trueVar}), expected: common.MapStr{ "type": "text", "doc_values": true, }, }, { - output: p.alias(&common.Field{Type: "alias", AliasPath: "a.c", MigrationAlias: false}), + output: p.alias(&mapping.Field{Type: "alias", AliasPath: "a.c", MigrationAlias: false}), expected: common.MapStr{"path": "a.c", "type": "alias"}, }, { - output: p.alias(&common.Field{Type: "alias", AliasPath: "a.d", MigrationAlias: true}), + output: p.alias(&mapping.Field{Type: "alias", AliasPath: "a.d", MigrationAlias: true}), expected: nil, }, { - output: migrationP.alias(&common.Field{Type: "alias", AliasPath: "a.e", MigrationAlias: false}), + output: migrationP.alias(&mapping.Field{Type: "alias", AliasPath: "a.e", MigrationAlias: false}), expected: common.MapStr{"path": "a.e", "type": "alias"}, }, { - output: migrationP.alias(&common.Field{Type: "alias", AliasPath: "a.f", MigrationAlias: true}), + output: migrationP.alias(&mapping.Field{Type: "alias", AliasPath: "a.f", MigrationAlias: true}), expected: common.MapStr{"path": "a.f", "type": "alias"}, }, } @@ -309,11 +310,11 @@ func TestProcessor(t *testing.T) { func TestDynamicTemplates(t *testing.T) { p := &Processor{} tests := []struct { - field common.Field + field mapping.Field expected []common.MapStr }{ { - field: common.Field{ + field: mapping.Field{ Type: "object", ObjectType: "keyword", Name: "context", }, @@ -328,7 +329,7 @@ func TestDynamicTemplates(t *testing.T) { }, }, { - field: common.Field{ + field: mapping.Field{ Type: "object", ObjectType: "long", ObjectTypeMappingType: "futuretype", Path: "language", Name: "english", }, @@ -343,7 +344,7 @@ func TestDynamicTemplates(t *testing.T) { }, }, { - field: common.Field{ + field: mapping.Field{ Type: "object", ObjectType: "long", ObjectTypeMappingType: "*", Path: "language", Name: "english", }, @@ -358,7 +359,7 @@ func TestDynamicTemplates(t *testing.T) { }, }, { - field: common.Field{ + field: mapping.Field{ Type: "object", ObjectType: "long", Path: "language", Name: "english", }, @@ -373,7 +374,7 @@ func TestDynamicTemplates(t *testing.T) { }, }, { - field: common.Field{ + field: mapping.Field{ Type: "object", ObjectType: "text", Path: "language", Name: "english", }, @@ -388,7 +389,7 @@ func TestDynamicTemplates(t *testing.T) { }, }, { - field: common.Field{ + field: mapping.Field{ Type: "object", ObjectType: "scaled_float", Name: "core.*.pct", }, @@ -406,7 +407,7 @@ func TestDynamicTemplates(t *testing.T) { }, }, { - field: common.Field{ + field: mapping.Field{ Type: "object", ObjectType: "scaled_float", Name: "core.*.pct", ScalingFactor: 100, ObjectTypeMappingType: "float", }, @@ -424,8 +425,8 @@ func TestDynamicTemplates(t *testing.T) { }, }, { - field: common.Field{ - Type: "object", ObjectTypeParams: []common.ObjectTypeCfg{ + field: mapping.Field{ + Type: "object", ObjectTypeParams: []mapping.ObjectTypeCfg{ {ObjectType: "float", ObjectTypeMappingType: "float"}, {ObjectType: "boolean"}, {ObjectType: "scaled_float", ScalingFactor: 10000}, @@ -460,10 +461,10 @@ func TestDynamicTemplates(t *testing.T) { for _, numericType := range []string{"byte", "double", "float", "long", "short", "boolean"} { gen := struct { - field common.Field + field mapping.Field expected []common.MapStr }{ - field: common.Field{ + field: mapping.Field{ Type: "object", ObjectType: numericType, Name: "somefield", ObjectTypeMappingType: "long", }, @@ -491,22 +492,22 @@ func TestDynamicTemplates(t *testing.T) { func TestPropertiesCombine(t *testing.T) { // Test common fields are combined even if they come from different objects - fields := common.Fields{ - common.Field{ + fields := mapping.Fields{ + mapping.Field{ Name: "test", Type: "group", - Fields: common.Fields{ - common.Field{ + Fields: mapping.Fields{ + mapping.Field{ Name: "one", Type: "text", }, }, }, - common.Field{ + mapping.Field{ Name: "test", Type: "group", - Fields: common.Fields{ - common.Field{ + Fields: mapping.Fields{ + mapping.Field{ Name: "two", Type: "text", }, @@ -541,20 +542,20 @@ func TestPropertiesCombine(t *testing.T) { func TestProcessNoName(t *testing.T) { // Test common fields are combined even if they come from different objects - fields := common.Fields{ - common.Field{ - Fields: common.Fields{ - common.Field{ + fields := mapping.Fields{ + mapping.Field{ + Fields: mapping.Fields{ + mapping.Field{ Name: "one", Type: "text", }, }, }, - common.Field{ + mapping.Field{ Name: "test", Type: "group", - Fields: common.Fields{ - common.Field{ + Fields: mapping.Fields{ + mapping.Field{ Name: "two", Type: "text", }, diff --git a/libbeat/template/template.go b/libbeat/template/template.go index 1e829d196b1..d0b7d7d3d12 100644 --- a/libbeat/template/template.go +++ b/libbeat/template/template.go @@ -26,6 +26,7 @@ import ( "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/common/cfgwarn" "github.com/elastic/beats/libbeat/common/fmtstr" + "github.com/elastic/beats/libbeat/mapping" "github.com/elastic/go-ucfg/yaml" ) @@ -129,7 +130,7 @@ func New( }, nil } -func (t *Template) load(fields common.Fields) (common.MapStr, error) { +func (t *Template) load(fields mapping.Fields) (common.MapStr, error) { // Locking to make sure dynamicTemplates and defaultFields is not accessed in parallel t.Lock() @@ -141,7 +142,7 @@ func (t *Template) load(fields common.Fields) (common.MapStr, error) { var err error if len(t.config.AppendFields) > 0 { cfgwarn.Experimental("append_fields is used.") - fields, err = common.ConcatFields(fields, t.config.AppendFields) + fields, err = mapping.ConcatFields(fields, t.config.AppendFields) if err != nil { return nil, err } @@ -160,7 +161,7 @@ func (t *Template) load(fields common.Fields) (common.MapStr, error) { // LoadFile loads the the template from the given file path func (t *Template) LoadFile(file string) (common.MapStr, error) { - fields, err := common.LoadFieldsYaml(file) + fields, err := mapping.LoadFieldsYaml(file) if err != nil { return nil, err } @@ -311,19 +312,19 @@ func buildIdxSettings(ver common.Version, userSettings common.MapStr) common.Map return indexSettings } -func loadYamlByte(data []byte) (common.Fields, error) { +func loadYamlByte(data []byte) (mapping.Fields, error) { cfg, err := yaml.NewConfig(data) if err != nil { return nil, err } - var keys []common.Field + var keys []mapping.Field err = cfg.Unpack(&keys) if err != nil { return nil, err } - fields := common.Fields{} + fields := mapping.Fields{} for _, key := range keys { fields = append(fields, key.Fields...) }