From 1448c630b193d71e0802a62adbc180f28f42b3c0 Mon Sep 17 00:00:00 2001 From: Sriranganathan Date: Sun, 8 Apr 2018 14:47:25 +0530 Subject: [PATCH] Add Condition to check if a list of fields exists (#6653) * Add Condition to check if a list of fields exists --- CHANGELOG.asciidoc | 1 + libbeat/docs/processors-using.asciidoc | 18 ++++++++++++++++ libbeat/processors/condition.go | 27 ++++++++++++++++++----- libbeat/processors/condition_test.go | 30 ++++++++++++++++++++++++++ libbeat/processors/config.go | 15 +++++++------ 5 files changed, 79 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index cbb7a7e6db6f..0d02b8770673 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -92,6 +92,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di - Add appender support to autodiscover {pull}6469[6469] - Add add_host_metadata processor {pull}5968[5968] - Retry configuration to load dashboards if Kibana is not reachable when the beat starts. {pull}6560[6560] +- Add `has_fields` conditional to filter events based on the existence of all the given fields. {issue}6285[6285] {pull}6653[6653] - Add support for spooling to disk to the beats event publishing pipeline. {pull}6581[6581] - Added logging of system info at Beat startup. {issue}5946[5946] diff --git a/libbeat/docs/processors-using.asciidoc b/libbeat/docs/processors-using.asciidoc index 908ee30f3d0f..5bc017f3ac9e 100644 --- a/libbeat/docs/processors-using.asciidoc +++ b/libbeat/docs/processors-using.asciidoc @@ -66,6 +66,7 @@ The supported conditions are: * <> * <> * <> +* <> * <> * <> * <> @@ -157,6 +158,23 @@ range: ------ +[float] +[[condition-has_fields]] +===== `has_fields` + +The `has_fields` condition checks if all the given fields exist in the +event. The condition accepts a list of string values denoting the field names. + +For example, the following condition checks if the `http.response.code` field +is present in the event. + + +[source,yaml] +------ +has_fields: ['http.response.code'] +------ + + [float] [[condition-or]] ===== `or` diff --git a/libbeat/processors/condition.go b/libbeat/processors/condition.go index fc3e0122eebf..7e8bfd9454ee 100644 --- a/libbeat/processors/condition.go +++ b/libbeat/processors/condition.go @@ -32,10 +32,11 @@ type Condition struct { name string filters map[string]match.Matcher } - rangexp map[string]RangeValue - or []Condition - and []Condition - not *Condition + hasfields []string + rangexp map[string]RangeValue + or []Condition + and []Condition + not *Condition } type WhenProcessor struct { @@ -82,6 +83,8 @@ func NewCondition(config *ConditionConfig) (*Condition, error) { c.matches.filters, err = compileMatches(config.Regexp.fields, match.Compile) case config.Range != nil: err = c.setRange(config.Range) + case config.HasFields != nil: + c.hasfields = config.HasFields case len(config.OR) > 0: c.or, err = NewConditionList(config.OR) case len(config.AND) > 0: @@ -226,7 +229,8 @@ func (c *Condition) Check(event ValuesMap) bool { return c.checkEquals(event) && c.checkMatches(event) && - c.checkRange(event) + c.checkRange(event) && + c.checkHasFields(event) } func (c *Condition) checkOR(event ValuesMap) bool { @@ -399,6 +403,16 @@ func (c *Condition) checkRange(event ValuesMap) bool { return true } +func (c *Condition) checkHasFields(event ValuesMap) bool { + for _, field := range c.hasfields { + _, err := event.GetValue(field) + if err != nil { + return false + } + } + return true +} + func (c Condition) String() string { s := "" @@ -411,6 +425,9 @@ func (c Condition) String() string { if len(c.rangexp) > 0 { s = s + fmt.Sprintf("range: %v", c.rangexp) } + if len(c.hasfields) > 0 { + s = s + fmt.Sprintf("has_fields: %v", c.hasfields) + } if len(c.or) > 0 { for _, cond := range c.or { s = s + cond.String() + " or " diff --git a/libbeat/processors/condition_test.go b/libbeat/processors/condition_test.go index 9570228571e4..f797b396b226 100644 --- a/libbeat/processors/condition_test.go +++ b/libbeat/processors/condition_test.go @@ -145,6 +145,24 @@ func TestCondition(t *testing.T) { }, result: true, }, + { + config: ConditionConfig{ + HasFields: []string{"proc.cmdline", "type"}, + }, + result: true, + }, + { + config: ConditionConfig{ + HasFields: []string{"cpu"}, + }, + result: false, + }, + { + config: ConditionConfig{ + HasFields: []string{"proc", "beat"}, + }, + result: false, + }, } event := &beat.Event{ @@ -581,6 +599,18 @@ func TestWhenProcessor(t *testing.T) { []common.MapStr{{"i": 10}}, 1, }, + { + "condition_matches", + config{"when.has_fields": []string{"i"}}, + []common.MapStr{{"i": 10}}, + 1, + }, + { + "condition_fails", + config{"when.has_fields": []string{"j"}}, + []common.MapStr{{"i": 10}}, + 0, + }, } for i, test := range tests { diff --git a/libbeat/processors/config.go b/libbeat/processors/config.go index a11038acb33f..b5668dfeabac 100644 --- a/libbeat/processors/config.go +++ b/libbeat/processors/config.go @@ -9,13 +9,14 @@ import ( ) type ConditionConfig struct { - Equals *ConditionFields `config:"equals"` - Contains *ConditionFields `config:"contains"` - Regexp *ConditionFields `config:"regexp"` - Range *ConditionFields `config:"range"` - OR []ConditionConfig `config:"or"` - AND []ConditionConfig `config:"and"` - NOT *ConditionConfig `config:"not"` + Equals *ConditionFields `config:"equals"` + Contains *ConditionFields `config:"contains"` + Regexp *ConditionFields `config:"regexp"` + Range *ConditionFields `config:"range"` + HasFields []string `config:"has_fields"` + OR []ConditionConfig `config:"or"` + AND []ConditionConfig `config:"and"` + NOT *ConditionConfig `config:"not"` } type ConditionFields struct {