Add Condition to check if a list of fields exists (elastic#6653)

* Add Condition to check if a list of fields exists

CHANGELOG.asciidoc

@@ -92,6 +92,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di
 - Add appender support to autodiscover {pull}6469[6469]
 - Add add_host_metadata processor {pull}5968[5968]
 - Retry configuration to load dashboards if Kibana is not reachable when the beat starts. {pull}6560[6560]
+- Add `has_fields` conditional to filter events based on the existence of all the given fields. {issue}6285[6285] {pull}6653[6653]
 - Add support for spooling to disk to the beats event publishing pipeline. {pull}6581[6581]
 - Added logging of system info at Beat startup. {issue}5946[5946]
 

libbeat/docs/processors-using.asciidoc

@@ -66,6 +66,7 @@ The supported conditions are:
 * <<condition-contains,`contains`>>
 * <<condition-regexp,`regexp`>>
 * <<condition-range, `range`>>
+* <<condition-has_fields, `has_fields`>>
 * <<condition-or, `or`>>
 * <<condition-and, `and`>>
 * <<condition-not, `not`>>
@@ -157,6 +158,23 @@ range:
 ------
 
 
+[float]
+[[condition-has_fields]]
+===== `has_fields`
+
+The `has_fields` condition checks if all the given fields exist in the
+event. The condition accepts a list of string values denoting the field names.
+
+For example, the following condition checks if the `http.response.code` field
+is present in the event.
+
+
+[source,yaml]
+------
+has_fields: ['http.response.code']
+------
+
+
 [float]
 [[condition-or]]
 ===== `or`

libbeat/processors/condition.go

@@ -32,10 +32,11 @@ type Condition struct {
 		name    string
 		filters map[string]match.Matcher
 	}
-	rangexp map[string]RangeValue
-	or      []Condition
-	and     []Condition
-	not     *Condition
+	hasfields []string
+	rangexp   map[string]RangeValue
+	or        []Condition
+	and       []Condition
+	not       *Condition
 }
 
 type WhenProcessor struct {
@@ -82,6 +83,8 @@ func NewCondition(config *ConditionConfig) (*Condition, error) {
 		c.matches.filters, err = compileMatches(config.Regexp.fields, match.Compile)
 	case config.Range != nil:
 		err = c.setRange(config.Range)
+	case config.HasFields != nil:
+		c.hasfields = config.HasFields
 	case len(config.OR) > 0:
 		c.or, err = NewConditionList(config.OR)
 	case len(config.AND) > 0:
@@ -226,7 +229,8 @@ func (c *Condition) Check(event ValuesMap) bool {
 
 	return c.checkEquals(event) &&
 		c.checkMatches(event) &&
-		c.checkRange(event)
+		c.checkRange(event) &&
+		c.checkHasFields(event)
 }
 
 func (c *Condition) checkOR(event ValuesMap) bool {
@@ -399,6 +403,16 @@ func (c *Condition) checkRange(event ValuesMap) bool {
 	return true
 }
 
+func (c *Condition) checkHasFields(event ValuesMap) bool {
+	for _, field := range c.hasfields {
+		_, err := event.GetValue(field)
+		if err != nil {
+			return false
+		}
+	}
+	return true
+}
+
 func (c Condition) String() string {
 	s := ""
 
@@ -411,6 +425,9 @@ func (c Condition) String() string {
 	if len(c.rangexp) > 0 {
 		s = s + fmt.Sprintf("range: %v", c.rangexp)
 	}
+	if len(c.hasfields) > 0 {
+		s = s + fmt.Sprintf("has_fields: %v", c.hasfields)
+	}
 	if len(c.or) > 0 {
 		for _, cond := range c.or {
 			s = s + cond.String() + " or "

libbeat/processors/condition_test.go

@@ -145,6 +145,24 @@ func TestCondition(t *testing.T) {
 			},
 			result: true,
 		},
+		{
+			config: ConditionConfig{
+				HasFields: []string{"proc.cmdline", "type"},
+			},
+			result: true,
+		},
+		{
+			config: ConditionConfig{
+				HasFields: []string{"cpu"},
+			},
+			result: false,
+		},
+		{
+			config: ConditionConfig{
+				HasFields: []string{"proc", "beat"},
+			},
+			result: false,
+		},
 	}
 
 	event := &beat.Event{
@@ -581,6 +599,18 @@ func TestWhenProcessor(t *testing.T) {
 			[]common.MapStr{{"i": 10}},
 			1,
 		},
+		{
+			"condition_matches",
+			config{"when.has_fields": []string{"i"}},
+			[]common.MapStr{{"i": 10}},
+			1,
+		},
+		{
+			"condition_fails",
+			config{"when.has_fields": []string{"j"}},
+			[]common.MapStr{{"i": 10}},
+			0,
+		},
 	}
 
 	for i, test := range tests {

libbeat/processors/config.go

@@ -9,13 +9,14 @@ import (
 )
 
 type ConditionConfig struct {
-	Equals   *ConditionFields  `config:"equals"`
-	Contains *ConditionFields  `config:"contains"`
-	Regexp   *ConditionFields  `config:"regexp"`
-	Range    *ConditionFields  `config:"range"`
-	OR       []ConditionConfig `config:"or"`
-	AND      []ConditionConfig `config:"and"`
-	NOT      *ConditionConfig  `config:"not"`
+	Equals    *ConditionFields  `config:"equals"`
+	Contains  *ConditionFields  `config:"contains"`
+	Regexp    *ConditionFields  `config:"regexp"`
+	Range     *ConditionFields  `config:"range"`
+	HasFields []string          `config:"has_fields"`
+	OR        []ConditionConfig `config:"or"`
+	AND       []ConditionConfig `config:"and"`
+	NOT       *ConditionConfig  `config:"not"`
 }
 
 type ConditionFields struct {