Add integration tests for suricata module

Makefile

@@ -130,7 +130,7 @@ notice: python-env
 .PHONY: python-env
 python-env:
 	@test -d $(PYTHON_ENV) || virtualenv $(VIRTUALENV_PARAMS) $(PYTHON_ENV)
-	@$(PYTHON_ENV)/bin/pip install -q --upgrade pip autopep8 six
+	@$(PYTHON_ENV)/bin/pip install -q --upgrade pip autopep8==1.3.5 six
 	@# Work around pip bug. See: https://github.com/pypa/pip/issues/4464
 	@find $(PYTHON_ENV) -type d -name dist-packages -exec sh -c "echo dist-packages > {}.pth" ';'
 

filebeat/_meta/fields.common.yml

@@ -185,3 +185,12 @@
             type: keyword
             description: >
               Region ISO code.
+
+    - name: user_agent
+      type: group
+      fields:
+      - name: os
+        type: group
+        fields:
+        - name: full_name
+          type: keyword

filebeat/docs/fields.asciidoc

@@ -4874,6 +4874,15 @@ Region ISO code.
 
 --
 
+
+
+*`user_agent.os.full_name`*::
++
+--
+type: keyword
+
+--
+
 [[exported-fields-logstash]]
 == logstash fields
 

filebeat/tests/system/filebeat.py

@@ -11,8 +11,10 @@ class BaseTest(TestCase):
 
     @classmethod
     def setUpClass(self):
-        self.beat_name = "filebeat"
-        self.beat_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../"))
+        if not hasattr(self, "beat_name"):
+            self.beat_name = "filebeat"
+        if not hasattr(self, "beat_path"):
+            self.beat_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../"))
 
         super(BaseTest, self).setUpClass()
 

filebeat/tests/system/test_modules.py

@@ -3,8 +3,8 @@
 import os
 import unittest
 import glob
-import shutil
 import subprocess
+
 from elasticsearch import Elasticsearch
 import json
 import logging
@@ -17,9 +17,10 @@ def load_fileset_test_cases():
     To execute tests for only 1 module, set the env variable TESTING_FILEBEAT_MODULES
     to the specific module name or a , separated lists of modules.
     """
-    current_dir = os.path.dirname(os.path.abspath(__file__))
-    modules_dir = os.path.join(current_dir, "..", "..", "module")
-
+    modules_dir = os.getenv("MODULES_PATH")
+    if not modules_dir:
+        current_dir = os.path.dirname(os.path.abspath(__file__))
+        modules_dir = os.path.join(current_dir, "..", "..", "module")
     modules = os.getenv("TESTING_FILEBEAT_MODULES")
     if modules:
         modules = modules.split(",")

libbeat/scripts/Makefile

@@ -208,6 +208,12 @@ system-tests: prepare-tests ${BEAT_NAME}.test python-env
 system-tests-environment:  ## @testing Runs the system tests inside a virtual environment. This can be run on any docker-machine (local, remote)
 system-tests-environment: prepare-tests build-image
 	${DOCKER_COMPOSE} run -e INTEGRATION_TESTS=1 -e TESTING_ENVIRONMENT=${TESTING_ENVIRONMENT} -e DOCKER_COMPOSE_PROJECT_NAME=${DOCKER_COMPOSE_PROJECT_NAME} beat make system-tests
+	#This is a hack to run x-pack/filebeat module tests
+	@XPACKBEAT="${ES_BEATS}/x-pack/${BEAT_NAME}" ; \
+	if [ -e "$$XPACKBEAT/tests/system" ] ; then \
+	    $(MAKE) -C ../x-pack/${BEAT_NAME} fields; \
+	    ${DOCKER_COMPOSE} run -e INTEGRATION_TESTS=1 -e MODULES_PATH="../../x-pack/${BEAT_NAME}/module" -e TESTING_ENVIRONMENT=${TESTING_ENVIRONMENT} -e DOCKER_COMPOSE_PROJECT_NAME=${DOCKER_COMPOSE_PROJECT_NAME} beat make -C "$$XPACKBEAT" ${BEAT_NAME}.test system-tests ; \
+	fi
 
 
 .PHONY: fast-system-tests

x-pack/filebeat/Makefile

@@ -0,0 +1,7 @@
+BEAT_NAME=filebeat
+ES_BEATS?=../..
+XPACK_BEAT_PATH?=github.com/elastic/beats/x-pack/${BEAT_NAME}
+GOPACKAGES?=$(shell go list ${BEAT_PATH}/... ${XPACK_BEAT_PATH}/... | grep -v /vendor/ | grep -v /scripts/cmd/ )
+
+# Include main filebeat Makefile
+include ${ES_BEATS}/${BEAT_NAME}/Makefile 

x-pack/filebeat/main_test.go

@@ -0,0 +1,27 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+package main
+
+// This file is mandatory as otherwise the filebeat.test binary is not generated correctly.
+import (
+	"flag"
+	"testing"
+
+	"github.com/elastic/beats/filebeat/cmd"
+)
+
+var systemTest *bool
+
+func init() {
+	systemTest = flag.Bool("systemTest", false, "Set to true when running system tests")
+	cmd.RootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("systemTest"))
+	cmd.RootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("test.coverprofile"))
+}
+
+// Test started when the test binary is started. Only calls main.
+func TestSystem(t *testing.T) {
+	if *systemTest {
+		main()
+	}
+}

x-pack/filebeat/module/suricata/eve/_meta/fields.yml

@@ -707,3 +707,7 @@
 
     - name: app_proto_expected
       type: keyword
+
+    - name: flags
+      type: group
+      fields: